From d805da906a1744c23e0b484b522b078102675770 Mon Sep 17 00:00:00 2001 From: Max Isom Date: Thu, 18 Nov 2021 20:50:44 -0500 Subject: [PATCH 1/7] Rename existing cache model --- src/models/index.ts | 4 ++-- src/models/{cache.ts => key-value-cache.ts} | 2 +- src/services/cache.ts | 6 +++--- src/utils/db.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) rename src/models/{cache.ts => key-value-cache.ts} (77%) diff --git a/src/models/index.ts b/src/models/index.ts index fec0c8e..d39a3dc 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -1,9 +1,9 @@ -import Cache from './cache.js'; +import KeyValueCache from './key-value-cache.js'; import Settings from './settings.js'; import Shortcut from './shortcut.js'; export { - Cache, + KeyValueCache, Settings, Shortcut, }; diff --git a/src/models/cache.ts b/src/models/key-value-cache.ts similarity index 77% rename from src/models/cache.ts rename to src/models/key-value-cache.ts index ebf8dad..795ff75 100644 --- a/src/models/cache.ts +++ b/src/models/key-value-cache.ts @@ -2,7 +2,7 @@ import {Table, Column, PrimaryKey, Model} from 'sequelize-typescript'; import sequelize from 'sequelize'; @Table -export default class Cache extends Model { +export default class KeyValueCache extends Model { @PrimaryKey @Column key!: string; diff --git a/src/services/cache.ts b/src/services/cache.ts index e877c56..95c6731 100644 --- a/src/services/cache.ts +++ b/src/services/cache.ts @@ -1,5 +1,5 @@ import {injectable} from 'inversify'; -import {Cache} from '../models/index.js'; +import {KeyValueCache} from '../models/index.js'; import debug from '../utils/debug.js'; type Seconds = number; @@ -29,7 +29,7 @@ export default class CacheProvider { throw new Error(`Cache key ${key} is too short.`); } - const cachedResult = await Cache.findByPk(key); + const cachedResult = await KeyValueCache.findByPk(key); if (cachedResult) { if (new Date() < cachedResult.expiresAt) { @@ -45,7 +45,7 @@ export default class CacheProvider { const result = await func(...options as any[]); // Save result - await Cache.upsert({ + await KeyValueCache.upsert({ key, value: JSON.stringify(result), expiresAt: futureTimeToDate(expiresIn), diff --git a/src/utils/db.ts b/src/utils/db.ts index be42013..f7d4fff 100644 --- a/src/utils/db.ts +++ b/src/utils/db.ts @@ -1,12 +1,12 @@ import {Sequelize} from 'sequelize-typescript'; import path from 'path'; import {DATA_DIR} from '../services/config.js'; -import {Cache, Settings, Shortcut} from '../models/index.js'; +import {KeyValueCache, Settings, Shortcut} from '../models/index.js'; export const sequelize = new Sequelize({ dialect: 'sqlite', database: 'muse', storage: path.join(DATA_DIR, 'db.sqlite'), - models: [Cache, Settings, Shortcut], + models: [KeyValueCache, Settings, Shortcut], logging: false, }); From 04c7e61fc076e2fa5ddf3d0faf6ebb73f4d52d82 Mon Sep 17 00:00:00 2001 From: Max Isom Date: Thu, 18 Nov 2021 20:55:57 -0500 Subject: [PATCH 2/7] Add FileCache model --- src/inversify.config.ts | 4 ++-- src/models/file-cache.ts | 14 ++++++++++++++ src/models/index.ts | 2 ++ src/services/get-songs.ts | 6 +++--- src/services/{cache.ts => key-value-cache.ts} | 2 +- src/types.ts | 3 ++- src/utils/db.ts | 4 ++-- 7 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 src/models/file-cache.ts rename src/services/{cache.ts => key-value-cache.ts} (96%) diff --git a/src/inversify.config.ts b/src/inversify.config.ts index b45c588..95ed234 100644 --- a/src/inversify.config.ts +++ b/src/inversify.config.ts @@ -28,7 +28,7 @@ import Shuffle from './commands/shuffle.js'; import Skip from './commands/skip.js'; import Unskip from './commands/unskip.js'; import ThirdParty from './services/third-party.js'; -import CacheProvider from './services/cache.js'; +import KeyValueCacheProvider from './services/key-value-cache.js'; const container = new Container(); @@ -76,6 +76,6 @@ container.bind(TYPES.Config).toConstantValue(new ConfigProvider()); // Static libraries container.bind(TYPES.ThirdParty).to(ThirdParty); -container.bind(TYPES.Cache).to(CacheProvider); +container.bind(TYPES.KeyValueCache).to(KeyValueCacheProvider); export default container; diff --git a/src/models/file-cache.ts b/src/models/file-cache.ts new file mode 100644 index 0000000..4846ba3 --- /dev/null +++ b/src/models/file-cache.ts @@ -0,0 +1,14 @@ +import {Table, Column, PrimaryKey, Model} from 'sequelize-typescript'; + +@Table +export default class FileCache extends Model { + @PrimaryKey + @Column + hash!: string; + + @Column + kbits!: number; + + @Column + accessedAt!: Date; +} diff --git a/src/models/index.ts b/src/models/index.ts index d39a3dc..e3f7c0a 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -1,8 +1,10 @@ +import FileCache from './file-cache.js'; import KeyValueCache from './key-value-cache.js'; import Settings from './settings.js'; import Shortcut from './shortcut.js'; export { + FileCache, KeyValueCache, Settings, Shortcut, diff --git a/src/services/get-songs.ts b/src/services/get-songs.ts index ba4401c..e7e3d5a 100644 --- a/src/services/get-songs.ts +++ b/src/services/get-songs.ts @@ -14,7 +14,7 @@ import {TYPES} from '../types.js'; import {cleanUrl} from '../utils/url.js'; import ThirdParty from './third-party.js'; import Config from './config.js'; -import CacheProvider from './cache.js'; +import KeyValueCacheProvider from './key-value-cache.js'; type QueuedSongWithoutChannel = Except; @@ -26,14 +26,14 @@ export default class { private readonly youtube: YouTube; private readonly youtubeKey: string; private readonly spotify: Spotify; - private readonly cache: CacheProvider; + private readonly cache: KeyValueCacheProvider; private readonly ytsrQueue: PQueue; constructor( @inject(TYPES.ThirdParty) thirdParty: ThirdParty, @inject(TYPES.Config) config: Config, - @inject(TYPES.Cache) cache: CacheProvider) { + @inject(TYPES.KeyValueCache) cache: KeyValueCacheProvider) { this.youtube = thirdParty.youtube; this.youtubeKey = config.YOUTUBE_API_KEY; this.spotify = thirdParty.spotify; diff --git a/src/services/cache.ts b/src/services/key-value-cache.ts similarity index 96% rename from src/services/cache.ts rename to src/services/key-value-cache.ts index 95c6731..7f1164d 100644 --- a/src/services/cache.ts +++ b/src/services/key-value-cache.ts @@ -12,7 +12,7 @@ type Options = { const futureTimeToDate = (time: Seconds) => new Date(new Date().getTime() + (time * 1000)); @injectable() -export default class CacheProvider { +export default class KeyValueCacheProvider { async wrap(func: (...options: any) => Promise, ...options: T): Promise { if (options.length === 0) { throw new Error('Missing cache options'); diff --git a/src/types.ts b/src/types.ts index 8bcd684..e6edd14 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,7 @@ export const TYPES = { Bot: Symbol('Bot'), - Cache: Symbol('Cache'), + KeyValueCache: Symbol('KeyValueCache'), + FileCache: Symbol('FileCache'), Client: Symbol('Client'), Config: Symbol('Config'), Command: Symbol('Command'), diff --git a/src/utils/db.ts b/src/utils/db.ts index f7d4fff..15a2d79 100644 --- a/src/utils/db.ts +++ b/src/utils/db.ts @@ -1,12 +1,12 @@ import {Sequelize} from 'sequelize-typescript'; import path from 'path'; import {DATA_DIR} from '../services/config.js'; -import {KeyValueCache, Settings, Shortcut} from '../models/index.js'; +import {FileCache, KeyValueCache, Settings, Shortcut} from '../models/index.js'; export const sequelize = new Sequelize({ dialect: 'sqlite', database: 'muse', storage: path.join(DATA_DIR, 'db.sqlite'), - models: [KeyValueCache, Settings, Shortcut], + models: [FileCache, KeyValueCache, Settings, Shortcut], logging: false, }); From 04d8f8d39000b711ec862043b687b8f47e454957 Mon Sep 17 00:00:00 2001 From: Max Isom Date: Fri, 19 Nov 2021 12:02:33 -0500 Subject: [PATCH 3/7] Better DX for `yarn dev` --- package.json | 11 ++- yarn.lock | 238 +++++++++++++++++++++++++++++++++------------------ 2 files changed, 163 insertions(+), 86 deletions(-) diff --git a/package.json b/package.json index c73ec3f..79235d3 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "watch": "tsc --watch", "prepack": "npm run clean && npm run build", "start": "node dist/index.js", - "dev": "nodemon", + "dev": "concurrently nodemon 'tsc --watch'", "docker-publish": "docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t codetheweb/muse:latest --push ." }, "devDependencies": { @@ -40,12 +40,14 @@ "@types/ws": "^8.2.0", "@typescript-eslint/eslint-plugin": "^4.31.1", "@typescript-eslint/parser": "^4.31.1", + "concurrently": "^6.4.0", "eslint": "^7.32.0", "eslint-config-xo": "^0.38.0", "eslint-config-xo-typescript": "^0.44.0", "husky": "^4.3.8", "nodemon": "^2.0.7", "ts-node": "^10.4.0", + "type-fest": "^2.5.4", "typescript": "^4.4.3" }, "eslintConfig": { @@ -79,7 +81,7 @@ "discord.js": "^13.3.0", "dotenv": "^8.5.1", "fluent-ffmpeg": "^2.1.2", - "fs-capacitor": "^6.2.0", + "fs-capacitor": "^7.0.1", "get-youtube-id": "^1.0.1", "got": "^11.8.2", "hasha": "^5.2.2", @@ -92,11 +94,12 @@ "p-limit": "^3.1.0", "p-queue": "^7.1.0", "reflect-metadata": "^0.1.13", - "sequelize": "^5.22.4", - "sequelize-typescript": "^1.1.0", + "sequelize": "^6.11.0", + "sequelize-typescript": "^2.1.1", "spotify-uri": "^2.2.0", "spotify-web-api-node": "^5.0.2", "sqlite3": "^5.0.2", + "xbytes": "^1.7.0", "youtube.ts": "^0.2.2", "ytdl-core": "^4.9.1", "ytsr": "^3.5.3" diff --git a/yarn.lock b/yarn.lock index 9a82bb8..93c2cd2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -610,11 +610,6 @@ block-stream@*: dependencies: inherits "~2.0.0" -bluebird@^3.5.0: - version "3.7.2" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" - integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== - boxen@^5.0.0: version "5.1.2" resolved "https://registry.yarnpkg.com/boxen/-/boxen-5.1.2.tgz#788cb686fc83c1f486dfa8a40c68fc2b831d2b50" @@ -750,6 +745,15 @@ cli-boxes@^2.2.1: resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" integrity sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw== +cliui@^7.0.2: + version "7.0.4" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.4.tgz#a0265ee655476fc807aea9df3df8df7783808b4f" + integrity sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ== + dependencies: + string-width "^4.2.0" + strip-ansi "^6.0.0" + wrap-ansi "^7.0.0" + clone-response@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/clone-response/-/clone-response-1.0.2.tgz#d1dc973920314df67fbeb94223b4ee350239e96b" @@ -757,14 +761,6 @@ clone-response@^1.0.2: dependencies: mimic-response "^1.0.0" -cls-bluebird@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/cls-bluebird/-/cls-bluebird-2.1.0.tgz#37ef1e080a8ffb55c2f4164f536f1919e7968aee" - integrity sha1-N+8eCAqP+1XC9BZPU28ZGeeWiu4= - dependencies: - is-bluebird "^1.0.2" - shimmer "^1.1.0" - code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" @@ -821,6 +817,20 @@ concat-map@0.0.1: resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= +concurrently@^6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/concurrently/-/concurrently-6.4.0.tgz#5387ee86be435a0eb51c292ade8a00acf479f170" + integrity sha512-HZ3D0RTQMH3oS4gvtYj1P+NBc6PzE2McEra6yEFcQKrUQ9HvtTGU4Dbne083F034p+LRb7kWU0tPRNvSGs1UCQ== + dependencies: + chalk "^4.1.0" + date-fns "^2.16.1" + lodash "^4.17.21" + rxjs "^6.6.3" + spawn-command "^0.0.2-1" + supports-color "^8.1.0" + tree-kill "^1.2.2" + yargs "^16.2.0" + configstore@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96" @@ -895,6 +905,11 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" +date-fns@^2.16.1: + version "2.25.0" + resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.25.0.tgz#8c5c8f1d958be3809a9a03f4b742eba894fc5680" + integrity sha512-ovYRFnTrbGPD4nqaEqescPEv1mNwvt+UTqI3Ay9SzNtey9NZnYu6E2qCcBBgJ6/2VF1zGGygpyTDITqpQQ5e+w== + debug@4, debug@^4.0.1, debug@^4.1.1, debug@^4.3.1: version "4.3.2" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" @@ -1072,6 +1087,11 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" +escalade@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" + integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== + escape-goat@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" @@ -1365,10 +1385,10 @@ formidable@^1.2.2: resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168" integrity sha512-KcpbcpuLNOwrEjnbpMC0gS+X8ciDoZE1kkqzat4a8vrprf+s9pKNQ/QIwWfbfs4ltgmFl3MD177SNTkve3BwGQ== -fs-capacitor@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/fs-capacitor/-/fs-capacitor-6.2.0.tgz#fa79ac6576629163cb84561995602d8999afb7f5" - integrity sha512-nKcE1UduoSKX27NSZlg879LdQc94OtbOsEmKMN2MBNudXREvijRKx2GEBsTMTfws+BrbkJoEuynbGSVRSpauvw== +fs-capacitor@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/fs-capacitor/-/fs-capacitor-7.0.1.tgz#351fa0224d74ae5ee17fdd165055082426ac60a3" + integrity sha512-YjxAAorsFA/pK3PTLuYJO+FlZ7wvGTIwGPbpGzVAJ+DUp6uof0zZjG6dcYsrGX864BMeUCj9R6lmkYZ5uY5lWQ== fs-minipass@^1.2.7: version "1.2.7" @@ -1443,6 +1463,11 @@ gauge@~2.7.3: strip-ansi "^3.0.1" wide-align "^1.1.0" +get-caller-file@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" + integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== + get-intrinsic@^1.0.2: version "1.1.1" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6" @@ -1485,19 +1510,7 @@ glob-parent@^5.1.2, glob-parent@~5.1.2: dependencies: is-glob "^4.0.1" -glob@7.1.2: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - integrity sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ== - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - -glob@^7.0.3, glob@^7.1.3: +glob@7.2.0, glob@^7.0.3, glob@^7.1.3: version "7.2.0" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== @@ -1720,10 +1733,10 @@ imurmurhash@^0.1.4: resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= -inflection@1.12.0: - version "1.12.0" - resolved "https://registry.yarnpkg.com/inflection/-/inflection-1.12.0.tgz#a200935656d6f5f6bc4dc7502e1aecb703228416" - integrity sha1-ogCTVlbW9fa8TcdQLhrstwMihBY= +inflection@1.13.1: + version "1.13.1" + resolved "https://registry.yarnpkg.com/inflection/-/inflection-1.13.1.tgz#c5cadd80888a90cf84c2e96e340d7edc85d5f0cb" + integrity sha512-dldYtl2WlN0QDkIDtg8+xFwOS2Tbmp12t1cHa5/YClU6ZQjTFm7B66UcVbh9NQB+HvT5BAd2t5+yKsBkw5pcqA== inflight@^1.0.4: version "1.0.6" @@ -1765,11 +1778,6 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" -is-bluebird@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-bluebird/-/is-bluebird-1.0.2.tgz#096439060f4aa411abee19143a84d6a55346d6e2" - integrity sha1-CWQ5Bg9KpBGr7hkUOoTWpVNG1uI= - is-ci@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c" @@ -2005,7 +2013,7 @@ lodash.truncate@^4.4.2: resolved "https://registry.yarnpkg.com/lodash.truncate/-/lodash.truncate-4.4.2.tgz#5a350da0b1113b837ecfffd5812cbe58d6eae193" integrity sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM= -lodash@^4.17.15, lodash@^4.17.21: +lodash@^4.17.20, lodash@^4.17.21: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -2151,14 +2159,14 @@ mkdirp@^1.0.3: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== -moment-timezone@^0.5.21: +moment-timezone@^0.5.31: version "0.5.34" resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.34.tgz#a75938f7476b88f155d3504a9343f7519d9a405c" integrity sha512-3zAEHh2hKUs3EXLESx/wsgw6IQdusOT8Bxm3D9UrHPQR7zlMmzwybC8zHEM1tQ4LJwP7fcxrWr8tuBg05fFCbg== dependencies: moment ">= 2.9.0" -"moment@>= 2.9.0", moment@^2.24.0: +"moment@>= 2.9.0", moment@^2.26.0: version "2.29.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== @@ -2533,6 +2541,11 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= +pg-connection-string@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" + integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== + picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3: version "2.3.0" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" @@ -2718,6 +2731,11 @@ request@^2.87.0: tunnel-agent "^0.6.0" uuid "^3.3.2" +require-directory@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" + integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= + require-from-string@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" @@ -2780,6 +2798,13 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" +rxjs@^6.6.3: + version "6.6.7" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9" + integrity sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ== + dependencies: + tslib "^1.9.0" + safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@^5.2.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" @@ -2839,38 +2864,37 @@ semver@~5.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" integrity sha1-myzl094C0XxgEq0yaqa00M9U+U8= -sequelize-pool@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/sequelize-pool/-/sequelize-pool-2.3.0.tgz#64f1fe8744228172c474f530604b6133be64993d" - integrity sha512-Ibz08vnXvkZ8LJTiUOxRcj1Ckdn7qafNZ2t59jYHMX1VIebTAOYefWdRYFt6z6+hy52WGthAHAoLc9hvk3onqA== +sequelize-pool@^6.0.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/sequelize-pool/-/sequelize-pool-6.1.0.tgz#caaa0c1e324d3c2c3a399fed2c7998970925d668" + integrity sha512-4YwEw3ZgK/tY/so+GfnSgXkdwIJJ1I32uZJztIEgZeAO6HMgj64OzySbWLgxj+tXhZCJnzRfkY9gINw8Ft8ZMg== -sequelize-typescript@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/sequelize-typescript/-/sequelize-typescript-1.1.0.tgz#d5c2945e7fbfe55a934917b27d84589858d79123" - integrity sha512-FAPEQPeAhIaFQNLAcf9Q2IWcqWhNcvn5OZZ7BzGB0CJMtImIsGg4E/EAb7huMmPaPwDArxJUWGqk1KurphTNRA== +sequelize-typescript@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/sequelize-typescript/-/sequelize-typescript-2.1.1.tgz#92445632062db868b760cd20215406403da737a2" + integrity sha512-4am/5O6dlAvtR/akH2KizcECm4rRAjWr+oc5mo9vFVMez8hrbOhQlDNzk0H6VMOQASCN7yBF+qOnSEN60V6/vA== dependencies: - glob "7.1.2" + glob "7.2.0" -sequelize@^5.22.4: - version "5.22.4" - resolved "https://registry.yarnpkg.com/sequelize/-/sequelize-5.22.4.tgz#4dbd8a1a735e98150880d43a95d45e9f46d151fa" - integrity sha512-xFQQ38HPg7EyDRDA+NdzMSRWbo9m6Z/RxpjnkBl3ggyQG+jRrup48x0jaw4Ox42h56wFnXOBC2NZOkTJfZeWCw== +sequelize@^6.11.0: + version "6.11.0" + resolved "https://registry.yarnpkg.com/sequelize/-/sequelize-6.11.0.tgz#11620b02ab2ca02af59ec652dd22bef7ccd348af" + integrity sha512-+j3N5lr+FR1eicMRGR3bRsGOl9HMY0UGb2PyB2i1yZ64XBgsz3xejMH0UD45LcUitj40soDGIa9CyvZG0dfzKg== dependencies: - bluebird "^3.5.0" - cls-bluebird "^2.1.0" debug "^4.1.1" dottie "^2.0.0" - inflection "1.12.0" - lodash "^4.17.15" - moment "^2.24.0" - moment-timezone "^0.5.21" + inflection "1.13.1" + lodash "^4.17.20" + moment "^2.26.0" + moment-timezone "^0.5.31" + pg-connection-string "^2.5.0" retry-as-promised "^3.2.0" - semver "^6.3.0" - sequelize-pool "^2.3.0" + semver "^7.3.2" + sequelize-pool "^6.0.0" toposort-class "^1.0.1" - uuid "^3.3.3" - validator "^10.11.0" - wkx "^0.4.8" + uuid "^8.1.0" + validator "^13.7.0" + wkx "^0.5.0" set-blocking@^2.0.0, set-blocking@~2.0.0: version "2.0.0" @@ -2889,11 +2913,6 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shimmer@^1.1.0: - version "1.2.1" - resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337" - integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw== - side-channel@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" @@ -2922,6 +2941,11 @@ slice-ansi@^4.0.0: astral-regex "^2.0.0" is-fullwidth-code-point "^3.0.0" +spawn-command@^0.0.2-1: + version "0.0.2-1" + resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2-1.tgz#62f5e9466981c1b796dc5929937e11c9c6921bd0" + integrity sha1-YvXpRmmBwbeW3Fkpk34RycaSG9A= + spotify-uri@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/spotify-uri/-/spotify-uri-2.2.0.tgz#8db641615cf6e122284874287fe39e89595922df" @@ -2981,7 +3005,7 @@ string-width@^1.0.1: is-fullwidth-code-point "^2.0.0" strip-ansi "^4.0.0" -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -3066,6 +3090,13 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" +supports-color@^8.1.0: + version "8.1.1" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" + integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== + dependencies: + has-flag "^4.0.0" + table@^6.0.9: version "6.7.3" resolved "https://registry.yarnpkg.com/table/-/table-6.7.3.tgz#255388439715a738391bd2ee4cbca89a4d05a9b7" @@ -3158,6 +3189,11 @@ tr46@~0.0.3: resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= +tree-kill@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" + integrity sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A== + ts-mixer@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/ts-mixer/-/ts-mixer-6.0.0.tgz#4e631d3a36e3fa9521b973b132e8353bc7267f9f" @@ -3181,7 +3217,7 @@ ts-node@^10.4.0: make-error "^1.1.1" yn "3.1.1" -tslib@^1.8.1: +tslib@^1.8.1, tslib@^1.9.0: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== @@ -3232,6 +3268,11 @@ type-fest@^1.2.1: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-1.4.0.tgz#e9fb813fe3bf1744ec359d55d1affefa76f14be1" integrity sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA== +type-fest@^2.5.4: + version "2.5.4" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-2.5.4.tgz#1613bf29a172ff1c66c29325466af9096fe505b5" + integrity sha512-zyPomVvb6u7+gJ/GPYUH6/nLDNiTtVOqXVUHtxFv5PmZQh6skgfeRtFYzWC01T5KeNWNIx5/0P111rKFLlkFvA== + typedarray-to-buffer@^3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" @@ -3295,11 +3336,16 @@ util-deprecate@^1.0.1, util-deprecate@~1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= -uuid@^3.3.2, uuid@^3.3.3: +uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== +uuid@^8.1.0: + version "8.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + v8-compile-cache@^2.0.3: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" @@ -3310,10 +3356,10 @@ vali-date@^1.0.0: resolved "https://registry.yarnpkg.com/vali-date/-/vali-date-1.0.0.tgz#1b904a59609fb328ef078138420934f6b86709a6" integrity sha1-G5BKWWCfsyjvB4E4Qgk09rhnCaY= -validator@^10.11.0: - version "10.11.0" - resolved "https://registry.yarnpkg.com/validator/-/validator-10.11.0.tgz#003108ea6e9a9874d31ccc9e5006856ccd76b228" - integrity sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw== +validator@^13.7.0: + version "13.7.0" + resolved "https://registry.yarnpkg.com/validator/-/validator-13.7.0.tgz#4f9658ba13ba8f3d82ee881d3516489ea85c0857" + integrity sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw== verror@1.10.0: version "1.10.0" @@ -3370,10 +3416,10 @@ widest-line@^3.1.0: dependencies: string-width "^4.0.0" -wkx@^0.4.8: - version "0.4.8" - resolved "https://registry.yarnpkg.com/wkx/-/wkx-0.4.8.tgz#a092cf088d112683fdc7182fd31493b2c5820003" - integrity sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ== +wkx@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/wkx/-/wkx-0.5.0.tgz#c6c37019acf40e517cc6b94657a25a3d4aa33e8c" + integrity sha512-Xng/d4Ichh8uN4l0FToV/258EjMGU9MGcA0HV2d9B/ZpZB3lqQm7nkOdZdm5GhKtLLhAE7PiVQwN4eN+2YJJUg== dependencies: "@types/node" "*" @@ -3411,11 +3457,21 @@ ws@^8.2.3: resolved "https://registry.yarnpkg.com/ws/-/ws-8.2.3.tgz#63a56456db1b04367d0b721a0b80cae6d8becbba" integrity sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA== +xbytes@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/xbytes/-/xbytes-1.7.0.tgz#a44ca476af66c54e79f744756a035326c697bb8d" + integrity sha512-iZglBXuHoC1F7jRz7746QhicNE167tEVq2H/iYQ1jIFdYIjqL8OfM86K52csfRZm+d83/VJO8bu37jN/G1ekKQ== + xdg-basedir@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q== +y18n@^5.0.5: + version "5.0.8" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" + integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== + yallist@^3.0.0, yallist@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" @@ -3431,6 +3487,24 @@ yaml@^1.10.0: resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== +yargs-parser@^20.2.2: + version "20.2.9" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" + integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== + +yargs@^16.2.0: + version "16.2.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" + integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== + dependencies: + cliui "^7.0.2" + escalade "^3.1.1" + get-caller-file "^2.0.5" + require-directory "^2.1.1" + string-width "^4.2.0" + y18n "^5.0.5" + yargs-parser "^20.2.2" + yn@3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" From f5149dfaba64c62f0a9ea6deab600b3d4d9b0f39 Mon Sep 17 00:00:00 2001 From: Max Isom Date: Fri, 19 Nov 2021 12:13:45 -0500 Subject: [PATCH 4/7] Move file caching logic to new FileCache service Also: removes the -re ffmpeg option. If this option is passed, ffmpeg won't write to fs-capacitor (and the cache file) as fast as possible. In other words, the cache file won't finish writing until the entire stream has been played. --- .env.example | 3 + README.md | 4 ++ src/index.ts | 3 + src/inversify.config.ts | 2 + src/managers/player.ts | 10 ++-- src/models/file-cache.ts | 4 +- src/models/key-value-cache.ts | 2 +- src/models/settings.ts | 2 +- src/models/shortcut.ts | 2 +- src/services/config.ts | 12 +++- src/services/file-cache.ts | 109 ++++++++++++++++++++++++++++++++++ src/services/player.ts | 53 ++++------------- 12 files changed, 154 insertions(+), 52 deletions(-) create mode 100644 src/services/file-cache.ts diff --git a/.env.example b/.env.example index 9d757da..db26bda 100644 --- a/.env.example +++ b/.env.example @@ -3,3 +3,6 @@ DATA_DIR= YOUTUBE_API_KEY= SPOTIFY_CLIENT_ID= SPOTIFY_CLIENT_SECRET= + +# Optional +# CACHE_LIMIT=2GB diff --git a/README.md b/README.md index 43ce9b9..ac6e0d3 100644 --- a/README.md +++ b/README.md @@ -69,3 +69,7 @@ services: 5. `yarn start` (or `npm run start`) **Note**: if you're on Windows, you may need to manually set the ffmpeg path. See [#345](https://github.com/codetheweb/muse/issues/345) for details. + +#### Advanced + +By default, Muse limits the total cache size to around 2 GB. If you want to change this, set the environment variable `CACHE_LIMIT`. For example, `CACHE_LIMIT=512MB` or `CACHE_LIMIT=10GB`. diff --git a/src/index.ts b/src/index.ts index 5bf5bab..383faef 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ import {TYPES} from './types.js'; import Bot from './bot.js'; import {sequelize} from './utils/db.js'; import Config from './services/config.js'; +import FileCacheProvider from './services/file-cache.js'; const bot = container.get(TYPES.Bot); @@ -18,5 +19,7 @@ const bot = container.get(TYPES.Bot); await sequelize.sync({alter: true}); + await container.get(TYPES.FileCache).cleanup(); + await bot.listen(); })(); diff --git a/src/inversify.config.ts b/src/inversify.config.ts index 95ed234..c277f7a 100644 --- a/src/inversify.config.ts +++ b/src/inversify.config.ts @@ -28,6 +28,7 @@ import Shuffle from './commands/shuffle.js'; import Skip from './commands/skip.js'; import Unskip from './commands/unskip.js'; import ThirdParty from './services/third-party.js'; +import FileCacheProvider from './services/file-cache.js'; import KeyValueCacheProvider from './services/key-value-cache.js'; const container = new Container(); @@ -76,6 +77,7 @@ container.bind(TYPES.Config).toConstantValue(new ConfigProvider()); // Static libraries container.bind(TYPES.ThirdParty).to(ThirdParty); +container.bind(TYPES.FileCache).to(FileCacheProvider); container.bind(TYPES.KeyValueCache).to(KeyValueCacheProvider); export default container; diff --git a/src/managers/player.ts b/src/managers/player.ts index 02e4ba0..5d816b8 100644 --- a/src/managers/player.ts +++ b/src/managers/player.ts @@ -2,25 +2,25 @@ import {inject, injectable} from 'inversify'; import {Client} from 'discord.js'; import {TYPES} from '../types.js'; import Player from '../services/player.js'; -import Config from '../services/config.js'; +import FileCacheProvider from '../services/file-cache.js'; @injectable() export default class { private readonly guildPlayers: Map; - private readonly cacheDir: string; private readonly discordClient: Client; + private readonly fileCache: FileCacheProvider; - constructor(@inject(TYPES.Config) config: Config, @inject(TYPES.Client) client: Client) { + constructor(@inject(TYPES.FileCache) fileCache: FileCacheProvider, @inject(TYPES.Client) client: Client) { this.guildPlayers = new Map(); - this.cacheDir = config.CACHE_DIR; this.discordClient = client; + this.fileCache = fileCache; } get(guildId: string): Player { let player = this.guildPlayers.get(guildId); if (!player) { - player = new Player(this.cacheDir, this.discordClient); + player = new Player(this.discordClient, this.fileCache); this.guildPlayers.set(guildId, player); } diff --git a/src/models/file-cache.ts b/src/models/file-cache.ts index 4846ba3..dbac6d3 100644 --- a/src/models/file-cache.ts +++ b/src/models/file-cache.ts @@ -1,13 +1,13 @@ import {Table, Column, PrimaryKey, Model} from 'sequelize-typescript'; @Table -export default class FileCache extends Model { +export default class FileCache extends Model { @PrimaryKey @Column hash!: string; @Column - kbits!: number; + bytes!: number; @Column accessedAt!: Date; diff --git a/src/models/key-value-cache.ts b/src/models/key-value-cache.ts index 795ff75..5072538 100644 --- a/src/models/key-value-cache.ts +++ b/src/models/key-value-cache.ts @@ -2,7 +2,7 @@ import {Table, Column, PrimaryKey, Model} from 'sequelize-typescript'; import sequelize from 'sequelize'; @Table -export default class KeyValueCache extends Model { +export default class KeyValueCache extends Model { @PrimaryKey @Column key!: string; diff --git a/src/models/settings.ts b/src/models/settings.ts index 29c2b3e..3318c47 100644 --- a/src/models/settings.ts +++ b/src/models/settings.ts @@ -1,7 +1,7 @@ import {Table, Column, PrimaryKey, Model, Default} from 'sequelize-typescript'; @Table -export default class Settings extends Model { +export default class Settings extends Model { @PrimaryKey @Column guildId!: string; diff --git a/src/models/shortcut.ts b/src/models/shortcut.ts index 7ce1177..4ec88ed 100644 --- a/src/models/shortcut.ts +++ b/src/models/shortcut.ts @@ -1,7 +1,7 @@ import {Table, Column, PrimaryKey, Model, AutoIncrement, Index} from 'sequelize-typescript'; @Table -export default class Shortcut extends Model { +export default class Shortcut extends Model { @PrimaryKey @AutoIncrement @Column diff --git a/src/services/config.ts b/src/services/config.ts index 759a27a3..96b161c 100644 --- a/src/services/config.ts +++ b/src/services/config.ts @@ -1,6 +1,8 @@ import dotenv from 'dotenv'; import {injectable} from 'inversify'; import path from 'path'; +import xbytes from 'xbytes'; +import {ConditionalKeys} from 'type-fest'; dotenv.config(); export const DATA_DIR = path.resolve(process.env.DATA_DIR ? process.env.DATA_DIR : './data'); @@ -12,6 +14,7 @@ const CONFIG_MAP = { SPOTIFY_CLIENT_SECRET: process.env.SPOTIFY_CLIENT_SECRET, DATA_DIR, CACHE_DIR: path.join(DATA_DIR, 'cache'), + CACHE_LIMIT_IN_BYTES: xbytes.parseSize(process.env.CACHE_LIMIT ?? '2GB'), } as const; @injectable() @@ -22,6 +25,7 @@ export default class Config { readonly SPOTIFY_CLIENT_SECRET!: string; readonly DATA_DIR!: string; readonly CACHE_DIR!: string; + readonly CACHE_LIMIT_IN_BYTES!: number; constructor() { for (const [key, value] of Object.entries(CONFIG_MAP)) { @@ -30,7 +34,13 @@ export default class Config { process.exit(1); } - this[key as keyof typeof CONFIG_MAP] = value; + if (typeof value === 'number') { + this[key as ConditionalKeys] = value; + } else if (typeof value === 'string') { + this[key as ConditionalKeys] = value; + } else { + throw new Error(`Unsupported type for ${key}`); + } } } } diff --git a/src/services/file-cache.ts b/src/services/file-cache.ts new file mode 100644 index 0000000..0ecf279 --- /dev/null +++ b/src/services/file-cache.ts @@ -0,0 +1,109 @@ +import {inject, injectable} from 'inversify'; +import sequelize from 'sequelize'; +import {FileCache} from '../models/index.js'; +import {TYPES} from '../types.js'; +import Config from './config.js'; +import {promises as fs, createWriteStream} from 'fs'; +import path from 'path'; + +@injectable() +export default class FileCacheProvider { + private readonly config: Config; + + constructor(@inject(TYPES.Config) config: Config) { + this.config = config; + } + + /** + * Returns path to cached file if it exists, otherwise throws an error. + * Updates the `accessedAt` property of the cached file. + * @param hash lookup key + */ + async getPathFor(hash: string): Promise { + const model = await FileCache.findByPk(hash); + + if (!model) { + throw new Error('File is not cached'); + } + + const resolvedPath = path.join(this.config.CACHE_DIR, hash); + + try { + await fs.access(resolvedPath); + } catch (_: unknown) { + await FileCache.destroy({where: {hash}}); + + throw new Error('File is not cached'); + } + + await model.update({accessedAt: new Date()}); + + return resolvedPath; + } + + /** + * Returns a write stream for the given hash key. + * The stream handles saving a new file and will + * update the database after the stream is finished. + * @param hash lookup key + */ + createWriteStream(hash: string) { + const tmpPath = path.join(this.config.CACHE_DIR, 'tmp', hash); + const finalPath = path.join(this.config.CACHE_DIR, hash); + + const stream = createWriteStream(tmpPath); + + stream.on('finish', async () => { + // Only move if size is non-zero (may have errored out) + const stats = await fs.stat(tmpPath); + + if (stats.size !== 0) { + await fs.rename(tmpPath, finalPath); + } + + await FileCache.create({hash, bytes: stats.size, accessedAt: new Date()}); + + await this.evictOldestIfNecessary(); + }); + + return stream; + } + + /** + * Deletes orphaned cache files and evicts files if + * necessary. Should be run on program startup so files + * will be evicted if the cache limit has changed. + */ + async cleanup() { + await this.removeOrphans(); + await this.evictOldestIfNecessary(); + } + + private async evictOldestIfNecessary() { + const [{dataValues: {totalSizeBytes}}] = await FileCache.findAll({ + attributes: [ + [sequelize.fn('sum', sequelize.col('bytes')), 'totalSizeBytes'], + ], + }) as unknown as [{dataValues: {totalSizeBytes: number}}]; + + if (totalSizeBytes > this.config.CACHE_LIMIT_IN_BYTES) { + const oldest = await FileCache.findOne({ + order: [ + ['accessedAt', 'ASC'], + ], + }); + + if (oldest) { + await oldest.destroy(); + await fs.unlink(path.join(this.config.CACHE_DIR, oldest.hash)); + } + + // Continue to evict until we're under the limit + await this.evictOldestIfNecessary(); + } + } + + private async removeOrphans() { + // TODO + } +} diff --git a/src/services/player.ts b/src/services/player.ts index c49e8c6..2738a91 100644 --- a/src/services/player.ts +++ b/src/services/player.ts @@ -1,7 +1,5 @@ import {VoiceChannel, Snowflake, Client, TextChannel} from 'discord.js'; -import {promises as fs, createWriteStream} from 'fs'; -import {Readable, PassThrough} from 'stream'; -import path from 'path'; +import {Readable} from 'stream'; import hasha from 'hasha'; import ytdl from 'ytdl-core'; import {WriteStream} from 'fs-capacitor'; @@ -9,6 +7,7 @@ import ffmpeg from 'fluent-ffmpeg'; import shuffle from 'array-shuffle'; import errorMsg from '../utils/error-msg.js'; import {AudioPlayer, AudioPlayerStatus, createAudioPlayer, createAudioResource, joinVoiceChannel, StreamType, VoiceConnection, VoiceConnectionStatus} from '@discordjs/voice'; +import FileCacheProvider from './file-cache.js'; export interface QueuedPlaylist { title: string; @@ -35,7 +34,6 @@ export default class { public voiceConnection: VoiceConnection | null = null; private queue: QueuedSong[] = []; private queuePosition = 0; - private readonly cacheDir: string; private audioPlayer: AudioPlayer | null = null; private nowPlaying: QueuedSong | null = null; private playPositionInterval: NodeJS.Timeout | undefined; @@ -44,10 +42,11 @@ export default class { private positionInSeconds = 0; private readonly discordClient: Client; + private readonly fileCache: FileCacheProvider; - constructor(cacheDir: string, client: Client) { - this.cacheDir = cacheDir; + constructor(client: Client, fileCache: FileCacheProvider) { this.discordClient = client; + this.fileCache = fileCache; } async connect(channel: VoiceChannel): Promise { @@ -283,40 +282,24 @@ export default class { return this.queueSize() === 0; } - private getCachedPath(url: string): string { - return path.join(this.cacheDir, hasha(url)); - } - - private getCachedPathTemp(url: string): string { - return path.join(this.cacheDir, 'tmp', hasha(url)); - } - - private async isCached(url: string): Promise { - try { - await fs.access(this.getCachedPath(url)); - - return true; - } catch (_: unknown) { - return false; - } + private getHashForCache(url: string): string { + return hasha(url); } private async getStream(url: string, options: {seek?: number} = {}): Promise { - const cachedPath = this.getCachedPath(url); - let ffmpegInput = ''; const ffmpegInputOptions: string[] = []; let shouldCacheVideo = false; let format: ytdl.videoFormat | undefined; - if (await this.isCached(url)) { - ffmpegInput = cachedPath; + try { + ffmpegInput = await this.fileCache.getPathFor(this.getHashForCache(url)); if (options.seek) { ffmpegInputOptions.push('-ss', options.seek.toString()); } - } else { + } catch { // Not yet cached, must download const info = await ytdl.getInfo(url); @@ -367,7 +350,6 @@ export default class { '1', '-reconnect_delay_max', '5', - '-re', ]); if (options.seek) { @@ -386,8 +368,7 @@ export default class { .on('error', error => { console.error(error); reject(error); - }) - .pipe() as PassThrough; + }); const capacitor = new WriteStream(); @@ -395,17 +376,7 @@ export default class { // Cache video if necessary if (shouldCacheVideo) { - const cacheTempPath = this.getCachedPathTemp(url); - const cacheStream = createWriteStream(cacheTempPath); - - cacheStream.on('finish', async () => { - // Only move if size is non-zero (may have errored out) - const stats = await fs.stat(cacheTempPath); - - if (stats.size !== 0) { - await fs.rename(cacheTempPath, cachedPath); - } - }); + const cacheStream = this.fileCache.createWriteStream(this.getHashForCache(url)); capacitor.createReadStream().pipe(cacheStream); } From 34e45d6273463e232168c2f5e273728d594e4d77 Mon Sep 17 00:00:00 2001 From: Max Isom Date: Fri, 19 Nov 2021 12:22:27 -0500 Subject: [PATCH 5/7] Add .removeOrphans() --- src/services/file-cache.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/services/file-cache.ts b/src/services/file-cache.ts index 0ecf279..6d9c23f 100644 --- a/src/services/file-cache.ts +++ b/src/services/file-cache.ts @@ -1,10 +1,10 @@ +import {promises as fs, createWriteStream} from 'fs'; +import path from 'path'; import {inject, injectable} from 'inversify'; import sequelize from 'sequelize'; import {FileCache} from '../models/index.js'; import {TYPES} from '../types.js'; import Config from './config.js'; -import {promises as fs, createWriteStream} from 'fs'; -import path from 'path'; @injectable() export default class FileCacheProvider { @@ -104,6 +104,14 @@ export default class FileCacheProvider { } private async removeOrphans() { - // TODO + for await (const dirent of await fs.opendir(this.config.CACHE_DIR)) { + if (dirent.isFile()) { + const model = await FileCache.findByPk(dirent.name); + + if (!model) { + await fs.unlink(path.join(this.config.CACHE_DIR, dirent.name)); + } + } + } } } From 49664be9e138b71eb920600cd24b4510f5f2e662 Mon Sep 17 00:00:00 2001 From: Max Isom Date: Wed, 24 Nov 2021 12:03:25 -0600 Subject: [PATCH 6/7] Change event to close --- src/services/file-cache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/file-cache.ts b/src/services/file-cache.ts index 6d9c23f..fa4f1f7 100644 --- a/src/services/file-cache.ts +++ b/src/services/file-cache.ts @@ -44,7 +44,7 @@ export default class FileCacheProvider { /** * Returns a write stream for the given hash key. * The stream handles saving a new file and will - * update the database after the stream is finished. + * update the database after the stream is closed. * @param hash lookup key */ createWriteStream(hash: string) { @@ -53,7 +53,7 @@ export default class FileCacheProvider { const stream = createWriteStream(tmpPath); - stream.on('finish', async () => { + stream.on('close', async () => { // Only move if size is non-zero (may have errored out) const stats = await fs.stat(tmpPath); From 9f9469f682e68c45d2c20251bfa5726e7c415b80 Mon Sep 17 00:00:00 2001 From: Max Isom Date: Wed, 24 Nov 2021 13:16:44 -0600 Subject: [PATCH 7/7] Add back -re option on uncached streams --- src/services/player.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/services/player.ts b/src/services/player.ts index 5f4d368..67b8b38 100644 --- a/src/services/player.ts +++ b/src/services/player.ts @@ -364,6 +364,17 @@ export default class { // Create stream and pipe to capacitor return new Promise((resolve, reject) => { + const capacitor = new WriteStream(); + + // Cache video if necessary + if (shouldCacheVideo) { + const cacheStream = this.fileCache.createWriteStream(this.getHashForCache(url)); + + capacitor.createReadStream().pipe(cacheStream); + } else { + ffmpegInputOptions.push('-re'); + } + const youtubeStream = ffmpeg(ffmpegInput) .inputOptions(ffmpegInputOptions) .noVideo() @@ -374,17 +385,8 @@ export default class { reject(error); }); - const capacitor = new WriteStream(); - youtubeStream.pipe(capacitor); - // Cache video if necessary - if (shouldCacheVideo) { - const cacheStream = this.fileCache.createWriteStream(this.getHashForCache(url)); - - capacitor.createReadStream().pipe(cacheStream); - } - resolve(capacitor.createReadStream()); }); }