Move to ESM, use ytsr, implement caching

Closes #315

src/services/cache.ts

@@ -0,0 +1,52 @@
+import {injectable} from 'inversify';
+import {Cache} from '../models/index.js';
+import debug from '../utils/debug.js';
+
+type Seconds = number;
+
+type Options = {
+  expiresIn: Seconds;
+  key?: string;
+};
+
+const futureTimeToDate = (time: Seconds) => new Date(new Date().getTime() + (time * 1000));
+
+@injectable()
+export default class CacheProvider {
+  async wrap<T extends [...any[], Options], F>(func: (...options: any) => Promise<F>, ...options: T): Promise<F> {
+    if (options.length === 0) {
+      throw new Error('Missing cache options');
+    }
+
+    const functionArgs = options.slice(0, options.length - 1);
+
+    const {
+      key = JSON.stringify(functionArgs),
+      expiresIn
+    } = options[options.length - 1] as Options;
+
+    const cachedResult = await Cache.findByPk(key);
+
+    if (cachedResult) {
+      if (new Date() < cachedResult.expiresAt) {
+        debug(`Cache hit: ${key}`);
+        return JSON.parse(cachedResult.value);
+      }
+
+      await cachedResult.destroy();
+    }
+
+    debug(`Cache miss: ${key}`);
+
+    const result = await func(...options as any[]);
+
+    // Save result
+    await Cache.upsert({
+      key,
+      value: JSON.stringify(result),
+      expiresAt: futureTimeToDate(expiresIn)
+    });
+
+    return result;
+  }
+}

src/services/get-songs.ts

@@ -2,37 +2,73 @@ import {URL} from 'url';
 import {inject, injectable} from 'inversify';
 import {toSeconds, parse} from 'iso8601-duration';
 import got from 'got';
+import ytsr, {Video} from 'ytsr';
 import spotifyURI from 'spotify-uri';
 import Spotify from 'spotify-web-api-node';
 import YouTube, {YoutubePlaylistItem} from 'youtube.ts';
-import pLimit from 'p-limit';
+import PQueue from 'p-queue';
 import shuffle from 'array-shuffle';
 import {Except} from 'type-fest';
-import {QueuedSong, QueuedPlaylist} from '../services/player';
-import {TYPES} from '../types';
-import {cleanUrl} from '../utils/url';
-import ThirdParty from './third-party';
-import Config from './config';
+import {QueuedSong, QueuedPlaylist} from '../services/player.js';
+import {TYPES} from '../types.js';
+import {cleanUrl} from '../utils/url.js';
+import ThirdParty from './third-party.js';
+import Config from './config.js';
+import CacheProvider from './cache.js';
 
 type QueuedSongWithoutChannel = Except<QueuedSong, 'addedInChannelId'>;
 
+const ONE_HOUR_IN_SECONDS = 60 * 60;
+const ONE_MINUTE_IN_SECONDS = 1 * 60;
+
 @injectable()
 export default class {
   private readonly youtube: YouTube;
   private readonly youtubeKey: string;
   private readonly spotify: Spotify;
+  private readonly cache: CacheProvider;
 
-  constructor(@inject(TYPES.ThirdParty) thirdParty: ThirdParty, @inject(TYPES.Config) config: Config) {
+  private readonly ytsrQueue: PQueue;
+
+  constructor(
+  @inject(TYPES.ThirdParty) thirdParty: ThirdParty,
+    @inject(TYPES.Config) config: Config,
+    @inject(TYPES.Cache) cache: CacheProvider) {
     this.youtube = thirdParty.youtube;
     this.youtubeKey = config.YOUTUBE_API_KEY;
     this.spotify = thirdParty.spotify;
+    this.cache = cache;
+
+    this.ytsrQueue = new PQueue({concurrency: 4});
   }
 
   async youtubeVideoSearch(query: string): Promise<QueuedSongWithoutChannel|null> {
     try {
-      const {items: [video]} = await this.youtube.videos.search({q: query, maxResults: 1, type: 'video'});
+      const {items} = await this.ytsrQueue.add(async () => this.cache.wrap(
+        ytsr,
+        query,
+        {
+          limit: 10
+        },
+        {
+          expiresIn: ONE_HOUR_IN_SECONDS
+        }
+      ));
 
-      return await this.youtubeVideo(video.id.videoId);
+      let firstVideo: Video | undefined;
+
+      for (const item of items) {
+        if (item.type === 'video') {
+          firstVideo = item;
+          break;
+        }
+      }
+
+      if (!firstVideo) {
+        throw new Error('No video found.');
+      }
+
+      return await this.youtubeVideo(firstVideo.id);
     } catch (_: unknown) {
       return null;
     }
@@ -40,7 +76,13 @@ export default class {
 
   async youtubeVideo(url: string): Promise<QueuedSongWithoutChannel|null> {
     try {
-      const videoDetails = await this.youtube.videos.get(cleanUrl(url));
+      const videoDetails = await this.cache.wrap(
+        this.youtube.videos.get,
+        cleanUrl(url),
+        {
+          expiresIn: ONE_HOUR_IN_SECONDS
+        }
+      );
 
       return {
         title: videoDetails.snippet.title,
@@ -57,7 +99,13 @@ export default class {
 
   async youtubePlaylist(listId: string): Promise<QueuedSongWithoutChannel[]> {
     // YouTube playlist
-    const playlist = await this.youtube.playlists.get(listId);
+    const playlist = await this.cache.wrap(
+      this.youtube.playlists.get,
+      listId,
+      {
+        expiresIn: ONE_MINUTE_IN_SECONDS
+      }
+    );
 
     interface VideoDetailsResponse {
       id: string;
@@ -75,7 +123,14 @@ export default class {
 
     while (playlistVideos.length !== playlist.contentDetails.itemCount) {
       // eslint-disable-next-line no-await-in-loop
-      const {items, nextPageToken} = await this.youtube.playlists.items(listId, {maxResults: '50', pageToken: nextToken});
+      const {items, nextPageToken} = await this.cache.wrap(
+        this.youtube.playlists.items,
+        listId,
+        {maxResults: '50', pageToken: nextToken},
+        {
+          expiresIn: ONE_MINUTE_IN_SECONDS
+        }
+      );
 
       nextToken = nextPageToken;
 
@@ -84,11 +139,24 @@ export default class {
       // Start fetching extra details about videos
       videoDetailsPromises.push((async () => {
         // Unfortunately, package doesn't provide a method for this
-        const {items: videoDetailItems}: {items: VideoDetailsResponse[]} = await got('https://www.googleapis.com/youtube/v3/videos', {searchParams: {
-          part: 'contentDetails',
-          id: items.map(item => item.contentDetails.videoId).join(','),
-          key: this.youtubeKey
-        }}).json();
+        const {items: videoDetailItems} = await this.cache.wrap(
+          () => {
+            return got(
+              'https://www.googleapis.com/youtube/v3/videos',
+              {
+                searchParams: {
+                  part: 'contentDetails',
+                  id: items.map(item => item.contentDetails.videoId).join(','),
+                  key: this.youtubeKey,
+                  responseType: 'json'
+                }
+              }
+            ).json();
+          },
+          {
+            expiresIn: ONE_MINUTE_IN_SECONDS
+          }
+        );
 
         videoDetails.push(...videoDetailItems);
       })());
@@ -193,9 +261,7 @@ export default class {
       tracks = shuffled.slice(0, 50);
     }
 
-    // Limit concurrency so hopefully we don't get banned for searching
-    const limit = pLimit(5);
-    let songs = await Promise.all(tracks.map(async track => limit(async () => this.spotifyToYouTube(track, playlist))));
+    let songs = await Promise.all(tracks.map(async track => this.spotifyToYouTube(track, playlist)));
 
     let nSongsNotFound = 0;
 
@@ -215,14 +281,7 @@ export default class {
 
   private async spotifyToYouTube(track: SpotifyApi.TrackObjectSimplified, _: QueuedPlaylist | null): Promise<QueuedSongWithoutChannel | null> {
     try {
-      const {items} = await this.youtube.videos.search({q: `"${track.name}" "${track.artists[0].name}"`, maxResults: 10});
-      const videoResult = items[0];
-
-      if (!videoResult) {
-        throw new Error('No video found for query.');
-      }
-
-      return await this.youtubeVideo(videoResult.id.videoId);
+      return await this.youtubeVideoSearch(`"${track.name}" "${track.artists[0].name}"`);
     } catch (_: unknown) {
       return null;
     }

src/services/natural-language-commands.ts

@@ -1,9 +1,9 @@
 import {inject, injectable} from 'inversify';
 import {Message, Guild, GuildMember} from 'discord.js';
-import {TYPES} from '../types';
-import PlayerManager from '../managers/player';
-import {QueuedSong} from '../services/player';
-import {getMostPopularVoiceChannel, getMemberVoiceChannel} from '../utils/channels';
+import {TYPES} from '../types.js';
+import PlayerManager from '../managers/player.js';
+import {QueuedSong} from '../services/player.js';
+import {getMostPopularVoiceChannel, getMemberVoiceChannel} from '../utils/channels.js';
 
 @injectable()
 export default class {

src/services/player.ts

@@ -7,7 +7,7 @@ import ytdl from 'ytdl-core';
 import {WriteStream} from 'fs-capacitor';
 import ffmpeg from 'fluent-ffmpeg';
 import shuffle from 'array-shuffle';
-import errorMsg from '../utils/error-msg';
+import errorMsg from '../utils/error-msg.js';
 
 export interface QueuedPlaylist {
   title: string;

src/services/third-party.ts

@@ -1,8 +1,8 @@
 import {inject, injectable} from 'inversify';
 import SpotifyWebApi from 'spotify-web-api-node';
-import Youtube from 'youtube.ts';
-import {TYPES} from '../types';
-import Config from './config';
+import Youtube from 'youtube.ts/dist/youtube.js';
+import {TYPES} from '../types.js';
+import Config from './config.js';
 
 @injectable()
 export default class ThirdParty {
@@ -12,7 +12,8 @@ export default class ThirdParty {
   private spotifyTokenTimerId?: NodeJS.Timeout;
 
   constructor(@inject(TYPES.Config) config: Config) {
-    this.youtube = new Youtube(config.YOUTUBE_API_KEY);
+    // Library is transpiled incorrectly
+    this.youtube = new ((Youtube as any).default)(config.YOUTUBE_API_KEY);
     this.spotify = new SpotifyWebApi({
       clientId: config.SPOTIFY_CLIENT_ID,
       clientSecret: config.SPOTIFY_CLIENT_SECRET