Improve YouTube API error handling (#1199)

Enhance YouTube API service with comprehensive error handling, retries, and
logging to improve reliability. Add typed YouTubeError interface, implement
exponential backoff with jitter, properly handle rate limits (429) and quota
exceeded (403), and improve cache key management.

src/services/youtube-api.ts

@@ -1,8 +1,10 @@
 import {inject, injectable} from 'inversify';
 import {toSeconds, parse} from 'iso8601-duration';
-import got, {Got} from 'got';
+import got, {Got, HTTPError, RequestError} from 'got';
 import ytsr, {Video} from '@distube/ytsr';
 import PQueue from 'p-queue';
+import pRetry from 'p-retry';
+import crypto from 'crypto';
 import {SongMetadata, QueuedPlaylist, MediaSource} from './player.js';
 import {TYPES} from '../types.js';
 import Config from './config.js';
@@ -10,6 +12,19 @@ import KeyValueCacheProvider from './key-value-cache.js';
 import {ONE_HOUR_IN_SECONDS, ONE_MINUTE_IN_SECONDS} from '../utils/constants.js';
 import {parseTime} from '../utils/time.js';
 import getYouTubeID from 'get-youtube-id';
+import debug from '../utils/debug.js';
+
+// Define structured error type for better error handling
+interface YouTubeError extends Error {
+  code: 'QUOTA_EXCEEDED' | 'RATE_LIMITED' | 'NOT_FOUND' | 'NETWORK_ERROR' | 'UNKNOWN';
+  status?: number;
+  retryable: boolean;
+}
+
+const YOUTUBE_MAX_RETRY_COUNT = 3;
+const YOUTUBE_BASE_RETRY_DELAY_MS = 1000;
+const YOUTUBE_SEARCH_CONCURRENCY = 4;
+const MAX_CACHE_KEY_LENGTH = 250;
 
 interface VideoDetailsResponse {
   id: string;
@@ -59,10 +74,13 @@ export default class {
   private readonly ytsrQueue: PQueue;
   private readonly got: Got;
 
-  constructor(@inject(TYPES.Config) config: Config, @inject(TYPES.KeyValueCache) cache: KeyValueCacheProvider) {
+  constructor(
+    @inject(TYPES.Config) config: Config,
+    @inject(TYPES.KeyValueCache) cache: KeyValueCacheProvider,
+  ) {
     this.youtubeKey = config.YOUTUBE_API_KEY;
     this.cache = cache;
-    this.ytsrQueue = new PQueue({concurrency: 4});
+    this.ytsrQueue = new PQueue({concurrency: YOUTUBE_SEARCH_CONCURRENCY});
 
     this.got = got.extend({
       prefixUrl: 'https://www.googleapis.com/youtube/v3/',
@@ -73,8 +91,10 @@ export default class {
     });
   }
 
-  async search(query: string, shouldSplitChapters: boolean): Promise<SongMetadata[]> {
+  public async search(query: string, shouldSplitChapters: boolean): Promise<SongMetadata[]> {
-    const {items} = await this.ytsrQueue.add(async () => this.cache.wrap(
+    try {
+      const {items} = await this.ytsrQueue.add(async () =>
+        this.cache.wrap(
           ytsr,
           query,
           {
@@ -82,11 +102,12 @@ export default class {
           },
           {
             expiresIn: ONE_HOUR_IN_SECONDS,
+            key: this.createCacheKey('youtube-search', query),
           },
-    ));
+        ),
+      );
 
       let firstVideo: Video | undefined;
-
       for (const item of items) {
         if (item.type === 'video') {
           firstVideo = item;
@@ -95,100 +116,258 @@ export default class {
       }
 
       if (!firstVideo) {
-      return [];
+        throw new Error('No matching videos found.');
       }
 
-    return this.getVideo(firstVideo.url, shouldSplitChapters);
+      return await this.getVideo(firstVideo.url, shouldSplitChapters);
+    } catch (error) {
+      debug('YouTube search error:', error);
+      throw new Error('Failed to search YouTube. Please try again.');
+    }
   }
 
-  async getVideo(url: string, shouldSplitChapters: boolean): Promise<SongMetadata[]> {
+  public async getVideo(url: string, shouldSplitChapters: boolean): Promise<SongMetadata[]> {
-    const result = await this.getVideosByID([String(getYouTubeID(url))]);
+    const videoId = getYouTubeID(url);
+    if (!videoId) {
+      throw new Error('Invalid YouTube URL.');
+    }
+
+    const result = await this.getVideosByID([videoId]);
     const video = result.at(0);
 
     if (!video) {
-      throw new Error('Video could not be found.');
+      throw new Error('Video could not be found or is unavailable.');
     }
 
     return this.getMetadataFromVideo({video, shouldSplitChapters});
   }
 
-  async getPlaylist(listId: string, shouldSplitChapters: boolean): Promise<SongMetadata[]> {
+  public async getPlaylist(listId: string, shouldSplitChapters: boolean): Promise<SongMetadata[]> {
+    try {
       const playlistParams = {
         searchParams: {
           part: 'id, snippet, contentDetails',
           id: listId,
         },
       };
+
       const {items: playlists} = await this.cache.wrap(
-      async () => this.got('playlists', playlistParams).json() as Promise<{items: PlaylistResponse[]}>,
+        async () => this.executeYouTubeRequest<{items: PlaylistResponse[]}>('playlists', playlistParams),
         playlistParams,
         {
           expiresIn: ONE_MINUTE_IN_SECONDS,
+          key: this.createCacheKey('youtube-playlist', listId),
         },
       );
 
-    const playlist = playlists.at(0)!;
+      const playlist = playlists.at(0);
-
       if (!playlist) {
         throw new Error('Playlist could not be found.');
       }
 
-    const playlistVideos: PlaylistItem[] = [];
+      // Helper function to fetch a single page of playlist items
-    const videoDetailsPromises: Array<Promise<void>> = [];
+      const fetchPlaylistPage = async (token?: string) => {
-    const videoDetails: VideoDetailsResponse[] = [];
-
-    let nextToken: string | undefined;
-
-    while (playlistVideos.length < playlist.contentDetails.itemCount) {
         const playlistItemsParams = {
           searchParams: {
             part: 'id, contentDetails',
             playlistId: listId,
             maxResults: '50',
-          pageToken: nextToken,
+            pageToken: token,
           },
         };
 
-      // eslint-disable-next-line no-await-in-loop
+        return this.cache.wrap(
-      const {items, nextPageToken} = await this.cache.wrap(
+          async () => this.executeYouTubeRequest<PlaylistItemsResponse>('playlistItems', playlistItemsParams),
-        async () => this.got('playlistItems', playlistItemsParams).json() as Promise<PlaylistItemsResponse>,
           playlistItemsParams,
           {
             expiresIn: ONE_MINUTE_IN_SECONDS,
+            key: this.createCacheKey('youtube-playlist-items', `${listId}-${token ?? 'initial'}`),
           },
         );
+      };
 
-      nextToken = nextPageToken;
+      // Recursively fetch all playlist pages
-      playlistVideos.push(...items);
+      const fetchAllPages = async (token?: string): Promise<PlaylistItem[]> => {
-
+        const {items, nextPageToken} = await fetchPlaylistPage(token);
-      // Start fetching extra details about videos
+        if (!nextPageToken || items.length >= playlist.contentDetails.itemCount) {
-      // PlaylistItem misses some details, eg. if the video is a livestream
+          return items;
-      videoDetailsPromises.push((async () => {
-        const videoDetailItems = await this.getVideosByID(items.map(item => item.contentDetails.videoId));
-        videoDetails.push(...videoDetailItems);
-      })());
         }
 
-    await Promise.all(videoDetailsPromises);
+        const nextItems = await fetchAllPages(nextPageToken);
+        return [...items, ...nextItems];
+      };
+
+      const playlistVideos = await fetchAllPages();
+
+      const videoDetailPromises = playlistVideos.map(async item =>
+        this.getVideosByID([item.contentDetails.videoId]),
+      );
+
+      const videoDetailChunks = await Promise.all(videoDetailPromises);
+      const videoDetails = videoDetailChunks.flat();
 
       const queuedPlaylist = {title: playlist.snippet.title, source: playlist.id};
-
       const songsToReturn: SongMetadata[] = [];
 
       for (const video of playlistVideos) {
         try {
-        songsToReturn.push(...this.getMetadataFromVideo({
+          const videoDetail = videoDetails.find(i => i.id === video.contentDetails.videoId);
-          video: videoDetails.find((i: {id: string}) => i.id === video.contentDetails.videoId)!,
+          if (videoDetail) {
+            songsToReturn.push(
+              ...this.getMetadataFromVideo({
+                video: videoDetail,
                 queuedPlaylist,
                 shouldSplitChapters,
-        }));
+              }),
-      } catch (_: unknown) {
+            );
-        // Private and deleted videos are sometimes in playlists, duration of these
+          }
-        // is not returned and they should not be added to the queue.
+        } catch (error) {
+          debug(`Skipping unavailable video in playlist: ${video.contentDetails.videoId}`);
         }
       }
 
+      if (songsToReturn.length === 0) {
+        throw new Error('No playable videos found in this playlist.');
+      }
+
       return songsToReturn;
+    } catch (error) {
+      debug('Playlist processing error:', error);
+
+      if (error instanceof Error) {
+        throw error;
+      }
+
+      throw new Error('Failed to process playlist. Please try again.');
+    }
+  }
+
+  private createYouTubeError(message: string, code: YouTubeError['code'], status?: number): YouTubeError {
+    const error = new Error(message) as YouTubeError;
+    error.code = code;
+    error.status = status;
+    error.retryable = code === 'NETWORK_ERROR' || (status ? status >= 500 : false);
+    return error;
+  }
+
+  private createCacheKey(prefix: string, key: string): string {
+    const fullKey = `${prefix}-${key}`;
+    if (fullKey.length <= MAX_CACHE_KEY_LENGTH) {
+      return fullKey;
+    }
+
+    const hash = crypto.createHash('sha1').update(key).digest('hex');
+    return `${prefix}-${key.slice(0, MAX_CACHE_KEY_LENGTH - prefix.length - 41)}-${hash}`;
+  }
+
+  private async executeYouTubeRequest<T>(endpoint: string, params: any): Promise<T> {
+    return pRetry(
+      async () => {
+        try {
+          const response = (await this.got(endpoint, params).json()) as T;
+
+          if (!response) {
+            throw this.createYouTubeError('Empty response from YouTube API', 'NETWORK_ERROR');
+          }
+
+          return response;
+        } catch (error) {
+          if (error instanceof HTTPError) {
+            const status = error.response.statusCode;
+
+            switch (status) {
+              case 403:
+                throw this.createYouTubeError(
+                  'YouTube API quota exceeded. Please try again later.',
+                  'QUOTA_EXCEEDED',
+                  status,
+                );
+              case 429:
+                throw this.createYouTubeError(
+                  'YouTube API rate limit reached. Please try again later.',
+                  'RATE_LIMITED',
+                  status,
+                );
+              case 404:
+                throw this.createYouTubeError('Resource not found on YouTube.', 'NOT_FOUND', status);
+              default:
+                if (status >= 500) {
+                  throw this.createYouTubeError('YouTube API is temporarily unavailable.', 'NETWORK_ERROR', status);
+                }
+
+                throw this.createYouTubeError('YouTube API request failed.', 'UNKNOWN', status);
+            }
+          }
+
+          if (error instanceof RequestError && error.code === 'ETIMEDOUT') {
+            throw this.createYouTubeError('YouTube API request timed out.', 'NETWORK_ERROR');
+          }
+
+          throw error;
+        }
+      },
+      {
+        retries: YOUTUBE_MAX_RETRY_COUNT,
+        minTimeout: YOUTUBE_BASE_RETRY_DELAY_MS,
+        factor: 2,
+        randomize: true,
+        onFailedAttempt: error => {
+          const youTubeError
+            = error.message && typeof error.message === 'object' && 'code' in error.message
+              ? (error.message as YouTubeError)
+              : null;
+
+          debug(
+            [
+              `YouTube API request failed (attempt ${error.attemptNumber}/${YOUTUBE_MAX_RETRY_COUNT + 1})`,
+              `Error code: ${youTubeError?.code ?? 'UNKNOWN'}`,
+              `Status: ${youTubeError?.status ?? 'N/A'}`,
+              `Message: ${error.message}`,
+              `Retries left: ${error.retriesLeft}`,
+            ].join('\n'),
+          );
+
+          if (youTubeError && !youTubeError.retryable) {
+            throw error;
+          }
+        },
+      },
+    );
+  }
+
+  private async getVideosByID(videoIDs: string[]): Promise<VideoDetailsResponse[]> {
+    if (videoIDs.length === 0) {
+      return [];
+    }
+
+    const params = {
+      searchParams: {
+        part: 'id, snippet, contentDetails',
+        id: videoIDs.join(','),
+      },
+    };
+
+    try {
+      const {items: videos} = await this.cache.wrap(
+        async () => this.executeYouTubeRequest<{items: VideoDetailsResponse[]}>('videos', params),
+        params,
+        {
+          expiresIn: ONE_HOUR_IN_SECONDS,
+          key: this.createCacheKey('youtube-videos', videoIDs.join(',')),
+        },
+      );
+
+      return videos;
+    } catch (error) {
+      if (error instanceof Error && 'code' in error) {
+        const youTubeError = error as YouTubeError;
+        if (youTubeError.code === 'QUOTA_EXCEEDED' || youTubeError.code === 'RATE_LIMITED') {
+          throw error;
+        }
+      }
+
+      throw new Error('Failed to fetch video information. Please try again.');
+    }
   }
 
   private getMetadataFromVideo({
@@ -196,7 +375,7 @@ export default class {
     queuedPlaylist,
     shouldSplitChapters,
   }: {
-    video: VideoDetailsResponse; // | YoutubePlaylistItem;
+    video: VideoDetailsResponse;
     queuedPlaylist?: QueuedPlaylist;
     shouldSplitChapters?: boolean;
   }): SongMetadata[] {
@@ -217,26 +396,22 @@ export default class {
     }
 
     const chapters = this.parseChaptersFromDescription(video.snippet.description, base.length);
-
     if (!chapters) {
       return [base];
     }
 
-    const tracks: SongMetadata[] = [];
+    return Array.from(chapters.entries()).map(([label, {offset, length}]) => ({
-
-    for (const [label, {offset, length}] of chapters) {
-      tracks.push({
       ...base,
       offset,
       length,
       title: `${label} (${base.title})`,
-      });
+    }));
   }
 
-    return tracks;
+  private parseChaptersFromDescription(
-  }
+    description: string,
-
+    videoDurationSeconds: number,
-  private parseChaptersFromDescription(description: string, videoDurationSeconds: number) {
+  ) {
     const map = new Map<string, {offset: number; length: number}>();
     let foundFirstTimestamp = false;
 
@@ -248,6 +423,7 @@ export default class {
       }
 
       if (!foundFirstTimestamp) {
+        // We expect the first timestamp to match something like "0:00" or "00:00"
         if (/0{1,2}:00/.test(timestamps[0][0])) {
           foundFirstTimestamp = true;
         } else {
@@ -265,34 +441,14 @@ export default class {
     for (const [i, {name, offset}] of foundTimestamps.entries()) {
       map.set(name, {
         offset,
-        length: i === foundTimestamps.length - 1
+        length:
+          i === foundTimestamps.length - 1
             ? videoDurationSeconds - offset
             : foundTimestamps[i + 1].offset - offset,
       });
     }
 
-    if (!map.size) {
+    return map.size > 0 ? map : null;
-      return null;
+  }
 }
 
-    return map;
-  }
-
-  private async getVideosByID(videoIDs: string[]): Promise<VideoDetailsResponse[]> {
-    const p = {
-      searchParams: {
-        part: 'id, snippet, contentDetails',
-        id: videoIDs.join(','),
-      },
-    };
-
-    const {items: videos} = await this.cache.wrap(
-      async () => this.got('videos', p).json() as Promise<{items: VideoDetailsResponse[]}>,
-      p,
-      {
-        expiresIn: ONE_HOUR_IN_SECONDS,
-      },
-    );
-    return videos;
-  }
-}