From 0fcd6d0a40b64d5972367f17272258b5d5875910 Mon Sep 17 00:00:00 2001 From: Michael <13820335+shiftybitshiftr@users.noreply.github.com> Date: Mon, 27 Jan 2025 17:25:28 -0800 Subject: [PATCH] Revert "Merge branch 'master' into dependencies/ytdl-core/upgrade/4-15-9" This reverts commit b1a3ae3aaa6e077f22cf8f0b9b8daf3bb2569115, reversing changes made to ebb5803653b7f43977e0094fda8b8319224ead87. --- src/services/youtube-api.ts | 434 ++++++++++++------------------------ 1 file changed, 141 insertions(+), 293 deletions(-) diff --git a/src/services/youtube-api.ts b/src/services/youtube-api.ts index a2affb0..47b91d8 100644 --- a/src/services/youtube-api.ts +++ b/src/services/youtube-api.ts @@ -1,10 +1,8 @@ import {inject, injectable} from 'inversify'; import {toSeconds, parse} from 'iso8601-duration'; -import got, {Got, HTTPError, RequestError} from 'got'; +import got, {Got} from 'got'; import ytsr, {Video} from '@distube/ytsr'; import PQueue from 'p-queue'; -import pRetry from 'p-retry'; -import crypto from 'crypto'; import {SongMetadata, QueuedPlaylist, MediaSource} from './player.js'; import {TYPES} from '../types.js'; import Config from './config.js'; @@ -12,19 +10,6 @@ import KeyValueCacheProvider from './key-value-cache.js'; import {ONE_HOUR_IN_SECONDS, ONE_MINUTE_IN_SECONDS} from '../utils/constants.js'; import {parseTime} from '../utils/time.js'; import getYouTubeID from 'get-youtube-id'; -import debug from '../utils/debug.js'; - -// Define structured error type for better error handling -interface YouTubeError extends Error { - code: 'QUOTA_EXCEEDED' | 'RATE_LIMITED' | 'NOT_FOUND' | 'NETWORK_ERROR' | 'UNKNOWN'; - status?: number; - retryable: boolean; -} - -const YOUTUBE_MAX_RETRY_COUNT = 3; -const YOUTUBE_BASE_RETRY_DELAY_MS = 1000; -const YOUTUBE_SEARCH_CONCURRENCY = 4; -const MAX_CACHE_KEY_LENGTH = 250; interface VideoDetailsResponse { id: string; @@ -74,13 +59,10 @@ export default class { private readonly ytsrQueue: PQueue; private readonly got: Got; - constructor( - @inject(TYPES.Config) config: Config, - @inject(TYPES.KeyValueCache) cache: KeyValueCacheProvider, - ) { + constructor(@inject(TYPES.Config) config: Config, @inject(TYPES.KeyValueCache) cache: KeyValueCacheProvider) { this.youtubeKey = config.YOUTUBE_API_KEY; this.cache = cache; - this.ytsrQueue = new PQueue({concurrency: YOUTUBE_SEARCH_CONCURRENCY}); + this.ytsrQueue = new PQueue({concurrency: 4}); this.got = got.extend({ prefixUrl: 'https://www.googleapis.com/youtube/v3/', @@ -91,283 +73,126 @@ export default class { }); } - public async search(query: string, shouldSplitChapters: boolean): Promise { - try { - const {items} = await this.ytsrQueue.add(async () => - this.cache.wrap( - ytsr, - query, - { - limit: 10, - }, - { - expiresIn: ONE_HOUR_IN_SECONDS, - key: this.createCacheKey('youtube-search', query), - }, - ), - ); + async search(query: string, shouldSplitChapters: boolean): Promise { + const result = await this.ytsrQueue.add(async () => this.cache.wrap( + ytsr, + query, + { + limit: 10, + }, + { + expiresIn: ONE_HOUR_IN_SECONDS, + }, + )); - let firstVideo: Video | undefined; - for (const item of items) { - if (item.type === 'video') { - firstVideo = item; - break; - } - } - - if (!firstVideo) { - throw new Error('No matching videos found.'); - } - - return await this.getVideo(firstVideo.url, shouldSplitChapters); - } catch (error) { - debug('YouTube search error:', error); - throw new Error('Failed to search YouTube. Please try again.'); + if (result === undefined) { + return []; } + + let firstVideo: Video | undefined; + + for (const item of result.items) { + if (item.type === 'video') { + firstVideo = item; + break; + } + } + + if (!firstVideo) { + return []; + } + + return this.getVideo(firstVideo.url, shouldSplitChapters); } - public async getVideo(url: string, shouldSplitChapters: boolean): Promise { - const videoId = getYouTubeID(url); - if (!videoId) { - throw new Error('Invalid YouTube URL.'); - } - - const result = await this.getVideosByID([videoId]); + async getVideo(url: string, shouldSplitChapters: boolean): Promise { + const result = await this.getVideosByID([String(getYouTubeID(url))]); const video = result.at(0); if (!video) { - throw new Error('Video could not be found or is unavailable.'); + throw new Error('Video could not be found.'); } return this.getMetadataFromVideo({video, shouldSplitChapters}); } - public async getPlaylist(listId: string, shouldSplitChapters: boolean): Promise { - try { - const playlistParams = { - searchParams: { - part: 'id, snippet, contentDetails', - id: listId, - }, - }; - - const {items: playlists} = await this.cache.wrap( - async () => this.executeYouTubeRequest<{items: PlaylistResponse[]}>('playlists', playlistParams), - playlistParams, - { - expiresIn: ONE_MINUTE_IN_SECONDS, - key: this.createCacheKey('youtube-playlist', listId), - }, - ); - - const playlist = playlists.at(0); - if (!playlist) { - throw new Error('Playlist could not be found.'); - } - - // Helper function to fetch a single page of playlist items - const fetchPlaylistPage = async (token?: string) => { - const playlistItemsParams = { - searchParams: { - part: 'id, contentDetails', - playlistId: listId, - maxResults: '50', - pageToken: token, - }, - }; - - return this.cache.wrap( - async () => this.executeYouTubeRequest('playlistItems', playlistItemsParams), - playlistItemsParams, - { - expiresIn: ONE_MINUTE_IN_SECONDS, - key: this.createCacheKey('youtube-playlist-items', `${listId}-${token ?? 'initial'}`), - }, - ); - }; - - // Recursively fetch all playlist pages - const fetchAllPages = async (token?: string): Promise => { - const {items, nextPageToken} = await fetchPlaylistPage(token); - if (!nextPageToken || items.length >= playlist.contentDetails.itemCount) { - return items; - } - - const nextItems = await fetchAllPages(nextPageToken); - return [...items, ...nextItems]; - }; - - const playlistVideos = await fetchAllPages(); - - const videoDetailPromises = playlistVideos.map(async item => - this.getVideosByID([item.contentDetails.videoId]), - ); - - const videoDetailChunks = await Promise.all(videoDetailPromises); - const videoDetails = videoDetailChunks.flat(); - - const queuedPlaylist = {title: playlist.snippet.title, source: playlist.id}; - const songsToReturn: SongMetadata[] = []; - - for (const video of playlistVideos) { - try { - const videoDetail = videoDetails.find(i => i.id === video.contentDetails.videoId); - if (videoDetail) { - songsToReturn.push( - ...this.getMetadataFromVideo({ - video: videoDetail, - queuedPlaylist, - shouldSplitChapters, - }), - ); - } - } catch (error) { - debug(`Skipping unavailable video in playlist: ${video.contentDetails.videoId}`); - } - } - - if (songsToReturn.length === 0) { - throw new Error('No playable videos found in this playlist.'); - } - - return songsToReturn; - } catch (error) { - debug('Playlist processing error:', error); - - if (error instanceof Error) { - throw error; - } - - throw new Error('Failed to process playlist. Please try again.'); - } - } - - private createYouTubeError(message: string, code: YouTubeError['code'], status?: number): YouTubeError { - const error = new Error(message) as YouTubeError; - error.code = code; - error.status = status; - error.retryable = code === 'NETWORK_ERROR' || (status ? status >= 500 : false); - return error; - } - - private createCacheKey(prefix: string, key: string): string { - const fullKey = `${prefix}-${key}`; - if (fullKey.length <= MAX_CACHE_KEY_LENGTH) { - return fullKey; - } - - const hash = crypto.createHash('sha1').update(key).digest('hex'); - return `${prefix}-${key.slice(0, MAX_CACHE_KEY_LENGTH - prefix.length - 41)}-${hash}`; - } - - private async executeYouTubeRequest(endpoint: string, params: any): Promise { - return pRetry( - async () => { - try { - const response = (await this.got(endpoint, params).json()) as T; - - if (!response) { - throw this.createYouTubeError('Empty response from YouTube API', 'NETWORK_ERROR'); - } - - return response; - } catch (error) { - if (error instanceof HTTPError) { - const status = error.response.statusCode; - - switch (status) { - case 403: - throw this.createYouTubeError( - 'YouTube API quota exceeded. Please try again later.', - 'QUOTA_EXCEEDED', - status, - ); - case 429: - throw this.createYouTubeError( - 'YouTube API rate limit reached. Please try again later.', - 'RATE_LIMITED', - status, - ); - case 404: - throw this.createYouTubeError('Resource not found on YouTube.', 'NOT_FOUND', status); - default: - if (status >= 500) { - throw this.createYouTubeError('YouTube API is temporarily unavailable.', 'NETWORK_ERROR', status); - } - - throw this.createYouTubeError('YouTube API request failed.', 'UNKNOWN', status); - } - } - - if (error instanceof RequestError && error.code === 'ETIMEDOUT') { - throw this.createYouTubeError('YouTube API request timed out.', 'NETWORK_ERROR'); - } - - throw error; - } - }, - { - retries: YOUTUBE_MAX_RETRY_COUNT, - minTimeout: YOUTUBE_BASE_RETRY_DELAY_MS, - factor: 2, - randomize: true, - onFailedAttempt: error => { - const youTubeError - = error.message && typeof error.message === 'object' && 'code' in error.message - ? (error.message as YouTubeError) - : null; - - debug( - [ - `YouTube API request failed (attempt ${error.attemptNumber}/${YOUTUBE_MAX_RETRY_COUNT + 1})`, - `Error code: ${youTubeError?.code ?? 'UNKNOWN'}`, - `Status: ${youTubeError?.status ?? 'N/A'}`, - `Message: ${error.message}`, - `Retries left: ${error.retriesLeft}`, - ].join('\n'), - ); - - if (youTubeError && !youTubeError.retryable) { - throw error; - } - }, - }, - ); - } - - private async getVideosByID(videoIDs: string[]): Promise { - if (videoIDs.length === 0) { - return []; - } - - const params = { + async getPlaylist(listId: string, shouldSplitChapters: boolean): Promise { + const playlistParams = { searchParams: { part: 'id, snippet, contentDetails', - id: videoIDs.join(','), + id: listId, }, }; + const {items: playlists} = await this.cache.wrap( + async () => this.got('playlists', playlistParams).json() as Promise<{items: PlaylistResponse[]}>, + playlistParams, + { + expiresIn: ONE_MINUTE_IN_SECONDS, + }, + ); - try { - const {items: videos} = await this.cache.wrap( - async () => this.executeYouTubeRequest<{items: VideoDetailsResponse[]}>('videos', params), - params, + const playlist = playlists.at(0)!; + + if (!playlist) { + throw new Error('Playlist could not be found.'); + } + + const playlistVideos: PlaylistItem[] = []; + const videoDetailsPromises: Array> = []; + const videoDetails: VideoDetailsResponse[] = []; + + let nextToken: string | undefined; + + while (playlistVideos.length < playlist.contentDetails.itemCount) { + const playlistItemsParams = { + searchParams: { + part: 'id, contentDetails', + playlistId: listId, + maxResults: '50', + pageToken: nextToken, + }, + }; + + // eslint-disable-next-line no-await-in-loop + const {items, nextPageToken} = await this.cache.wrap( + async () => this.got('playlistItems', playlistItemsParams).json() as Promise, + playlistItemsParams, { - expiresIn: ONE_HOUR_IN_SECONDS, - key: this.createCacheKey('youtube-videos', videoIDs.join(',')), + expiresIn: ONE_MINUTE_IN_SECONDS, }, ); - return videos; - } catch (error) { - if (error instanceof Error && 'code' in error) { - const youTubeError = error as YouTubeError; - if (youTubeError.code === 'QUOTA_EXCEEDED' || youTubeError.code === 'RATE_LIMITED') { - throw error; - } - } + nextToken = nextPageToken; + playlistVideos.push(...items); - throw new Error('Failed to fetch video information. Please try again.'); + // Start fetching extra details about videos + // PlaylistItem misses some details, eg. if the video is a livestream + videoDetailsPromises.push((async () => { + const videoDetailItems = await this.getVideosByID(items.map(item => item.contentDetails.videoId)); + videoDetails.push(...videoDetailItems); + })()); } + + await Promise.all(videoDetailsPromises); + + const queuedPlaylist = {title: playlist.snippet.title, source: playlist.id}; + + const songsToReturn: SongMetadata[] = []; + + for (const video of playlistVideos) { + try { + songsToReturn.push(...this.getMetadataFromVideo({ + video: videoDetails.find((i: {id: string}) => i.id === video.contentDetails.videoId)!, + queuedPlaylist, + shouldSplitChapters, + })); + } catch (_: unknown) { + // Private and deleted videos are sometimes in playlists, duration of these + // is not returned and they should not be added to the queue. + } + } + + return songsToReturn; } private getMetadataFromVideo({ @@ -375,7 +200,7 @@ export default class { queuedPlaylist, shouldSplitChapters, }: { - video: VideoDetailsResponse; + video: VideoDetailsResponse; // | YoutubePlaylistItem; queuedPlaylist?: QueuedPlaylist; shouldSplitChapters?: boolean; }): SongMetadata[] { @@ -396,22 +221,26 @@ export default class { } const chapters = this.parseChaptersFromDescription(video.snippet.description, base.length); + if (!chapters) { return [base]; } - return Array.from(chapters.entries()).map(([label, {offset, length}]) => ({ - ...base, - offset, - length, - title: `${label} (${base.title})`, - })); + const tracks: SongMetadata[] = []; + + for (const [label, {offset, length}] of chapters) { + tracks.push({ + ...base, + offset, + length, + title: `${label} (${base.title})`, + }); + } + + return tracks; } - private parseChaptersFromDescription( - description: string, - videoDurationSeconds: number, - ) { + private parseChaptersFromDescription(description: string, videoDurationSeconds: number) { const map = new Map(); let foundFirstTimestamp = false; @@ -423,7 +252,6 @@ export default class { } if (!foundFirstTimestamp) { - // We expect the first timestamp to match something like "0:00" or "00:00" if (/0{1,2}:00/.test(timestamps[0][0])) { foundFirstTimestamp = true; } else { @@ -441,14 +269,34 @@ export default class { for (const [i, {name, offset}] of foundTimestamps.entries()) { map.set(name, { offset, - length: - i === foundTimestamps.length - 1 - ? videoDurationSeconds - offset - : foundTimestamps[i + 1].offset - offset, + length: i === foundTimestamps.length - 1 + ? videoDurationSeconds - offset + : foundTimestamps[i + 1].offset - offset, }); } - return map.size > 0 ? map : null; + if (!map.size) { + return null; + } + + return map; + } + + private async getVideosByID(videoIDs: string[]): Promise { + const p = { + searchParams: { + part: 'id, snippet, contentDetails', + id: videoIDs.join(','), + }, + }; + + const {items: videos} = await this.cache.wrap( + async () => this.got('videos', p).json() as Promise<{items: VideoDetailsResponse[]}>, + p, + { + expiresIn: ONE_HOUR_IN_SECONDS, + }, + ); + return videos; } } -