refactor: Use patch-package to patch ytdl-core (#1006)

Dockerfile

@@ -9,7 +9,6 @@ RUN apt-get update \
     tini \
     openssl \
     ca-certificates \
-    git \
     && apt-get autoclean \
     && apt-get autoremove \
     && rm -rf /var/lib/apt/lists/*
@@ -20,6 +19,7 @@ FROM base AS dependencies
 WORKDIR /usr/app
 
 COPY package.json .
+COPY patches ./patches
 COPY yarn.lock .
 
 RUN yarn install --prod

package.json

@@ -28,7 +28,8 @@
     "prisma:generate": "prisma generate",
     "env:set-database-url": "tsx src/scripts/run-with-database-url.ts",
     "release": "release-it",
-    "build": "tsc"
+    "build": "tsc",
+    "postinstall": "patch-package"
   },
   "devDependencies": {
     "@release-it/keep-a-changelog": "^2.3.0",
@@ -107,6 +108,8 @@
     "p-retry": "4.6.2",
     "pagination.djs": "^4.0.10",
     "parse-duration": "1.0.2",
+    "patch-package": "^8.0.0",
+    "postinstall-postinstall": "^2.1.0",
     "read-pkg": "7.1.0",
     "reflect-metadata": "^0.1.13",
     "spotify-uri": "^3.0.2",
@@ -114,7 +117,7 @@
     "sync-fetch": "^0.3.1",
     "tsx": "3.8.2",
     "xbytes": "^1.7.0",
-    "ytdl-core": "git+https://github.com/khlevon/node-ytdl-core.git#v4.11.4-patch.2",
+    "ytdl-core": "^4.11.5",
     "ytsr": "^3.8.4"
   },
   "resolutions": {

patches/ytdl-core+4.11.5.patch

@@ -0,0 +1,172 @@
+diff --git a/node_modules/ytdl-core/lib/sig.js b/node_modules/ytdl-core/lib/sig.js
+index eb7bfaa..b2eee87 100644
+--- a/node_modules/ytdl-core/lib/sig.js
++++ b/node_modules/ytdl-core/lib/sig.js
+@@ -3,6 +3,9 @@ const Cache = require('./cache');
+ const utils = require('./utils');
+ const vm = require('vm');
+ 
++
++let nTransformWarning = false;
++
+ // A shared cache to keep track of html5player js functions.
+ exports.cache = new Cache();
+ 
+@@ -23,6 +26,49 @@ exports.getFunctions = (html5playerfile, options) => exports.cache.getOrSet(html
+   return functions;
+ });
+ 
++// eslint-disable-next-line max-len
++// https://github.com/TeamNewPipe/NewPipeExtractor/blob/41c8dce452aad278420715c00810b1fed0109adf/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java#L816
++const DECIPHER_REGEXPS = [
++  '(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)' +
++  '\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*""\\s*\\)',
++  '\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)',
++  '\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)',
++  '([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(""\\)\\s*;',
++  '\\b([\\w$]{2,})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(""\\)\\s*;',
++  '\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(',
++];
++
++const DECIPHER_ARGUMENT = 'sig';
++const N_ARGUMENT = 'ncode';
++
++const matchGroup1 = (regex, str) => {
++  const match = str.match(new RegExp(regex));
++  if (!match) throw new Error(`Could not match ${regex}`);
++  return match[1];
++};
++
++const getFuncName = (body, regexps) => {
++  try {
++    let fn;
++    for (const regex of regexps) {
++      try {
++        fn = matchGroup1(regex, body);
++        const idx = fn.indexOf('[0]');
++        if (idx > -1) fn = matchGroup1(`${fn.slice(0, 3)}=\\[([a-zA-Z0-9$\\[\\]]{2,})\\]`, body);
++      } catch (err) {
++        continue;
++      }
++    }
++    if (!fn || fn.includes('[')) throw Error("Couldn't find fn name");
++    return fn;
++  } catch (e) {
++    throw Error(`Please open an issue on ytdl-core GitHub: ${e.message}`);
++  }
++};
++
++const getDecipherFuncName = body => getFuncName(body, DECIPHER_REGEXPS);
++
++
+ /**
+  * Extracts the actions that should be taken to decipher a signature
+  * and tranform the n parameter
+@@ -31,44 +77,45 @@ exports.getFunctions = (html5playerfile, options) => exports.cache.getOrSet(html
+  * @returns {Array.<string>}
+  */
+ exports.extractFunctions = body => {
++  body = body.replace(/\n|\r/g, '');
+   const functions = [];
+-  const extractManipulations = caller => {
+-    const functionName = utils.between(caller, `a=a.split("");`, `.`);
+-    if (!functionName) return '';
+-    const functionStart = `var ${functionName}={`;
+-    const ndx = body.indexOf(functionStart);
+-    if (ndx < 0) return '';
+-    const subBody = body.slice(ndx + functionStart.length - 1);
+-    return `var ${functionName}=${utils.cutAfterJS(subBody)}`;
+-  };
++  // This is required function, so we can't continue if it's not found.
+   const extractDecipher = () => {
+-    const functionName = utils.between(body, `a.set("alr","yes");c&&(c=`, `(decodeURIC`);
+-    if (functionName && functionName.length) {
+-      const functionStart = `${functionName}=function(a)`;
+-      const ndx = body.indexOf(functionStart);
+-      if (ndx >= 0) {
+-        const subBody = body.slice(ndx + functionStart.length);
+-        let functionBody = `var ${functionStart}${utils.cutAfterJS(subBody)}`;
+-        functionBody = `${extractManipulations(functionBody)};${functionBody};${functionName}(sig);`;
+-        functions.push(functionBody);
+-      }
++    const decipherFuncName = getDecipherFuncName(body);
++    try {
++      const functionPattern = `(${decipherFuncName.replace(/\$/g, '\\$')}=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})`;
++      const decipherFunction = `var ${matchGroup1(functionPattern, body)};`;
++      const helperObjectName = matchGroup1(';([A-Za-z0-9_\\$]{2,})\\.\\w+\\(', decipherFunction)
++          .replace(/\$/g, '\\$');
++      const helperPattern = `(var ${helperObjectName}=\\{[\\s\\S]+?\\}\\};)`;
++      const helperObject = matchGroup1(helperPattern, body);
++      const callerFunction = `${decipherFuncName}(${DECIPHER_ARGUMENT});`;
++      const resultFunction = helperObject + decipherFunction + callerFunction;
++      functions.push(resultFunction);
++    } catch (err) {
++      throw Error(`Could not parse decipher function: ${err}`);
+     }
+   };
+-  const extractNCode = () => {
+-    let functionName = utils.between(body, `&&(b=a.get("n"))&&(b=`, `(b)`);
+-    if (functionName.includes('[')) functionName = utils.between(body, `var ${functionName.split('[')[0]}=[`, `]`);
+-    if (functionName && functionName.length) {
+-      const functionStart = `${functionName}=function(a)`;
+-      const ndx = body.indexOf(functionStart);
+-      if (ndx >= 0) {
+-        const subBody = body.slice(ndx + functionStart.length);
+-        const functionBody = `var ${functionStart}${utils.cutAfterJS(subBody)};${functionName}(ncode);`;
+-        functions.push(functionBody);
++  // This is optional, so we can continue if it's not found, but it will bottleneck the download.
++  const extractNTransform = () => {
++    let nFuncName = utils.between(body, `(b=a.get("n"))&&(b=`, `(b)`);
++    if (nFuncName.includes('[')) nFuncName = utils.between(body, `${nFuncName.split('[')[0]}=[`, `]`);
++    if (nFuncName && nFuncName.length) {
++      const nBegin = `${nFuncName}=function(a)`;
++      const nEnd = '.join("")};';
++      const nFunction = utils.between(body, nBegin, nEnd);
++      if (nFunction) {
++        const callerFunction = `${nFuncName}(${N_ARGUMENT});`;
++        const resultFunction = nBegin + nFunction + nEnd + callerFunction;
++        functions.push(resultFunction);
++      } else if (!nTransformWarning) {
++        console.warn('Could not parse n transform function, please report it on @distube/ytdl-core GitHub.');
++        nTransformWarning = true;
+       }
+     }
+   };
+   extractDecipher();
+-  extractNCode();
++  extractNTransform();
+   return functions;
+ };
+ 
+@@ -82,22 +129,25 @@ exports.extractFunctions = body => {
+ exports.setDownloadURL = (format, decipherScript, nTransformScript) => {
+   const decipher = url => {
+     const args = querystring.parse(url);
+-    if (!args.s || !decipherScript) return args.url;
++    if (!args.s) return args.url;
+     const components = new URL(decodeURIComponent(args.url));
+-    components.searchParams.set(args.sp ? args.sp : 'signature',
+-      decipherScript.runInNewContext({ sig: decodeURIComponent(args.s) }));
++    const context = {};
++    context[DECIPHER_ARGUMENT] = decodeURIComponent(args.s);
++    components.searchParams.set(args.sp || 'sig', decipherScript.runInNewContext(context));
+     return components.toString();
+   };
+-  const ncode = url => {
++  const nTransform = url => {
+     const components = new URL(decodeURIComponent(url));
+     const n = components.searchParams.get('n');
+     if (!n || !nTransformScript) return url;
+-    components.searchParams.set('n', nTransformScript.runInNewContext({ ncode: n }));
++    const context = {};
++    context[N_ARGUMENT] = n;
++    components.searchParams.set('n', nTransformScript.runInNewContext(context));
+     return components.toString();
+   };
+   const cipher = !format.url;
+   const url = format.url || format.signatureCipher || format.cipher;
+-  format.url = cipher ? ncode(decipher(url)) : ncode(url);
++  format.url = cipher ? nTransform(decipher(url)) : nTransform(url);
+   delete format.signatureCipher;
+   delete format.cipher;
+ };