From 28d3e1a395c614b30568d796265573ac57e5752b Mon Sep 17 00:00:00 2001 From: WardPearce Date: Wed, 2 Apr 2025 15:38:59 +1300 Subject: [PATCH] Completed android potoken minting --- materialious/capacitor.config.ts | 4 + materialious/electron/capacitor.config.ts | 2 +- materialious/package-lock.json | 68 +++--- .../src/lib/android/http/androidRequests.ts | 6 +- .../src/lib/android/http/capacitorFetch.ts | 217 ------------------ .../src/lib/android/youtube/minter.ts | 57 +++++ materialious/src/lib/components/Player.svelte | 1 - materialious/src/lib/patches/youtubejs.ts | 85 +------ materialious/static/nodejs-android/index.js | 4 +- 9 files changed, 108 insertions(+), 336 deletions(-) delete mode 100644 materialious/src/lib/android/http/capacitorFetch.ts create mode 100644 materialious/src/lib/android/youtube/minter.ts diff --git a/materialious/capacitor.config.ts b/materialious/capacitor.config.ts index e432084b..0847dd5c 100644 --- a/materialious/capacitor.config.ts +++ b/materialious/capacitor.config.ts @@ -9,6 +9,10 @@ const config: CapacitorConfig = { nodeDir: 'nodejs-android', } }, + server: { + hostname: 'www.youtube.com', + androidScheme: 'https' + } }; export default config; diff --git a/materialious/electron/capacitor.config.ts b/materialious/electron/capacitor.config.ts index ab267b2c..e432084b 100644 --- a/materialious/electron/capacitor.config.ts +++ b/materialious/electron/capacitor.config.ts @@ -6,7 +6,7 @@ const config: CapacitorConfig = { webDir: 'build', plugins: { CapacitorNodeJS: { - nodeDir: 'nodejs-android' + nodeDir: 'nodejs-android', } }, }; diff --git a/materialious/package-lock.json b/materialious/package-lock.json index 1f01886e..44ab5ac9 100644 --- a/materialious/package-lock.json +++ b/materialious/package-lock.json @@ -1787,9 +1787,9 @@ } }, "node_modules/@capacitor/android": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/@capacitor/android/-/android-6.2.0.tgz", - "integrity": "sha512-3YIDPylV0Q2adEQ/H568p496QdYG0jK/XGMdx7OGSqdBZen92ciAsYdyhLtyl91UVsN1lBhDi5H6j3T2KS6aJg==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/@capacitor/android/-/android-6.2.1.tgz", + "integrity": "sha512-8gd4CIiQO5LAIlPIfd5mCuodBRxMMdZZEdj8qG8m+dQ1sQ2xyemVpzHmRK8qSCHorsBUCg3D62j2cp6bEBAkdw==", "license": "MIT", "peerDependencies": { "@capacitor/core": "^6.2.0" @@ -1957,9 +1957,9 @@ } }, "node_modules/@capacitor/core": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/@capacitor/core/-/core-6.2.0.tgz", - "integrity": "sha512-B9IlJtDpUqhhYb+T8+cp2Db/3RETX36STgjeU2kQZBs/SLAcFiMama227o+msRjLeo3DO+7HJjWVA1+XlyyPEg==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/@capacitor/core/-/core-6.2.1.tgz", + "integrity": "sha512-urZwxa7hVE/BnA18oCFAdizXPse6fCKanQyEqpmz6cBJ2vObwMpyJDG5jBeoSsgocS9+Ax+9vb4ducWJn0y2qQ==", "license": "MIT", "dependencies": { "tslib": "^2.1.0" @@ -9408,31 +9408,6 @@ } } }, - "node_modules/node-fetch/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-fetch/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true, - "license": "BSD-2-Clause" - }, - "node_modules/node-fetch/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/node-html-parser": { "version": "5.4.2", "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-5.4.2.tgz", @@ -12076,9 +12051,9 @@ } }, "node_modules/svelte": { - "version": "5.25.3", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.25.3.tgz", - "integrity": "sha512-J9rcZ/xVJonAoESqVGHHZhrNdVbrCfkdB41BP6eiwHMoFShD9it3yZXApVYMHdGfCshBsZCKsajwJeBbS/M1zg==", + "version": "5.25.5", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.25.5.tgz", + "integrity": "sha512-ULi9rkVWQJyJYZSpy6SIgSTchWadyWG1QYAUx3JAXL2gXrnhdXtoB20KmXGSNdtNyquq3eYd/gkwAkLcL5PGWw==", "license": "MIT", "dependencies": { "@ampproject/remapping": "^2.3.0", @@ -12488,6 +12463,13 @@ "node": ">=6" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, "node_modules/tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", @@ -13531,6 +13513,13 @@ "defaults": "^1.0.3" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, "node_modules/webrtc-adapter": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.1.tgz", @@ -13544,6 +13533,17 @@ "npm": ">=3.10.0" } }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/materialious/src/lib/android/http/androidRequests.ts b/materialious/src/lib/android/http/androidRequests.ts index 367d9aea..ff355a23 100644 --- a/materialious/src/lib/android/http/androidRequests.ts +++ b/materialious/src/lib/android/http/androidRequests.ts @@ -4,13 +4,9 @@ import { Capacitor } from "@capacitor/core"; if (Capacitor.getPlatform() === 'android') { const originalFetch = window.fetch; - const currentOrigin: string = window.location.protocol + '//' + window.location.host; - function needsProxying(target: string): boolean { if (!target.startsWith('http')) return false; - - const targetOriginMatch = /^https?:\/\/([^\/]+)/i.exec(target); - return (targetOriginMatch && targetOriginMatch[0].toLowerCase()) !== currentOrigin; + return true; } const corsProxyUrl: string = 'http://localhost:3000/'; diff --git a/materialious/src/lib/android/http/capacitorFetch.ts b/materialious/src/lib/android/http/capacitorFetch.ts deleted file mode 100644 index 26dea032..00000000 --- a/materialious/src/lib/android/http/capacitorFetch.ts +++ /dev/null @@ -1,217 +0,0 @@ -import type { HttpResponse } from '@capacitor/core'; -import { CapacitorHttp } from '@capacitor/core'; -import type { CapFormDataEntry, WindowCapacitor } from '@capacitor/core/types/definitions-internal'; - -// Rip from https://github.com/ionic-team/capacitor/blob/912d532cf6c7424d180b185120e7a9ba9c1ae050/core/native-bridge.ts -// with some modifications. - -const CAPACITOR_HTTP_INTERCEPTOR = '/_capacitor_http_interceptor_'; -const CAPACITOR_HTTP_INTERCEPTOR_URL_PARAM = 'u'; - -const isRelativeOrProxyUrl = (url: string | undefined): boolean => - !url || - !(url.startsWith('http:') || url.startsWith('https:')) || - url.indexOf(CAPACITOR_HTTP_INTERCEPTOR) > -1; - -const createProxyUrl = (url: string, win: WindowCapacitor): string => { - if (isRelativeOrProxyUrl(url)) return url; - const bridgeUrl = new URL(win.Capacitor?.getServerUrl() ?? ''); - bridgeUrl.pathname = CAPACITOR_HTTP_INTERCEPTOR; - bridgeUrl.searchParams.append(CAPACITOR_HTTP_INTERCEPTOR_URL_PARAM, url); - - return bridgeUrl.toString(); -}; - -const readFileAsBase64 = (file: File): Promise => - new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onloadend = () => { - const data = reader.result as string; - resolve(btoa(data)); - }; - reader.onerror = reject; - - reader.readAsBinaryString(file); - }); - -const convertFormData = async (formData: FormData): Promise => { - const newFormData: CapFormDataEntry[] = []; - for (const pair of formData.entries()) { - const [key, value] = pair; - if (value instanceof File) { - const base64File = await readFileAsBase64(value); - newFormData.push({ - key, - value: base64File, - type: 'base64File', - contentType: value.type, - fileName: value.name, - }); - } else { - newFormData.push({ key, value, type: 'string' }); - } - } - - return newFormData; -}; - -const convertBody = async ( - body: Document | XMLHttpRequestBodyInit | ReadableStream | undefined, - contentType?: string, -): Promise => { - if (body instanceof ReadableStream || body instanceof Uint8Array) { - let encodedData; - if (body instanceof ReadableStream) { - const reader = body.getReader(); - const chunks: any[] = []; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - } - const concatenated = new Uint8Array( - chunks.reduce((acc, chunk) => acc + chunk.length, 0), - ); - let position = 0; - for (const chunk of chunks) { - concatenated.set(chunk, position); - position += chunk.length; - } - encodedData = concatenated; - } else { - encodedData = body; - } - - let data = new TextDecoder().decode(encodedData); - let type; - if (contentType === 'application/json') { - try { - data = JSON.parse(data); - } catch (ignored) { - // ignore - } - type = 'json'; - } else if (contentType === 'multipart/form-data') { - type = 'formData'; - } else if (contentType?.startsWith('image')) { - type = 'image'; - } else if (contentType === 'application/octet-stream') { - type = 'binary'; - } else { - type = 'text'; - } - - return { - data, - type, - headers: { 'Content-Type': contentType || 'application/octet-stream' }, - }; - } else if (body instanceof URLSearchParams) { - return { - data: body.toString(), - type: 'text', - }; - } else if (body instanceof FormData) { - const formData = await convertFormData(body); - return { - data: formData, - type: 'formData', - }; - } else if (body instanceof File) { - const fileData = await readFileAsBase64(body); - return { - data: fileData, - type: 'file', - headers: { 'Content-Type': body.type }, - }; - } - - return { data: body, type: 'json' }; -}; - -export const capacitorFetch = async ( - resource: RequestInfo | URL, - options?: RequestInit, -) => { - const request = new Request(resource, options); - const { method } = request; - if ( - method.toLocaleUpperCase() === 'GET' || - method.toLocaleUpperCase() === 'HEAD' || - method.toLocaleUpperCase() === 'OPTIONS' || - method.toLocaleUpperCase() === 'TRACE' - ) { - if (typeof resource === 'string') { - return await fetch( - createProxyUrl(resource, window as WindowCapacitor), - options, - ); - } else if (resource instanceof Request) { - const modifiedRequest = new Request( - createProxyUrl(resource.url, window as WindowCapacitor), - resource, - ); - return await fetch(modifiedRequest, options); - } - } - - const tag = `CapacitorHttp fetch ${Date.now()} ${resource}`; - console.time(tag); - - try { - const { body } = request; - const optionHeaders = Object.fromEntries(request.headers.entries()); - const { - data: requestData, - type, - headers, - } = await convertBody( - options?.body || body || undefined, - optionHeaders['Content-Type'] || optionHeaders['content-type'], - ); - - const nativeResponse: HttpResponse = await CapacitorHttp.request({ - url: request.url, - method: method, - data: requestData, - dataType: type, - headers: { - ...headers, - ...optionHeaders, - }, - }); - - const contentType = - nativeResponse.headers['Content-Type'] || - nativeResponse.headers['content-type']; - let data = contentType?.startsWith('application/json') - ? JSON.stringify(nativeResponse.data) - : nativeResponse.data; - - // use null data for 204 No Content HTTP response - if (nativeResponse.status === 204) { - data = null; - } - - // intercept & parse response before returning - const response = new Response(data, { - headers: nativeResponse.headers, - status: nativeResponse.status, - }); - - /* - * copy url to response, `cordova-plugin-ionic` uses this url from the response - * we need `Object.defineProperty` because url is an inherited getter on the Response - * see: https://stackoverflow.com/a/57382543 - * */ - Object.defineProperty(response, 'url', { - value: nativeResponse.url, - }); - - console.timeEnd(tag); - return response; - } catch (error) { - console.timeEnd(tag); - return Promise.reject(error); - } -}; \ No newline at end of file diff --git a/materialious/src/lib/android/youtube/minter.ts b/materialious/src/lib/android/youtube/minter.ts new file mode 100644 index 00000000..7a362bf0 --- /dev/null +++ b/materialious/src/lib/android/youtube/minter.ts @@ -0,0 +1,57 @@ +import { BG, buildURL, GOOG_API_KEY, type WebPoSignalOutput } from 'bgutils-js'; +import { Innertube } from 'youtubei.js'; + +type WebPoMinter = { + sessionPoToken: string; + contentPoToken: string; +}; + +export async function androidPoTokenMinter(youtube: Innertube, videoId: string): Promise { + const requestKey = 'O43z0dpjhgX20SCx4KAo'; + + const challengeResponse = await youtube.getAttestationChallenge('ENGAGEMENT_TYPE_UNBOUND'); + + if (!challengeResponse.bg_challenge) + throw new Error('Yt.js: Could not get challenge'); + + const interpreterUrl = challengeResponse.bg_challenge.interpreter_url.private_do_not_access_or_else_trusted_resource_url_wrapped_value; + const bgScriptResponse = await fetch(`https:${interpreterUrl}`); + const interpreterJavascript = await bgScriptResponse.text(); + + if (interpreterJavascript) { + new Function(interpreterJavascript)(); + } else throw new Error('Could not load VM'); + + const botguardClient = await BG.BotGuardClient.create({ + program: challengeResponse.bg_challenge.program, + globalName: challengeResponse.bg_challenge.global_name, + globalObj: globalThis + }); + + const webPoSignalOutput: WebPoSignalOutput = []; + const botguardResponse = await botguardClient.snapshot({ webPoSignalOutput }); + + const integrityTokenResponse = await fetch(buildURL('GenerateIT', true), { + method: 'POST', + headers: { + 'content-type': 'application/json+protobuf', + 'x-goog-api-key': GOOG_API_KEY, + 'x-user-agent': 'grpc-web-javascript/0.1', + }, + body: JSON.stringify([requestKey, botguardResponse]) + }); + + const integrityTokenResponseData = await integrityTokenResponse.json(); + + if (typeof integrityTokenResponseData[0] !== 'string') + throw new Error('Yt.js: Could not get integrity token'); + + const integrityToken = integrityTokenResponseData[0]; + + const integrityTokenBasedMinter = await BG.WebPoMinter.create({ integrityToken }, webPoSignalOutput); + + return { + sessionPoToken: await integrityTokenBasedMinter.mintAsWebsafeString(youtube.session.context.client.visitorData ?? ''), + contentPoToken: await integrityTokenBasedMinter.mintAsWebsafeString(videoId) + }; +} \ No newline at end of file diff --git a/materialious/src/lib/components/Player.svelte b/materialious/src/lib/components/Player.svelte index c3d0c271..2eeecb88 100644 --- a/materialious/src/lib/components/Player.svelte +++ b/materialious/src/lib/components/Player.svelte @@ -153,7 +153,6 @@ url.searchParams.set('srfvp', '1'); const cachedPoToken = get(poTokenCacheStore); - if (cachedPoToken) url.searchParams.set('pot', cachedPoToken); delete request.headers.Range; diff --git a/materialious/src/lib/patches/youtubejs.ts b/materialious/src/lib/patches/youtubejs.ts index 8dab0136..8f05cd93 100644 --- a/materialious/src/lib/patches/youtubejs.ts +++ b/materialious/src/lib/patches/youtubejs.ts @@ -1,71 +1,13 @@ +import { androidPoTokenMinter } from '$lib/android/youtube/minter'; import type { AdaptiveFormats, Captions, Image, StoryBoard, Thumbnail, VideoBase, VideoPlay } from '$lib/api/model'; import { interfaceRegionStore, poTokenCacheStore } from '$lib/store'; import { numberWithCommas } from '$lib/time'; import { Capacitor } from '@capacitor/core'; -import { BG, buildURL, GOOG_API_KEY, type WebPoSignalOutput } from 'bgutils-js'; +import { USER_AGENT } from 'bgutils-js'; import { Buffer } from 'buffer'; import { get } from 'svelte/store'; import { Innertube, UniversalCache, YT, YTNodes } from 'youtubei.js'; -type WebPoMinter = { - integrityTokenBasedMinter?: BG.WebPoMinter; - botguardClient?: BG.BotGuardClient; -}; - -async function getWebPoMinter(youtube: Innertube): Promise { - const requestKey = 'O43z0dpjhgX20SCx4KAo'; - - const challengeResponse = await youtube.getAttestationChallenge('ENGAGEMENT_TYPE_UNBOUND'); - - if (!challengeResponse.bg_challenge) - throw new Error('Yt.js: Could not get challenge'); - - const interpreterUrl = challengeResponse.bg_challenge.interpreter_url.private_do_not_access_or_else_trusted_resource_url_wrapped_value; - const bgScriptResponse = await fetch(`https:${interpreterUrl}`); - const interpreterJavascript = await bgScriptResponse.text(); - - if (interpreterJavascript) { - new Function(interpreterJavascript)(); - } else throw new Error('Yt.js: Could not load VM'); - - const botguardClient = await BG.BotGuardClient.create({ - program: challengeResponse.bg_challenge.program, - globalName: challengeResponse.bg_challenge.global_name, - globalObj: globalThis - }); - - const webPoSignalOutput: WebPoSignalOutput = []; - const botguardResponse = await botguardClient.snapshot({ webPoSignalOutput }); - - console.log('botguardResponse', botguardResponse); - - const integrityTokenResponse = await fetch(buildURL('GenerateIT', true), { - method: 'POST', - headers: { - 'content-type': 'application/json+protobuf', - 'x-goog-api-key': GOOG_API_KEY, - 'x-user-agent': 'grpc-web-javascript/0.1', - }, - body: JSON.stringify([requestKey, botguardResponse]) - }); - - const integrityTokenResponseData = await integrityTokenResponse.json(); - - console.log(integrityTokenResponseData); - - if (typeof integrityTokenResponseData[0] !== 'string') - throw new Error('Yt.js: Could not get integrity token'); - - const integrityToken = integrityTokenResponseData[0] as string | undefined; - - const integrityTokenBasedMinter = await BG.WebPoMinter.create({ integrityToken }, webPoSignalOutput); - - return { - integrityTokenBasedMinter, - botguardClient - }; -} - export async function patchYoutubeJs(videoId: string): Promise { if (!Capacitor.isNativePlatform()) { throw new Error('Yt.js: Platform not supported'); @@ -73,20 +15,15 @@ export async function patchYoutubeJs(videoId: string): Promise { const youtube = await Innertube.create({ fetch: fetch, - generate_session_locally: true, cache: new UniversalCache(false), location: get(interfaceRegionStore), + user_agent: USER_AGENT, + enable_session_cache: false }); + const { sessionPoToken, contentPoToken } = await androidPoTokenMinter(youtube, videoId); - let sessionWebPo: string | undefined; - const { integrityTokenBasedMinter } = await getWebPoMinter(youtube); - - if (integrityTokenBasedMinter) { - sessionWebPo = await integrityTokenBasedMinter.mintAsWebsafeString(youtube.session.context.client.visitorData ?? ''); - } - - poTokenCacheStore.set(sessionWebPo); + poTokenCacheStore.set(sessionPoToken); const extraArgs: Record = { playbackContext: { @@ -97,17 +34,13 @@ export async function patchYoutubeJs(videoId: string): Promise { signatureTimestamp: youtube.session.player?.sts } }, + serviceIntegrityDimensions: { + poToken: contentPoToken + }, contentCheckOk: true, racyCheckOk: true }; - // Generate content WebPO token. - if (integrityTokenBasedMinter) { - extraArgs.serviceIntegrityDimensions = { - poToken: await integrityTokenBasedMinter.mintAsWebsafeString(videoId) - }; - } - const watchEndpoint = new YTNodes.NavigationEndpoint({ watchEndpoint: { videoId } }); const rawPlayerResponse = await watchEndpoint.call(youtube.actions, extraArgs); const rawNextResponse = await watchEndpoint.call(youtube.actions, { diff --git a/materialious/static/nodejs-android/index.js b/materialious/static/nodejs-android/index.js index b8240c9d..f3c41640 100644 --- a/materialious/static/nodejs-android/index.js +++ b/materialious/static/nodejs-android/index.js @@ -6,7 +6,7 @@ const PORT = 3000; const MAX_REDIRECTS = 10; const CORS_HEADERS = 'Origin, X-Requested-With, Content-Type, Accept, Authorization, x-goog-visitor-id, x-goog-api-key, x-origin, x-youtube-client-version, x-youtube-client-name, x-goog-api-format-version, x-user-agent, Accept-Language, Range, Referer' -const CORS_ORIGIN = 'https://localhost' +const CORS_ORIGIN = 'https://www.youtube.com' const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36(KHTML, like Gecko)' function setCorsHeaders(res) { @@ -67,7 +67,7 @@ const server = http.createServer(async (req, res) => { let parsedTarget; try { // Ensure protocol (http) is added if missing - if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) { + if (!targetUrl.startsWith('http')) { targetUrl = 'http://' + targetUrl; } parsedTarget = new URL(targetUrl);