From 52126e53664f2f7cdd7309d518f40640e0ced52b Mon Sep 17 00:00:00 2001 From: WardPearce Date: Tue, 1 Apr 2025 13:07:59 +1300 Subject: [PATCH] Update local fallback to latest logic --- materialious/capacitor.config.ts | 2 +- materialious/electron/src/setup.ts | 5 + .../src/lib/android/http/androidRequests.ts | 26 ++- materialious/src/lib/components/Player.svelte | 6 +- materialious/src/lib/patches/youtubejs.ts | 154 ++++++++++++------ materialious/src/lib/store.ts | 2 +- materialious/static/nodejs-android/index.js | 13 +- 7 files changed, 144 insertions(+), 64 deletions(-) diff --git a/materialious/capacitor.config.ts b/materialious/capacitor.config.ts index ab267b2c..e432084b 100644 --- a/materialious/capacitor.config.ts +++ b/materialious/capacitor.config.ts @@ -6,7 +6,7 @@ const config: CapacitorConfig = { webDir: 'build', plugins: { CapacitorNodeJS: { - nodeDir: 'nodejs-android' + nodeDir: 'nodejs-android', } }, }; diff --git a/materialious/electron/src/setup.ts b/materialious/electron/src/setup.ts index 17ec8dc0..49b8dfb3 100644 --- a/materialious/electron/src/setup.ts +++ b/materialious/electron/src/setup.ts @@ -229,7 +229,12 @@ export function setupContentSecurityPolicy(customScheme: string): void { }); session.defaultSession.webRequest.onBeforeSendHeaders((details, callback) => { + const uri = new URL(details.url); + details.requestHeaders['User-Agent'] = USER_AGENT; + details.requestHeaders['origin'] = uri.origin; + details.requestHeaders['host'] = uri.host; + callback({ requestHeaders: details.requestHeaders }); }); } diff --git a/materialious/src/lib/android/http/androidRequests.ts b/materialious/src/lib/android/http/androidRequests.ts index 1bc95d1f..367d9aea 100644 --- a/materialious/src/lib/android/http/androidRequests.ts +++ b/materialious/src/lib/android/http/androidRequests.ts @@ -7,6 +7,8 @@ if (Capacitor.getPlatform() === 'android') { const currentOrigin: string = window.location.protocol + '//' + window.location.host; function needsProxying(target: string): boolean { + if (!target.startsWith('http')) return false; + const targetOriginMatch = /^https?:\/\/([^\/]+)/i.exec(target); return (targetOriginMatch && targetOriginMatch[0].toLowerCase()) !== currentOrigin; } @@ -14,16 +16,28 @@ if (Capacitor.getPlatform() === 'android') { const corsProxyUrl: string = 'http://localhost:3000/'; window.fetch = async (requestInput: string | URL | Request, requestOptions?: RequestInit): Promise => { - const uri = requestInput.toString(); - - console.log(uri, needsProxying(uri)); + const uri = requestInput instanceof Request ? requestInput.url : requestInput.toString(); if (needsProxying(uri)) { - requestInput = corsProxyUrl + uri; + if (requestInput instanceof Request) { + requestInput = new Request(corsProxyUrl + uri, { + method: requestInput.method, + headers: requestInput.headers, + body: requestInput.body, + mode: requestInput.mode, + credentials: requestInput.credentials, + cache: requestInput.cache, + redirect: requestInput.redirect, + referrer: requestInput.referrer, + integrity: requestInput.integrity, + keepalive: requestInput.keepalive, + ...(requestInput.body ? { duplex: "half" } : {}) + }); + } else { + requestInput = corsProxyUrl + uri; + } } - console.log(requestInput); - // Use the original fetch with the proxied URL and options return originalFetch(requestInput, requestOptions); }; diff --git a/materialious/src/lib/components/Player.svelte b/materialious/src/lib/components/Player.svelte index 45f2bcef..c3d0c271 100644 --- a/materialious/src/lib/components/Player.svelte +++ b/materialious/src/lib/components/Player.svelte @@ -151,7 +151,11 @@ url.searchParams.set('range', request.headers.Range.split('=')[1]); url.searchParams.set('ump', '1'); url.searchParams.set('srfvp', '1'); - url.searchParams.set('pot', get(poTokenCacheStore).poToken); + + const cachedPoToken = get(poTokenCacheStore); + + if (cachedPoToken) url.searchParams.set('pot', cachedPoToken); + delete request.headers.Range; } diff --git a/materialious/src/lib/patches/youtubejs.ts b/materialious/src/lib/patches/youtubejs.ts index dfdc8a52..8dab0136 100644 --- a/materialious/src/lib/patches/youtubejs.ts +++ b/materialious/src/lib/patches/youtubejs.ts @@ -2,72 +2,124 @@ import type { AdaptiveFormats, Captions, Image, StoryBoard, Thumbnail, VideoBase import { interfaceRegionStore, poTokenCacheStore } from '$lib/store'; import { numberWithCommas } from '$lib/time'; import { Capacitor } from '@capacitor/core'; -import { BG, type BgConfig } from 'bgutils-js'; +import { BG, buildURL, GOOG_API_KEY, type WebPoSignalOutput } from 'bgutils-js'; import { Buffer } from 'buffer'; import { get } from 'svelte/store'; -import { Innertube, UniversalCache } from 'youtubei.js'; +import { Innertube, UniversalCache, YT, YTNodes } from 'youtubei.js'; + +type WebPoMinter = { + integrityTokenBasedMinter?: BG.WebPoMinter; + botguardClient?: BG.BotGuardClient; +}; + +async function getWebPoMinter(youtube: Innertube): Promise { + const requestKey = 'O43z0dpjhgX20SCx4KAo'; + + const challengeResponse = await youtube.getAttestationChallenge('ENGAGEMENT_TYPE_UNBOUND'); + + if (!challengeResponse.bg_challenge) + throw new Error('Yt.js: Could not get challenge'); + + const interpreterUrl = challengeResponse.bg_challenge.interpreter_url.private_do_not_access_or_else_trusted_resource_url_wrapped_value; + const bgScriptResponse = await fetch(`https:${interpreterUrl}`); + const interpreterJavascript = await bgScriptResponse.text(); + + if (interpreterJavascript) { + new Function(interpreterJavascript)(); + } else throw new Error('Yt.js: Could not load VM'); + + const botguardClient = await BG.BotGuardClient.create({ + program: challengeResponse.bg_challenge.program, + globalName: challengeResponse.bg_challenge.global_name, + globalObj: globalThis + }); + + const webPoSignalOutput: WebPoSignalOutput = []; + const botguardResponse = await botguardClient.snapshot({ webPoSignalOutput }); + + console.log('botguardResponse', botguardResponse); + + const integrityTokenResponse = await fetch(buildURL('GenerateIT', true), { + method: 'POST', + headers: { + 'content-type': 'application/json+protobuf', + 'x-goog-api-key': GOOG_API_KEY, + 'x-user-agent': 'grpc-web-javascript/0.1', + }, + body: JSON.stringify([requestKey, botguardResponse]) + }); + + const integrityTokenResponseData = await integrityTokenResponse.json(); + + console.log(integrityTokenResponseData); + + if (typeof integrityTokenResponseData[0] !== 'string') + throw new Error('Yt.js: Could not get integrity token'); + + const integrityToken = integrityTokenResponseData[0] as string | undefined; + + const integrityTokenBasedMinter = await BG.WebPoMinter.create({ integrityToken }, webPoSignalOutput); + + return { + integrityTokenBasedMinter, + botguardClient + }; +} export async function patchYoutubeJs(videoId: string): Promise { if (!Capacitor.isNativePlatform()) { - throw new Error('Platform not supported'); + throw new Error('Yt.js: Platform not supported'); } - let youtube: Innertube; - - if (!get(poTokenCacheStore)) { - youtube = await Innertube.create({ retrieve_player: false, fetch: fetch }); - - const requestKey = 'O43z0dpjhgX20SCx4KAo'; - const visitorData = youtube.session.context.client.visitorData; - - if (!visitorData) - throw new Error('Could not get visitor data'); - - const bgConfig: BgConfig = { - fetch: (input: string | URL | Request, init?: RequestInit) => fetch(input, init), - globalObj: globalThis, - identifier: visitorData, - requestKey - }; - - const bgChallenge = await BG.Challenge.create(bgConfig); - - if (!bgChallenge) - throw new Error('Could not get challenge'); - - const interpreterJavascript = bgChallenge.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue; - - if (interpreterJavascript) { - new Function(interpreterJavascript)(); - } else throw new Error('Could not load VM'); - - const poTokenResult = await BG.PoToken.generate({ - program: bgChallenge.program, - globalName: bgChallenge.globalName, - bgConfig - }); - - poTokenCacheStore.set({ - poToken: poTokenResult.poToken, - visitorData: visitorData - }); - } - - const cachedPoToken = get(poTokenCacheStore); - - youtube = await Innertube.create({ + const youtube = await Innertube.create({ fetch: fetch, generate_session_locally: true, cache: new UniversalCache(false), location: get(interfaceRegionStore), - visitor_data: cachedPoToken.visitorData, - po_token: cachedPoToken.poToken }); - const video = await youtube.getInfo(videoId); + + let sessionWebPo: string | undefined; + const { integrityTokenBasedMinter } = await getWebPoMinter(youtube); + + if (integrityTokenBasedMinter) { + sessionWebPo = await integrityTokenBasedMinter.mintAsWebsafeString(youtube.session.context.client.visitorData ?? ''); + } + + poTokenCacheStore.set(sessionWebPo); + + const extraArgs: Record = { + playbackContext: { + contentPlaybackContext: { + vis: 0, + splay: false, + lactMilliseconds: '-1', + signatureTimestamp: youtube.session.player?.sts + } + }, + contentCheckOk: true, + racyCheckOk: true + }; + + // Generate content WebPO token. + if (integrityTokenBasedMinter) { + extraArgs.serviceIntegrityDimensions = { + poToken: await integrityTokenBasedMinter.mintAsWebsafeString(videoId) + }; + } + + const watchEndpoint = new YTNodes.NavigationEndpoint({ watchEndpoint: { videoId } }); + const rawPlayerResponse = await watchEndpoint.call(youtube.actions, extraArgs); + const rawNextResponse = await watchEndpoint.call(youtube.actions, { + override_endpoint: '/next', + racyCheckOk: true, + contentCheckOk: true + }); + + const video = new YT.VideoInfo([rawPlayerResponse, rawNextResponse], youtube!.actions, ''); if (!video.primary_info || !video.secondary_info) { - throw new Error('Unable to pull video info from youtube.js'); + throw new Error('Yt.js: Unable to pull video info from youtube.js'); } let dashUri: string = ''; diff --git a/materialious/src/lib/store.ts b/materialious/src/lib/store.ts index 1fc7f1c1..186151d1 100644 --- a/materialious/src/lib/store.ts +++ b/materialious/src/lib/store.ts @@ -101,6 +101,6 @@ export const playlistSettingsStore: Writable = writable(); +export const poTokenCacheStore: Writable = writable(); export const searchHistoryStore: Writable = persisted('searchHistory', []); diff --git a/materialious/static/nodejs-android/index.js b/materialious/static/nodejs-android/index.js index 3621258c..b8240c9d 100644 --- a/materialious/static/nodejs-android/index.js +++ b/materialious/static/nodejs-android/index.js @@ -5,10 +5,14 @@ const HOST = 'localhost'; const PORT = 3000; const MAX_REDIRECTS = 10; +const CORS_HEADERS = 'Origin, X-Requested-With, Content-Type, Accept, Authorization, x-goog-visitor-id, x-goog-api-key, x-origin, x-youtube-client-version, x-youtube-client-name, x-goog-api-format-version, x-user-agent, Accept-Language, Range, Referer' +const CORS_ORIGIN = 'https://localhost' +const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36(KHTML, like Gecko)' + function setCorsHeaders(res) { - res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Origin', CORS_ORIGIN); res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS'); - res.setHeader('Access-Control-Allow-Headers', '*'); + res.setHeader('Access-Control-Allow-Headers', CORS_HEADERS); res.setHeader('Access-Control-Max-Age', '86400') res.setHeader('Access-Control-Allow-Credentials', 'true'); } @@ -89,6 +93,7 @@ const server = http.createServer(async (req, res) => { options.headers.host = parsedTarget.host; options.headers.origin = parsedTarget.origin; + options.headers['user-agent'] = USER_AGENT; // For POST and PUT methods, pass the body to the outgoing request if (req.method === 'POST' || req.method === 'PUT') { @@ -106,9 +111,9 @@ const server = http.createServer(async (req, res) => { res.writeHead(proxyRes.statusCode, { ...proxyRes.headers, - 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Origin': CORS_ORIGIN, 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS', - 'Access-Control-Allow-Headers': '*', + 'Access-Control-Allow-Headers': CORS_HEADERS, 'Access-Control-Allow-Credentials': 'true', });