From 76a372bd986b95fbb1b9829fc985c0e7962ccf81 Mon Sep 17 00:00:00 2001 From: WardPearce Date: Tue, 1 Oct 2024 13:46:32 +1300 Subject: [PATCH] Improve po token generation --- materialious/android/app/build.gradle | 4 +- materialious/electron/package.json | 5 +- materialious/electron/src/preload.ts | 5 - materialious/electron/src/setup.ts | 12 +- materialious/package-lock.json | 11 ++ materialious/package.json | 3 +- .../src/lib/patches/poTokenAndroid.ts | 123 ------------------ materialious/src/lib/patches/youtubejs.ts | 68 ++++++++-- update_versions.py | 2 +- 9 files changed, 76 insertions(+), 157 deletions(-) delete mode 100644 materialious/src/lib/patches/poTokenAndroid.ts diff --git a/materialious/android/app/build.gradle b/materialious/android/app/build.gradle index 5433d296..26fd293e 100644 --- a/materialious/android/app/build.gradle +++ b/materialious/android/app/build.gradle @@ -7,8 +7,8 @@ android { applicationId "us.materialio.app" minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion - versionCode 50 - versionName "1.5.9" + versionCode 51 + versionName "1.6.0" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" aaptOptions { // Files and dirs to omit from the packaged assets dir, modified to accommodate modern web apps. diff --git a/materialious/electron/package.json b/materialious/electron/package.json index 64207f21..b7302002 100644 --- a/materialious/electron/package.json +++ b/materialious/electron/package.json @@ -1,6 +1,6 @@ { "name": "Materialious", - "version": "1.5.9", + "version": "1.6.0", "description": "Modern material design for Invidious.", "author": { "name": "Ward Pearce", @@ -28,8 +28,7 @@ "electron-unhandled": "^4.0.1", "electron-updater": "^6.3.4", "electron-window-state": "^5.0.3", - "jsonfile": "^6.1.0", - "youtube-po-token-generator": "^0.2.0" + "jsonfile": "^6.1.0" }, "devDependencies": { "electron": "^32.1.0", diff --git a/materialious/electron/src/preload.ts b/materialious/electron/src/preload.ts index 92641d9c..d3107168 100644 --- a/materialious/electron/src/preload.ts +++ b/materialious/electron/src/preload.ts @@ -1,6 +1 @@ require('./rt/electron-rt'); -const { contextBridge, ipcRenderer } = require('electron'); - -contextBridge.exposeInMainWorld('electron', { - generatePoToken: () => ipcRenderer.invoke('generate-po-token') -}); diff --git a/materialious/electron/src/setup.ts b/materialious/electron/src/setup.ts index 2e265b8d..dbfe60b2 100644 --- a/materialious/electron/src/setup.ts +++ b/materialious/electron/src/setup.ts @@ -6,12 +6,11 @@ import { } from '@capacitor-community/electron'; import chokidar from 'chokidar'; import type { MenuItemConstructorOptions } from 'electron'; -import { app, BrowserWindow, ipcMain, Menu, MenuItem, nativeImage, session, Tray } from 'electron'; +import { app, BrowserWindow, Menu, MenuItem, nativeImage, session, Tray } from 'electron'; import electronIsDev from 'electron-is-dev'; import electronServe from 'electron-serve'; import windowStateKeeper from 'electron-window-state'; import { join } from 'path'; -import { generate } from 'youtube-po-token-generator'; // Define components for a watcher to detect when the webapp is changed so we can reload in Dev mode. const reloadWatcher = { @@ -215,15 +214,6 @@ export class ElectronCapacitorApp { CapElectronEventEmitter.emit('CAPELECTRON_DeeplinkListenerInitialized', ''); }, 400); }); - - ipcMain.handle('generate-po-token', async () => { - try { - return await generate(); - } catch (error) { - console.error('Error generating token:', error); - return { error: error.message }; // Return error to renderer - } - }); } } diff --git a/materialious/package-lock.json b/materialious/package-lock.json index 673287a0..766ab121 100644 --- a/materialious/package-lock.json +++ b/materialious/package-lock.json @@ -19,6 +19,7 @@ "@ffmpeg/ffmpeg": "^0.12.10", "@mediagrid/capacitor-native-audio": "^1.0.0", "beercss": "^3.7.8", + "bgutils-js": "^2.2.0", "capacitor-nodejs": "https://github.com/EdenwareApps/Capacitor-NodeJS/releases/download/v1.0.0-beta.7/capacitor6-nodejs.tgz", "dashjs": "^4.7.4", "fuse.js": "^7.0.0", @@ -4306,6 +4307,15 @@ "material-dynamic-colors": "^1.1.2" } }, + "node_modules/bgutils-js": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/bgutils-js/-/bgutils-js-2.2.0.tgz", + "integrity": "sha512-6IRDTZIdHxoyfttEHlXfCVHdje8yNoxcgp8Os5pu6uaWdeZMrN7iiXFxk6EGVbcaQTDZEUcH5li8qMK7sqWS0g==", + "funding": [ + "https://github.com/sponsors/LuanRT" + ], + "license": "MIT" + }, "node_modules/big-integer": { "version": "1.6.52", "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", @@ -9189,6 +9199,7 @@ "write-file-atomic" ], "dev": true, + "license": "Artistic-2.0", "workspaces": [ "docs", "smoke-tests", diff --git a/materialious/package.json b/materialious/package.json index 9b6e24f9..7f313a25 100644 --- a/materialious/package.json +++ b/materialious/package.json @@ -1,6 +1,6 @@ { "name": "materialious", - "version": "1.5.9", + "version": "1.6.0", "private": true, "scripts": { "dev": "vite dev", @@ -55,6 +55,7 @@ "@ffmpeg/ffmpeg": "^0.12.10", "@mediagrid/capacitor-native-audio": "^1.0.0", "beercss": "^3.7.8", + "bgutils-js": "^2.2.0", "capacitor-nodejs": "https://github.com/EdenwareApps/Capacitor-NodeJS/releases/download/v1.0.0-beta.7/capacitor6-nodejs.tgz", "dashjs": "^4.7.4", "fuse.js": "^7.0.0", diff --git a/materialious/src/lib/patches/poTokenAndroid.ts b/materialious/src/lib/patches/poTokenAndroid.ts deleted file mode 100644 index d003e2a0..00000000 --- a/materialious/src/lib/patches/poTokenAndroid.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { InAppBrowser } from '@capgo/inappbrowser'; -import { minify } from 'terser'; - -export interface PoTokens { - visitor_data: string; - po_token: string; -} - -const headers = { - 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', - 'accept-language': 'en-US;q=0.9', - 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko)', -}; - -// Inspired by https://github.com/YunzheZJU/youtube-po-token-generator & https://github.com/YunzheZJU/youtube-po-token-generator -export async function getPoToken(): Promise { - let visitorData: string; - - return new Promise((resolve, reject) => { - const messageListener = async (data: any) => { - - if (!data.po_token) { - visitorData = data.visitor_data; - const injectPlayer = await fetch('/po-token-youtube/inject.js'); - - await InAppBrowser.executeScript({ - code: (await minify(` -window.onPoToken = (poToken) => { - window.mobileApp.postMessage({po_token: poToken}); -} - -ytcfg.set({ - 'INNERTUBE_CONTEXT': { - ...ytcfg.get('INNERTUBE_CONTEXT'), - client: { - ...ytcfg.get('INNERTUBE_CONTEXT').client, - visitorData: "${visitorData}", - }, - }, - 'VISITOR_DATA': "${visitorData}", - 'IDENTITY_MEMENTO': { - 'visitor_data': "${visitorData}", - }, -}) - -${await injectPlayer.text()}`)).code as string - }); - } else if (visitorData && data.po_token) { - await InAppBrowser.removeAllListeners(); - await InAppBrowser.close(); - resolve({ visitor_data: visitorData, po_token: data.po_token }); - } - }; - - const closeListener = async () => { - await InAppBrowser.removeAllListeners(); - }; - - const urlChangeListener = async () => { - // Code must be minified to ensure runs correctly. - await InAppBrowser.executeScript({ - code: (await minify(`const headers = {"accept": "${headers.accept}", "accept-language": "${headers['accept-language']}", "user-agent": "${headers['user-agent']}"}; -Object.defineProperty(window.navigator, 'userAgent', { value: headers["user-agent"], writable: false }); - -const originalOpen = XMLHttpRequest.prototype.open; -const originalSend = XMLHttpRequest.prototype.send; - -XMLHttpRequest.prototype.open = function (method, url) { - this._url = url; - originalOpen.apply(this, arguments); -}; - -XMLHttpRequest.prototype.send = function (body) { - const xhr = this; - if (this._url.includes("/youtubei/v1/player")) { - for (const [key, value] of Object.entries(headers)) { - this.setRequestHeader(key, value); - } - - this.addEventListener("load", function () { - try { - const postJson = JSON.parse(body); - window.mobileApp.postMessage({visitor_data: postJson.context.client.visitorData}); - } catch (error) { - console.error("Error parsing request body:", error); - } - }); - } - originalSend.apply(this, arguments); -}; - -function attemptClickPlayButton() { - const interval = setInterval(() => { - const playButtons = document.getElementsByClassName("ytp-play-button"); - if (playButtons.length > 0) { - playButtons[0].click(); - clearInterval(interval); - } - }, 100); -} - -attemptClickPlayButton(); -`)).code as string - }); - }; - - InAppBrowser.addListener('messageFromWebview', messageListener); - InAppBrowser.addListener('closeEvent', closeListener); - InAppBrowser.addListener('urlChangeEvent', urlChangeListener); - - setTimeout(async () => { - reject(new Error('Timeout trying to pull Po tokens')); - await InAppBrowser.removeAllListeners(); - await InAppBrowser.close(); - }, 60000); - - InAppBrowser.openWebView({ - url: 'https://www.youtube.com/embed/jNQXAC9IVRw', - title: 'Pulling po tokens (This may take a moment)', - headers: headers, - }); - }); -} diff --git a/materialious/src/lib/patches/youtubejs.ts b/materialious/src/lib/patches/youtubejs.ts index 2a2dd545..83dfb88d 100644 --- a/materialious/src/lib/patches/youtubejs.ts +++ b/materialious/src/lib/patches/youtubejs.ts @@ -2,31 +2,75 @@ import type { Image, Thumbnail, VideoBase, VideoPlay } from '$lib/Api/model'; import { numberWithCommas } from '$lib/misc'; import { poTokenCacheStore } from '$lib/store'; import { Capacitor } from '@capacitor/core'; +import { BG } from 'bgutils-js'; import { get } from 'svelte/store'; +import { Innertube, ProtoUtils, UniversalCache, Utils } from 'youtubei.js'; import { capacitorFetch } from './capacitorFetch'; -import { type PoTokens } from './poTokenAndroid'; + + +export interface PoTokens { + visitor_data: string; + po_token: string; +} + +async function getPo(identifier: string): Promise { + const requestKey = 'O43z0dpjhgX20SCx4KAo'; + + const bgConfig = { + fetch: fetch, + globalObj: window, + requestKey, + identifier + }; + + const challenge = await BG.Challenge.create(bgConfig); + + if (!challenge) + throw new Error('Could not get challenge'); + + if (challenge.script) { + const script = challenge.script.find((sc) => sc !== null); + if (script) + new Function(script)(); + } + + const poToken = await BG.PoToken.generate({ + program: challenge.challenge, + globalName: challenge.globalName, + bgConfig + }); + + return poToken; +} export async function patchYoutubeJs(videoId: string): Promise { - const innertube = (await import('youtubei.js')).Innertube; - let tokens: PoTokens; + if (!Capacitor.isNativePlatform()) { + throw new Error('Platform not supported'); + } + const poTokenCache = get(poTokenCacheStore); if (!poTokenCache) { - if (Capacitor.getPlatform() === 'electron') { - tokens = await (window as any).electron.generatePoToken(); - } else if (Capacitor.getPlatform() === 'android') { - tokens = await (await import('../../lib/patches/poTokenAndroid')).getPoToken(); - } else { - throw new Error('This platform cant generate po tokens'); + const visitorData = ProtoUtils.encodeVisitorData(Utils.generateRandomString(11), Math.floor(Date.now() / 1000)); + const poToken = await getPo(visitorData); + + if (!poToken) { + throw new Error('Unable to generate PO token'); } + tokens = { + visitor_data: visitorData, + po_token: poToken + }; poTokenCacheStore.set(tokens); } else { tokens = poTokenCache; } - const youtube = await innertube.create({ + console.log(tokens); + + const youtube = await Innertube.create({ visitor_data: tokens.visitor_data, po_token: tokens.po_token, fetch: async (input: RequestInfo | URL, init?: RequestInit) => { @@ -35,7 +79,9 @@ export async function patchYoutubeJs(videoId: string): Promise { } else { return fetch(input, init); } - } + }, + generate_session_locally: true, + cache: new UniversalCache(false) }); const video = await youtube.getInfo(videoId); diff --git a/update_versions.py b/update_versions.py index 2644a2b6..cf0de12c 100644 --- a/update_versions.py +++ b/update_versions.py @@ -5,7 +5,7 @@ import json import os import re -LATEST_VERSION = "1.5.9" +LATEST_VERSION = "1.6.0" WORKING_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "materialious") ROOT_PACKAGE = os.path.join(WORKING_DIR, "package.json")