From 09737e93f537093b23bd6ce276f449ee5e118952 Mon Sep 17 00:00:00 2001 From: Sommerwiesel Date: Wed, 18 Jun 2025 15:40:12 +0200 Subject: [PATCH] Production: Enable redis, change HAPRoxy settings --- etc/haproxy/haproxy.cfg | 72 ++++++--- patches/002-multiple-companion-backends.patch | 11 +- patches/010-use-redis-for-video-cache.patch.d | 151 ------------------ ...14-rewrite-obnoxious-blocked-message.patch | 2 +- 4 files changed, 54 insertions(+), 182 deletions(-) delete mode 100644 patches/010-use-redis-for-video-cache.patch.d diff --git a/etc/haproxy/haproxy.cfg b/etc/haproxy/haproxy.cfg index 7dcbfc6..b810cd2 100644 --- a/etc/haproxy/haproxy.cfg +++ b/etc/haproxy/haproxy.cfg @@ -41,27 +41,55 @@ frontend invidious bind 192.42.6.1:8118 mode http - default_backend invidious_proxy + acl use_proxy_1 req.cook(COMPANION_ID) 0 + acl use_proxy_2 req.cook(COMPANION_ID) 1 + acl use_proxy_3 req.cook(COMPANION_ID) 2 + acl use_proxy_4 req.cook(COMPANION_ID) 3 + acl use_proxy_5 req.cook(COMPANION_ID) 4 + acl use_proxy_6 req.cook(COMPANION_ID) 5 + acl use_proxy_7 req.cook(COMPANION_ID) 6 + acl use_proxy_8 req.cook(COMPANION_ID) 7 -backend invidious_proxy - mode http - cookie COMPANION_ID insert indirect nocache httponly secure - balance leastconn - #balance hdr(X-Forwarded-For) - hash-type consistent + use_backend proxy_1 if use_proxy_1 + use_backend proxy_2 if use_proxy_2 + use_backend proxy_3 if use_proxy_3 + use_backend proxy_4 if use_proxy_4 + use_backend proxy_5 if use_proxy_5 + use_backend proxy_6 if use_proxy_6 + use_backend proxy_7 if use_proxy_7 + use_backend proxy_8 if use_proxy_8 + + default_backend proxy_1 + +backend proxy_1 + mode http + server server1 192.42.6.11:8282 + +backend proxy_2 + mode http + server server2 192.42.6.12:8282 + +backend proxy_3 + mode http + server server3 192.42.6.13:8282 + +backend proxy_4 + mode http + server server4 192.42.6.14:8282 + +backend proxy_5 + mode http + server server5 192.42.6.15:8282 + +backend proxy_6 + mode http + server server6 192.42.6.16:8282 + +backend proxy_7 + mode http + server server7 192.42.6.17:8282 + +backend proxy_8 + mode http + server server8 192.42.6.18:8282 - retry-on all-retryable-errors - - # Ipv6 - server proxy_1 192.42.6.11:8282 cookie 0 - server proxy_2 192.42.6.12:8282 cookie 1 - server proxy_3 192.42.6.13:8282 cookie 2 - server proxy_4 192.42.6.14:8282 cookie 3 - server proxy_5 192.42.6.15:8282 cookie 4 - server proxy_6 192.42.6.16:8282 cookie 5 - server proxy_7 192.42.6.17:8282 cookie 6 - server proxy_8 192.42.6.18:8282 cookie 7 - - # IPv4 - #server proxy_1 192.42.6.200:8282 cookie 10 - #server proxy_2 192.42.6.201:8282 cookie 11 diff --git a/patches/002-multiple-companion-backends.patch b/patches/002-multiple-companion-backends.patch index d0d1130..bd896f5 100644 --- a/patches/002-multiple-companion-backends.patch +++ b/patches/002-multiple-companion-backends.patch @@ -358,17 +358,12 @@ index 654efc15..affcfdca 100644 + # Browsers will reject the cookie if it contains the port number. This is + # because `example.com:3000` is not the same as `example.com` on a cookie. + domain = domain.split(":")[0] -+ # Not secure if it's being accessed from I2P -+ # Browsers expect the domain to include https. On I2P there is no HTTPS -+ if domain.not_nil!.split(".").last == "i2p" -+ @@secure = false -+ end + return HTTP::Cookie.new( + name: CONFIG.server_id_cookie_name, + domain: domain, + path: "/", + value: server_id.to_s, -+ secure: @@secure, ++ secure: true, + http_only: true, + samesite: HTTP::Cookie::SameSite::Lax + ) @@ -468,7 +463,7 @@ index 9904b4fc..ad6c1c0e 100644 + companion_switched = env.get?("companion_switched") + status = BackendInfo.get_status + %> -+
++
+ Backend
+ <% current_page = env.get("current_page") %> + <% CONFIG.invidious_companion.each_with_index do | companion, index | %> @@ -487,7 +482,7 @@ index 9904b4fc..ad6c1c0e 100644 + <% end %> + + <% if companion_switched %> -+
++
+

<%= translate(locale, "backend_unavailable") %>

+
+ <% end %> diff --git a/patches/010-use-redis-for-video-cache.patch.d b/patches/010-use-redis-for-video-cache.patch.d deleted file mode 100644 index c810b62..0000000 --- a/patches/010-use-redis-for-video-cache.patch.d +++ /dev/null @@ -1,151 +0,0 @@ -From 7c4c24f829b4eeb8ca035b4a9d77eef53ca8aefa Mon Sep 17 00:00:00 2001 -From: Emilien Devos -Date: Sat, 10 Jun 2023 22:21:38 +0200 -Subject: [PATCH 1/1] use redis for video cache - ---- - shard.lock | 8 ++++++++ - shard.yml | 2 ++ - src/invidious.cr | 8 +++++++- - src/invidious/config.cr | 3 +++ - src/invidious/database/videos.cr | 16 +++++++++++++--- - src/invidious/videos.cr | 2 +- - 6 files changed, 34 insertions(+), 5 deletions(-) - -diff --git a/shard.lock b/shard.lock -index 397bd8bc..8c599d1b 100644 ---- a/shard.lock -+++ b/shard.lock -@@ -36,6 +36,10 @@ shards: - git: https://github.com/will/crystal-pg.git - version: 0.28.0 - -+ pool: -+ git: https://github.com/ysbaddaden/pool.git -+ version: 0.2.4 -+ - protodec: - git: https://github.com/iv-org/protodec.git - version: 0.1.5 -@@ -44,6 +48,10 @@ shards: - git: https://github.com/luislavena/radix.git - version: 0.4.1 - -+ redis: -+ git: https://github.com/stefanwille/crystal-redis.git -+ version: 2.9.1 -+ - spectator: - git: https://github.com/icy-arctic-fox/spectator.git - version: 0.10.6 -diff --git a/shard.yml b/shard.yml -index 367f7c73..c46e1c42 100644 ---- a/shard.yml -+++ b/shard.yml -@@ -31,6 +31,8 @@ dependencies: - http_proxy: - github: mamantoha/http_proxy - version: ~> 0.10.3 -+ redis: -+ github: stefanwille/crystal-redis - - development_dependencies: - spectator: -diff --git a/src/invidious.cr b/src/invidious.cr -index 3804197e..e2d40597 100644 ---- a/src/invidious.cr -+++ b/src/invidious.cr -@@ -31,6 +31,7 @@ require "xml" - require "yaml" - require "compress/zip" - require "protodec/utils" -+require "redis" - - require "./invidious/database/*" - require "./invidious/database/migrations/*" -@@ -60,7 +61,12 @@ alias IV = Invidious - CONFIG = Config.load - HMAC_KEY = CONFIG.hmac_key - --PG_DB = DB.open CONFIG.database_url -+PG_DB = DB.open CONFIG.database_url -+REDIS_DB = Redis::PooledClient.new(unixsocket: CONFIG.redis_socket || nil, url: CONFIG.redis_url || nil, database: CONFIG.redis_db || 0, password: CONFIG.redis_auth || nil) -+ -+if REDIS_DB.ping -+ puts "Connected to redis" -+end - ARCHIVE_URL = URI.parse("https://archive.org") - PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") - REDDIT_URL = URI.parse("https://www.reddit.com") -diff --git a/src/invidious/config.cr b/src/invidious/config.cr -index c4ddcdb3..5b6c576c 100644 ---- a/src/invidious/config.cr -+++ b/src/invidious/config.cr -@@ -77,6 +77,11 @@ class Config - # Used for crawling channels: threads should check all videos uploaded by a channel - property full_refresh : Bool = false - -+ property redis_url : String? -+ property redis_socket : String? -+ property redis_auth : String? -+ property redis_db : Int32? -+ - # Jobs config structure. See jobs.cr and jobs/base_job.cr - property jobs = Invidious::Jobs::JobsConfig.new - -diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr -index 695f5b33..776ef5b1 100644 ---- a/src/invidious/database/videos.cr -+++ b/src/invidious/database/videos.cr -@@ -10,7 +10,8 @@ module Invidious::Database::Videos - ON CONFLICT (id) DO NOTHING - SQL - -- PG_DB.exec(request, video.id, video.info.to_json, video.updated) -+ REDIS_DB.set(video.id, video.info.to_json, ex: 3600) -+ REDIS_DB.set(video.id + ":time", video.updated, ex: 3600) - end - - def delete(id) -@@ -19,7 +20,8 @@ module Invidious::Database::Videos - WHERE id = $1 - SQL - -- PG_DB.exec(request, id) -+ REDIS_DB.del(id) -+ REDIS_DB.del(id + ":time") - end - - def delete_expired -@@ -47,6 +49,14 @@ module Invidious::Database::Videos - WHERE id = $1 - SQL - -- return PG_DB.query_one?(request, id, as: Video) -+ if ((info = REDIS_DB.get(id)) && (time = REDIS_DB.get(id + ":time"))) -+ return Video.new({ -+ id: id, -+ info: JSON.parse(info).as_h, -+ updated: Time.parse(time, "%Y-%m-%d %H:%M:%S %z", Time::Location::UTC), -+ }) -+ else -+ return nil -+ end - end - end -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index 6d0cf9ba..dde35291 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -400,7 +400,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) - video.schema_version != Video::SCHEMA_VERSION # cache control - begin - video = fetch_video(id, region, env) -- Invidious::Database::Videos.update(video) -+ Invidious::Database::Videos.insert(video) - rescue ex - Invidious::Database::Videos.delete(id) - raise ex --- -2.46.0 - diff --git a/patches/014-rewrite-obnoxious-blocked-message.patch b/patches/014-rewrite-obnoxious-blocked-message.patch index 7dd93ec..2a4ba47 100644 --- a/patches/014-rewrite-obnoxious-blocked-message.patch +++ b/patches/014-rewrite-obnoxious-blocked-message.patch @@ -8,7 +8,7 @@ index b2df682d..7534ef31 100644 - error_message = translate(locale, message) + if message.includes?("Error while communicating with Invidious companion") || message.includes?("YouTube is currently trying to block Invidious instances") || message.includes?("This helps protect our community") || message.includes?("Watch on the latest version of YouTube") || message.includes?("Sign in to confirm you’re not a bot") || message.includes?("Please sign in") || message.includes?("This content isn't available, try again later") -+ error_message = "

Error loading video

Original error message: " + message + "

Please try reloading the page or try switching to another backend. This will fix it almost all of the time.

If the error persists (after multiple tries, using different backends and trying different videos, please open an issue on my github.

" ++ error_message = "

Error loading video

Original error message: " + message + "

Please try reloading the page or try switching to another backend. This will fix it almost all of the time.

If the error persists (after multiple tries, using different backends and trying different videos), please open an issue on my github.

" + else + error_message = translate(locale, message) + end