mirror of
https://git.nerdvpn.de/NerdVPN.de/invidious
synced 2026-02-14 22:51:42 +01:00
Production: Enable redis, change HAPRoxy settings
This commit is contained in:
Sommerwiesel
+50
-22
@@ -41,27 +41,55 @@ frontend invidious
|
||||
bind 192.42.6.1:8118
|
||||
mode http
|
||||
|
||||
default_backend invidious_proxy
|
||||
acl use_proxy_1 req.cook(COMPANION_ID) 0
|
||||
acl use_proxy_2 req.cook(COMPANION_ID) 1
|
||||
acl use_proxy_3 req.cook(COMPANION_ID) 2
|
||||
acl use_proxy_4 req.cook(COMPANION_ID) 3
|
||||
acl use_proxy_5 req.cook(COMPANION_ID) 4
|
||||
acl use_proxy_6 req.cook(COMPANION_ID) 5
|
||||
acl use_proxy_7 req.cook(COMPANION_ID) 6
|
||||
acl use_proxy_8 req.cook(COMPANION_ID) 7
|
||||
|
||||
backend invidious_proxy
|
||||
mode http
|
||||
cookie COMPANION_ID insert indirect nocache httponly secure
|
||||
balance leastconn
|
||||
#balance hdr(X-Forwarded-For)
|
||||
hash-type consistent
|
||||
use_backend proxy_1 if use_proxy_1
|
||||
use_backend proxy_2 if use_proxy_2
|
||||
use_backend proxy_3 if use_proxy_3
|
||||
use_backend proxy_4 if use_proxy_4
|
||||
use_backend proxy_5 if use_proxy_5
|
||||
use_backend proxy_6 if use_proxy_6
|
||||
use_backend proxy_7 if use_proxy_7
|
||||
use_backend proxy_8 if use_proxy_8
|
||||
|
||||
default_backend proxy_1
|
||||
|
||||
backend proxy_1
|
||||
mode http
|
||||
server server1 192.42.6.11:8282
|
||||
|
||||
backend proxy_2
|
||||
mode http
|
||||
server server2 192.42.6.12:8282
|
||||
|
||||
backend proxy_3
|
||||
mode http
|
||||
server server3 192.42.6.13:8282
|
||||
|
||||
backend proxy_4
|
||||
mode http
|
||||
server server4 192.42.6.14:8282
|
||||
|
||||
backend proxy_5
|
||||
mode http
|
||||
server server5 192.42.6.15:8282
|
||||
|
||||
backend proxy_6
|
||||
mode http
|
||||
server server6 192.42.6.16:8282
|
||||
|
||||
backend proxy_7
|
||||
mode http
|
||||
server server7 192.42.6.17:8282
|
||||
|
||||
backend proxy_8
|
||||
mode http
|
||||
server server8 192.42.6.18:8282
|
||||
|
||||
retry-on all-retryable-errors
|
||||
|
||||
# Ipv6
|
||||
server proxy_1 192.42.6.11:8282 cookie 0
|
||||
server proxy_2 192.42.6.12:8282 cookie 1
|
||||
server proxy_3 192.42.6.13:8282 cookie 2
|
||||
server proxy_4 192.42.6.14:8282 cookie 3
|
||||
server proxy_5 192.42.6.15:8282 cookie 4
|
||||
server proxy_6 192.42.6.16:8282 cookie 5
|
||||
server proxy_7 192.42.6.17:8282 cookie 6
|
||||
server proxy_8 192.42.6.18:8282 cookie 7
|
||||
|
||||
# IPv4
|
||||
#server proxy_1 192.42.6.200:8282 cookie 10
|
||||
#server proxy_2 192.42.6.201:8282 cookie 11
|
||||
|
||||
@@ -358,17 +358,12 @@ index 654efc15..affcfdca 100644
|
||||
+ # Browsers will reject the cookie if it contains the port number. This is
|
||||
+ # because `example.com:3000` is not the same as `example.com` on a cookie.
|
||||
+ domain = domain.split(":")[0]
|
||||
+ # Not secure if it's being accessed from I2P
|
||||
+ # Browsers expect the domain to include https. On I2P there is no HTTPS
|
||||
+ if domain.not_nil!.split(".").last == "i2p"
|
||||
+ @@secure = false
|
||||
+ end
|
||||
+ return HTTP::Cookie.new(
|
||||
+ name: CONFIG.server_id_cookie_name,
|
||||
+ domain: domain,
|
||||
+ path: "/",
|
||||
+ value: server_id.to_s,
|
||||
+ secure: @@secure,
|
||||
+ secure: true,
|
||||
+ http_only: true,
|
||||
+ samesite: HTTP::Cookie::SameSite::Lax
|
||||
+ )
|
||||
@@ -468,7 +463,7 @@ index 9904b4fc..ad6c1c0e 100644
|
||||
+ companion_switched = env.get?("companion_switched")
|
||||
+ status = BackendInfo.get_status
|
||||
+ %>
|
||||
+ <div class="h-box" style="margin-bottom: 10px; font-size:0.85em; text-align: center;">
|
||||
+ <div id="backend-selector" class="h-box">
|
||||
+ <b>Backend</b><br />
|
||||
+ <% current_page = env.get("current_page") %>
|
||||
+ <% CONFIG.invidious_companion.each_with_index do | companion, index | %>
|
||||
@@ -487,7 +482,7 @@ index 9904b4fc..ad6c1c0e 100644
|
||||
+ <% end %>
|
||||
+
|
||||
+ <% if companion_switched %>
|
||||
+ <div class="h-box">
|
||||
+ <div id="backend-info" class="h-box">
|
||||
+ <p><%= translate(locale, "backend_unavailable") %></p>
|
||||
+ </div>
|
||||
+ <% end %>
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
From 7c4c24f829b4eeb8ca035b4a9d77eef53ca8aefa Mon Sep 17 00:00:00 2001
|
||||
From: Emilien Devos <contact@emiliendevos.be>
|
||||
Date: Sat, 10 Jun 2023 22:21:38 +0200
|
||||
Subject: [PATCH 1/1] use redis for video cache
|
||||
|
||||
---
|
||||
shard.lock | 8 ++++++++
|
||||
shard.yml | 2 ++
|
||||
src/invidious.cr | 8 +++++++-
|
||||
src/invidious/config.cr | 3 +++
|
||||
src/invidious/database/videos.cr | 16 +++++++++++++---
|
||||
src/invidious/videos.cr | 2 +-
|
||||
6 files changed, 34 insertions(+), 5 deletions(-)
|
||||
|
||||
diff --git a/shard.lock b/shard.lock
|
||||
index 397bd8bc..8c599d1b 100644
|
||||
--- a/shard.lock
|
||||
+++ b/shard.lock
|
||||
@@ -36,6 +36,10 @@ shards:
|
||||
git: https://github.com/will/crystal-pg.git
|
||||
version: 0.28.0
|
||||
|
||||
+ pool:
|
||||
+ git: https://github.com/ysbaddaden/pool.git
|
||||
+ version: 0.2.4
|
||||
+
|
||||
protodec:
|
||||
git: https://github.com/iv-org/protodec.git
|
||||
version: 0.1.5
|
||||
@@ -44,6 +48,10 @@ shards:
|
||||
git: https://github.com/luislavena/radix.git
|
||||
version: 0.4.1
|
||||
|
||||
+ redis:
|
||||
+ git: https://github.com/stefanwille/crystal-redis.git
|
||||
+ version: 2.9.1
|
||||
+
|
||||
spectator:
|
||||
git: https://github.com/icy-arctic-fox/spectator.git
|
||||
version: 0.10.6
|
||||
diff --git a/shard.yml b/shard.yml
|
||||
index 367f7c73..c46e1c42 100644
|
||||
--- a/shard.yml
|
||||
+++ b/shard.yml
|
||||
@@ -31,6 +31,8 @@ dependencies:
|
||||
http_proxy:
|
||||
github: mamantoha/http_proxy
|
||||
version: ~> 0.10.3
|
||||
+ redis:
|
||||
+ github: stefanwille/crystal-redis
|
||||
|
||||
development_dependencies:
|
||||
spectator:
|
||||
diff --git a/src/invidious.cr b/src/invidious.cr
|
||||
index 3804197e..e2d40597 100644
|
||||
--- a/src/invidious.cr
|
||||
+++ b/src/invidious.cr
|
||||
@@ -31,6 +31,7 @@ require "xml"
|
||||
require "yaml"
|
||||
require "compress/zip"
|
||||
require "protodec/utils"
|
||||
+require "redis"
|
||||
|
||||
require "./invidious/database/*"
|
||||
require "./invidious/database/migrations/*"
|
||||
@@ -60,7 +61,12 @@ alias IV = Invidious
|
||||
CONFIG = Config.load
|
||||
HMAC_KEY = CONFIG.hmac_key
|
||||
|
||||
-PG_DB = DB.open CONFIG.database_url
|
||||
+PG_DB = DB.open CONFIG.database_url
|
||||
+REDIS_DB = Redis::PooledClient.new(unixsocket: CONFIG.redis_socket || nil, url: CONFIG.redis_url || nil, database: CONFIG.redis_db || 0, password: CONFIG.redis_auth || nil)
|
||||
+
|
||||
+if REDIS_DB.ping
|
||||
+ puts "Connected to redis"
|
||||
+end
|
||||
ARCHIVE_URL = URI.parse("https://archive.org")
|
||||
PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
|
||||
REDDIT_URL = URI.parse("https://www.reddit.com")
|
||||
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
|
||||
index c4ddcdb3..5b6c576c 100644
|
||||
--- a/src/invidious/config.cr
|
||||
+++ b/src/invidious/config.cr
|
||||
@@ -77,6 +77,11 @@ class Config
|
||||
# Used for crawling channels: threads should check all videos uploaded by a channel
|
||||
property full_refresh : Bool = false
|
||||
|
||||
+ property redis_url : String?
|
||||
+ property redis_socket : String?
|
||||
+ property redis_auth : String?
|
||||
+ property redis_db : Int32?
|
||||
+
|
||||
# Jobs config structure. See jobs.cr and jobs/base_job.cr
|
||||
property jobs = Invidious::Jobs::JobsConfig.new
|
||||
|
||||
diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr
|
||||
index 695f5b33..776ef5b1 100644
|
||||
--- a/src/invidious/database/videos.cr
|
||||
+++ b/src/invidious/database/videos.cr
|
||||
@@ -10,7 +10,8 @@ module Invidious::Database::Videos
|
||||
ON CONFLICT (id) DO NOTHING
|
||||
SQL
|
||||
|
||||
- PG_DB.exec(request, video.id, video.info.to_json, video.updated)
|
||||
+ REDIS_DB.set(video.id, video.info.to_json, ex: 3600)
|
||||
+ REDIS_DB.set(video.id + ":time", video.updated, ex: 3600)
|
||||
end
|
||||
|
||||
def delete(id)
|
||||
@@ -19,7 +20,8 @@ module Invidious::Database::Videos
|
||||
WHERE id = $1
|
||||
SQL
|
||||
|
||||
- PG_DB.exec(request, id)
|
||||
+ REDIS_DB.del(id)
|
||||
+ REDIS_DB.del(id + ":time")
|
||||
end
|
||||
|
||||
def delete_expired
|
||||
@@ -47,6 +49,14 @@ module Invidious::Database::Videos
|
||||
WHERE id = $1
|
||||
SQL
|
||||
|
||||
- return PG_DB.query_one?(request, id, as: Video)
|
||||
+ if ((info = REDIS_DB.get(id)) && (time = REDIS_DB.get(id + ":time")))
|
||||
+ return Video.new({
|
||||
+ id: id,
|
||||
+ info: JSON.parse(info).as_h,
|
||||
+ updated: Time.parse(time, "%Y-%m-%d %H:%M:%S %z", Time::Location::UTC),
|
||||
+ })
|
||||
+ else
|
||||
+ return nil
|
||||
+ end
|
||||
end
|
||||
end
|
||||
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
|
||||
index 6d0cf9ba..dde35291 100644
|
||||
--- a/src/invidious/videos.cr
|
||||
+++ b/src/invidious/videos.cr
|
||||
@@ -400,7 +400,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
|
||||
video.schema_version != Video::SCHEMA_VERSION # cache control
|
||||
begin
|
||||
video = fetch_video(id, region, env)
|
||||
- Invidious::Database::Videos.update(video)
|
||||
+ Invidious::Database::Videos.insert(video)
|
||||
rescue ex
|
||||
Invidious::Database::Videos.delete(id)
|
||||
raise ex
|
||||
--
|
||||
2.46.0
|
||||
|
||||
@@ -8,7 +8,7 @@ index b2df682d..7534ef31 100644
|
||||
|
||||
- error_message = translate(locale, message)
|
||||
+ if message.includes?("Error while communicating with Invidious companion") || message.includes?("YouTube is currently trying to block Invidious instances") || message.includes?("This helps protect our community") || message.includes?("Watch on the latest version of YouTube") || message.includes?("Sign in to confirm you’re not a bot") || message.includes?("Please sign in") || message.includes?("This content isn't available, try again later")
|
||||
+ error_message = "<h2>Error loading video</h2><p style='margin: -20px 0 20px 0;'>Original error message: " + message + "<br /><br /><b>Please try reloading the page or try switching to another backend. This will fix it almost all of the time.</b><br /><br />If the error persists (after multiple tries, using different backends and trying different videos, please <a href='https://github.com/Sommerwiesel/invidious-nerdvpn/issues/new?template=bug_report.md' target='_blank'>open an issue on my github</a>.</p>"
|
||||
+ error_message = "<h2>Error loading video</h2><p style='margin: -20px 0 20px 0;'>Original error message: " + message + "<br /><br /><b>Please try reloading the page or try switching to another backend. This will fix it almost all of the time.</b><br /><br />If the error persists (after multiple tries, using different backends and trying different videos), please <a href='https://github.com/Sommerwiesel/invidious-nerdvpn/issues/new?template=bug_report.md' target='_blank'>open an issue on my github</a>.</p>"
|
||||
+ else
|
||||
+ error_message = translate(locale, message)
|
||||
+ end
|
||||
|
||||
Reference in New Issue
Block a user