NerdVPN.de/invidious
1
0
Fork 0
mirror of https://git.nerdvpn.de/NerdVPN.de/invidious synced 2026-02-14 22:51:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

Forgot to add modified files

Browse Source
This commit is contained in:
Sommerwiesel
2025-06-19 11:05:11 +02:00
parent 8863033022
commit d38142df42
2 changed files with 362 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/003-pubsub-innertube.patch
+56
View File
@@ -0,0 +1,56 @@
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index 4d69854c..59cbaa08 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -117,6 +117,7 @@ class Config
property domain : String?
# Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
property use_pubsub_feeds : Bool | Int32 = false
+ property use_innertube_for_feeds : Bool = true
property popular_enabled : Bool = true
property captcha_enabled : Bool = true
property login_enabled : Bool = true
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index 070c96eb..439fe462 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -419,24 +419,28 @@ module Invidious::Routes::Feeds
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
-
- begin
- video = get_video(id, force_refresh: true)
- rescue
- next # skip this video since it raised an exception (e.g. it is a scheduled live event)
+ ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
+ title = entry.xpath_node("default:title", namespaces).not_nil!.content
+
+ if CONFIG.use_innertube_for_feeds
+ begin
+ video_ = get_video(id, force_refresh: true)
+ rescue
+ next
+ end
end
video = ChannelVideo.new({
id: id,
- title: video.title,
+ title: title,
published: published,
updated: updated,
- ucid: video.ucid,
+ ucid: ucid,
author: author,
- length_seconds: video.length_seconds,
- live_now: video.live_now,
- premiere_timestamp: video.premiere_timestamp,
- views: video.views,
+ length_seconds: video_.try &.length_seconds || 0,
+ live_now: video_.try &.live_now || false,
+ premiere_timestamp: video_.try &.premiere_timestamp || nil,
+ views: video_.try &.views || nil,
})
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
patches/011-remove-psql-mat-views.patch
+306
View File
@@ -0,0 +1,306 @@
From e476dbe25baddcbfd5ee61d27c706e16e7771f68 Mon Sep 17 00:00:00 2001
From: Emilien Devos <4016501+unixfox@users.noreply.github.com>
Date: Wed, 26 Jul 2023 15:43:43 +0200
Subject: [PATCH] limit feeds and delete materialized views
---
config/migrate-scripts/migrate-db-8bc91ce.sh | 6 ++
config/sql/channel_videos.sql | 8 +--
src/invidious.cr | 12 ----
src/invidious/config.cr | 2 -
src/invidious/jobs/refresh_feeds_job.cr | 75 --------------------
src/invidious/routes/account.cr | 2 -
src/invidious/routes/login.cr | 3 -
src/invidious/search/processors.cr | 18 ++---
src/invidious/users.cr | 39 +++++-----
9 files changed, 41 insertions(+), 124 deletions(-)
create mode 100644 config/migrate-scripts/migrate-db-8bc91ce.sh
delete mode 100644 src/invidious/jobs/refresh_feeds_job.cr
diff --git a/config/migrate-scripts/migrate-db-8bc91ce.sh b/config/migrate-scripts/migrate-db-8bc91ce.sh
new file mode 100644
index 000000000..04388175e
--- /dev/null
+++ b/config/migrate-scripts/migrate-db-8bc91ce.sh
@@ -0,0 +1,6 @@
+CREATE INDEX channel_videos_ucid_published_idx
+ ON public.channel_videos
+ USING btree
+ (ucid COLLATE pg_catalog."default", published);
+
+DROP INDEX channel_videos_ucid_idx;
\ No newline at end of file
diff --git a/config/sql/channel_videos.sql b/config/sql/channel_videos.sql
index cd4e0ffdb..f2ac4876c 100644
--- a/config/sql/channel_videos.sql
+++ b/config/sql/channel_videos.sql
@@ -19,12 +19,12 @@ CREATE TABLE IF NOT EXISTS public.channel_videos
GRANT ALL ON TABLE public.channel_videos TO current_user;
--- Index: public.channel_videos_ucid_idx
+-- Index: public.channel_videos_ucid_published_idx
--- DROP INDEX public.channel_videos_ucid_idx;
+-- DROP INDEX public.channel_videos_ucid_published_idx;
-CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx
+CREATE INDEX IF NOT EXISTS channel_videos_ucid_published_idx
ON public.channel_videos
USING btree
- (ucid COLLATE pg_catalog."default");
+ (ucid COLLATE pg_catalog."default", published);
diff --git a/src/invidious.cr b/src/invidious.cr
index 3804197e3..961ae8721 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -103,14 +103,6 @@ Kemal.config.extra_options do |parser|
exit
end
end
- parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number|
- begin
- CONFIG.feed_threads = number.to_i
- rescue ex
- puts "THREADS must be integer"
- exit
- end
- end
parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output|
CONFIG.output = output
end
@@ -168,10 +160,6 @@ if CONFIG.channel_threads > 0
Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB)
end
-if CONFIG.feed_threads > 0
- Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB)
-end
-
if CONFIG.statistics_enabled
Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE)
end
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index c4ddcdb36..2e6df47a1 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -62,8 +62,6 @@ class Config
# Time interval between two executions of the job that crawls channel videos (subscriptions update).
@[YAML::Field(converter: Preferences::TimeSpanConverter)]
property channel_refresh_interval : Time::Span = 30.minutes
- # Number of threads to use for updating feeds
- property feed_threads : Int32 = 1
# Log file path or STDOUT
property output : String = "STDOUT"
# Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr
deleted file mode 100644
index 4f8130dfd..000000000
--- a/src/invidious/jobs/refresh_feeds_job.cr
+++ /dev/null
@@ -1,75 +0,0 @@
-class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
- private getter db : DB::Database
-
- def initialize(@db)
- end
-
- def begin
- max_fibers = CONFIG.feed_threads
- active_fibers = 0
- active_channel = ::Channel(Bool).new
-
- loop do
- db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
- rs.each do
- email = rs.read(String)
- view_name = "subscriptions_#{sha256(email)}"
-
- if active_fibers >= max_fibers
- if active_channel.receive
- active_fibers -= 1
- end
- end
-
- active_fibers += 1
- spawn do
- begin
- # Drop outdated views
- column_array = Invidious::Database.get_column_array(db, view_name)
- ChannelVideo.type_array.each_with_index do |name, i|
- if name != column_array[i]?
- LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}")
- db.exec("DROP MATERIALIZED VIEW #{view_name}")
- raise "view does not exist"
- end
- end
-
- if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))"
- LOGGER.info("RefreshFeedsJob: Materialized view #{view_name} is out-of-date, recreating...")
- db.exec("DROP MATERIALIZED VIEW #{view_name}")
- end
-
- db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
- rescue ex
- # Rename old views
- begin
- legacy_view_name = "subscriptions_#{sha256(email)[0..7]}"
-
- db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0")
- LOGGER.info("RefreshFeedsJob: RENAME MATERIALIZED VIEW #{legacy_view_name}")
- db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}")
- rescue ex
- begin
- # While iterating through, we may have an email stored from a deleted account
- if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
- LOGGER.info("RefreshFeedsJob: CREATE #{view_name}")
- db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}")
- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
- end
- rescue ex
- LOGGER.error("RefreshFeedJobs: REFRESH #{email} : #{ex.message}")
- end
- end
- end
-
- active_channel.send(true)
- end
- end
- end
-
- sleep 5.seconds
- Fiber.yield
- end
- end
-end
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr
index dd65e7a60..8086a54e5 100644
--- a/src/invidious/routes/account.cr
+++ b/src/invidious/routes/account.cr
@@ -123,10 +123,8 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
- view_name = "subscriptions_#{sha256(user.email)}"
Invidious::Database::Users.delete(user)
Invidious::Database::SessionIDs.delete(email: user.email)
- PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}")
env.request.cookies.each do |cookie|
cookie.expires = Time.utc(1990, 1, 1)
diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr
index d0f7ac229..add9f75d3 100644
--- a/src/invidious/routes/login.cr
+++ b/src/invidious/routes/login.cr
@@ -160,9 +160,6 @@ module Invidious::Routes::Login
Invidious::Database::Users.insert(user)
Invidious::Database::SessionIDs.insert(sid, email)
- view_name = "subscriptions_#{sha256(user.email)}"
- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
-
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
if env.request.cookies["PREFS"]?
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index 25edb9362..10b81c59e 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -37,18 +37,18 @@ module Invidious::Search
# Search inside of user subscriptions
def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
- view_name = "subscriptions_#{sha256(user.email)}"
-
return PG_DB.query_all("
SELECT id,title,published,updated,ucid,author,length_seconds
FROM (
- SELECT *,
- to_tsvector(#{view_name}.title) ||
- to_tsvector(#{view_name}.author)
- as document
- FROM #{view_name}
- ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
- query.text, (query.page - 1) * 20,
+ SELECT cv.*,
+ to_tsvector(cv.title) ||
+ to_tsvector(cv.author) AS document
+ FROM channel_videos cv
+ JOIN users ON cv.ucid = any(users.subscriptions)
+ WHERE users.email = $1 AND published > now() - interval '1 month'
+ ORDER BY published
+ ) v_search WHERE v_search.document @@ plainto_tsquery($2) LIMIT 20 OFFSET $3;",
+ user.email, query.text, (query.page - 1) * 20,
as: ChannelVideo
)
end
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index 65566d207..0b2d1ef5f 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -27,7 +27,6 @@ def get_subscription_feed(user, max_results = 40, page = 1)
offset = (page - 1) * limit
notifications = Invidious::Database::Users.select_notifications(user)
- view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
@@ -53,33 +52,39 @@ def get_subscription_feed(user, max_results = 40, page = 1)
# Show latest video from a channel that a user hasn't watched
# "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
- if user.watched.empty?
- values = "'{}'"
- else
- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
- end
- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
+ # "SELECT cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = $1 AND published > now() - interval '1 month' ORDER BY published DESC"
+ # "SELECT DISTINCT ON (cv.ucid) cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = ? AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' ORDER BY ucid, published DESC"
+ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
+ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
else
# Show latest video from each channel
- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
+ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND published > now() - interval '1 month' " \
+ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
end
videos.sort_by!(&.published).reverse!
else
if user.preferences.unseen_only
# Only show unwatched
-
- if user.watched.empty?
- values = "'{}'"
- else
- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
- end
- videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
+ videos = PG_DB.query_all("SELECT cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
+ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
else
# Sort subscriptions as normal
-
- videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
+ videos = PG_DB.query_all("SELECT cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND published > now() - interval '1 month' " \
+ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
end
end