From a2d13fd1d1b5c87fdddc7c31809631b7684ff5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20Devos?= Date: Fri, 15 Oct 2021 07:58:35 +0000 Subject: [PATCH] Initial commit. --- .github/workflows/docker-image.yml | 80 +++++++ .gitignore | 1 + Dockerfile | 35 +++ Dockerfile.new | 39 +++ docker/APKBUILD-boringssl | 46 ++++ docker/APKBUILD-lsquic | 43 ++++ patches/1-2469.patch | 372 +++++++++++++++++++++++++++++ patches/2-2111.patch.disable | 45 ++++ patches/2-2506.patch | 55 +++++ patches/5-donate-page.patch | 95 ++++++++ 10 files changed, 811 insertions(+) create mode 100644 .github/workflows/docker-image.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Dockerfile.new create mode 100644 docker/APKBUILD-boringssl create mode 100644 docker/APKBUILD-lsquic create mode 100644 patches/1-2469.patch create mode 100644 patches/2-2111.patch.disable create mode 100644 patches/2-2506.patch create mode 100644 patches/5-donate-page.patch diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..057a690 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,80 @@ +name: Docker Image CI + +on: + push: + branches: + - master + + pull_request: + branches: + - master + + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 0 * * *' + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + - name: Check Out Repo + uses: actions/checkout@v2 + + - name: Check Out Repo Invidious + uses: actions/checkout@v2 + with: + repository: "iv-org/invidious" + ref: 'master' + path: 'invidious' + + - run: | + cd invidious + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git am ../patches/*.patch + sed -i 's/https:\/\/invidious.io\/donate\//\/donate/g' src/invidious/views/template.ecr + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Set up build timestamp + run: echo "timestamp=$(date +%Y%m%d)" >> $GITHUB_ENV + + - name: Build and push alpine docker image + id: docker_build_new + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./Dockerfile.new + build-args: | + SENTRY_DSN=$SENTRY_DSN + release=1 + platforms: linux/amd64,linux/arm64/v8 + push: true + tags: unixfox/invidious-custom:new-latest, unixfox/invidious-custom:new-build-${{ env.timestamp }} + + - name: Build and push official crystal docker image + id: docker_build + uses: docker/build-push-action@v2 + with: + context: ./ + build-args: | + SENTRY_DSN=$SENTRY_DSN + release=1 + file: ./Dockerfile + push: true + tags: unixfox/invidious-custom:latest, unixfox/invidious-custom:build-${{ env.timestamp }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1213661 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +invidious/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3499234 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM crystallang/crystal:1.1.1-alpine AS builder +RUN apk add --no-cache sqlite-static yaml-static + +WORKDIR /invidious +COPY ./invidious/shard.yml ./shard.yml +COPY ./invidious/shard.lock ./shard.lock +RUN shards install + +COPY --from=quay.io/invidious/lsquic-compiled /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a + +COPY ./invidious/src/ ./src/ +# TODO: .git folder is required for building – this is destructive. +# See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION. +COPY ./invidious/.git/ ./.git/ +RUN crystal build --release ./src/invidious.cr \ + --static --warnings all \ + --link-flags "-lxml2 -llzma" + +FROM alpine:latest +RUN apk add --no-cache librsvg ttf-opensans +WORKDIR /invidious +RUN addgroup -g 1000 -S invidious && \ + adduser -u 1000 -S invidious -G invidious +COPY ./invidious/assets/ ./assets/ +COPY --chown=invidious ./invidious/config/config.* ./config/ +RUN mv -n config/config.example.yml config/config.yml +RUN sed -i 's/host: \(127.0.0.1\|localhost\)/host: postgres/' config/config.yml +COPY ./invidious/config/sql/ ./config/sql/ +COPY ./invidious/locales/ ./locales/ +COPY --from=builder /invidious/invidious . +RUN chmod o+rX -R ./assets ./config ./locales + +EXPOSE 3000 +USER invidious +CMD [ "/invidious/invidious" ] \ No newline at end of file diff --git a/Dockerfile.new b/Dockerfile.new new file mode 100644 index 0000000..a5e1205 --- /dev/null +++ b/Dockerfile.new @@ -0,0 +1,39 @@ +FROM alpine:edge AS builder +RUN apk add --no-cache 'crystal=1.1.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev yq + +WORKDIR /invidious +COPY ./invidious/shard.yml ./shard.yml +COPY ./invidious/shard.lock ./shard.lock +RUN yq e -i '.dependencies.raven.github = "Sija/raven.cr"' shard.yml +RUN yq e -i '.targets.sentry_crash_handler.main = "lib/raven/src/crash_handler.cr"' shard.yml +RUN shards install + +COPY --from=quay.io/invidious/lsquic-compiled /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a + +COPY ./invidious/src/ ./src/ +# TODO: .git folder is required for building – this is destructive. +# See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION. +COPY ./invidious/.git/ ./.git/ +RUN crystal build --release ./src/invidious.cr \ + --static --warnings all \ + --link-flags "-lxml2 -llzma" +RUN shards build --release --static sentry_crash_handler + +FROM alpine:edge +RUN apk add --no-cache librsvg ttf-opensans +WORKDIR /invidious +RUN addgroup -g 1000 -S invidious && \ + adduser -u 1000 -S invidious -G invidious +COPY ./invidious/assets/ ./assets/ +COPY --chown=invidious ./invidious/config/config.* ./config/ +RUN mv -n config/config.example.yml config/config.yml +RUN sed -i 's/host: \(127.0.0.1\|localhost\)/host: postgres/' config/config.yml +COPY ./invidious/config/sql/ ./config/sql/ +COPY ./invidious/locales/ ./locales/ +COPY --from=builder /invidious/invidious . +COPY --from=builder /invidious/bin/sentry_crash_handler . +RUN chmod o+rX -R ./assets ./config ./locales + +EXPOSE 3000 +USER invidious +CMD [ "/invidious/sentry_crash_handler", "/invidious/invidious" ] diff --git a/docker/APKBUILD-boringssl b/docker/APKBUILD-boringssl new file mode 100644 index 0000000..61caa4f --- /dev/null +++ b/docker/APKBUILD-boringssl @@ -0,0 +1,46 @@ +# Based on https://aur.archlinux.org/packages/boringssl-git/ +# Maintainer: Omar Roth +pkgname=boringssl +pkgver=1.1.0 +pkgrel=0 +pkgdesc="BoringSSL is a fork of OpenSSL that is designed to meet Google's needs" +url="https://boringssl.googlesource.com/boringssl" +arch="all" +license="MIT" +replaces="openssl libressl" +depends="!openssl-libs-static" +makedepends_host="linux-headers" +makedepends="cmake git go perl" +subpackages="$pkgname-static $pkgname-dev $pkgname-doc" +source="251b516.tar.gz::https://github.com/google/boringssl/tarball/251b516" +builddir="$srcdir/google-boringssl-251b516" + +prepare() { + : +} + +build() { + cmake -DCMAKE_BUILD_TYPE=Release . + make ssl crypto +} + +check() { + make all_tests +} + +package() { + for i in *.md ; do + install -Dm644 $i "$pkgdir/usr/share/doc/$pkgname/$i" + done + install -d "$pkgdir/usr/lib" + install -d "$pkgdir/usr/include" + cp -R include/openssl "$pkgdir/usr/include" + + install -Dm755 crypto/libcrypto.a "$pkgdir/usr/lib/libcrypto.a" + install -Dm755 ssl/libssl.a "$pkgdir/usr/lib/libssl.a" +# install -Dm755 decrepit/libdecrepit.a "$pkgdir/usr/lib/libdecrepit.a" +# install -Dm755 libboringssl_gtest.a "$pkgdir/usr/lib/libboringssl_gtest.a" +} +sha512sums=" +b1d42ed188cf0cce89d40061fa05de85b387ee4244f1236ea488a431536a2c6b657b4f03daed0ac9328c7f5c4c9330499283b8a67f1444dcf9ba5e97e1199c4e 251b516.tar.gz +" diff --git a/docker/APKBUILD-lsquic b/docker/APKBUILD-lsquic new file mode 100644 index 0000000..51630a0 --- /dev/null +++ b/docker/APKBUILD-lsquic @@ -0,0 +1,43 @@ +# Maintainer: Omar Roth +pkgname=lsquic +pkgver=2.18.1 +pkgrel=0 +pkgdesc="LiteSpeed QUIC and HTTP/3 Library" +url="https://github.com/litespeedtech/lsquic" +arch="all" +license="MIT" +depends="boringssl-dev boringssl-static zlib-static libevent-static" +makedepends="cmake git go perl bsd-compat-headers linux-headers" +subpackages="$pkgname-static" +source="v$pkgver.tar.gz::https://github.com/litespeedtech/lsquic/tarball/v2.18.1 +ls-qpack-$pkgver.tar.gz::https://github.com/litespeedtech/ls-qpack/tarball/a8ae6ef +ls-hpack-$pkgver.tar.gz::https://github.com/litespeedtech/ls-hpack/tarball/bd5d589" +builddir="$srcdir/litespeedtech-$pkgname-692a910" + +prepare() { + cp -r -T "$srcdir/litespeedtech-ls-qpack-a8ae6ef" "$builddir/src/liblsquic/ls-qpack" + cp -r -T "$srcdir/litespeedtech-ls-hpack-bd5d589" "$builddir/src/lshpack" +} + +build() { + cmake \ + -DCMAKE_BUILD_TYPE=None \ + -DBORINGSSL_INCLUDE=/usr/include/openssl \ + -DBORINGSSL_LIB_crypto=/usr/lib \ + -DBORINGSSL_LIB_ssl=/usr/lib . + make lsquic +} + +check() { + make tests +} + +package() { + install -d "$pkgdir/usr/lib" + install -Dm755 src/liblsquic/liblsquic.a "$pkgdir/usr/lib/liblsquic.a" +} +sha512sums=" +d015a72f1e88750ecb364768a40f532678f11ded09c6447a2e698b20f43fa499ef143a53f4c92a5938dfece0e39e687dc9df4aea97c618faee0c63da771561c3 v2.18.1.tar.gz +c5629085a3881815fb0b72a321eeba8de093eff9417b8ac7bde1ee1264971be0dca6d61d74799b02ae03a4c629b2a9cf21387deeb814935339a8a2503ea33fee ls-qpack-2.18.1.tar.gz +1b9f7ce4c82dadfca8154229a415b0335a61761eba698f814d4b94195c708003deb5cb89318a1ab78ac8fa88b141bc9df283fb1c6e40b3ba399660feaae353a0 ls-hpack-2.18.1.tar.gz +" diff --git a/patches/1-2469.patch b/patches/1-2469.patch new file mode 100644 index 0000000..4c2afd6 --- /dev/null +++ b/patches/1-2469.patch @@ -0,0 +1,372 @@ +From 4968e2bbe6df1c3b87397d64e14b22a78e2c581a Mon Sep 17 00:00:00 2001 +From: Andrew Zhao +Date: Sat, 1 May 2021 13:09:57 -0400 +Subject: [PATCH] limit feeds and delete materialized views rebased by unixfox + +--- + config/migrate-scripts/migrate-db-8bc91ce.sh | 6 ++ + config/sql/channel_videos.sql | 8 +-- + docker-compose.yml | 1 - + kubernetes/values.yaml | 1 - + src/invidious.cr | 14 ---- + src/invidious/helpers/helpers.cr | 1 - + src/invidious/jobs/refresh_feeds_job.cr | 75 -------------------- + src/invidious/routes/login.cr | 3 - + src/invidious/routes/search.cr | 2 + + src/invidious/search.cr | 19 ++--- + src/invidious/users.cr | 51 ++++++------- + 11 files changed, 41 insertions(+), 140 deletions(-) + create mode 100644 config/migrate-scripts/migrate-db-8bc91ce.sh + delete mode 100644 src/invidious/jobs/refresh_feeds_job.cr + +diff --git a/config/migrate-scripts/migrate-db-8bc91ce.sh b/config/migrate-scripts/migrate-db-8bc91ce.sh +new file mode 100644 +index 000000000..04388175e +--- /dev/null ++++ b/config/migrate-scripts/migrate-db-8bc91ce.sh +@@ -0,0 +1,6 @@ ++CREATE INDEX channel_videos_ucid_published_idx ++ ON public.channel_videos ++ USING btree ++ (ucid COLLATE pg_catalog."default", published); ++ ++DROP INDEX channel_videos_ucid_idx; +\ No newline at end of file +diff --git a/config/sql/channel_videos.sql b/config/sql/channel_videos.sql +index cd4e0ffdb..f2ac4876c 100644 +--- a/config/sql/channel_videos.sql ++++ b/config/sql/channel_videos.sql +@@ -19,12 +19,12 @@ CREATE TABLE IF NOT EXISTS public.channel_videos + + GRANT ALL ON TABLE public.channel_videos TO current_user; + +--- Index: public.channel_videos_ucid_idx ++-- Index: public.channel_videos_ucid_published_idx + +--- DROP INDEX public.channel_videos_ucid_idx; ++-- DROP INDEX public.channel_videos_ucid_published_idx; + +-CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx ++CREATE INDEX IF NOT EXISTS channel_videos_ucid_published_idx + ON public.channel_videos + USING btree +- (ucid COLLATE pg_catalog."default"); ++ (ucid COLLATE pg_catalog."default", published); + +diff --git a/docker-compose.yml b/docker-compose.yml +index b94f98137..2da5755f2 100644 +--- a/docker-compose.yml ++++ b/docker-compose.yml +@@ -25,7 +25,6 @@ services: + INVIDIOUS_CONFIG: | + channel_threads: 1 + check_tables: true +- feed_threads: 1 + db: + user: kemal + password: kemal +diff --git a/kubernetes/values.yaml b/kubernetes/values.yaml +index f241970c9..e8792ca6f 100644 +--- a/kubernetes/values.yaml ++++ b/kubernetes/values.yaml +@@ -44,7 +44,6 @@ postgresql: + # Adapted from ../config/config.yml + config: + channel_threads: 1 +- feed_threads: 1 + db: + user: kemal + password: kemal +diff --git a/src/invidious.cr b/src/invidious.cr +index 9e67e216a..15a3d16d5 100644 +--- a/src/invidious.cr ++++ b/src/invidious.cr +@@ -80,14 +80,6 @@ Kemal.config.extra_options do |parser| + exit + end + end +- parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{CONFIG.feed_threads})") do |number| +- begin +- CONFIG.feed_threads = number.to_i +- rescue ex +- puts "THREADS must be integer" +- exit +- end +- end + parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: #{CONFIG.output})") do |output| + CONFIG.output = output + end +@@ -132,10 +124,6 @@ if CONFIG.channel_threads > 0 + Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB) + end + +-if CONFIG.feed_threads > 0 +- Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB) +-end +- + DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling) + if CONFIG.decrypt_polling + Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new +@@ -1005,10 +993,8 @@ post "/delete_account" do |env| + next error_template(400, ex) + end + +- view_name = "subscriptions_#{sha256(user.email)}" + PG_DB.exec("DELETE FROM users * WHERE email = $1", user.email) + PG_DB.exec("DELETE FROM session_ids * WHERE email = $1", user.email) +- PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") + + env.request.cookies.each do |cookie| + cookie.expires = Time.utc(1990, 1, 1) +diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr +index 99adcd304..81eb4c050 100644 +--- a/src/invidious/helpers/helpers.cr ++++ b/src/invidious/helpers/helpers.cr +@@ -69,7 +69,6 @@ class Config + include YAML::Serializable + + property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) +- property feed_threads : Int32 = 1 # Number of threads to use for updating feeds + property output : String = "STDOUT" # Log file path or STDOUT + property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr + property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) +diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr +deleted file mode 100644 +index 926c27fa6..000000000 +--- a/src/invidious/jobs/refresh_feeds_job.cr ++++ /dev/null +@@ -1,75 +0,0 @@ +-class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob +- private getter db : DB::Database +- +- def initialize(@db) +- end +- +- def begin +- max_fibers = CONFIG.feed_threads +- active_fibers = 0 +- active_channel = Channel(Bool).new +- +- loop do +- db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs| +- rs.each do +- email = rs.read(String) +- view_name = "subscriptions_#{sha256(email)}" +- +- if active_fibers >= max_fibers +- if active_channel.receive +- active_fibers -= 1 +- end +- end +- +- active_fibers += 1 +- spawn do +- begin +- # Drop outdated views +- column_array = get_column_array(db, view_name) +- ChannelVideo.type_array.each_with_index do |name, i| +- if name != column_array[i]? +- LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}") +- db.exec("DROP MATERIALIZED VIEW #{view_name}") +- raise "view does not exist" +- end +- end +- +- if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))" +- LOGGER.info("RefreshFeedsJob: Materialized view #{view_name} is out-of-date, recreating...") +- db.exec("DROP MATERIALIZED VIEW #{view_name}") +- end +- +- db.exec("REFRESH MATERIALIZED VIEW #{view_name}") +- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) +- rescue ex +- # Rename old views +- begin +- legacy_view_name = "subscriptions_#{sha256(email)[0..7]}" +- +- db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0") +- LOGGER.info("RefreshFeedsJob: RENAME MATERIALIZED VIEW #{legacy_view_name}") +- db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}") +- rescue ex +- begin +- # While iterating through, we may have an email stored from a deleted account +- if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool) +- LOGGER.info("RefreshFeedsJob: CREATE #{view_name}") +- db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}") +- db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) +- end +- rescue ex +- LOGGER.error("RefreshFeedJobs: REFRESH #{email} : #{ex.message}") +- end +- end +- end +- +- active_channel.send(true) +- end +- end +- end +- +- sleep 5.seconds +- Fiber.yield +- end +- end +-end +diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr +index e7aef289f..b9787ee2c 100644 +--- a/src/invidious/routes/login.cr ++++ b/src/invidious/routes/login.cr +@@ -451,9 +451,6 @@ module Invidious::Routes::Login + PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) + PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + +- view_name = "subscriptions_#{sha256(user.email)}" +- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") +- + if Kemal.config.ssl || CONFIG.https_only + secure = true + else +diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr +index 3f1e219f2..cd40d3913 100644 +--- a/src/invidious/routes/search.cr ++++ b/src/invidious/routes/search.cr +@@ -53,6 +53,8 @@ module Invidious::Routes::Search + + user = env.get? "user" + ++ user = user ? user.as(User) : nil ++ + begin + search_query, count, videos, operators = process_search_query(query, page, user, region: region) + rescue ex +diff --git a/src/invidious/search.cr b/src/invidious/search.cr +index d95d802ea..93197b6f0 100644 +--- a/src/invidious/search.cr ++++ b/src/invidious/search.cr +@@ -170,11 +170,6 @@ def produce_channel_search_continuation(ucid, query, page) + end + + def process_search_query(query, page, user, region) +- if user +- user = user.as(User) +- view_name = "subscriptions_#{sha256(user.email)}" +- end +- + channel = nil + content_type = "all" + date = "" +@@ -212,14 +207,14 @@ def process_search_query(query, page, user, region) + if channel + count, items = channel_search(search_query, page, channel) + elsif subscriptions +- if view_name ++ if user + items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM ( +- SELECT *, +- to_tsvector(#{view_name}.title) || +- to_tsvector(#{view_name}.author) +- as document +- FROM #{view_name} +- ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) ++ SELECT cv.*, to_tsvector(cv.title) || to_tsvector(cv.author) AS document ++ FROM channel_videos cv ++ JOIN users ON cv.ucid = any(users.subscriptions) ++ WHERE users.email = $1 AND published > now() - interval '1 month' ++ ORDER BY published ++ ) v_search WHERE v_search.document @@ plainto_tsquery($2) LIMIT 20 OFFSET $3;", user.email, search_query, (page - 1) * 20, as: ChannelVideo) + count = items.size + else + items = [] of ChannelVideo +diff --git a/src/invidious/users.cr b/src/invidious/users.cr +index aff76b536..b35d07b5f 100644 +--- a/src/invidious/users.cr ++++ b/src/invidious/users.cr +@@ -302,12 +302,6 @@ def get_user(sid, headers, db, refresh = true) + + db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ + ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) +- +- begin +- view_name = "subscriptions_#{sha256(user.email)}" +- db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") +- rescue ex +- end + end + else + user, sid = fetch_user(sid, headers, db) +@@ -320,12 +314,6 @@ def get_user(sid, headers, db, refresh = true) + + db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ + ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) +- +- begin +- view_name = "subscriptions_#{sha256(user.email)}" +- db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") +- rescue ex +- end + end + + return user, sid +@@ -496,7 +484,6 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) + + notifications = db.query_one("SELECT notifications FROM users WHERE email = $1", user.email, + as: Array(String)) +- view_name = "subscriptions_#{sha256(user.email)}" + + if user.preferences.notifications_only && !notifications.empty? + # Only show notifications +@@ -525,33 +512,39 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) + # Show latest video from a channel that a user hasn't watched + # "unseen_only" isn't really correct here, more accurate would be "unwatched_only" + +- if user.watched.empty? +- values = "'{}'" +- else +- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" +- end +- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo) ++ # "SELECT cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = $1 AND published > now() - interval '1 month' ORDER BY published DESC" ++ # "SELECT DISTINCT ON (cv.ucid) cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = ? AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' ORDER BY ucid, published DESC" ++ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \ ++ "FROM channel_videos cv " \ ++ "JOIN users ON cv.ucid = any(users.subscriptions) " \ ++ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \ ++ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo) + else + # Show latest video from each channel + +- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo) ++ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \ ++ "FROM channel_videos cv " \ ++ "JOIN users ON cv.ucid = any(users.subscriptions) " \ ++ "WHERE users.email = $1 AND published > now() - interval '1 month' " \ ++ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo) + end + + videos.sort_by! { |video| video.published }.reverse! + else + if user.preferences.unseen_only + # Only show unwatched +- +- if user.watched.empty? +- values = "'{}'" +- else +- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" +- end +- videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) ++ videos = PG_DB.query_all("SELECT cv.* " \ ++ "FROM channel_videos cv " \ ++ "JOIN users ON cv.ucid = any(users.subscriptions) " \ ++ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \ ++ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo) + else + # Sort subscriptions as normal +- +- videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) ++ videos = PG_DB.query_all("SELECT cv.* " \ ++ "FROM channel_videos cv " \ ++ "JOIN users ON cv.ucid = any(users.subscriptions) " \ ++ "WHERE users.email = $1 AND published > now() - interval '1 month' " \ ++ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo) + end + end + diff --git a/patches/2-2111.patch.disable b/patches/2-2111.patch.disable new file mode 100644 index 0000000..9ad24ac --- /dev/null +++ b/patches/2-2111.patch.disable @@ -0,0 +1,45 @@ +From 78773d732672d8985795fb040a39dd7e946c7b7c Mon Sep 17 00:00:00 2001 +From: Emilien Devos +Date: Sat, 22 May 2021 17:42:23 +0200 +Subject: [PATCH] add the ability to listen on unix sockets + +--- + src/invidious.cr | 15 ++++++++++++--- + src/invidious/helpers/helpers.cr | 1 + + 2 files changed, 13 insertions(+), 3 deletions(-) + +diff --git a/src/invidious.cr b/src/invidious.cr +index ae20e13e5..65b1091bb 100644 +--- a/src/invidious.cr ++++ b/src/invidious.cr +@@ -3917,6 +3917,15 @@ add_context_storage_type(Preferences) + add_context_storage_type(User) + + Kemal.config.logger = LOGGER +-Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding +-Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port +-Kemal.run ++ ++Kemal.run do |config| ++ if CONFIG.bind_unix ++ if File.exists?(CONFIG.bind_unix.not_nil!) ++ File.delete(CONFIG.bind_unix.not_nil!) ++ end ++ config.server.not_nil!.bind_unix CONFIG.bind_unix.not_nil! ++ else ++ config.host_binding = config.host_binding != "0.0.0.0" ? config.host_binding : CONFIG.host_binding ++ config.port = config.port != 3000 ? config.port : CONFIG.port ++ end ++end +diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr +index e1d877b78..6a5789a0e 100644 +--- a/src/invidious/helpers/helpers.cr ++++ b/src/invidious/helpers/helpers.cr +@@ -98,6 +98,7 @@ class Config + property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) + property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) + property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) ++ property bind_unix : String? = nil # Make Invidious listening on UNIX sockets - Example: /tmp/invidious.sock + property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + property use_quic : Bool = true # Use quic transport for youtube api + diff --git a/patches/2-2506.patch b/patches/2-2506.patch new file mode 100644 index 0000000..887ed5b --- /dev/null +++ b/patches/2-2506.patch @@ -0,0 +1,55 @@ +From b49b5fbda90a7fc9923af27a870504be976d55e9 Mon Sep 17 00:00:00 2001 +From: syeopite +Date: Wed, 13 Oct 2021 11:32:27 -0700 +Subject: [PATCH 1/2] Support empty categories + +--- + src/invidious/helpers/extractors.cr | 12 +++++++----- + 1 file changed, 7 insertions(+), 5 deletions(-) + +diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/helpers/extractors.cr +index c8a6cd4a4..73b07fd25 100644 +--- a/src/invidious/helpers/extractors.cr ++++ b/src/invidious/helpers/extractors.cr +@@ -321,11 +321,13 @@ private module Parsers + content_container = item_contents["contents"] + end + +- raw_contents = content_container["items"].as_a +- raw_contents.each do |item| +- result = extract_item(item) +- if !result.nil? +- contents << result ++ raw_contents = content_container["items"]?.try &.as_a ++ if !raw_contents.nil? ++ raw_contents.each do |item| ++ result = extract_item(item) ++ if !result.nil? ++ contents << result ++ end + end + end + + +From 4d44b2c3a422abb3e989a9010c082cfb6a52fb12 Mon Sep 17 00:00:00 2001 +From: syeopite +Date: Wed, 13 Oct 2021 11:33:04 -0700 +Subject: [PATCH 2/2] Handle YT tabs without any content + +--- + src/invidious/helpers/extractors.cr | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/helpers/extractors.cr +index 73b07fd25..6af40de5a 100644 +--- a/src/invidious/helpers/extractors.cr ++++ b/src/invidious/helpers/extractors.cr +@@ -401,7 +401,7 @@ private module Extractors + items_container = renderer_container_contents + end + +- items_container["items"].as_a.each do |item| ++ items_container["items"]?.try &.as_a.each do |item| + raw_items << item + end + end diff --git a/patches/5-donate-page.patch b/patches/5-donate-page.patch new file mode 100644 index 0000000..706ea92 --- /dev/null +++ b/patches/5-donate-page.patch @@ -0,0 +1,95 @@ +From a7bfbd84d9da7f15e6f7b8bde2aafde19a232b97 Mon Sep 17 00:00:00 2001 +From: Emilien Devos +Date: Thu, 12 Aug 2021 23:16:22 +0200 +Subject: [PATCH] donate page + +--- + src/invidious.cr | 1 + + src/invidious/routes/misc.cr | 5 +++ + src/invidious/views/components/feed_menu.ecr | 3 ++ + src/invidious/views/donate.ecr | 34 ++++++++++++++++++++ + src/invidious/views/template.ecr | 4 +-- + 5 files changed, 45 insertions(+), 2 deletions(-) + create mode 100644 src/invidious/views/donate.ecr + +diff --git a/src/invidious.cr b/src/invidious.cr +index 9e67e216..a73e6645 100644 +--- a/src/invidious.cr ++++ b/src/invidious.cr +@@ -314,6 +314,7 @@ end + + {% unless flag?(:api_only) %} + Invidious::Routing.get "/", Invidious::Routes::Misc, :home ++ Invidious::Routing.get "/donate", Invidious::Routes::Misc, :donate + Invidious::Routing.get "/privacy", Invidious::Routes::Misc, :privacy + Invidious::Routing.get "/licenses", Invidious::Routes::Misc, :licenses + +diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr +index 0e6356d0..cbf5215d 100644 +--- a/src/invidious/routes/misc.cr ++++ b/src/invidious/routes/misc.cr +@@ -33,6 +33,11 @@ module Invidious::Routes::Misc + templated "privacy" + end + ++ def self.donate(env) ++ locale = LOCALES[env.get("preferences").as(Preferences).locale]? ++ templated "donate" ++ end ++ + def self.licenses(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + rendered "licenses" +diff --git a/src/invidious/views/components/feed_menu.ecr b/src/invidious/views/components/feed_menu.ecr +index 3dbeaf37..d84d03d0 100644 +--- a/src/invidious/views/components/feed_menu.ecr ++++ b/src/invidious/views/components/feed_menu.ecr +@@ -1,4 +1,7 @@ +
++ ++ Donate ++ + <% feed_menu = env.get("preferences").as(Preferences).feed_menu.dup %> + <% if !env.get?("user") %> + <% feed_menu.reject! {|item| {"Subscriptions", "Playlists"}.includes? item} %> +diff --git a/src/invidious/views/donate.ecr b/src/invidious/views/donate.ecr +new file mode 100644 +index 00000000..59e6f7f9 +--- /dev/null ++++ b/src/invidious/views/donate.ecr +@@ -0,0 +1,34 @@ ++<% content_for "header" do %> ++Donate - Invidious ++<% end %> ++ ++
++

Donate to YewTube

++

Why donating?

++

Every day YewTube handle 40 million requests and 2TB (2000GB) of bandwidth. All this traffic has a cost.

++

By donating, you are helping me cover the costs and also improve the user experience because I'll be able to rent more powerful servers.

++

I truly thank you if you consider donating to YewTube.

++

Payment methods

++

Credit/debit card and bank transfer

++ ++

Cryptocurrency

++ ++

Is your prefered payment method not listed?

++

Feel ask me by email at donate[at]yewtu.be to add it.

++
+\ No newline at end of file