diff --git a/patches/006-disable-notifications.patch b/patches/006-disable-notifications.patch deleted file mode 100644 index 200de9c..0000000 --- a/patches/006-disable-notifications.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 09e382465cebd61aa7a8b67b0a87a939e8ab5a03 Mon Sep 17 00:00:00 2001 -From: Emilien Devos -Date: Wed, 26 Jan 2022 21:47:00 +0100 -Subject: [PATCH 1/1] disable notifications - ---- - src/invidious/database/users.cr | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr -index 53724dbf..3752a213 100644 ---- a/src/invidious/database/users.cr -+++ b/src/invidious/database/users.cr -@@ -127,7 +127,6 @@ module Invidious::Database::Users - WHERE $2 = ANY(subscriptions) - SQL - -- PG_DB.exec(request, video.id, video.ucid) - end - - def remove_notification(user : User, vid : String) -@@ -137,7 +136,6 @@ module Invidious::Database::Users - WHERE email = $2 - SQL - -- PG_DB.exec(request, vid, user.email) - end - - def clear_notifications(user : User) --- -2.34.1 - diff --git a/patches/010-3419.patch b/patches/010-3419.patch new file mode 100644 index 0000000..e6ba28f --- /dev/null +++ b/patches/010-3419.patch @@ -0,0 +1,2663 @@ +From 99bf5197812b470f17e66aec2ca6eccc55ca6e15 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Thu, 1 Dec 2022 20:09:31 +0100 +Subject: [PATCH 01/16] shards: Bump protodec to v0.1.5 + +--- + shard.lock | 2 +- + shard.yml | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/shard.lock b/shard.lock +index cdce11604..235e4c259 100644 +--- a/shard.lock ++++ b/shard.lock +@@ -34,7 +34,7 @@ shards: + + protodec: + git: https://github.com/iv-org/protodec.git +- version: 0.1.4 ++ version: 0.1.5 + + radix: + git: https://github.com/luislavena/radix.git +diff --git a/shard.yml b/shard.yml +index 9c9b0d37f..7ee0bb2a9 100644 +--- a/shard.yml ++++ b/shard.yml +@@ -24,7 +24,7 @@ dependencies: + version: ~> 0.6.1 + protodec: + github: iv-org/protodec +- version: ~> 0.1.4 ++ version: ~> 0.1.5 + lsquic: + github: iv-org/lsquic.cr + version: ~> 2.18.1-2 + +From fbcce57ce29b05c234c0c31b5f179d861e143260 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Fri, 11 Nov 2022 01:31:32 +0100 +Subject: [PATCH 02/16] channel: use extractor utils to parse tabs (+ code + cleaning) + +--- + src/invidious/channels/about.cr | 62 ++++++++++++++++++--------------- + 1 file changed, 34 insertions(+), 28 deletions(-) + +diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr +index 4c442959d..bb9bd8c71 100644 +--- a/src/invidious/channels/about.cr ++++ b/src/invidious/channels/about.cr +@@ -100,34 +100,40 @@ def get_about_info(ucid, locale) : AboutChannel + total_views = 0_i64 + joined = Time.unix(0) + +- tabs = [] of String +- +- tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a? +- if !tabs_json.nil? +- # Retrieve information from the tabs array. The index we are looking for varies between channels. +- tabs_json.each do |node| +- # Try to find the about section which is located in only one of the tabs. +- channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]? +- .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]? +- .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]? +- +- if !channel_about_meta.nil? +- total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 +- +- # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. +- joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s } +- .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) +- +- # Normal Auto-generated channels +- # https://support.google.com/youtube/answer/2579942 +- # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] +- if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) && +- (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube" +- auto_generated = true +- end +- end ++ tab_names = [] of String ++ ++ if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? ++ # Get the name of the tabs available on this channel ++ tab_names = tabs_json.as_a ++ .compact_map(&.dig?("tabRenderer", "title").try &.as_s.downcase) ++ ++ # Get the currently active tab ("About") ++ about_tab = extract_selected_tab(tabs_json) ++ ++ # Try to find the about metadata section ++ channel_about_meta = about_tab.dig?( ++ "content", ++ "sectionListRenderer", "contents", 0, ++ "itemSectionRenderer", "contents", 0, ++ "channelAboutFullMetadataRenderer" ++ ) ++ ++ if !channel_about_meta.nil? ++ total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 ++ ++ # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. ++ joined = extract_text(channel_about_meta["joinedDateText"]?) ++ .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) ++ ++ # Normal Auto-generated channels ++ # https://support.google.com/youtube/answer/2579942 ++ # For auto-generated channels, channel_about_meta only has ++ # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] ++ auto_generated = ( ++ (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ ++ extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" ++ ) + end +- tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase) + end + + sub_count = initdata +@@ -148,7 +154,7 @@ def get_about_info(ucid, locale) : AboutChannel + joined: joined, + is_family_friendly: is_family_friendly, + allowed_regions: allowed_regions, +- tabs: tabs, ++ tabs: tab_names, + verified: author_verified || false, + ) + end + +From 9588fcb5d1dd90e8591ed53a342727a0df6923c4 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Sat, 3 Dec 2022 22:39:24 +0100 +Subject: [PATCH 03/16] frontend: remove paging on channel videos + +--- + src/invidious/routes/channels.cr | 5 +---- + src/invidious/views/channel.ecr | 18 +++--------------- + 2 files changed, 4 insertions(+), 19 deletions(-) + +diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr +index c6e02cbd6..f26f29f5d 100644 +--- a/src/invidious/routes/channels.cr ++++ b/src/invidious/routes/channels.cr +@@ -12,9 +12,6 @@ module Invidious::Routes::Channels + end + locale, user, subscriptions, continuation, ucid, channel = data + +- page = env.params.query["page"]?.try &.to_i? +- page ||= 1 +- + sort_by = env.params.query["sort_by"]?.try &.downcase + + if channel.auto_generated +@@ -35,7 +32,7 @@ module Invidious::Routes::Channels + sort_options = {"newest", "oldest", "popular"} + sort_by ||= "newest" + +- count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) ++ count, items = get_60_videos(channel.ucid, channel.author, 1, channel.auto_generated, sort_by) + end + + templated "channel" +diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr +index dea86abec..878587d44 100644 +--- a/src/invidious/views/channel.ecr ++++ b/src/invidious/views/channel.ecr +@@ -90,7 +90,7 @@ + <% if sort_by == sort %> + <%= translate(locale, sort) %> + <% else %> +- ++ + <%= translate(locale, sort) %> + + <% end %> +@@ -111,19 +111,7 @@ + + +
+-
+- <% if page > 1 %> +- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> +- <%= translate(locale, "Previous page") %> +- +- <% end %> +-
++
+
+-
+- <% if count == 60 %> +- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> +- <%= translate(locale, "Next page") %> +- +- <% end %> +-
++
+
+ +From bdc51cd20fd2df99c2fe5ddc281aada86000a783 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Thu, 10 Nov 2022 23:32:51 +0100 +Subject: [PATCH 04/16] extractors: separate 'extract' and 'parse' logic + +--- + src/invidious/channels/playlists.cr | 2 +- + src/invidious/search/processors.cr | 2 +- + src/invidious/yt_backend/extractors.cr | 54 +++++++++++++++----------- + 3 files changed, 33 insertions(+), 25 deletions(-) + +diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr +index d5628f6ab..e6c0a1d51 100644 +--- a/src/invidious/channels/playlists.cr ++++ b/src/invidious/channels/playlists.cr +@@ -8,7 +8,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) + + items = [] of SearchItem + continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| +- extract_item(item, author, ucid).try { |t| items << t } ++ parse_item(item, author, ucid).try { |t| items << t } + } + + continuation = continuation_items.as_a.last["continuationItemRenderer"]? +diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr +index d1409c06c..683a4a7e2 100644 +--- a/src/invidious/search/processors.cr ++++ b/src/invidious/search/processors.cr +@@ -37,7 +37,7 @@ module Invidious::Search + + items = [] of SearchItem + continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| +- extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } ++ parse_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } + end + + return items +diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr +index edc722cf1..a4b20d047 100644 +--- a/src/invidious/yt_backend/extractors.cr ++++ b/src/invidious/yt_backend/extractors.cr +@@ -20,6 +20,8 @@ private ITEM_PARSERS = { + Parsers::ReelItemRendererParser, + } + ++private alias InitialData = Hash(String, JSON::Any) ++ + record AuthorFallback, name : String, id : String + + # Namespace for logic relating to parsing InnerTube data into various datastructs. +@@ -348,7 +350,7 @@ private module Parsers + raw_contents = content_container["items"]?.try &.as_a + if !raw_contents.nil? + raw_contents.each do |item| +- result = extract_item(item) ++ result = parse_item(item) + if !result.nil? + contents << result + end +@@ -510,7 +512,7 @@ private module Extractors + # }] + # + module YouTubeTabs +- def self.process(initial_data : Hash(String, JSON::Any)) ++ def self.process(initial_data : InitialData) + if target = initial_data["twoColumnBrowseResultsRenderer"]? + self.extract(target) + end +@@ -575,7 +577,7 @@ private module Extractors + # } + # + module SearchResults +- def self.process(initial_data : Hash(String, JSON::Any)) ++ def self.process(initial_data : InitialData) + if target = initial_data["twoColumnSearchResultsRenderer"]? + self.extract(target) + end +@@ -608,8 +610,8 @@ private module Extractors + # The way they are structured is too varied to be accurately written down here. + # However, they all eventually lead to an array of parsable items after traversing + # through the JSON structure. +- module Continuation +- def self.process(initial_data : Hash(String, JSON::Any)) ++ module ContinuationContent ++ def self.process(initial_data : InitialData) + if target = initial_data["continuationContents"]? + self.extract(target) + elsif target = initial_data["appendContinuationItemsAction"]? +@@ -691,8 +693,7 @@ end + + # Parses an item from Youtube's JSON response into a more usable structure. + # The end result can either be a SearchVideo, SearchPlaylist or SearchChannel. +-def extract_item(item : JSON::Any, author_fallback : String? = "", +- author_id_fallback : String? = "") ++def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "") + # We "allow" nil values but secretly use empty strings instead. This is to save us the + # hassle of modifying every author_fallback and author_id_fallback arg usage + # which is more often than not nil. +@@ -702,24 +703,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "", + # Each parser automatically validates the data given to see if the data is + # applicable to itself. If not nil is returned and the next parser is attempted. + ITEM_PARSERS.each do |parser| +- LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)") ++ LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)") + + if result = parser.process(item, author_fallback) +- LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}") +- ++ LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}") + return result + else +- LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...") ++ LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...") + end + end + end + + # Parses multiple items from YouTube's initial JSON response into a more usable structure. + # The end result is an array of SearchItem. +-def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, +- author_id_fallback : String? = nil) : Array(SearchItem) +- items = [] of SearchItem +- ++# ++# This function yields the container so that items can be parsed separately. ++# ++def extract_items(initial_data : InitialData, &block) + if unpackaged_data = initial_data["contents"]?.try &.as_h + elsif unpackaged_data = initial_data["response"]?.try &.as_h + elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h +@@ -727,24 +727,32 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri + unpackaged_data = initial_data + end + +- # This is identical to the parser cycling of extract_item(). ++ # This is identical to the parser cycling of parse_item(). + ITEM_CONTAINER_EXTRACTOR.each do |extractor| + LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)") + + if container = extractor.process(unpackaged_data) + LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"") + # Extract items in container +- container.each do |item| +- if parsed_result = extract_item(item, author_fallback, author_id_fallback) +- items << parsed_result +- end +- end +- +- break ++ container.each { |item| yield item } + else + LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...") + end + end ++end ++ ++# Wrapper using the block function above ++def extract_items( ++ initial_data : InitialData, ++ author_fallback : String? = nil, ++ author_id_fallback : String? = nil ++) : Array(SearchItem) ++ items = [] of SearchItem ++ ++ extract_items(initial_data) do |item| ++ parsed = parse_item(item, author_fallback, author_id_fallback) ++ items << parsed if !parsed.nil? ++ end + + return items + end + +From ce7db8d2cb87111af15de2de9faf12aae38283bb Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Sat, 5 Nov 2022 18:56:35 +0100 +Subject: [PATCH 05/16] extractors: Add continuation token parser + +--- + spec/invidious/hashtag_spec.cr | 4 +- + src/invidious/channels/playlists.cr | 16 +----- + src/invidious/hashtag.cr | 3 +- + src/invidious/helpers/serialized_yt_data.cr | 7 +++ + src/invidious/search/processors.cr | 14 ++--- + src/invidious/yt_backend/extractors.cr | 54 +++++++++++++++----- + src/invidious/yt_backend/extractors_utils.cr | 27 ++-------- + 7 files changed, 63 insertions(+), 62 deletions(-) + +diff --git a/spec/invidious/hashtag_spec.cr b/spec/invidious/hashtag_spec.cr +index 776768780..266ec57bc 100644 +--- a/spec/invidious/hashtag_spec.cr ++++ b/spec/invidious/hashtag_spec.cr +@@ -4,7 +4,7 @@ Spectator.describe Invidious::Hashtag do + it "parses richItemRenderer containers (test 1)" do + # Enable mock + test_content = load_mock("hashtag/martingarrix_page1") +- videos = extract_items(test_content) ++ videos, _ = extract_items(test_content) + + expect(typeof(videos)).to eq(Array(SearchItem)) + expect(videos.size).to eq(60) +@@ -57,7 +57,7 @@ Spectator.describe Invidious::Hashtag do + it "parses richItemRenderer containers (test 2)" do + # Enable mock + test_content = load_mock("hashtag/martingarrix_page2") +- videos = extract_items(test_content) ++ videos, _ = extract_items(test_content) + + expect(typeof(videos)).to eq(Array(SearchItem)) + expect(videos.size).to eq(60) +diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr +index e6c0a1d51..0d46499a0 100644 +--- a/src/invidious/channels/playlists.cr ++++ b/src/invidious/channels/playlists.cr +@@ -1,18 +1,7 @@ + def fetch_channel_playlists(ucid, author, continuation, sort_by) + if continuation + response_json = YoutubeAPI.browse(continuation) +- continuation_items = response_json["onResponseReceivedActions"]? +- .try &.[0]["appendContinuationItemsAction"]["continuationItems"] +- +- return [] of SearchItem, nil if !continuation_items +- +- items = [] of SearchItem +- continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| +- parse_item(item, author, ucid).try { |t| items << t } +- } +- +- continuation = continuation_items.as_a.last["continuationItemRenderer"]? +- .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s ++ items, continuation = extract_items(response_json, author, ucid) + else + url = "/channel/#{ucid}/playlists?flow=list&view=1" + +@@ -30,8 +19,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) + initial_data = extract_initial_data(response.body) + return [] of SearchItem, nil if !initial_data + +- items = extract_items(initial_data, author, ucid) +- continuation = response.body.match(/"token":"(?[^"]+)"/).try &.["continuation"]? ++ items, continuation = extract_items(initial_data, author, ucid) + end + + return items, continuation +diff --git a/src/invidious/hashtag.cr b/src/invidious/hashtag.cr +index afe31a364..bc329205a 100644 +--- a/src/invidious/hashtag.cr ++++ b/src/invidious/hashtag.cr +@@ -8,7 +8,8 @@ module Invidious::Hashtag + client_config = YoutubeAPI::ClientConfig.new(region: region) + response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config) + +- return extract_items(response) ++ items, _ = extract_items(response) ++ return items + end + + def generate_continuation(hashtag : String, cursor : Int) +diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr +index c52e2a0dc..635f09844 100644 +--- a/src/invidious/helpers/serialized_yt_data.cr ++++ b/src/invidious/helpers/serialized_yt_data.cr +@@ -265,4 +265,11 @@ class Category + end + end + ++struct Continuation ++ getter token ++ ++ def initialize(@token : String) ++ end ++end ++ + alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category +diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr +index 683a4a7e2..7e9095903 100644 +--- a/src/invidious/search/processors.cr ++++ b/src/invidious/search/processors.cr +@@ -9,7 +9,8 @@ module Invidious::Search + client_config = YoutubeAPI::ClientConfig.new(region: query.region) + initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) + +- return extract_items(initial_data) ++ items, _ = extract_items(initial_data) ++ return items + end + + # Search a youtube channel +@@ -30,16 +31,7 @@ module Invidious::Search + continuation = produce_channel_search_continuation(ucid, query.text, query.page) + response_json = YoutubeAPI.browse(continuation) + +- continuation_items = response_json["onResponseReceivedActions"]? +- .try &.[0]["appendContinuationItemsAction"]["continuationItems"] +- +- return [] of SearchItem if !continuation_items +- +- items = [] of SearchItem +- continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| +- parse_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } +- end +- ++ items, _ = extract_items(response_json, "", ucid) + return items + end + +diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr +index a4b20d047..baf521181 100644 +--- a/src/invidious/yt_backend/extractors.cr ++++ b/src/invidious/yt_backend/extractors.cr +@@ -7,7 +7,7 @@ require "../helpers/serialized_yt_data" + private ITEM_CONTAINER_EXTRACTOR = { + Extractors::YouTubeTabs, + Extractors::SearchResults, +- Extractors::Continuation, ++ Extractors::ContinuationContent, + } + + private ITEM_PARSERS = { +@@ -18,6 +18,7 @@ private ITEM_PARSERS = { + Parsers::CategoryRendererParser, + Parsers::RichItemRendererParser, + Parsers::ReelItemRendererParser, ++ Parsers::ContinuationItemRendererParser, + } + + private alias InitialData = Hash(String, JSON::Any) +@@ -347,14 +348,9 @@ private module Parsers + content_container = item_contents["contents"] + end + +- raw_contents = content_container["items"]?.try &.as_a +- if !raw_contents.nil? +- raw_contents.each do |item| +- result = parse_item(item) +- if !result.nil? +- contents << result +- end +- end ++ content_container["items"]?.try &.as_a.each do |item| ++ result = parse_item(item, author_fallback.name, author_fallback.id) ++ contents << result if result.is_a?(SearchItem) + end + + Category.new({ +@@ -477,6 +473,35 @@ private module Parsers + return {{@type.name}} + end + end ++ ++ # Parses an InnerTube continuationItemRenderer into a Continuation. ++ # Returns nil when the given object isn't a continuationItemRenderer. ++ # ++ # continuationItemRenderer contains various metadata ued to load more ++ # content (i.e when the user scrolls down). The interesting bit is the ++ # protobuf object known as the "continutation token". Previously, those ++ # were generated from sratch, but recent (as of 11/2022) Youtube changes ++ # are forcing us to extract them from replies. ++ # ++ module ContinuationItemRendererParser ++ def self.process(item : JSON::Any, author_fallback : AuthorFallback) ++ if item_contents = item["continuationItemRenderer"]? ++ return self.parse(item_contents) ++ end ++ end ++ ++ private def self.parse(item_contents) ++ token = item_contents ++ .dig?("continuationEndpoint", "continuationCommand", "token") ++ .try &.as_s ++ ++ return Continuation.new(token) if token ++ end ++ ++ def self.parser_name ++ return {{@type.name}} ++ end ++ end + end + + # The following are the extractors for extracting an array of items from +@@ -746,13 +771,18 @@ def extract_items( + initial_data : InitialData, + author_fallback : String? = nil, + author_id_fallback : String? = nil +-) : Array(SearchItem) ++) : {Array(SearchItem), String?} + items = [] of SearchItem ++ continuation = nil + + extract_items(initial_data) do |item| + parsed = parse_item(item, author_fallback, author_id_fallback) +- items << parsed if !parsed.nil? ++ ++ case parsed ++ when .is_a?(Continuation) then continuation = parsed.token ++ when .is_a?(SearchItem) then items << parsed ++ end + end + +- return items ++ return items, continuation + end +diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr +index f82451607..0cb3c0797 100644 +--- a/src/invidious/yt_backend/extractors_utils.cr ++++ b/src/invidious/yt_backend/extractors_utils.cr +@@ -68,10 +68,10 @@ rescue ex + return false + end + +-def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) +- extracted = extract_items(initial_data, author_fallback, author_id_fallback) ++def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo) ++ extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback) + +- target = [] of SearchItem ++ target = [] of (SearchItem | Continuation) + extracted.each do |i| + if i.is_a?(Category) + i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } +@@ -79,28 +79,11 @@ def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : Str + target << i + end + end +- return target.select(SearchVideo).map(&.as(SearchVideo)) ++ ++ return target.select(SearchVideo) + end + + def extract_selected_tab(tabs) + # Extract the selected tab from the array of tabs Youtube returns + return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"] + end +- +-def fetch_continuation_token(items : Array(JSON::Any)) +- # Fetches the continuation token from an array of items +- return items.last["continuationItemRenderer"]? +- .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s +-end +- +-def fetch_continuation_token(initial_data : Hash(String, JSON::Any)) +- # Fetches the continuation token from initial data +- if initial_data["onResponseReceivedActions"]? +- continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"] +- else +- tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) +- continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"] +- end +- +- return fetch_continuation_token(continuation_items.as_a) +-end + +From 8e8ca4fcc5cfcb7bebc3f29440d6abc1de770513 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Sat, 12 Nov 2022 00:04:27 +0100 +Subject: [PATCH 06/16] Prepare to create a 'Channel' module + +--- + src/invidious.cr | 9 ++++++++- + src/invidious/jobs/notification_job.cr | 4 ++-- + src/invidious/jobs/refresh_channels_job.cr | 2 +- + src/invidious/jobs/refresh_feeds_job.cr | 2 +- + src/invidious/jobs/subscribe_to_feeds_job.cr | 2 +- + 5 files changed, 13 insertions(+), 6 deletions(-) + +diff --git a/src/invidious.cr b/src/invidious.cr +index 2874cc712..5064f0b85 100644 +--- a/src/invidious.cr ++++ b/src/invidious.cr +@@ -48,6 +48,13 @@ require "./invidious/search/*" + require "./invidious/routes/**" + require "./invidious/jobs/**" + ++# Declare the base namespace for invidious ++module Invidious ++end ++ ++# Simple alias to make code easier to read ++alias IV = Invidious ++ + CONFIG = Config.load + HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) + +@@ -172,7 +179,7 @@ if CONFIG.popular_enabled + Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) + end + +-CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32) ++CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32) + Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url) + + Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new +diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr +index 2f525e084..b445107b4 100644 +--- a/src/invidious/jobs/notification_job.cr ++++ b/src/invidious/jobs/notification_job.cr +@@ -1,12 +1,12 @@ + class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob +- private getter connection_channel : Channel({Bool, Channel(PQ::Notification)}) ++ private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)}) + private getter pg_url : URI + + def initialize(@connection_channel, @pg_url) + end + + def begin +- connections = [] of Channel(PQ::Notification) ++ connections = [] of ::Channel(PQ::Notification) + + PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) } + +diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr +index 92681408d..80812a63c 100644 +--- a/src/invidious/jobs/refresh_channels_job.cr ++++ b/src/invidious/jobs/refresh_channels_job.cr +@@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob + max_fibers = CONFIG.channel_threads + lim_fibers = max_fibers + active_fibers = 0 +- active_channel = Channel(Bool).new ++ active_channel = ::Channel(Bool).new + backoff = 2.minutes + + loop do +diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr +index 4b52c9596..4f8130dfd 100644 +--- a/src/invidious/jobs/refresh_feeds_job.cr ++++ b/src/invidious/jobs/refresh_feeds_job.cr +@@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob + def begin + max_fibers = CONFIG.feed_threads + active_fibers = 0 +- active_channel = Channel(Bool).new ++ active_channel = ::Channel(Bool).new + + loop do + db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs| +diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr +index a431a48af..8584fb9cf 100644 +--- a/src/invidious/jobs/subscribe_to_feeds_job.cr ++++ b/src/invidious/jobs/subscribe_to_feeds_job.cr +@@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob + end + + active_fibers = 0 +- active_channel = Channel(Bool).new ++ active_channel = ::Channel(Bool).new + + loop do + db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs| + +From c5ee2bfc0f5e485f91e53dedc879312c3e729be8 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Fri, 11 Nov 2022 00:44:24 +0100 +Subject: [PATCH 07/16] channel: use YT API to fetch playlist items + +--- + src/invidious/channels/playlists.cr | 40 +++++++++++++++-------------- + 1 file changed, 21 insertions(+), 19 deletions(-) + +diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr +index 0d46499a0..8fdac3a7e 100644 +--- a/src/invidious/channels/playlists.cr ++++ b/src/invidious/channels/playlists.cr +@@ -1,28 +1,30 @@ + def fetch_channel_playlists(ucid, author, continuation, sort_by) + if continuation +- response_json = YoutubeAPI.browse(continuation) +- items, continuation = extract_items(response_json, author, ucid) ++ initial_data = YoutubeAPI.browse(continuation) + else +- url = "/channel/#{ucid}/playlists?flow=list&view=1" ++ params = ++ case sort_by ++ when "last", "last_added" ++ # Equivalent to "&sort=lad" ++ # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1} ++ "EglwbGF5bGlzdHMYBCABMAE%3D" ++ when "oldest", "oldest_created" ++ # formerly "&sort=da" ++ # Not available anymore :c or maybe ?? ++ # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1} ++ "EglwbGF5bGlzdHMYAiABMAE%3D" ++ # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1} ++ # "EglwbGF5bGlzdHMYASABMAE%3D" ++ when "newest", "newest_created" ++ # Formerly "&sort=dd" ++ # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1} ++ "EglwbGF5bGlzdHMYAyABMAE%3D" ++ end + +- case sort_by +- when "last", "last_added" +- # +- when "oldest", "oldest_created" +- url += "&sort=da" +- when "newest", "newest_created" +- url += "&sort=dd" +- else nil # Ignore +- end +- +- response = YT_POOL.client &.get(url) +- initial_data = extract_initial_data(response.body) +- return [] of SearchItem, nil if !initial_data +- +- items, continuation = extract_items(initial_data, author, ucid) ++ initial_data = YoutubeAPI.browse(ucid, params: params || "") + end + +- return items, continuation ++ return extract_items(initial_data, ucid, author) + end + + # ## NOTE: DEPRECATED + +From 2903e896ecf2404bf932438a33432125a6ad1fca Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Fri, 11 Nov 2022 20:26:34 +0100 +Subject: [PATCH 08/16] channel: use YT API + extractors to fetch videos + +--- + src/invidious/channels/channels.cr | 67 +++++++++---------- + src/invidious/channels/playlists.cr | 2 +- + src/invidious/channels/videos.cr | 88 ++++++++++++++++++------- + src/invidious/routes/api/v1/channels.cr | 66 ++++++++----------- + src/invidious/routes/channels.cr | 4 +- + 5 files changed, 125 insertions(+), 102 deletions(-) + +diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr +index e3d3d9eec..27369f127 100644 +--- a/src/invidious/channels/channels.cr ++++ b/src/invidious/channels/channels.cr +@@ -180,11 +180,16 @@ def fetch_channel(ucid, pull_all_videos : Bool) + + LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") + +- page = 1 ++ channel = InvidiousChannel.new({ ++ id: ucid, ++ author: author, ++ updated: Time.utc, ++ deleted: false, ++ subscribed: nil, ++ }) + + LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") +- initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) +- videos = extract_videos(initial_data, author, ucid) ++ videos, continuation = IV::Channel::Tabs.get_videos(channel) + + LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") + rss.xpath_nodes("//feed/entry").each do |entry| +@@ -197,7 +202,9 @@ def fetch_channel(ucid, pull_all_videos : Bool) + views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64? + views ||= 0_i64 + +- channel_video = videos.select { |video| video.id == video_id }[0]? ++ channel_video = videos ++ .select(SearchVideo) ++ .select(&.id.== video_id)[0]? + + length_seconds = channel_video.try &.length_seconds + length_seconds ||= 0 +@@ -235,30 +242,25 @@ def fetch_channel(ucid, pull_all_videos : Bool) + end + + if pull_all_videos +- page += 1 +- +- ids = [] of String +- + loop do +- initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) +- videos = extract_videos(initial_data, author, ucid) +- +- count = videos.size +- videos = videos.map { |video| ChannelVideo.new({ +- id: video.id, +- title: video.title, +- published: video.published, +- updated: Time.utc, +- ucid: video.ucid, +- author: video.author, +- length_seconds: video.length_seconds, +- live_now: video.live_now, +- premiere_timestamp: video.premiere_timestamp, +- views: video.views, +- }) } +- +- videos.each do |video| +- ids << video.id ++ # Keep fetching videos using the continuation token retrieved earlier ++ videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) ++ ++ count = 0 ++ videos.select(SearchVideo).each do |video| ++ count += 1 ++ video = ChannelVideo.new({ ++ id: video.id, ++ title: video.title, ++ published: video.published, ++ updated: Time.utc, ++ ucid: video.ucid, ++ author: video.author, ++ length_seconds: video.length_seconds, ++ live_now: video.live_now, ++ premiere_timestamp: video.premiere_timestamp, ++ views: video.views, ++ }) + + # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, + # so since they don't provide a published date here we can safely ignore them. +@@ -269,17 +271,10 @@ def fetch_channel(ucid, pull_all_videos : Bool) + end + + break if count < 25 +- page += 1 ++ sleep 500.milliseconds + end + end + +- channel = InvidiousChannel.new({ +- id: ucid, +- author: author, +- updated: Time.utc, +- deleted: false, +- subscribed: nil, +- }) +- ++ channel.updated = Time.utc + return channel + end +diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr +index 8fdac3a7e..772eecb95 100644 +--- a/src/invidious/channels/playlists.cr ++++ b/src/invidious/channels/playlists.cr +@@ -24,7 +24,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) + initial_data = YoutubeAPI.browse(ucid, params: params || "") + end + +- return extract_items(initial_data, ucid, author) ++ return extract_items(initial_data, author, ucid) + end + + # ## NOTE: DEPRECATED +diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr +index b495e597f..23ad4e028 100644 +--- a/src/invidious/channels/videos.cr ++++ b/src/invidious/channels/videos.cr +@@ -16,6 +16,14 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + ++ sort_by_numerical = ++ case sort_by ++ when "newest" then 1_i64 ++ when "popular" then 2_i64 ++ when "oldest" then 3_i64 # Broken as of 10/2022 :c ++ else 1_i64 # Fallback to "newest" ++ end ++ + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { +@@ -24,7 +32,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so + "1:string" => object_inner_2_encoded, + "2:string" => "00000000-0000-0000-0000-000000000000", + }, +- "3:varint" => 1_i64, ++ "3:varint" => sort_by_numerical, + }, + }, + }, +@@ -52,34 +60,66 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so + return continuation + end + +-def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") +- continuation = produce_channel_videos_continuation(ucid, page, +- auto_generated: auto_generated, sort_by: sort_by, v2: true) +- +- return YoutubeAPI.browse(continuation) ++# Used in bypass_captcha_job.cr ++def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) ++ continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) ++ return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + end + +-def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") +- videos = [] of SearchVideo ++module Invidious::Channel::Tabs ++ extend self + +- # 2.times do |i| +- # initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) +- initial_data = get_channel_videos_response(ucid, 1, auto_generated: auto_generated, sort_by: sort_by) +- videos = extract_videos(initial_data, author, ucid) +- # end ++ # ------------------- ++ # Regular videos ++ # ------------------- + +- return videos.size, videos +-end ++ def make_initial_video_ctoken(ucid, sort_by) : String ++ return produce_channel_videos_continuation(ucid, sort_by: sort_by) ++ end + +-def get_latest_videos(ucid) +- initial_data = get_channel_videos_response(ucid) +- author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s ++ # Wrapper for AboutChannel, as we still need to call get_videos with ++ # an author name and ucid directly (e.g in RSS feeds). ++ # TODO: figure out how to get rid of that ++ def get_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") ++ return get_videos( ++ channel.author, channel.ucid, ++ continuation: continuation, sort_by: sort_by ++ ) ++ end + +- return extract_videos(initial_data, author, ucid) +-end ++ # Wrapper for InvidiousChannel, as we still need to call get_videos with ++ # an author name and ucid directly (e.g in RSS feeds). ++ # TODO: figure out how to get rid of that ++ def get_videos(channel : InvidiousChannel, *, continuation : String? = nil, sort_by = "newest") ++ return get_videos( ++ channel.author, channel.id, ++ continuation: continuation, sort_by: sort_by ++ ) ++ end + +-# Used in bypass_captcha_job.cr +-def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) +- continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) +- return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" ++ def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") ++ continuation ||= make_initial_video_ctoken(ucid, sort_by) ++ initial_data = YoutubeAPI.browse(continuation: continuation) ++ ++ return extract_items(initial_data, author, ucid) ++ end ++ ++ def get_60_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") ++ if continuation.nil? ++ # Fetch the first "page" of video ++ items, next_continuation = get_videos(channel, sort_by: sort_by) ++ else ++ # Fetch a "page" of videos using the given continuation token ++ items, next_continuation = get_videos(channel, continuation: continuation) ++ end ++ ++ # If there is more to load, then load a second "page" ++ # and replace the previous continuation token ++ if !next_continuation.nil? ++ items_2, next_continuation = get_videos(channel, continuation: next_continuation) ++ items.concat items_2 ++ end ++ ++ return items, next_continuation ++ end + end +diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr +index 6b81c5461..72d9ae5f3 100644 +--- a/src/invidious/routes/api/v1/channels.cr ++++ b/src/invidious/routes/api/v1/channels.cr +@@ -5,8 +5,6 @@ module Invidious::Routes::API::V1::Channels + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] +- sort_by = env.params.query["sort_by"]?.try &.downcase +- sort_by ||= "newest" + + begin + channel = get_about_info(ucid, locale) +@@ -19,16 +17,13 @@ module Invidious::Routes::API::V1::Channels + return error_json(500, ex) + end + +- page = 1 +- if channel.auto_generated +- videos = [] of SearchVideo +- count = 0 +- else +- begin +- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) +- rescue ex +- return error_json(500, ex) +- end ++ # Retrieve "sort by" setting from URL parameters ++ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" ++ ++ begin ++ videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) ++ rescue ex ++ return error_json(500, ex) + end + + JSON.build do |json| +@@ -134,25 +129,11 @@ module Invidious::Routes::API::V1::Channels + end + + def self.latest(env) +- locale = env.get("preferences").as(Preferences).locale +- +- env.response.content_type = "application/json" +- +- ucid = env.params.url["ucid"] +- +- begin +- videos = get_latest_videos(ucid) +- rescue ex +- return error_json(500, ex) +- end ++ # Remove parameters that could affect this endpoint's behavior ++ env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by") ++ env.params.query.delete("continuation") if env.params.query.has_key?("continuation") + +- JSON.build do |json| +- json.array do +- videos.each do |video| +- video.to_json(locale, json) +- end +- end +- end ++ return self.videos(env) + end + + def self.videos(env) +@@ -161,11 +142,6 @@ module Invidious::Routes::API::V1::Channels + env.response.content_type = "application/json" + + ucid = env.params.url["ucid"] +- page = env.params.query["page"]?.try &.to_i? +- page ||= 1 +- sort_by = env.params.query["sort"]?.try &.downcase +- sort_by ||= env.params.query["sort_by"]?.try &.downcase +- sort_by ||= "newest" + + begin + channel = get_about_info(ucid, locale) +@@ -178,17 +154,27 @@ module Invidious::Routes::API::V1::Channels + return error_json(500, ex) + end + ++ # Retrieve some URL parameters ++ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" ++ continuation = env.params.query["continuation"]? ++ + begin +- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) ++ videos, next_continuation = Channel::Tabs.get_60_videos( ++ channel, continuation: continuation, sort_by: sort_by ++ ) + rescue ex + return error_json(500, ex) + end + +- JSON.build do |json| +- json.array do +- videos.each do |video| +- video.to_json(locale, json) ++ return JSON.build do |json| ++ json.object do ++ json.field "videos" do ++ json.array do ++ videos.each &.to_json(locale, json) ++ end + end ++ ++ json.field "continuation", next_continuation if next_continuation + end + end + end +diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr +index f26f29f5d..2773deb78 100644 +--- a/src/invidious/routes/channels.cr ++++ b/src/invidious/routes/channels.cr +@@ -32,7 +32,9 @@ module Invidious::Routes::Channels + sort_options = {"newest", "oldest", "popular"} + sort_by ||= "newest" + +- count, items = get_60_videos(channel.ucid, channel.author, 1, channel.auto_generated, sort_by) ++ items, continuation = Channel::Tabs.get_60_videos( ++ channel, continuation: continuation, sort_by: sort_by ++ ) + end + + templated "channel" + +From 52ef89f02d0ab29fd0f218abc4051328b3d96809 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Sat, 12 Nov 2022 00:09:03 +0100 +Subject: [PATCH 09/16] channel: Add support for shorts and livestreams + (backend only) + +--- + src/invidious/channels/videos.cr | 50 ++++++++++++++++++++++++++++++++ + 1 file changed, 50 insertions(+) + +diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr +index 23ad4e028..bea406c1a 100644 +--- a/src/invidious/channels/videos.cr ++++ b/src/invidious/channels/videos.cr +@@ -122,4 +122,54 @@ module Invidious::Channel::Tabs + + return items, next_continuation + end ++ ++ # ------------------- ++ # Shorts ++ # ------------------- ++ ++ def get_shorts(channel : AboutChannel, continuation : String? = nil) ++ if continuation.nil? ++ # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" ++ # TODO: try to extract the continuation tokens that allows other sorting options ++ initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") ++ else ++ initial_data = YoutubeAPI.browse(continuation: continuation) ++ end ++ ++ return extract_items(initial_data, channel.author, channel.ucid) ++ end ++ ++ # ------------------- ++ # Livestreams ++ # ------------------- ++ ++ def get_livestreams(channel : AboutChannel, continuation : String? = nil) ++ if continuation.nil? ++ # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" ++ initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") ++ else ++ initial_data = YoutubeAPI.browse(continuation: continuation) ++ end ++ ++ return extract_items(initial_data, channel.author, channel.ucid) ++ end ++ ++ def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) ++ if continuation.nil? ++ # Fetch the first "page" of streams ++ items, next_continuation = get_livestreams(channel) ++ else ++ # Fetch a "page" of streams using the given continuation token ++ items, next_continuation = get_livestreams(channel, continuation: continuation) ++ end ++ ++ # If there is more to load, then load a second "page" ++ # and replace the previous continuation token ++ if !next_continuation.nil? ++ items_2, next_continuation = get_livestreams(channel, continuation: next_continuation) ++ items.concat items_2 ++ end ++ ++ return items, next_continuation ++ end + end + +From 5d6abd5301b14c24475bf7ad477a43c60ff78993 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Thu, 1 Dec 2022 23:01:31 +0100 +Subject: [PATCH 10/16] extractors: Fix ReelItemRendererParser + +--- + src/invidious/yt_backend/extractors.cr | 32 ++++++++++++++++---------- + 1 file changed, 20 insertions(+), 12 deletions(-) + +diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr +index baf521181..bca0dcbd5 100644 +--- a/src/invidious/yt_backend/extractors.cr ++++ b/src/invidious/yt_backend/extractors.cr +@@ -382,7 +382,9 @@ private module Parsers + end + + private def self.parse(item_contents, author_fallback) +- return VideoRendererParser.process(item_contents, author_fallback) ++ child = VideoRendererParser.process(item_contents, author_fallback) ++ child ||= ReelItemRendererParser.process(item_contents, author_fallback) ++ return child + end + + def self.parser_name +@@ -406,12 +408,18 @@ private module Parsers + private def self.parse(item_contents, author_fallback) + video_id = item_contents["videoId"].as_s + +- video_details_container = item_contents.dig( +- "navigationEndpoint", "reelWatchEndpoint", +- "overlay", "reelPlayerOverlayRenderer", +- "reelPlayerHeaderSupportedRenderers", +- "reelPlayerHeaderRenderer" +- ) ++ begin ++ video_details_container = item_contents.dig( ++ "navigationEndpoint", "reelWatchEndpoint", ++ "overlay", "reelPlayerOverlayRenderer", ++ "reelPlayerHeaderSupportedRenderers", ++ "reelPlayerHeaderRenderer" ++ ) ++ rescue ex : KeyError ++ # Extract key name from original message ++ key = /"([^"]+)"/.match(ex.message || "").try &.[1]? ++ raise BrokenTubeException.new(key || "reelPlayerOverlayRenderer") ++ end + + # Author infos + +@@ -434,9 +442,9 @@ private module Parsers + + # View count + +- view_count_text = video_details_container.dig?("viewCountText", "simpleText") +- view_count_text ||= video_details_container +- .dig?("viewCountText", "accessibility", "accessibilityData", "label") ++ # View count used to be in the reelWatchEndpoint, but that changed? ++ view_count_text = item_contents.dig?("viewCountText", "simpleText") ++ view_count_text ||= video_details_container.dig?("viewCountText", "simpleText") + + view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 + +@@ -448,8 +456,8 @@ private module Parsers + + regex_match = /- (?\d+ minutes? )?(?\d+ seconds?)+ -/.match(a11y_data) + +- minutes = regex_match.try &.["min"].to_i(strict: false) || 0 +- seconds = regex_match.try &.["sec"].to_i(strict: false) || 0 ++ minutes = regex_match.try &.["min"]?.try &.to_i(strict: false) || 0 ++ seconds = regex_match.try &.["sec"]?.try &.to_i(strict: false) || 0 + + duration = (minutes*60 + seconds) + + +From 6c9754e66316d903ed4f89d2cd59cd82940509f5 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Wed, 30 Nov 2022 00:29:48 +0100 +Subject: [PATCH 11/16] frontend: Add support for shorts and livestreams + +--- + locales/en-US.json | 9 +++-- + src/invidious/channels/about.cr | 10 ++++- + src/invidious/frontend/channel_page.cr | 43 ++++++++++++++++++++ + src/invidious/routes/channels.cr | 54 ++++++++++++++++++++++++-- + src/invidious/routing.cr | 4 +- + src/invidious/views/channel.ecr | 30 +++++--------- + src/invidious/views/community.ecr | 15 +------ + src/invidious/views/playlists.ecr | 14 +------ + 8 files changed, 124 insertions(+), 55 deletions(-) + create mode 100644 src/invidious/frontend/channel_page.cr + +diff --git a/locales/en-US.json b/locales/en-US.json +index 5554b9286..44b40c249 100644 +--- a/locales/en-US.json ++++ b/locales/en-US.json +@@ -404,9 +404,7 @@ + "`x` marked it with a ❤": "`x` marked it with a ❤", + "Audio mode": "Audio mode", + "Video mode": "Video mode", +- "Videos": "Videos", + "Playlists": "Playlists", +- "Community": "Community", + "search_filters_title": "Filters", + "search_filters_date_label": "Upload date", + "search_filters_date_option_none": "Any date", +@@ -472,5 +470,10 @@ + "crash_page_read_the_faq": "read the Frequently Asked Questions (FAQ)", + "crash_page_search_issue": "searched for existing issues on GitHub", + "crash_page_report_issue": "If none of the above helped, please open a new issue on GitHub (preferably in English) and include the following text in your message (do NOT translate that text):", +- "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. Click here for the playlist home page." ++ "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. Click here for the playlist home page.", ++ "channel_tab_videos_label": "Videos", ++ "channel_tab_shorts_label": "Shorts", ++ "channel_tab_streams_label": "Livestreams", ++ "channel_tab_playlists_label": "Playlists", ++ "channel_tab_community_label": "Community" + } +diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr +index bb9bd8c71..09c3427a3 100644 +--- a/src/invidious/channels/about.cr ++++ b/src/invidious/channels/about.cr +@@ -104,8 +104,14 @@ def get_about_info(ucid, locale) : AboutChannel + + if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? + # Get the name of the tabs available on this channel +- tab_names = tabs_json.as_a +- .compact_map(&.dig?("tabRenderer", "title").try &.as_s.downcase) ++ tab_names = tabs_json.as_a.compact_map do |entry| ++ name = entry.dig?("tabRenderer", "title").try &.as_s.downcase ++ ++ # This is a small fix to not add extra code on the HTML side ++ # I.e, the URL for the "live" tab is .../streams, so use "streams" ++ # everywhere for the sake of simplicity ++ (name == "live") ? "streams" : name ++ end + + # Get the currently active tab ("About") + about_tab = extract_selected_tab(tabs_json) +diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr +new file mode 100644 +index 000000000..7ac0e071c +--- /dev/null ++++ b/src/invidious/frontend/channel_page.cr +@@ -0,0 +1,43 @@ ++module Invidious::Frontend::ChannelPage ++ extend self ++ ++ enum TabsAvailable ++ Videos ++ Shorts ++ Streams ++ Playlists ++ Community ++ end ++ ++ def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable) ++ return String.build(1500) do |str| ++ base_url = "/channel/#{channel.ucid}" ++ ++ TabsAvailable.each do |tab| ++ # Ignore playlists, as it is not supported for auto-generated channels yet ++ next if (tab.playlists? && channel.auto_generated) ++ ++ tab_name = tab.to_s.downcase ++ ++ if channel.tabs.includes? tab_name ++ str << %(
\n) ++ ++ if tab == selected_tab ++ str << "\t" ++ str << translate(locale, "channel_tab_#{tab_name}_label") ++ str << "\n" ++ else ++ # Video tab doesn't have the last path component ++ url = tab.videos? ? base_url : "#{base_url}/#{tab_name}" ++ ++ str << %(\t) ++ str << translate(locale, "channel_tab_#{tab_name}_label") ++ str << "\n" ++ end ++ ++ str << "
" ++ end ++ end ++ end ++ end ++end +diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr +index 2773deb78..78b383419 100644 +--- a/src/invidious/routes/channels.cr ++++ b/src/invidious/routes/channels.cr +@@ -18,7 +18,7 @@ module Invidious::Routes::Channels + sort_options = {"last", "oldest", "newest"} + sort_by ||= "last" + +- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) ++ items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + items.uniq! do |item| + if item.responds_to?(:title) + item.title +@@ -32,11 +32,59 @@ module Invidious::Routes::Channels + sort_options = {"newest", "oldest", "popular"} + sort_by ||= "newest" + +- items, continuation = Channel::Tabs.get_60_videos( ++ # Fetch items and continuation token ++ items, next_continuation = Channel::Tabs.get_videos( + channel, continuation: continuation, sort_by: sort_by + ) + end + ++ selected_tab = Frontend::ChannelPage::TabsAvailable::Videos ++ templated "channel" ++ end ++ ++ def self.shorts(env) ++ data = self.fetch_basic_information(env) ++ return data if !data.is_a?(Tuple) ++ ++ locale, user, subscriptions, continuation, ucid, channel = data ++ ++ if !channel.tabs.includes? "shorts" ++ return env.redirect "/channel/#{channel.ucid}" ++ end ++ ++ # TODO: support sort option for shorts ++ sort_by = "" ++ sort_options = [] of String ++ ++ # Fetch items and continuation token ++ items, next_continuation = Channel::Tabs.get_shorts( ++ channel, continuation: continuation ++ ) ++ ++ selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts ++ templated "channel" ++ end ++ ++ def self.streams(env) ++ data = self.fetch_basic_information(env) ++ return data if !data.is_a?(Tuple) ++ ++ locale, user, subscriptions, continuation, ucid, channel = data ++ ++ if !channel.tabs.includes? "streams" ++ return env.redirect "/channel/#{channel.ucid}" ++ end ++ ++ # TODO: support sort option for livestreams ++ sort_by = "" ++ sort_options = [] of String ++ ++ # Fetch items and continuation token ++ items, next_continuation = Channel::Tabs.get_60_livestreams( ++ channel, continuation: continuation ++ ) ++ ++ selected_tab = Frontend::ChannelPage::TabsAvailable::Streams + templated "channel" + end + +@@ -124,7 +172,7 @@ module Invidious::Routes::Channels + end + + selected_tab = env.request.path.split("/")[-1] +- if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab ++ if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab + url = "/channel/#{ucid}/#{selected_tab}" + else + url = "/channel/#{ucid}" +diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr +index f409f13c8..08739c3d3 100644 +--- a/src/invidious/routing.cr ++++ b/src/invidious/routing.cr +@@ -115,6 +115,8 @@ module Invidious::Routing + get "/channel/:ucid", Routes::Channels, :home + get "/channel/:ucid/home", Routes::Channels, :home + get "/channel/:ucid/videos", Routes::Channels, :videos ++ get "/channel/:ucid/shorts", Routes::Channels, :shorts ++ get "/channel/:ucid/streams", Routes::Channels, :streams + get "/channel/:ucid/playlists", Routes::Channels, :playlists + get "/channel/:ucid/community", Routes::Channels, :community + get "/channel/:ucid/about", Routes::Channels, :about +@@ -122,7 +124,7 @@ module Invidious::Routing + get "/user/:user/live", Routes::Channels, :live + get "/c/:user/live", Routes::Channels, :live + +- ["", "/videos", "/playlists", "/community", "/about"].each do |path| ++ {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path| + # /c/LinusTechTips + get "/c/:user#{path}", Routes::Channels, :brand_redirect + # /user/linustechtips | Not always the same as /c/ +diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr +index 878587d44..f6cc3340f 100644 +--- a/src/invidious/views/channel.ecr ++++ b/src/invidious/views/channel.ecr +@@ -64,23 +64,8 @@ + <%= translate(locale, "Switch Invidious Instance") %> + <% end %> + +- <% if !channel.auto_generated %> +-
+- <%= translate(locale, "Videos") %> +-
+- <% end %> +-
+- <% if channel.auto_generated %> +- <%= translate(locale, "Playlists") %> +- <% else %> +- <%= translate(locale, "Playlists") %> +- <% end %> +-
+-
+- <% if channel.tabs.includes? "community" %> +- <%= translate(locale, "Community") %> +- <% end %> +-
++ ++ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %> + +
+
+@@ -111,7 +96,12 @@ +
+ +
+-
+-
+-
++
++
++ <% if next_continuation %> ++ &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> ++ <%= translate(locale, "Next page") %> ++ ++ <% end %> ++
+
+diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr +index 3bc29e552..e467a679c 100644 +--- a/src/invidious/views/community.ecr ++++ b/src/invidious/views/community.ecr +@@ -50,19 +50,8 @@ + <%= translate(locale, "Switch Invidious Instance") %> + <% end %> + +- <% if !channel.auto_generated %> +-
+- <%= translate(locale, "Videos") %> +-
+- <% end %> +-
+- <%= translate(locale, "Playlists") %> +-
+-
+- <% if channel.tabs.includes? "community" %> +- <%= translate(locale, "Community") %> +- <% end %> +-
++ ++ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Community) %> + +
+ +diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr +index c8718e7b7..56d25ef5d 100644 +--- a/src/invidious/views/playlists.ecr ++++ b/src/invidious/views/playlists.ecr +@@ -54,19 +54,7 @@ + <% end %> + + +-
+- <%= translate(locale, "Videos") %> +-
+-
+- <% if !channel.auto_generated %> +- <%= translate(locale, "Playlists") %> +- <% end %> +-
+-
+- <% if channel.tabs.includes? "community" %> +- <%= translate(locale, "Community") %> +- <% end %> +-
++ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Playlists) %> + +
+
+ +From 40c666cab22693cf9d31895978ae4b4356e6579b Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Sun, 4 Dec 2022 19:24:51 +0100 +Subject: [PATCH 12/16] api: Add support for shorts and livestreams + +--- + src/invidious/routes/api/v1/channels.cr | 114 ++++++++++++++++++------ + src/invidious/routing.cr | 3 + + 2 files changed, 90 insertions(+), 27 deletions(-) + +diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr +index 72d9ae5f3..4e92b54e5 100644 +--- a/src/invidious/routes/api/v1/channels.cr ++++ b/src/invidious/routes/api/v1/channels.cr +@@ -1,11 +1,7 @@ + module Invidious::Routes::API::V1::Channels +- def self.home(env) +- locale = env.get("preferences").as(Preferences).locale +- +- env.response.content_type = "application/json" +- +- ucid = env.params.url["ucid"] +- ++ # Macro to avoid duplicating some code below ++ # This sets the `channel` variable, or handles Exceptions. ++ private macro get_channel + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect +@@ -16,6 +12,17 @@ module Invidious::Routes::API::V1::Channels + rescue ex + return error_json(500, ex) + end ++ end ++ ++ def self.home(env) ++ locale = env.get("preferences").as(Preferences).locale ++ ucid = env.params.url["ucid"] ++ ++ env.response.content_type = "application/json" ++ ++ # Use the private macro defined above. ++ channel = nil # Make the compiler happy ++ get_channel() + + # Retrieve "sort by" setting from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" +@@ -138,29 +145,89 @@ module Invidious::Routes::API::V1::Channels + + def self.videos(env) + locale = env.get("preferences").as(Preferences).locale ++ ucid = env.params.url["ucid"] + + env.response.content_type = "application/json" + ++ # Use the private macro defined above. ++ channel = nil # Make the compiler happy ++ get_channel() ++ ++ # Retrieve some URL parameters ++ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" ++ continuation = env.params.query["continuation"]? ++ ++ begin ++ videos, next_continuation = Channel::Tabs.get_60_videos( ++ channel, continuation: continuation, sort_by: sort_by ++ ) ++ rescue ex ++ return error_json(500, ex) ++ end ++ ++ return JSON.build do |json| ++ json.object do ++ json.field "videos" do ++ json.array do ++ videos.each &.to_json(locale, json) ++ end ++ end ++ ++ json.field "continuation", next_continuation if next_continuation ++ end ++ end ++ end ++ ++ def self.shorts(env) ++ locale = env.get("preferences").as(Preferences).locale + ucid = env.params.url["ucid"] + ++ env.response.content_type = "application/json" ++ ++ # Use the private macro defined above. ++ channel = nil # Make the compiler happy ++ get_channel() ++ ++ # Retrieve continuation from URL parameters ++ continuation = env.params.query["continuation"]? ++ + begin +- channel = get_about_info(ucid, locale) +- rescue ex : ChannelRedirect +- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) +- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) +- rescue ex : NotFoundException +- return error_json(404, ex) ++ videos, next_continuation = Channel::Tabs.get_shorts( ++ channel, continuation: continuation ++ ) + rescue ex + return error_json(500, ex) + end + +- # Retrieve some URL parameters +- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" ++ return JSON.build do |json| ++ json.object do ++ json.field "videos" do ++ json.array do ++ videos.each &.to_json(locale, json) ++ end ++ end ++ ++ json.field "continuation", next_continuation if next_continuation ++ end ++ end ++ end ++ ++ def self.streams(env) ++ locale = env.get("preferences").as(Preferences).locale ++ ucid = env.params.url["ucid"] ++ ++ env.response.content_type = "application/json" ++ ++ # Use the private macro defined above. ++ channel = nil # Make the compiler happy ++ get_channel() ++ ++ # Retrieve continuation from URL parameters + continuation = env.params.query["continuation"]? + + begin +- videos, next_continuation = Channel::Tabs.get_60_videos( +- channel, continuation: continuation, sort_by: sort_by ++ videos, next_continuation = Channel::Tabs.get_60_livestreams( ++ channel, continuation: continuation + ) + rescue ex + return error_json(500, ex) +@@ -190,16 +257,9 @@ module Invidious::Routes::API::V1::Channels + env.params.query["sort_by"]?.try &.downcase || + "last" + +- begin +- channel = get_about_info(ucid, locale) +- rescue ex : ChannelRedirect +- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) +- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) +- rescue ex : NotFoundException +- return error_json(404, ex) +- rescue ex +- return error_json(500, ex) +- end ++ # Use the macro defined above ++ channel = nil # Make the compiler happy ++ get_channel() + + items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) + +diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr +index 08739c3d3..0e6fba214 100644 +--- a/src/invidious/routing.cr ++++ b/src/invidious/routing.cr +@@ -222,6 +222,9 @@ module Invidious::Routing + + # Channels + get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home ++ get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts ++ get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams ++ + {% for route in {"videos", "latest", "playlists", "community", "search"} %} + get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} + get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} + +From b6a4de66a5414f8ae790033fc3fc9e9fda70a860 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Sun, 4 Dec 2022 23:19:25 +0100 +Subject: [PATCH 13/16] frontend: Unify the various channel pages + +--- + src/invidious/routes/channels.cr | 33 ++++--- + src/invidious/views/channel.ecr | 93 +++++------------- + src/invidious/views/community.ecr | 65 +++---------- + .../views/components/channel_info.ecr | 60 ++++++++++++ + src/invidious/views/playlists.ecr | 96 ------------------- + 5 files changed, 115 insertions(+), 232 deletions(-) + create mode 100644 src/invidious/views/components/channel_info.ecr + delete mode 100644 src/invidious/views/playlists.ecr + +diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr +index 78b383419..77d309fba 100644 +--- a/src/invidious/routes/channels.cr ++++ b/src/invidious/routes/channels.cr +@@ -7,18 +7,19 @@ module Invidious::Routes::Channels + + def self.videos(env) + data = self.fetch_basic_information(env) +- if !data.is_a?(Tuple) +- return data +- end ++ return data if !data.is_a?(Tuple) ++ + locale, user, subscriptions, continuation, ucid, channel = data + + sort_by = env.params.query["sort_by"]?.try &.downcase + + if channel.auto_generated + sort_options = {"last", "oldest", "newest"} +- sort_by ||= "last" + +- items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) ++ items, next_continuation = fetch_channel_playlists( ++ channel.ucid, channel.author, continuation, (sort_by || "last") ++ ) ++ + items.uniq! do |item| + if item.responds_to?(:title) + item.title +@@ -30,11 +31,10 @@ module Invidious::Routes::Channels + items.each(&.author = "") + else + sort_options = {"newest", "oldest", "popular"} +- sort_by ||= "newest" + + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_videos( +- channel, continuation: continuation, sort_by: sort_by ++ channel, continuation: continuation, sort_by: (sort_by || "newest") + ) + end + +@@ -90,24 +90,26 @@ module Invidious::Routes::Channels + + def self.playlists(env) + data = self.fetch_basic_information(env) +- if !data.is_a?(Tuple) +- return data +- end ++ return data if !data.is_a?(Tuple) ++ + locale, user, subscriptions, continuation, ucid, channel = data + + sort_options = {"last", "oldest", "newest"} + sort_by = env.params.query["sort_by"]?.try &.downcase +- sort_by ||= "last" + + if channel.auto_generated + return env.redirect "/channel/#{channel.ucid}" + end + +- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) ++ items, next_continuation = fetch_channel_playlists( ++ channel.ucid, channel.author, continuation, (sort_by || "last") ++ ) ++ + items = items.select(SearchPlaylist).map(&.as(SearchPlaylist)) + items.each(&.author = "") + +- templated "playlists" ++ selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists ++ templated "channel" + end + + def self.community(env) +@@ -121,12 +123,15 @@ module Invidious::Routes::Channels + thin_mode = thin_mode == "true" + + continuation = env.params.query["continuation"]? +- # sort_by = env.params.query["sort_by"]?.try &.downcase + + if !channel.tabs.includes? "community" + return env.redirect "/channel/#{channel.ucid}" + end + ++ # TODO: support sort options for community posts ++ sort_by = "" ++ sort_options = [] of String ++ + begin + items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) + rescue ex : InfoException +diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr +index f6cc3340f..039f8752c 100644 +--- a/src/invidious/views/channel.ecr ++++ b/src/invidious/views/channel.ecr +@@ -1,8 +1,23 @@ +-<% ucid = channel.ucid %> +-<% author = HTML.escape(channel.author) %> +-<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %> ++<%- ++ ucid = channel.ucid ++ author = HTML.escape(channel.author) ++ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target ++ ++ relative_url = ++ case selected_tab ++ when .shorts? then "/channel/#{ucid}/shorts" ++ when .streams? then "/channel/#{ucid}/streams" ++ when .playlists? then "/channel/#{ucid}/playlists" ++ else ++ "/channel/#{ucid}" ++ end ++ ++ youtube_url = "https://www.youtube.com#{relative_url}" ++ redirect_url = Invidious::Frontend::Misc.redirect_url(env) ++-%> + + <% content_for "header" do %> ++<%- if selected_tab.videos? -%> + + + +@@ -14,76 +29,14 @@ + + + +- +-<%= author %> - Invidious + +-<% end %> +- +-<% if channel.banner %> +-
+- "> +-
++<%- end -%> + +-
+-
+-
++ ++<%= author %> - Invidious + <% end %> + +-
+-
+-
+- +- <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %> +-
+-
+-
+-

+- +-

+-
+-
+- +-
+-
+-

<%= channel.description_html %>

+-
+-
+- +-
+- <% sub_count_text = number_to_short_text(channel.sub_count) %> +- <%= rendered "components/subscribe_widget" %> +-
+- +-
+-
+- <%= translate(locale, "View channel on YouTube") %> +-
+- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> +- "><%= translate(locale, "Switch Invidious Instance") %> +- <% else %> +- <%= translate(locale, "Switch Invidious Instance") %> +- <% end %> +-
+- +- <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %> +-
+-
+-
+-
+- <% sort_options.each do |sort| %> +-
+- <% if sort_by == sort %> +- <%= translate(locale, sort) %> +- <% else %> +- +- <%= translate(locale, sort) %> +- +- <% end %> +-
+- <% end %> +-
+-
+-
++<%= rendered "components/channel_info" %> + +
+
+@@ -99,7 +52,7 @@ +
+
+ <% if next_continuation %> +- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> ++ + <%= translate(locale, "Next page") %> + + <% end %> +diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr +index e467a679c..9e11d5622 100644 +--- a/src/invidious/views/community.ecr ++++ b/src/invidious/views/community.ecr +@@ -1,60 +1,21 @@ +-<% ucid = channel.ucid %> +-<% author = HTML.escape(channel.author) %> ++<%- ++ ucid = channel.ucid ++ author = HTML.escape(channel.author) ++ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target + +-<% content_for "header" do %> +-<%= author %> - Invidious +-<% end %> ++ relative_url = "/channel/#{ucid}/community" ++ youtube_url = "https://www.youtube.com#{relative_url}" ++ redirect_url = Invidious::Frontend::Misc.redirect_url(env) + +-<% if channel.banner %> +-
+- "> +-
++ selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community ++-%> + +-
+-
+-
++<% content_for "header" do %> ++ ++<%= author %> - Invidious + <% end %> + +-
+-
+-
+- +- <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %> +-
+-
+-
+-

+- +-

+-
+-
+- +-
+-
+-

<%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %>

+-
+-
+- +-
+- <% sub_count_text = number_to_short_text(channel.sub_count) %> +- <%= rendered "components/subscribe_widget" %> +-
+- +-
+-
+- <%= translate(locale, "View channel on YouTube") %> +-
+- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> +- "><%= translate(locale, "Switch Invidious Instance") %> +- <% else %> +- <%= translate(locale, "Switch Invidious Instance") %> +- <% end %> +-
+- +- <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Community) %> +-
+-
+-
++<%= rendered "components/channel_info" %> + +
+
+diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr +new file mode 100644 +index 000000000..f216359f3 +--- /dev/null ++++ b/src/invidious/views/components/channel_info.ecr +@@ -0,0 +1,60 @@ ++<% if channel.banner %> ++
++ "> ++
++ ++
++
++
++<% end %> ++ ++
++
++
++ ++ <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %> ++
++
++
++

++ ++

++
++
++ ++
++
++

<%= channel.description_html %>

++
++
++ ++
++ <% sub_count_text = number_to_short_text(channel.sub_count) %> ++ <%= rendered "components/subscribe_widget" %> ++
++ ++
++
++
++ <%= translate(locale, "View channel on YouTube") %> ++
++
++ <%= translate(locale, "Switch Invidious Instance") %> ++
++ ++ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %> ++
++
++
++ <% sort_options.each do |sort| %> ++
++ <% if sort_by == sort %> ++ <%= translate(locale, sort) %> ++ <% else %> ++ <%= translate(locale, sort) %> ++ <% end %> ++
++ <% end %> ++
++
++
+diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr +deleted file mode 100644 +index 56d25ef5d..000000000 +--- a/src/invidious/views/playlists.ecr ++++ /dev/null +@@ -1,96 +0,0 @@ +-<% ucid = channel.ucid %> +-<% author = HTML.escape(channel.author) %> +- +-<% content_for "header" do %> +-<%= author %> - Invidious +-<% end %> +- +-<% if channel.banner %> +-
+- "> +-
+- +-
+-
+-
+-<% end %> +- +-
+-
+-
+- +- <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %> +-
+-
+-
+-

+- +-

+-
+-
+- +-
+-
+-

<%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %>

+-
+-
+- +-
+- <% sub_count_text = number_to_short_text(channel.sub_count) %> +- <%= rendered "components/subscribe_widget" %> +-
+- +-
+-
+-
+- <%= translate(locale, "View channel on YouTube") %> +-
+- +-
+- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%> +- "><%= translate(locale, "Switch Invidious Instance") %> +- <% else %> +- <%= translate(locale, "Switch Invidious Instance") %> +- <% end %> +-
+- +- <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, Invidious::Frontend::ChannelPage::TabsAvailable::Playlists) %> +-
+-
+-
+-
+- <% {"last", "oldest", "newest"}.each do |sort| %> +-
+- <% if sort_by == sort %> +- <%= translate(locale, sort) %> +- <% else %> +- +- <%= translate(locale, sort) %> +- +- <% end %> +-
+- <% end %> +-
+-
+-
+- +-
+-
+-
+- +-
+-<% items.each do |item| %> +- <%= rendered "components/item" %> +-<% end %> +-
+- +-
+-
+-
+- <% if continuation %> +- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>"> +- <%= translate(locale, "Next page") %> +- +- <% end %> +-
+-
+ +From 4e3a9306260b737e2d13c6a763899b946a6ecfbb Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Mon, 5 Dec 2022 00:50:04 +0100 +Subject: [PATCH 14/16] frontend: Add support for the "featured channels" page + +--- + locales/en-US.json | 3 +- + src/invidious/channels/about.cr | 50 +++++-------------------- + src/invidious/frontend/channel_page.cr | 1 + + src/invidious/routes/api/v1/channels.cr | 24 ++---------- + src/invidious/routes/channels.cr | 20 ++++++++++ + src/invidious/routing.cr | 1 + + src/invidious/views/channel.ecr | 1 + + 7 files changed, 37 insertions(+), 63 deletions(-) + +diff --git a/locales/en-US.json b/locales/en-US.json +index 44b40c249..129556654 100644 +--- a/locales/en-US.json ++++ b/locales/en-US.json +@@ -475,5 +475,6 @@ + "channel_tab_shorts_label": "Shorts", + "channel_tab_streams_label": "Livestreams", + "channel_tab_playlists_label": "Playlists", +- "channel_tab_community_label": "Community" ++ "channel_tab_community_label": "Community", ++ "channel_tab_channels_label": "Channels" + } +diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr +index 09c3427a3..0054f8f24 100644 +--- a/src/invidious/channels/about.cr ++++ b/src/invidious/channels/about.cr +@@ -16,12 +16,6 @@ record AboutChannel, + tabs : Array(String), + verified : Bool + +-record AboutRelatedChannel, +- ucid : String, +- author : String, +- author_url : String, +- author_thumbnail : String +- + def get_about_info(ucid, locale) : AboutChannel + begin + # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} +@@ -165,41 +159,15 @@ def get_about_info(ucid, locale) : AboutChannel + ) + end + +-def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel) +- # params is {"2:string":"channels"} encoded +- channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") +- +- tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any +- tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels")) +- +- return [] of AboutRelatedChannel if tab.nil? +- +- items = tab.dig?( +- "tabRenderer", "content", +- "sectionListRenderer", "contents", 0, +- "itemSectionRenderer", "contents", 0, +- "gridRenderer", "items" +- ).try &.as_a? +- +- related = [] of AboutRelatedChannel +- return related if (items.nil? || items.empty?) +- +- items.each do |item| +- renderer = item["gridChannelRenderer"]? +- next if !renderer +- +- related_id = renderer.dig("channelId").as_s +- related_title = renderer.dig("title", "simpleText").as_s +- related_author_url = renderer.dig("navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s +- related_author_thumbnail = HelperExtractors.get_thumbnails(renderer) +- +- related << AboutRelatedChannel.new( +- ucid: related_id, +- author: related_title, +- author_url: related_author_url, +- author_thumbnail: related_author_thumbnail, +- ) ++def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?} ++ if continuation.nil? ++ # params is {"2:string":"channels"} encoded ++ initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") ++ else ++ initial_data = YoutubeAPI.browse(continuation) + end + +- return related ++ items, continuation = extract_items(initial_data) ++ ++ return items.select(SearchChannel), continuation + end +diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr +index 7ac0e071c..53745dd5a 100644 +--- a/src/invidious/frontend/channel_page.cr ++++ b/src/invidious/frontend/channel_page.cr +@@ -7,6 +7,7 @@ module Invidious::Frontend::ChannelPage + Streams + Playlists + Community ++ Channels + end + + def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable) +diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr +index 4e92b54e5..28ccdab94 100644 +--- a/src/invidious/routes/api/v1/channels.cr ++++ b/src/invidious/routes/api/v1/channels.cr +@@ -102,31 +102,13 @@ module Invidious::Routes::API::V1::Channels + json.array do + # Fetch related channels + begin +- related_channels = fetch_related_channels(channel) ++ related_channels, _ = fetch_related_channels(channel) + rescue ex +- related_channels = [] of AboutRelatedChannel ++ related_channels = [] of SearchChannel + end + + related_channels.each do |related_channel| +- json.object do +- json.field "author", related_channel.author +- json.field "authorId", related_channel.ucid +- json.field "authorUrl", related_channel.author_url +- +- json.field "authorThumbnails" do +- json.array do +- qualities = {32, 48, 76, 100, 176, 512} +- +- qualities.each do |quality| +- json.object do +- json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") +- json.field "width", quality +- json.field "height", quality +- end +- end +- end +- end +- end ++ related_channel.to_json(locale, json) + end + end + end # relatedChannels +diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr +index 77d309fba..d3969d297 100644 +--- a/src/invidious/routes/channels.cr ++++ b/src/invidious/routes/channels.cr +@@ -147,6 +147,26 @@ module Invidious::Routes::Channels + templated "community" + end + ++ def self.channels(env) ++ data = self.fetch_basic_information(env) ++ return data if !data.is_a?(Tuple) ++ ++ locale, user, subscriptions, continuation, ucid, channel = data ++ ++ if channel.auto_generated ++ return env.redirect "/channel/#{channel.ucid}" ++ end ++ ++ items, next_continuation = fetch_related_channels(channel, continuation) ++ ++ # Featured/related channels can't be sorted ++ sort_options = [] of String ++ sort_by = nil ++ ++ selected_tab = Frontend::ChannelPage::TabsAvailable::Channels ++ templated "channel" ++ end ++ + def self.about(env) + data = self.fetch_basic_information(env) + if !data.is_a?(Tuple) +diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr +index 0e6fba214..84dbed5bf 100644 +--- a/src/invidious/routing.cr ++++ b/src/invidious/routing.cr +@@ -119,6 +119,7 @@ module Invidious::Routing + get "/channel/:ucid/streams", Routes::Channels, :streams + get "/channel/:ucid/playlists", Routes::Channels, :playlists + get "/channel/:ucid/community", Routes::Channels, :community ++ get "/channel/:ucid/channels", Routes::Channels, :channels + get "/channel/:ucid/about", Routes::Channels, :about + get "/channel/:ucid/live", Routes::Channels, :live + get "/user/:user/live", Routes::Channels, :live +diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr +index 039f8752c..a29315ef8 100644 +--- a/src/invidious/views/channel.ecr ++++ b/src/invidious/views/channel.ecr +@@ -8,6 +8,7 @@ + when .shorts? then "/channel/#{ucid}/shorts" + when .streams? then "/channel/#{ucid}/streams" + when .playlists? then "/channel/#{ucid}/playlists" ++ when .channels? then "/channel/#{ucid}/channels" + else + "/channel/#{ucid}" + end + +From 69b8e0919fd0a410d35f5f5fccc4753f79faf940 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Thu, 22 Dec 2022 17:26:30 +0100 +Subject: [PATCH 15/16] api: Add support for the "featured channels" endpoint + +--- + src/invidious/routes/api/v1/channels.cr | 31 +++++++++++++++++++++++++ + src/invidious/routing.cr | 1 + + 2 files changed, 32 insertions(+) + +diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr +index 28ccdab94..ca2b27342 100644 +--- a/src/invidious/routes/api/v1/channels.cr ++++ b/src/invidious/routes/api/v1/channels.cr +@@ -283,6 +283,37 @@ module Invidious::Routes::API::V1::Channels + end + end + ++ def self.channels(env) ++ locale = env.get("preferences").as(Preferences).locale ++ ucid = env.params.url["ucid"] ++ ++ env.response.content_type = "application/json" ++ ++ # Use the macro defined above ++ channel = nil # Make the compiler happy ++ get_channel() ++ ++ continuation = env.params.query["continuation"]? ++ ++ begin ++ items, next_continuation = fetch_related_channels(channel, continuation) ++ rescue ex ++ return error_json(500, ex) ++ end ++ ++ JSON.build do |json| ++ json.object do ++ json.field "relatedChannels" do ++ json.array do ++ items.each &.to_json(locale, json) ++ end ++ end ++ ++ json.field "continuation", next_continuation if next_continuation ++ end ++ end ++ end ++ + def self.search(env) + locale = env.get("preferences").as(Preferences).locale + region = env.params.query["region"]? +diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr +index 84dbed5bf..54bd82a45 100644 +--- a/src/invidious/routing.cr ++++ b/src/invidious/routing.cr +@@ -225,6 +225,7 @@ module Invidious::Routing + get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts + get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams ++ get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels + + {% for route in {"videos", "latest", "playlists", "community", "search"} %} + get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} + +From f9eb839c7ae2c29e641495c4a2affd384445bf97 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Thu, 22 Dec 2022 13:05:13 +0100 +Subject: [PATCH 16/16] channel: remove dead playlists code + +--- + spec/invidious/helpers_spec.cr | 6 ---- + src/invidious/channels/playlists.cr | 55 ----------------------------- + 2 files changed, 61 deletions(-) + +diff --git a/spec/invidious/helpers_spec.cr b/spec/invidious/helpers_spec.cr +index ab361770d..f81cd29ac 100644 +--- a/spec/invidious/helpers_spec.cr ++++ b/spec/invidious/helpers_spec.cr +@@ -23,12 +23,6 @@ Spectator.describe "Helper" do + end + end + +- describe "#produce_channel_playlists_url" do +- it "correctly produces a /browse_ajax URL with the given UCID and cursor" do +- expect(produce_channel_playlists_url("UCCj956IF62FbT7Gouszaj9w", "AIOkY9EQpi_gyn1_QrFuZ1reN81_MMmI1YmlBblw8j7JHItEFG5h7qcJTNd4W9x5Quk_CVZ028gW")).to eq("/browse_ajax?continuation=4qmFsgLNARIYVUNDajk1NklGNjJGYlQ3R291c3phajl3GrABRWdsd2JHRjViR2x6ZEhNd0FqZ0JZQUZxQUxnQkFIcG1VVlZzVUdFeGF6VlNWa1ozWVZZNWJtVlhOSGhZTVVaNVVtNVdZVTFZU214VWFtZDRXREF4VG1KVmEzaFhWekZ6VVcxS2MyUjZhSEZPTUhCSlUxaFNSbEpyWXpGaFJHUjRXVEJ3VlZSdFVUQldlbXcwVGxaR01XRXhPVVJXYkc5M1RXcG9ibFozSUFFWUF3PT0%3D&gl=US&hl=en") +- end +- end +- + describe "#produce_comment_continuation" do + it "correctly produces a continuation token for comments" do + expect(produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA")).to eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyjAMK9gJBRFNKX2kycXZKZUZ0TDBodG1TNV9LNUN0ajNlR0ZWQk1XTDlXZDQybzNrbVVMNl9tQXpkTHA4NS1saVFaTDBtWXJfMTZCaGFnZ1VxWDY1MlN2OUpxVjZWWGluU2hTUC1aVDZyTDROb2xQQmFQWFZ0SnNPNV9yQV9xRTNHdWJBdUxGdzl1eklJWFUyLUhucFhiZGdQTFdURmF2ZlgyMDZocVdtbXBId1VPcm14UVZfT1g2dFlrTTN1eDNyUEFLQ0RyVDhlV0w3TVUzYkxpTmNuYmdrVzhvMGg4S1lMTF84QlBhOExjSGJUdjhwQW9Oa2plcmxYMXg3SzRwcXhhWFBveXo4OXFObG5oNnJSeDZBWGdBenpvSEgxZG1jeVE4Q0lCZU9IZy1tNGk4WnhkWDRkUDg4WFdySUZnLWpKR2hwR1A4SlVNRGdaZ2F2eFZ4MjI1aFVFWVpNeXJMR2xlcjVlbTRGZ2JHNjJZV0M1MW1vTERMZVlFQSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D") +diff --git a/src/invidious/channels/playlists.cr b/src/invidious/channels/playlists.cr +index 772eecb95..8dc824b23 100644 +--- a/src/invidious/channels/playlists.cr ++++ b/src/invidious/channels/playlists.cr +@@ -26,58 +26,3 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) + + return extract_items(initial_data, author, ucid) + end +- +-# ## NOTE: DEPRECATED +-# Reason -> Unstable +-# The Protobuf object must be provided with an id of the last playlist from the current "page" +-# in order to fetch the next one accurately +-# (if the id isn't included, entries shift around erratically between pages, +-# leading to repetitions and skip overs) +-# +-# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user, +-# it's better to stick to continuation tokens provided by the first request and onward +-def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false) +- object = { +- "80226972:embedded" => { +- "2:string" => ucid, +- "3:base64" => { +- "2:string" => "playlists", +- "6:varint" => 2_i64, +- "7:varint" => 1_i64, +- "12:varint" => 1_i64, +- "13:string" => "", +- "23:varint" => 0_i64, +- }, +- }, +- } +- +- if cursor +- cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated +- object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor +- end +- +- if auto_generated +- object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64 +- else +- object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64 +- case sort +- when "oldest", "oldest_created" +- object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64 +- when "newest", "newest_created" +- object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64 +- when "last", "last_added" +- object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64 +- else nil # Ignore +- end +- end +- +- object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) +- object["80226972:embedded"].delete("3:base64") +- +- continuation = object.try { |i| Protodec::Any.cast_json(i) } +- .try { |i| Protodec::Any.from_json(i) } +- .try { |i| Base64.urlsafe_encode(i) } +- .try { |i| URI.encode_www_form(i) } +- +- return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" +-end