diff --git a/patches/10-2659.patch b/patches/10-2659.patch new file mode 100644 index 0000000..94d1c2b --- /dev/null +++ b/patches/10-2659.patch @@ -0,0 +1,153 @@ +From 80a513baa5e595f62b08d7eed1ac709533fde838 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Wed, 24 Nov 2021 01:22:09 +0100 +Subject: [PATCH] Use new techniques to get (dis)likes back + +--- + src/invidious/videos.cr | 114 ++++++++++++++++++++++++++++------------ + 1 file changed, 79 insertions(+), 35 deletions(-) + +diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr +index d3e5800cc..1f9a6bc98 100644 +--- a/src/invidious/videos.cr ++++ b/src/invidious/videos.cr +@@ -877,42 +877,84 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ + } + ).try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) + +- primary_results = player_response.try &.["contents"]?.try &.["twoColumnWatchNextResults"]?.try &.["results"]? +- .try &.["results"]?.try &.["contents"]? +- sentiment_bar = primary_results.try &.as_a.select(&.["videoPrimaryInfoRenderer"]?)[0]? +- .try &.["videoPrimaryInfoRenderer"]? +- .try &.["sentimentBar"]? +- .try &.["sentimentBarRenderer"]? +- .try &.["tooltip"]? +- .try &.as_s +- +- likes, dislikes = sentiment_bar.try &.split(" / ", 2).map &.gsub(/\D/, "").to_i64 || {0_i64, 0_i64} +- params["likes"] = JSON::Any.new(likes) +- params["dislikes"] = JSON::Any.new(dislikes) +- +- params["descriptionHtml"] = JSON::Any.new(primary_results.try &.as_a.select(&.["videoSecondaryInfoRenderer"]?)[0]? +- .try &.["videoSecondaryInfoRenderer"]?.try &.["description"]?.try &.["runs"]? +- .try &.as_a.try { |t| content_to_comment_html(t).gsub("\n", "
") } || "

") +- +- metadata = primary_results.try &.as_a.select(&.["videoSecondaryInfoRenderer"]?)[0]? +- .try &.["videoSecondaryInfoRenderer"]? +- .try &.["metadataRowContainer"]? +- .try &.["metadataRowContainerRenderer"]? +- .try &.["rows"]? +- .try &.as_a ++ # Top level elements ++ ++ primary_results = player_response ++ .dig?("contents", "twoColumnWatchNextResults", "results", "results", "contents") ++ ++ video_primary_renderer = primary_results ++ .try &.as_a.find(&.["videoPrimaryInfoRenderer"]?) ++ .try &.["videoPrimaryInfoRenderer"] ++ ++ video_secondary_renderer = primary_results ++ .try &.as_a.find(&.["videoSecondaryInfoRenderer"]?) ++ .try &.["videoSecondaryInfoRenderer"] ++ ++ # Likes/dislikes ++ ++ toplevel_buttons = video_primary_renderer ++ .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") ++ ++ if toplevel_buttons ++ likes_button = toplevel_buttons.as_a ++ .find(&.["toggleButtonRenderer"]["defaultIcon"]["iconType"].as_s.== "LIKE") ++ .try &.["toggleButtonRenderer"] ++ ++ if likes_button ++ likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) ++ .try &.dig?("accessibility", "accessibilityData", "label") ++ likes = likes_txt.as_s.gsub(/\D/, "").to_i64 if likes_txt ++ ++ LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") ++ LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes ++ end ++ ++ dislikes_button = toplevel_buttons.as_a ++ .find(&.["toggleButtonRenderer"]["defaultIcon"]["iconType"].as_s.== "DISLIKE") ++ .try &.["toggleButtonRenderer"] ++ ++ if dislikes_button ++ dislikes_txt = (dislikes_button["defaultText"] || dislikes_button["toggledText"]?) ++ .try &.dig?("accessibility", "accessibilityData", "label") ++ dislikes = dislikes_txt.as_s.gsub(/\D/, "").to_i64 if dislikes_txt ++ ++ LOGGER.trace("extract_video_info: Found \"dislikes\" button. Button text is \"#{dislikes_txt}\"") ++ LOGGER.debug("extract_video_info: Dislikes count is #{dislikes}") if dislikes ++ end ++ end ++ ++ if likes && likes != 0_i64 && (!dislikes || dislikes == 0_i64) ++ if rating = player_response.dig?("videoDetails", "averageRating").try &.as_f ++ dislikes = (likes * ((5 - rating)/(rating - 1))).round.to_i64 ++ LOGGER.debug("extract_video_info: Dislikes count (using fallback method) is #{dislikes}") ++ end ++ end ++ ++ params["likes"] = JSON::Any.new(likes || 0_i64) ++ params["dislikes"] = JSON::Any.new(dislikes || 0_i64) ++ ++ # Description ++ ++ description_html = video_secondary_renderer.try &.dig?("description", "runs") ++ .try &.as_a.try { |t| content_to_comment_html(t).gsub("\n", "
") } ++ ++ params["descriptionHtml"] = JSON::Any.new(description_html || "

") ++ ++ # Video metadata ++ ++ metadata = video_secondary_renderer ++ .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") ++ .try &.as_a + + params["genre"] = params["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["category"]? || JSON::Any.new("") + params["genreUrl"] = JSON::Any.new(nil) + + metadata.try &.each do |row| + title = row["metadataRowRenderer"]?.try &.["title"]?.try &.["simpleText"]?.try &.as_s +- contents = row["metadataRowRenderer"]? +- .try &.["contents"]? +- .try &.as_a[0]? ++ contents = row.dig?("metadataRowRenderer", "contents", 0) + + if title.try &.== "Category" +- contents = contents.try &.["runs"]? +- .try &.as_a[0]? ++ contents = contents.try &.["runs"]?.try &.as_a[0]? + + params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "") + params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]? +@@ -927,17 +969,19 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ + end + end + +- author_info = primary_results.try &.as_a.select(&.["videoSecondaryInfoRenderer"]?)[0]? +- .try &.["videoSecondaryInfoRenderer"]?.try &.["owner"]?.try &.["videoOwnerRenderer"]? ++ # Author infos + +- params["authorThumbnail"] = JSON::Any.new(author_info.try &.["thumbnail"]? +- .try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"]? +- .try &.as_s || "") ++ author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") ++ author_thumbnail = author_info.try &.dig?("thumbnail", "thumbnails", 0, "url") ++ ++ params["authorThumbnail"] = JSON::Any.new(author_thumbnail.try &.as_s || "") + + params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]? +- .try { |t| t["simpleText"]? || t["runs"]?.try &.[0]?.try &.["text"]? }.try &.as_s.split(" ", 2)[0] || "-") ++ .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }.try &.as_s.split(" ", 2)[0] || "-") ++ ++ # Return data + +- params ++ return params + end + + def get_video(id, db, refresh = true, region = nil, force_refresh = false) diff --git a/patches/11-2656.patch b/patches/11-2656.patch new file mode 100644 index 0000000..32d38b4 --- /dev/null +++ b/patches/11-2656.patch @@ -0,0 +1,33 @@ +From 319587e2f11abe53c507d2363ca551d8321b203f Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Sun, 21 Nov 2021 17:34:17 +0100 +Subject: [PATCH] extract_video_info: make sure that the Android player + response is valid + +--- + src/invidious/videos.cr | 12 ++++++++++-- + 1 file changed, 10 insertions(+), 2 deletions(-) + +diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr +index d3e5800cc..4406284f0 100644 +--- a/src/invidious/videos.cr ++++ b/src/invidious/videos.cr +@@ -857,8 +857,16 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ + else + client_config.client_type = YoutubeAPI::ClientType::Android + end +- stream_data = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) +- params["streamingData"] = stream_data["streamingData"]? || JSON::Any.new("") ++ android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) ++ ++ # Sometime, the video is available from the web client, but not on Android, so check ++ # that here, and fallback to the streaming data from the web client if needed. ++ # See: https://github.com/iv-org/invidious/issues/2549 ++ if android_player["playabilityStatus"]["status"] == "OK" ++ params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("") ++ else ++ params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("") ++ end + end + + {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| diff --git a/patches/9-2623.patch b/patches/9-2623.patch new file mode 100644 index 0000000..9df422b --- /dev/null +++ b/patches/9-2623.patch @@ -0,0 +1,177 @@ +From 2eac23a0b37082eebbfa1436c2e6200210b6e0a9 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Tue, 16 Nov 2021 13:24:13 +0100 +Subject: [PATCH 1/4] Temporary fix for #2612 + +Don't rely on the auto compression/decompression provided by the crystal stdlib. +--- + src/invidious/yt_backend/youtube_api.cr | 30 +++++++++++++++---------- + 1 file changed, 18 insertions(+), 12 deletions(-) + +diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr +index 27f250363..0403cb562 100644 +--- a/src/invidious/yt_backend/youtube_api.cr ++++ b/src/invidious/yt_backend/youtube_api.cr +@@ -404,19 +404,10 @@ module YoutubeAPI + url = "#{endpoint}?key=#{client_config.api_key}" + + headers = HTTP::Headers{ +- "Content-Type" => "application/json; charset=UTF-8", ++ "Content-Type" => "application/json; charset=UTF-8", ++ "Accept-Encoding" => "gzip, deflate", + } + +- # The normal HTTP client automatically applies accept-encoding: gzip, +- # and decompresses. However, explicitly applying it will remove this functionality. +- # +- # https://github.com/crystal-lang/crystal/issues/11252#issuecomment-929594741 +- {% unless flag?(:disable_quic) %} +- if CONFIG.use_quic +- headers["Accept-Encoding"] = "gzip" +- end +- {% end %} +- + # Logging + LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") + LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") +@@ -434,8 +425,23 @@ module YoutubeAPI + ) + end + ++ # Decompress the body ourselves, given that auto-decompress is ++ # broken in the Crystal stdlib. ++ # Read more: ++ # - https://github.com/iv-org/invidious/issues/2612 ++ # - https://github.com/crystal-lang/crystal/issues/11354 ++ # ++ case headers["Content-Encoding"]? ++ when "gzip" ++ body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) ++ when "deflate" ++ body = Compress::Deflate::Reader.new(response.body_io, sync_close: true) ++ else ++ body = response.body ++ end ++ + # Convert result to Hash +- initial_data = JSON.parse(response.body).as_h ++ initial_data = JSON.parse(body).as_h + + # Error handling + if initial_data.has_key?("error") + +From dad8f9a0ce576ce3e938fbaa578d12179ca63553 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Tue, 16 Nov 2021 20:39:26 +0100 +Subject: [PATCH 2/4] Fix typo + +Should be checking the returned headers, not the sent ones. +--- + src/invidious/yt_backend/youtube_api.cr | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr +index 0403cb562..4973e4de6 100644 +--- a/src/invidious/yt_backend/youtube_api.cr ++++ b/src/invidious/yt_backend/youtube_api.cr +@@ -431,7 +431,7 @@ module YoutubeAPI + # - https://github.com/iv-org/invidious/issues/2612 + # - https://github.com/crystal-lang/crystal/issues/11354 + # +- case headers["Content-Encoding"]? ++ case response.headers["Content-Encoding"]? + when "gzip" + body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) + when "deflate" + +From 2c447a42f29a05b8067444338248e344b3705cd0 Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Tue, 16 Nov 2021 21:40:35 +0100 +Subject: [PATCH 3/4] Make sure to only apply fix if QUIC is disabled + +--- + src/invidious/yt_backend/youtube_api.cr | 28 ++++++++++++++----------- + 1 file changed, 16 insertions(+), 12 deletions(-) + +diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr +index 4973e4de6..b26af8d12 100644 +--- a/src/invidious/yt_backend/youtube_api.cr ++++ b/src/invidious/yt_backend/youtube_api.cr +@@ -425,19 +425,23 @@ module YoutubeAPI + ) + end + +- # Decompress the body ourselves, given that auto-decompress is +- # broken in the Crystal stdlib. +- # Read more: +- # - https://github.com/iv-org/invidious/issues/2612 +- # - https://github.com/crystal-lang/crystal/issues/11354 +- # +- case response.headers["Content-Encoding"]? +- when "gzip" +- body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) +- when "deflate" +- body = Compress::Deflate::Reader.new(response.body_io, sync_close: true) +- else ++ if {{ !flag?(:disable_quic) }} && CONFIG.use_quic + body = response.body ++ else ++ # Decompress the body ourselves, when using HTTP::Client given that ++ # auto-decompress is broken in the Crystal stdlib. ++ # Read more: ++ # - https://github.com/iv-org/invidious/issues/2612 ++ # - https://github.com/crystal-lang/crystal/issues/11354 ++ # ++ case response.headers["Content-Encoding"]? ++ when "gzip" ++ body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) ++ when "deflate" ++ body = Compress::Deflate::Reader.new(response.body_io, sync_close: true) ++ else ++ body = response.body ++ end + end + + # Convert result to Hash + +From ba48f68fc30990437331791848efe896559f49cd Mon Sep 17 00:00:00 2001 +From: Samantaz Fox +Date: Sun, 21 Nov 2021 18:16:05 +0100 +Subject: [PATCH 4/4] allow multiple, successive content-encodings + +--- + src/invidious/yt_backend/youtube_api.cr | 21 ++++++++++++++++----- + 1 file changed, 16 insertions(+), 5 deletions(-) + +diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr +index b26af8d12..977aea04f 100644 +--- a/src/invidious/yt_backend/youtube_api.cr ++++ b/src/invidious/yt_backend/youtube_api.cr +@@ -434,11 +434,22 @@ module YoutubeAPI + # - https://github.com/iv-org/invidious/issues/2612 + # - https://github.com/crystal-lang/crystal/issues/11354 + # +- case response.headers["Content-Encoding"]? +- when "gzip" +- body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) +- when "deflate" +- body = Compress::Deflate::Reader.new(response.body_io, sync_close: true) ++ if encodings = response.headers["Content-Encoding"]? ++ io = response.body_io ++ ++ # Multiple encodings can be combined, and are listed in the order ++ # in which they were applied. E.g: "deflate, gzip" means that the ++ # content must be first "gunzipped", then "defated". ++ encodings.split(',').reverse.each do |enc| ++ case enc.strip(' ') ++ when "gzip" ++ io = Compress::Gzip::Reader.new(io, sync_close: true) ++ when "deflate" ++ io = Compress::Deflate::Reader.new(io, sync_close: true) ++ end ++ end ++ ++ body = io.gets_to_end + else + body = response.body + end