From b3a102d85ab19aa6636789414b24a14e7ad43c8e Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Wed, 26 Jul 2023 21:35:22 +0200 Subject: [PATCH] 4001 PR patch --- patches/012-4001.patch | 574 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 574 insertions(+) create mode 100644 patches/012-4001.patch diff --git a/patches/012-4001.patch b/patches/012-4001.patch new file mode 100644 index 0000000..9eb380e --- /dev/null +++ b/patches/012-4001.patch @@ -0,0 +1,574 @@ +From 2e67b90540d35ede212866e1fb597fd57ced35d5 Mon Sep 17 00:00:00 2001 +From: syeopite +Date: Sat, 22 Jul 2023 23:55:05 -0700 +Subject: [PATCH 1/6] Add method to query /youtubei/v1/get_transcript + +--- + src/invidious/yt_backend/youtube_api.cr | 24 ++++++++++++++++++++++++ + 1 file changed, 24 insertions(+) + +diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr +index 3dd9e9d83..f8aca04d9 100644 +--- a/src/invidious/yt_backend/youtube_api.cr ++++ b/src/invidious/yt_backend/youtube_api.cr +@@ -557,6 +557,30 @@ module YoutubeAPI + return self._post_json("/youtubei/v1/search", data, client_config) + end + ++ #################################################################### ++ # transcript(params) ++ # ++ # Requests the youtubei/v1/get_transcript endpoint with the required headers ++ # and POST data in order to get a JSON reply. ++ # ++ # The requested data is a specially encoded protobuf string that denotes the specific language requested. ++ # ++ # An optional ClientConfig parameter can be passed, too (see ++ # `struct ClientConfig` above for more details). ++ # ++ ++ def transcript( ++ params : String, ++ client_config : ClientConfig | Nil = nil ++ ) : Hash(String, JSON::Any) ++ data = { ++ "context" => self.make_context(client_config), ++ "params" => params, ++ } ++ ++ return self._post_json("/youtubei/v1/get_transcript", data, client_config) ++ end ++ + #################################################################### + # _post_json(endpoint, data, client_config?) + # + +From 7e5935a9da5355bbdd4c047edf692b0ce57722c7 Mon Sep 17 00:00:00 2001 +From: syeopite +Date: Sun, 23 Jul 2023 00:54:43 -0700 +Subject: [PATCH 2/6] Rename Caption struct to CaptionMetadata + +The Caption object does not actually store any text lines for the +subtitles. Instead it stores the metadata needed to display and fetch +the actual captions from the YT timedtext API. + +Therefore it may be wiser to rename the struct to be more reflective of +its current usage as well as the future usage once the current caption +retrival system is replaced via InnerTube's transcript API +--- + src/invidious/frontend/watch_page.cr | 2 +- + src/invidious/videos.cr | 6 +++--- + src/invidious/videos/caption.cr | 8 ++++---- + src/invidious/views/user/preferences.ecr | 2 +- + 4 files changed, 9 insertions(+), 9 deletions(-) + +diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr +index e32144693..b860dba77 100644 +--- a/src/invidious/frontend/watch_page.cr ++++ b/src/invidious/frontend/watch_page.cr +@@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage + getter full_videos : Array(Hash(String, JSON::Any)) + getter video_streams : Array(Hash(String, JSON::Any)) + getter audio_streams : Array(Hash(String, JSON::Any)) +- getter captions : Array(Invidious::Videos::Caption) ++ getter captions : Array(Invidious::Videos::CaptionMetadata) + + def initialize( + @full_videos, +diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr +index f38b33e5b..2b1d26033 100644 +--- a/src/invidious/videos.cr ++++ b/src/invidious/videos.cr +@@ -24,7 +24,7 @@ struct Video + property updated : Time + + @[DB::Field(ignore: true)] +- @captions = [] of Invidious::Videos::Caption ++ @captions = [] of Invidious::Videos::CaptionMetadata + + @[DB::Field(ignore: true)] + property adaptive_fmts : Array(Hash(String, JSON::Any))? +@@ -215,9 +215,9 @@ struct Video + keywords.includes? "YouTube Red" + end + +- def captions : Array(Invidious::Videos::Caption) ++ def captions : Array(Invidious::Videos::CaptionMetadata) + if @captions.empty? && @info.has_key?("captions") +- @captions = Invidious::Videos::Caption.from_yt_json(info["captions"]) ++ @captions = Invidious::Videos::CaptionMetadata.from_yt_json(info["captions"]) + end + + return @captions +diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr +index 13f81a317..c85b46c3b 100644 +--- a/src/invidious/videos/caption.cr ++++ b/src/invidious/videos/caption.cr +@@ -1,7 +1,7 @@ + require "json" + + module Invidious::Videos +- struct Caption ++ struct CaptionMetadata + property name : String + property language_code : String + property base_url : String +@@ -10,12 +10,12 @@ module Invidious::Videos + end + + # Parse the JSON structure from Youtube +- def self.from_yt_json(container : JSON::Any) : Array(Caption) ++ def self.from_yt_json(container : JSON::Any) : Array(CaptionMetadata) + caption_tracks = container + .dig?("playerCaptionsTracklistRenderer", "captionTracks") + .try &.as_a + +- captions_list = [] of Caption ++ captions_list = [] of CaptionMetadata + return captions_list if caption_tracks.nil? + + caption_tracks.each do |caption| +@@ -25,7 +25,7 @@ module Invidious::Videos + language_code = caption["languageCode"].to_s + base_url = caption["baseUrl"].to_s + +- captions_list << Caption.new(name, language_code, base_url) ++ captions_list << CaptionMetadata.new(name, language_code, base_url) + end + + return captions_list +diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr +index dfda14341..b1061ee82 100644 +--- a/src/invidious/views/user/preferences.ecr ++++ b/src/invidious/views/user/preferences.ecr +@@ -89,7 +89,7 @@ + + <% preferences.captions.each_with_index do |caption, index| %> + + +From 8e18d445a7adf9a0c0887249003a7b84f0fb95af Mon Sep 17 00:00:00 2001 +From: syeopite +Date: Sun, 23 Jul 2023 01:52:53 -0700 +Subject: [PATCH 3/6] Add method to generate params for transcripts api + +--- + src/invidious/videos/transcript.cr | 34 ++++++++++++++++++++++++++++++ + 1 file changed, 34 insertions(+) + create mode 100644 src/invidious/videos/transcript.cr + +diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr +new file mode 100644 +index 000000000..c50f7569d +--- /dev/null ++++ b/src/invidious/videos/transcript.cr +@@ -0,0 +1,34 @@ ++module Invidious::Videos ++ # Namespace for methods primarily relating to Transcripts ++ module Transcript ++ def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String ++ if !auto_generated ++ is_auto_generated = "" ++ elsif is_auto_generated = "asr" ++ end ++ ++ object = { ++ "1:0:string" => video_id, ++ ++ "2:base64" => { ++ "1:string" => is_auto_generated, ++ "2:string" => language_code, ++ "3:string" => "", ++ }, ++ ++ "3:varint" => 1_i64, ++ "5:string" => "engagement-panel-searchable-transcript-search-panel", ++ "6:varint" => 1_i64, ++ "7:varint" => 1_i64, ++ "8:varint" => 1_i64, ++ } ++ ++ params = object.try { |i| Protodec::Any.cast_json(i) } ++ .try { |i| Protodec::Any.from_json(i) } ++ .try { |i| Base64.urlsafe_encode(i) } ++ .try { |i| URI.encode_www_form(i) } ++ ++ return params ++ end ++ end ++end + +From 4b3ac1a757a5ee14919e83a84de31a3d0bd14a4c Mon Sep 17 00:00:00 2001 +From: syeopite +Date: Sun, 23 Jul 2023 03:22:19 -0700 +Subject: [PATCH 4/6] Add method to parse transcript JSON into structs + +--- + src/invidious/videos/transcript.cr | 37 ++++++++++++++++++++++++++++++ + 1 file changed, 37 insertions(+) + +diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr +index c50f7569d..0d8b0b253 100644 +--- a/src/invidious/videos/transcript.cr ++++ b/src/invidious/videos/transcript.cr +@@ -1,6 +1,8 @@ + module Invidious::Videos + # Namespace for methods primarily relating to Transcripts + module Transcript ++ record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String ++ + def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String + if !auto_generated + is_auto_generated = "" +@@ -30,5 +32,40 @@ module Invidious::Videos + + return params + end ++ ++ def self.convert_transcripts_to_vtt(initial_data : JSON::Any, target_language : String) : String ++ # Convert into TranscriptLine ++ ++ vtt = String.build do |vtt| ++ result << <<-END_VTT ++ WEBVTT ++ Kind: captions ++ Language: #{tlang} ++ ++ ++ END_VTT ++ ++ vtt << "\n\n" ++ end ++ end ++ ++ def self.parse(initial_data : Hash(String, JSON::Any)) ++ body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", ++ "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", ++ "initialSegments").as_a ++ ++ lines = [] of TranscriptLine ++ body.each do |line| ++ line = line["transcriptSegmentRenderer"] ++ start_ms = line["startMs"].as_s.to_i.millisecond ++ end_ms = line["endMs"].as_s.to_i.millisecond ++ ++ text = extract_text(line["snippet"]) || "" ++ ++ lines << TranscriptLine.new(start_ms, end_ms, text) ++ end ++ ++ return lines ++ end + end + end + +From caac7e21668dd88eaf3d57ddc300427885af0a23 Mon Sep 17 00:00:00 2001 +From: syeopite +Date: Sun, 23 Jul 2023 03:52:26 -0700 +Subject: [PATCH 5/6] Add method to convert transcripts response to vtt + +--- + src/invidious/videos/transcript.cr | 39 ++++++++++++++++++++++++++---- + 1 file changed, 34 insertions(+), 5 deletions(-) + +diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr +index 0d8b0b253..ec9908839 100644 +--- a/src/invidious/videos/transcript.cr ++++ b/src/invidious/videos/transcript.cr +@@ -33,23 +33,52 @@ module Invidious::Videos + return params + end + +- def self.convert_transcripts_to_vtt(initial_data : JSON::Any, target_language : String) : String +- # Convert into TranscriptLine ++ def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String ++ # Convert into array of TranscriptLine ++ lines = self.parse(initial_data) + ++ # Taken from Invidious::Videos::CaptionMetadata.timedtext_to_vtt() + vtt = String.build do |vtt| +- result << <<-END_VTT ++ vtt << <<-END_VTT + WEBVTT + Kind: captions +- Language: #{tlang} ++ Language: #{target_language} + + + END_VTT + + vtt << "\n\n" ++ ++ lines.each do |line| ++ start_time = line.start_ms ++ end_time = line.end_ms ++ ++ # start_time ++ vtt << start_time.hours.to_s.rjust(2, '0') ++ vtt << ':' << start_time.minutes.to_s.rjust(2, '0') ++ vtt << ':' << start_time.seconds.to_s.rjust(2, '0') ++ vtt << '.' << start_time.milliseconds.to_s.rjust(3, '0') ++ ++ vtt << " --> " ++ ++ # end_time ++ vtt << end_time.hours.to_s.rjust(2, '0') ++ vtt << ':' << end_time.minutes.to_s.rjust(2, '0') ++ vtt << ':' << end_time.seconds.to_s.rjust(2, '0') ++ vtt << '.' << end_time.milliseconds.to_s.rjust(3, '0') ++ ++ vtt << "\n" ++ vtt << line.line ++ ++ vtt << "\n" ++ vtt << "\n" ++ end + end ++ ++ return vtt + end + +- def self.parse(initial_data : Hash(String, JSON::Any)) ++ private def self.parse(initial_data : Hash(String, JSON::Any)) + body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", + "initialSegments").as_a + +From e4942b188f5c192d5693687698db9b106571332c Mon Sep 17 00:00:00 2001 +From: syeopite +Date: Sun, 23 Jul 2023 05:02:02 -0700 +Subject: [PATCH 6/6] Integrate transcript captions into captions API + +--- + config/config.example.yml | 13 +++ + src/invidious/config.cr | 3 + + src/invidious/routes/api/v1/videos.cr | 114 ++++++++++++++------------ + src/invidious/videos/caption.cr | 11 ++- + src/invidious/videos/transcript.cr | 6 ++ + 5 files changed, 92 insertions(+), 55 deletions(-) + +diff --git a/config/config.example.yml b/config/config.example.yml +index 34070fe51..51beab895 100644 +--- a/config/config.example.yml ++++ b/config/config.example.yml +@@ -182,6 +182,19 @@ https_only: false + #force_resolve: + + ++## ++## Use Innertube's transcripts API instead of timedtext for closed captions ++## ++## Useful for larger instances as InnerTube is **not ratelimited**. See https://github.com/iv-org/invidious/issues/2567 ++## ++## Subtitle experience may differ slightly on Invidious. ++## ++## Accepted values: true, false ++## Default: false ++## ++# use_innertube_for_captions: false ++ ++ + # ----------------------------- + # Logging + # ----------------------------- +diff --git a/src/invidious/config.cr b/src/invidious/config.cr +index e5f1e8222..c88a48370 100644 +--- a/src/invidious/config.cr ++++ b/src/invidious/config.cr +@@ -129,6 +129,9 @@ class Config + # Use quic transport for youtube api + property use_quic : Bool = false + ++ # Use Innertube's transcripts API instead of timedtext for closed captions ++ property use_innertube_for_captions : Bool = false ++ + # Saved cookies in "name1=value1; name2=value2..." format + @[YAML::Field(converter: Preferences::StringToCookies)] + property cookies : HTTP::Cookies = HTTP::Cookies.new +diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr +index af4fc806a..000e64b91 100644 +--- a/src/invidious/routes/api/v1/videos.cr ++++ b/src/invidious/routes/api/v1/videos.cr +@@ -87,70 +87,78 @@ module Invidious::Routes::API::V1::Videos + caption = caption[0] + end + +- url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target +- +- # Auto-generated captions often have cues that aren't aligned properly with the video, +- # as well as some other markup that makes it cumbersome, so we try to fix that here +- if caption.name.includes? "auto-generated" +- caption_xml = YT_POOL.client &.get(url).body +- +- if caption_xml.starts_with?(" i + 1 ++ end_time = caption_nodes[i + 1]["start"].to_f.seconds ++ else ++ end_time = start_time + duration ++ end + +- END_VTT ++ start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" ++ end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" + +- caption_nodes = caption_xml.xpath_nodes("//transcript/text") +- caption_nodes.each_with_index do |node, i| +- start_time = node["start"].to_f.seconds +- duration = node["dur"]?.try &.to_f.seconds +- duration ||= start_time ++ text = HTML.unescape(node.content) ++ text = text.gsub(//, "") ++ text = text.gsub(/<\/font>/, "") ++ if md = text.match(/(?.*) : (?.*)/) ++ text = "#{md["text"]}" ++ end + +- if caption_nodes.size > i + 1 +- end_time = caption_nodes[i + 1]["start"].to_f.seconds +- else +- end_time = start_time + duration +- end ++ str << <<-END_CUE ++ #{start_time} --> #{end_time} ++ #{text} + +- start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}" +- end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}" + +- text = HTML.unescape(node.content) +- text = text.gsub(//, "") +- text = text.gsub(/<\/font>/, "") +- if md = text.match(/(?.*) : (?.*)/) +- text = "#{md["text"]}" ++ END_CUE + end +- +- str << <<-END_CUE +- #{start_time} --> #{end_time} +- #{text} +- +- +- END_CUE + end + end +- end +- else +- # Some captions have "align:[start/end]" and "position:[num]%" +- # attributes. Those are causing issues with VideoJS, which is unable +- # to properly align the captions on the video, so we remove them. +- # +- # See: https://github.com/iv-org/invidious/issues/2391 +- webvtt = YT_POOL.client &.get("#{url}&format=vtt").body +- if webvtt.starts_with?(" [0-9:.]{12}).+/, "\\1") ++ if webvtt.starts_with?(" [0-9:.]{12}).+/, "\\1") ++ end + end + end + +diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr +index c85b46c3b..1e2abde91 100644 +--- a/src/invidious/videos/caption.cr ++++ b/src/invidious/videos/caption.cr +@@ -6,7 +6,9 @@ module Invidious::Videos + property language_code : String + property base_url : String + +- def initialize(@name, @language_code, @base_url) ++ property auto_generated : Bool ++ ++ def initialize(@name, @language_code, @base_url, @auto_generated) + end + + # Parse the JSON structure from Youtube +@@ -25,7 +27,12 @@ module Invidious::Videos + language_code = caption["languageCode"].to_s + base_url = caption["baseUrl"].to_s + +- captions_list << CaptionMetadata.new(name, language_code, base_url) ++ auto_generated = false ++ if caption["kind"]? && caption["kind"] == "asr" ++ auto_generated = true ++ end ++ ++ captions_list << CaptionMetadata.new(name, language_code, base_url, auto_generated) + end + + return captions_list +diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr +index ec9908839..ba2728cd4 100644 +--- a/src/invidious/videos/transcript.cr ++++ b/src/invidious/videos/transcript.cr +@@ -85,7 +85,13 @@ module Invidious::Videos + + lines = [] of TranscriptLine + body.each do |line| ++ # Transcript section headers. They are not apart of the captions and as such we can safely skip them. ++ if line.as_h.has_key?("transcriptSectionHeaderRenderer") ++ next ++ end ++ + line = line["transcriptSegmentRenderer"] ++ + start_ms = line["startMs"].as_s.to_i.millisecond + end_ms = line["endMs"].as_s.to_i.millisecond +