NerdVPN.de/invidious
1
0
Fork 0
mirror of https://git.nerdvpn.de/NerdVPN.de/invidious synced 2026-02-14 22:51:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

4001 PR patch

Browse Source
This commit is contained in:
Emilien Devos
2023-07-26 21:35:22 +02:00
parent ae2b0c8624
commit b3a102d85a
1 changed files with 574 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/012-4001.patch
+574
View File
@@ -0,0 +1,574 @@
From 2e67b90540d35ede212866e1fb597fd57ced35d5 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sat, 22 Jul 2023 23:55:05 -0700
Subject: [PATCH 1/6] Add method to query /youtubei/v1/get_transcript
---
src/invidious/yt_backend/youtube_api.cr | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr
index 3dd9e9d83..f8aca04d9 100644
--- a/src/invidious/yt_backend/youtube_api.cr
+++ b/src/invidious/yt_backend/youtube_api.cr
@@ -557,6 +557,30 @@ module YoutubeAPI
return self._post_json("/youtubei/v1/search", data, client_config)
end
+ ####################################################################
+ # transcript(params)
+ #
+ # Requests the youtubei/v1/get_transcript endpoint with the required headers
+ # and POST data in order to get a JSON reply.
+ #
+ # The requested data is a specially encoded protobuf string that denotes the specific language requested.
+ #
+ # An optional ClientConfig parameter can be passed, too (see
+ # `struct ClientConfig` above for more details).
+ #
+
+ def transcript(
+ params : String,
+ client_config : ClientConfig | Nil = nil
+ ) : Hash(String, JSON::Any)
+ data = {
+ "context" => self.make_context(client_config),
+ "params" => params,
+ }
+
+ return self._post_json("/youtubei/v1/get_transcript", data, client_config)
+ end
+
####################################################################
# _post_json(endpoint, data, client_config?)
#
From 7e5935a9da5355bbdd4c047edf692b0ce57722c7 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sun, 23 Jul 2023 00:54:43 -0700
Subject: [PATCH 2/6] Rename Caption struct to CaptionMetadata
The Caption object does not actually store any text lines for the
subtitles. Instead it stores the metadata needed to display and fetch
the actual captions from the YT timedtext API.
Therefore it may be wiser to rename the struct to be more reflective of
its current usage as well as the future usage once the current caption
retrival system is replaced via InnerTube's transcript API
---
src/invidious/frontend/watch_page.cr | 2 +-
src/invidious/videos.cr | 6 +++---
src/invidious/videos/caption.cr | 8 ++++----
src/invidious/views/user/preferences.ecr | 2 +-
4 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr
index e32144693..b860dba77 100644
--- a/src/invidious/frontend/watch_page.cr
+++ b/src/invidious/frontend/watch_page.cr
@@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage
getter full_videos : Array(Hash(String, JSON::Any))
getter video_streams : Array(Hash(String, JSON::Any))
getter audio_streams : Array(Hash(String, JSON::Any))
- getter captions : Array(Invidious::Videos::Caption)
+ getter captions : Array(Invidious::Videos::CaptionMetadata)
def initialize(
@full_videos,
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index f38b33e5b..2b1d26033 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -24,7 +24,7 @@ struct Video
property updated : Time
@[DB::Field(ignore: true)]
- @captions = [] of Invidious::Videos::Caption
+ @captions = [] of Invidious::Videos::CaptionMetadata
@[DB::Field(ignore: true)]
property adaptive_fmts : Array(Hash(String, JSON::Any))?
@@ -215,9 +215,9 @@ struct Video
keywords.includes? "YouTube Red"
end
- def captions : Array(Invidious::Videos::Caption)
+ def captions : Array(Invidious::Videos::CaptionMetadata)
if @captions.empty? && @info.has_key?("captions")
- @captions = Invidious::Videos::Caption.from_yt_json(info["captions"])
+ @captions = Invidious::Videos::CaptionMetadata.from_yt_json(info["captions"])
end
return @captions
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
index 13f81a317..c85b46c3b 100644
--- a/src/invidious/videos/caption.cr
+++ b/src/invidious/videos/caption.cr
@@ -1,7 +1,7 @@
require "json"
module Invidious::Videos
- struct Caption
+ struct CaptionMetadata
property name : String
property language_code : String
property base_url : String
@@ -10,12 +10,12 @@ module Invidious::Videos
end
# Parse the JSON structure from Youtube
- def self.from_yt_json(container : JSON::Any) : Array(Caption)
+ def self.from_yt_json(container : JSON::Any) : Array(CaptionMetadata)
caption_tracks = container
.dig?("playerCaptionsTracklistRenderer", "captionTracks")
.try &.as_a
- captions_list = [] of Caption
+ captions_list = [] of CaptionMetadata
return captions_list if caption_tracks.nil?
caption_tracks.each do |caption|
@@ -25,7 +25,7 @@ module Invidious::Videos
language_code = caption["languageCode"].to_s
base_url = caption["baseUrl"].to_s
- captions_list << Caption.new(name, language_code, base_url)
+ captions_list << CaptionMetadata.new(name, language_code, base_url)
end
return captions_list
diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr
index dfda14341..b1061ee82 100644
--- a/src/invidious/views/user/preferences.ecr
+++ b/src/invidious/views/user/preferences.ecr
@@ -89,7 +89,7 @@
<label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label>
<% preferences.captions.each_with_index do |caption, index| %>
<select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]">
- <% Invidious::Videos::Caption::LANGUAGES.each do |option| %>
+ <% Invidious::Videos::CaptionMetadata::LANGUAGES.each do |option| %>
<option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option>
<% end %>
</select>
From 8e18d445a7adf9a0c0887249003a7b84f0fb95af Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sun, 23 Jul 2023 01:52:53 -0700
Subject: [PATCH 3/6] Add method to generate params for transcripts api
---
src/invidious/videos/transcript.cr | 34 ++++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
create mode 100644 src/invidious/videos/transcript.cr
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
new file mode 100644
index 000000000..c50f7569d
--- /dev/null
+++ b/src/invidious/videos/transcript.cr
@@ -0,0 +1,34 @@
+module Invidious::Videos
+ # Namespace for methods primarily relating to Transcripts
+ module Transcript
+ def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
+ if !auto_generated
+ is_auto_generated = ""
+ elsif is_auto_generated = "asr"
+ end
+
+ object = {
+ "1:0:string" => video_id,
+
+ "2:base64" => {
+ "1:string" => is_auto_generated,
+ "2:string" => language_code,
+ "3:string" => "",
+ },
+
+ "3:varint" => 1_i64,
+ "5:string" => "engagement-panel-searchable-transcript-search-panel",
+ "6:varint" => 1_i64,
+ "7:varint" => 1_i64,
+ "8:varint" => 1_i64,
+ }
+
+ params = object.try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+ .try { |i| URI.encode_www_form(i) }
+
+ return params
+ end
+ end
+end
From 4b3ac1a757a5ee14919e83a84de31a3d0bd14a4c Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sun, 23 Jul 2023 03:22:19 -0700
Subject: [PATCH 4/6] Add method to parse transcript JSON into structs
---
src/invidious/videos/transcript.cr | 37 ++++++++++++++++++++++++++++++
1 file changed, 37 insertions(+)
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
index c50f7569d..0d8b0b253 100644
--- a/src/invidious/videos/transcript.cr
+++ b/src/invidious/videos/transcript.cr
@@ -1,6 +1,8 @@
module Invidious::Videos
# Namespace for methods primarily relating to Transcripts
module Transcript
+ record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String
+
def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
if !auto_generated
is_auto_generated = ""
@@ -30,5 +32,40 @@ module Invidious::Videos
return params
end
+
+ def self.convert_transcripts_to_vtt(initial_data : JSON::Any, target_language : String) : String
+ # Convert into TranscriptLine
+
+ vtt = String.build do |vtt|
+ result << <<-END_VTT
+ WEBVTT
+ Kind: captions
+ Language: #{tlang}
+
+
+ END_VTT
+
+ vtt << "\n\n"
+ end
+ end
+
+ def self.parse(initial_data : Hash(String, JSON::Any))
+ body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
+ "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer",
+ "initialSegments").as_a
+
+ lines = [] of TranscriptLine
+ body.each do |line|
+ line = line["transcriptSegmentRenderer"]
+ start_ms = line["startMs"].as_s.to_i.millisecond
+ end_ms = line["endMs"].as_s.to_i.millisecond
+
+ text = extract_text(line["snippet"]) || ""
+
+ lines << TranscriptLine.new(start_ms, end_ms, text)
+ end
+
+ return lines
+ end
end
end
From caac7e21668dd88eaf3d57ddc300427885af0a23 Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sun, 23 Jul 2023 03:52:26 -0700
Subject: [PATCH 5/6] Add method to convert transcripts response to vtt
---
src/invidious/videos/transcript.cr | 39 ++++++++++++++++++++++++++----
1 file changed, 34 insertions(+), 5 deletions(-)
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
index 0d8b0b253..ec9908839 100644
--- a/src/invidious/videos/transcript.cr
+++ b/src/invidious/videos/transcript.cr
@@ -33,23 +33,52 @@ module Invidious::Videos
return params
end
- def self.convert_transcripts_to_vtt(initial_data : JSON::Any, target_language : String) : String
- # Convert into TranscriptLine
+ def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String
+ # Convert into array of TranscriptLine
+ lines = self.parse(initial_data)
+ # Taken from Invidious::Videos::CaptionMetadata.timedtext_to_vtt()
vtt = String.build do |vtt|
- result << <<-END_VTT
+ vtt << <<-END_VTT
WEBVTT
Kind: captions
- Language: #{tlang}
+ Language: #{target_language}
END_VTT
vtt << "\n\n"
+
+ lines.each do |line|
+ start_time = line.start_ms
+ end_time = line.end_ms
+
+ # start_time
+ vtt << start_time.hours.to_s.rjust(2, '0')
+ vtt << ':' << start_time.minutes.to_s.rjust(2, '0')
+ vtt << ':' << start_time.seconds.to_s.rjust(2, '0')
+ vtt << '.' << start_time.milliseconds.to_s.rjust(3, '0')
+
+ vtt << " --> "
+
+ # end_time
+ vtt << end_time.hours.to_s.rjust(2, '0')
+ vtt << ':' << end_time.minutes.to_s.rjust(2, '0')
+ vtt << ':' << end_time.seconds.to_s.rjust(2, '0')
+ vtt << '.' << end_time.milliseconds.to_s.rjust(3, '0')
+
+ vtt << "\n"
+ vtt << line.line
+
+ vtt << "\n"
+ vtt << "\n"
+ end
end
+
+ return vtt
end
- def self.parse(initial_data : Hash(String, JSON::Any))
+ private def self.parse(initial_data : Hash(String, JSON::Any))
body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
"content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer",
"initialSegments").as_a
From e4942b188f5c192d5693687698db9b106571332c Mon Sep 17 00:00:00 2001
From: syeopite <syeopite@syeopite.dev>
Date: Sun, 23 Jul 2023 05:02:02 -0700
Subject: [PATCH 6/6] Integrate transcript captions into captions API
---
config/config.example.yml | 13 +++
src/invidious/config.cr | 3 +
src/invidious/routes/api/v1/videos.cr | 114 ++++++++++++++------------
src/invidious/videos/caption.cr | 11 ++-
src/invidious/videos/transcript.cr | 6 ++
5 files changed, 92 insertions(+), 55 deletions(-)
diff --git a/config/config.example.yml b/config/config.example.yml
index 34070fe51..51beab895 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -182,6 +182,19 @@ https_only: false
#force_resolve:
+##
+## Use Innertube's transcripts API instead of timedtext for closed captions
+##
+## Useful for larger instances as InnerTube is **not ratelimited**. See https://github.com/iv-org/invidious/issues/2567
+##
+## Subtitle experience may differ slightly on Invidious.
+##
+## Accepted values: true, false
+## Default: false
+##
+# use_innertube_for_captions: false
+
+
# -----------------------------
# Logging
# -----------------------------
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index e5f1e8222..c88a48370 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -129,6 +129,9 @@ class Config
# Use quic transport for youtube api
property use_quic : Bool = false
+ # Use Innertube's transcripts API instead of timedtext for closed captions
+ property use_innertube_for_captions : Bool = false
+
# Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new
diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index af4fc806a..000e64b91 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -87,70 +87,78 @@ module Invidious::Routes::API::V1::Videos
caption = caption[0]
end
- url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
-
- # Auto-generated captions often have cues that aren't aligned properly with the video,
- # as well as some other markup that makes it cumbersome, so we try to fix that here
- if caption.name.includes? "auto-generated"
- caption_xml = YT_POOL.client &.get(url).body
-
- if caption_xml.starts_with?("<?xml")
- webvtt = caption.timedtext_to_vtt(caption_xml, tlang)
- else
- caption_xml = XML.parse(caption_xml)
-
- webvtt = String.build do |str|
- str << <<-END_VTT
- WEBVTT
- Kind: captions
- Language: #{tlang || caption.language_code}
+ if CONFIG.use_innertube_for_captions
+ params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
+ initial_data = YoutubeAPI.transcript(params.to_s)
+ webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code)
+ else
+ # Timedtext API handling
+ url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
+
+ # Auto-generated captions often have cues that aren't aligned properly with the video,
+ # as well as some other markup that makes it cumbersome, so we try to fix that here
+ if caption.name.includes? "auto-generated"
+ caption_xml = YT_POOL.client &.get(url).body
+
+ if caption_xml.starts_with?("<?xml")
+ webvtt = caption.timedtext_to_vtt(caption_xml, tlang)
+ else
+ caption_xml = XML.parse(caption_xml)
+
+ webvtt = String.build do |str|
+ str << <<-END_VTT
+ WEBVTT
+ Kind: captions
+ Language: #{tlang || caption.language_code}
+
+
+ END_VTT
+
+ caption_nodes = caption_xml.xpath_nodes("//transcript/text")
+ caption_nodes.each_with_index do |node, i|
+ start_time = node["start"].to_f.seconds
+ duration = node["dur"]?.try &.to_f.seconds
+ duration ||= start_time
+
+ if caption_nodes.size > i + 1
+ end_time = caption_nodes[i + 1]["start"].to_f.seconds
+ else
+ end_time = start_time + duration
+ end
- END_VTT
+ start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
+ end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
- caption_nodes = caption_xml.xpath_nodes("//transcript/text")
- caption_nodes.each_with_index do |node, i|
- start_time = node["start"].to_f.seconds
- duration = node["dur"]?.try &.to_f.seconds
- duration ||= start_time
+ text = HTML.unescape(node.content)
+ text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "")
+ text = text.gsub(/<\/font>/, "")
+ if md = text.match(/(?<name>.*) : (?<text>.*)/)
+ text = "<v #{md["name"]}>#{md["text"]}</v>"
+ end
- if caption_nodes.size > i + 1
- end_time = caption_nodes[i + 1]["start"].to_f.seconds
- else
- end_time = start_time + duration
- end
+ str << <<-END_CUE
+ #{start_time} --> #{end_time}
+ #{text}
- start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
- end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
- text = HTML.unescape(node.content)
- text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "")
- text = text.gsub(/<\/font>/, "")
- if md = text.match(/(?<name>.*) : (?<text>.*)/)
- text = "<v #{md["name"]}>#{md["text"]}</v>"
+ END_CUE
end
-
- str << <<-END_CUE
- #{start_time} --> #{end_time}
- #{text}
-
-
- END_CUE
end
end
- end
- else
- # Some captions have "align:[start/end]" and "position:[num]%"
- # attributes. Those are causing issues with VideoJS, which is unable
- # to properly align the captions on the video, so we remove them.
- #
- # See: https://github.com/iv-org/invidious/issues/2391
- webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
- if webvtt.starts_with?("<?xml")
- webvtt = caption.timedtext_to_vtt(webvtt)
else
+ # Some captions have "align:[start/end]" and "position:[num]%"
+ # attributes. Those are causing issues with VideoJS, which is unable
+ # to properly align the captions on the video, so we remove them.
+ #
+ # See: https://github.com/iv-org/invidious/issues/2391
webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
- .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
+ if webvtt.starts_with?("<?xml")
+ webvtt = caption.timedtext_to_vtt(webvtt)
+ else
+ webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
+ .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
+ end
end
end
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
index c85b46c3b..1e2abde91 100644
--- a/src/invidious/videos/caption.cr
+++ b/src/invidious/videos/caption.cr
@@ -6,7 +6,9 @@ module Invidious::Videos
property language_code : String
property base_url : String
- def initialize(@name, @language_code, @base_url)
+ property auto_generated : Bool
+
+ def initialize(@name, @language_code, @base_url, @auto_generated)
end
# Parse the JSON structure from Youtube
@@ -25,7 +27,12 @@ module Invidious::Videos
language_code = caption["languageCode"].to_s
base_url = caption["baseUrl"].to_s
- captions_list << CaptionMetadata.new(name, language_code, base_url)
+ auto_generated = false
+ if caption["kind"]? && caption["kind"] == "asr"
+ auto_generated = true
+ end
+
+ captions_list << CaptionMetadata.new(name, language_code, base_url, auto_generated)
end
return captions_list
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
index ec9908839..ba2728cd4 100644
--- a/src/invidious/videos/transcript.cr
+++ b/src/invidious/videos/transcript.cr
@@ -85,7 +85,13 @@ module Invidious::Videos
lines = [] of TranscriptLine
body.each do |line|
+ # Transcript section headers. They are not apart of the captions and as such we can safely skip them.
+ if line.as_h.has_key?("transcriptSectionHeaderRenderer")
+ next
+ end
+
line = line["transcriptSegmentRenderer"]
+
start_ms = line["startMs"].as_s.to_i.millisecond
end_ms = line["endMs"].as_s.to_i.millisecond