diff --git a/patches/010-3238.patch b/patches/010-3238.patch deleted file mode 100644 index 1a30749..0000000 --- a/patches/010-3238.patch +++ /dev/null @@ -1,4003 +0,0 @@ -From 6250039405a0a9762a228066df16dd2e8579f4f3 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Tue, 23 Aug 2022 18:23:53 +0200 -Subject: [PATCH 01/16] videos: move regions list to a dedicated file - ---- - src/invidious/videos.cr | 2 -- - src/invidious/videos/regions.cr | 27 +++++++++++++++++++++++++++ - 2 files changed, 27 insertions(+), 2 deletions(-) - create mode 100644 src/invidious/videos/regions.cr - -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index c0ed6e85f..97b4f4b83 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -131,8 +131,6 @@ CAPTION_LANGUAGES = { - "Zulu", - } - --REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"} -- - # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 - VIDEO_FORMATS = { - "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, -diff --git a/src/invidious/videos/regions.cr b/src/invidious/videos/regions.cr -new file mode 100644 -index 000000000..575f8c25d ---- /dev/null -+++ b/src/invidious/videos/regions.cr -@@ -0,0 +1,27 @@ -+# List of geographical regions that Youtube recognizes. -+# This is used to determine if a video is either restricted to a list -+# of allowed regions (= whitelisted) or if it can't be watched in -+# a set of regions (= blacklisted). -+REGIONS = { -+ "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", -+ "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", -+ "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", -+ "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", -+ "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", -+ "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", -+ "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", -+ "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", -+ "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", -+ "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", -+ "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", -+ "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", -+ "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", -+ "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", -+ "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", -+ "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", -+ "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", -+ "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", -+ "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", -+ "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", -+ "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW", -+} - -From 88141c459c553bcca053643c0b1a2ae3338f75b9 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Mon, 23 May 2022 21:54:48 +0200 -Subject: [PATCH 02/16] videos: move formats structure to a separate - file/module - ---- - src/invidious.cr | 1 + - src/invidious/frontend/watch_page.cr | 2 +- - src/invidious/videos.cr | 119 +-------------------------- - src/invidious/videos/formats.cr | 116 ++++++++++++++++++++++++++ - 4 files changed, 120 insertions(+), 118 deletions(-) - create mode 100644 src/invidious/videos/formats.cr - -diff --git a/src/invidious.cr b/src/invidious.cr -index 58adaa354..8df0c0cdb 100644 ---- a/src/invidious.cr -+++ b/src/invidious.cr -@@ -37,6 +37,7 @@ require "./invidious/database/migrations/*" - require "./invidious/helpers/*" - require "./invidious/yt_backend/*" - require "./invidious/frontend/*" -+require "./invidious/videos/*" - - require "./invidious/*" - require "./invidious/channels/*" -diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr -index 80b676415..9212eb2fd 100644 ---- a/src/invidious/frontend/watch_page.cr -+++ b/src/invidious/frontend/watch_page.cr -@@ -50,7 +50,7 @@ module Invidious::Frontend::WatchPage - video_assets.full_videos.each do |option| - mimetype = option["mimeType"].as_s.split(";")[0] - -- height = itag_to_metadata?(option["itag"]).try &.["height"]? -+ height = Invidious::Videos::Formats.itag_to_metadata?(option["itag"]).try &.["height"]? - - value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json - -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index 97b4f4b83..3a71f1635 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -131,117 +131,6 @@ CAPTION_LANGUAGES = { - "Zulu", - } - --# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 --VIDEO_FORMATS = { -- "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, -- "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, -- "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, -- "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, -- "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, -- "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -- "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -- "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -- -- "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, -- "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -- "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -- "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, -- "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, -- "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, -- "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, -- "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -- "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -- -- # 3D videos -- "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -- "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -- "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -- "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -- "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, -- "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, -- "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, -- -- # Apple HTTP Live Streaming -- "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, -- "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, -- "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -- "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -- "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, -- "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, -- "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, -- "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, -- -- # DASH mp4 video -- "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, -- "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, -- "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, -- "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, -- "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, -- "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559) -- "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, -- "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, -- "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, -- "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, -- "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, -- "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, -- -- # Dash mp4 audio -- "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, -- "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, -- "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, -- "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, -- "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, -- "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, -- "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, -- -- # Dash webm -- "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -- "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -- "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -- "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -- "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -- "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -- "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, -- "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, -- "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, -- "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, -- "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, -- "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, -- "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, -- "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, -- "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, -- # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) -- "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, -- "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, -- "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -- -- # Dash webm audio -- "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, -- "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, -- -- # Dash webm audio with opus inside -- "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, -- "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, -- "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, -- -- # av01 video only formats sometimes served with "unknown" codecs -- "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"}, -- "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"}, -- "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"}, -- "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, --} -- - struct VideoPreferences - include JSON::Serializable - -@@ -390,7 +279,7 @@ struct Video - json.field "lmt", fmt["lastModified"] - json.field "projectionType", fmt["projectionType"] - -- if fmt_info = itag_to_metadata?(fmt["itag"]) -+ if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 - json.field "fps", fps - json.field "container", fmt_info["ext"] -@@ -437,7 +326,7 @@ struct Video - json.field "type", fmt["mimeType"] - json.field "quality", fmt["quality"] - -- fmt_info = itag_to_metadata?(fmt["itag"]) -+ fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) - if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 - json.field "fps", fps -@@ -1164,10 +1053,6 @@ def fetch_video(id, region) - return video - end - --def itag_to_metadata?(itag : JSON::Any) -- return VIDEO_FORMATS[itag.to_s]? --end -- - def process_continuation(query, plid, id) - continuation = nil - if plid -diff --git a/src/invidious/videos/formats.cr b/src/invidious/videos/formats.cr -new file mode 100644 -index 000000000..e98e7257c ---- /dev/null -+++ b/src/invidious/videos/formats.cr -@@ -0,0 +1,116 @@ -+module Invidious::Videos::Formats -+ def self.itag_to_metadata?(itag : JSON::Any) -+ return FORMATS[itag.to_s]? -+ end -+ -+ # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 -+ private FORMATS = { -+ "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, -+ "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, -+ "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, -+ "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, -+ "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, -+ "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -+ "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -+ "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -+ -+ "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, -+ "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -+ "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -+ "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, -+ "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, -+ "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, -+ "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, -+ "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -+ "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -+ -+ # 3D videos -+ "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -+ "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -+ "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -+ "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, -+ "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, -+ "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, -+ "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, -+ -+ # Apple HTTP Live Streaming -+ "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, -+ "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, -+ "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -+ "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, -+ "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, -+ "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, -+ "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, -+ "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, -+ -+ # DASH mp4 video -+ "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, -+ "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, -+ "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, -+ "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, -+ "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, -+ "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559) -+ "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, -+ "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, -+ "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, -+ "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, -+ "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, -+ "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, -+ -+ # Dash mp4 audio -+ "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, -+ "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, -+ "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, -+ "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, -+ "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, -+ "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, -+ "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, -+ -+ # Dash webm -+ "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -+ "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -+ "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -+ "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -+ "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -+ "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, -+ "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, -+ "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, -+ "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, -+ "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, -+ "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, -+ "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, -+ "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, -+ "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, -+ "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, -+ # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) -+ "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, -+ "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, -+ "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, -+ -+ # Dash webm audio -+ "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, -+ "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, -+ -+ # Dash webm audio with opus inside -+ "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, -+ "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, -+ "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, -+ -+ # av01 video only formats sometimes served with "unknown" codecs -+ "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"}, -+ "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"}, -+ "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"}, -+ "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, -+ } -+end - -From 9baaef412fe1615eac4fd1508ef879de6b7a8805 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Mon, 23 May 2022 22:29:10 +0200 -Subject: [PATCH 03/16] videos: move 'VideoPreferences' and its associated - function to a separate file - -This will require some rework later. ---- - src/invidious/videos.cr | 157 ---------------------- - src/invidious/videos/video_preferences.cr | 156 +++++++++++++++++++++ - 2 files changed, 156 insertions(+), 157 deletions(-) - create mode 100644 src/invidious/videos/video_preferences.cr - -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index 3a71f1635..f40126669 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -131,34 +131,6 @@ CAPTION_LANGUAGES = { - "Zulu", - } - --struct VideoPreferences -- include JSON::Serializable -- -- property annotations : Bool -- property autoplay : Bool -- property comments : Array(String) -- property continue : Bool -- property continue_autoplay : Bool -- property controls : Bool -- property listen : Bool -- property local : Bool -- property preferred_captions : Array(String) -- property player_style : String -- property quality : String -- property quality_dash : String -- property raw : Bool -- property region : String? -- property related_videos : Bool -- property speed : Float32 | Float64 -- property video_end : Float64 | Int32 -- property video_loop : Bool -- property extend_desc : Bool -- property video_start : Float64 | Int32 -- property volume : Int32 -- property vr_mode : Bool -- property save_player_pos : Bool --end -- - struct Video - include DB::Serializable - -@@ -1067,135 +1039,6 @@ def process_continuation(query, plid, id) - continuation - end - --def process_video_params(query, preferences) -- annotations = query["iv_load_policy"]?.try &.to_i? -- autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- comments = query["comments"]?.try &.split(",").map(&.downcase) -- continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- player_style = query["player_style"]? -- preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) -- quality = query["quality"]? -- quality_dash = query["quality_dash"]? -- region = query["region"]? -- related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- speed = query["speed"]?.try &.rchop("x").to_f? -- video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- volume = query["volume"]?.try &.to_i? -- vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } -- -- if preferences -- # region ||= preferences.region -- annotations ||= preferences.annotations.to_unsafe -- autoplay ||= preferences.autoplay.to_unsafe -- comments ||= preferences.comments -- continue ||= preferences.continue.to_unsafe -- continue_autoplay ||= preferences.continue_autoplay.to_unsafe -- listen ||= preferences.listen.to_unsafe -- local ||= preferences.local.to_unsafe -- player_style ||= preferences.player_style -- preferred_captions ||= preferences.captions -- quality ||= preferences.quality -- quality_dash ||= preferences.quality_dash -- related_videos ||= preferences.related_videos.to_unsafe -- speed ||= preferences.speed -- video_loop ||= preferences.video_loop.to_unsafe -- extend_desc ||= preferences.extend_desc.to_unsafe -- volume ||= preferences.volume -- vr_mode ||= preferences.vr_mode.to_unsafe -- save_player_pos ||= preferences.save_player_pos.to_unsafe -- end -- -- annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe -- autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe -- comments ||= CONFIG.default_user_preferences.comments -- continue ||= CONFIG.default_user_preferences.continue.to_unsafe -- continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe -- listen ||= CONFIG.default_user_preferences.listen.to_unsafe -- local ||= CONFIG.default_user_preferences.local.to_unsafe -- player_style ||= CONFIG.default_user_preferences.player_style -- preferred_captions ||= CONFIG.default_user_preferences.captions -- quality ||= CONFIG.default_user_preferences.quality -- quality_dash ||= CONFIG.default_user_preferences.quality_dash -- related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe -- speed ||= CONFIG.default_user_preferences.speed -- video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe -- extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe -- volume ||= CONFIG.default_user_preferences.volume -- vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe -- save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe -- -- annotations = annotations == 1 -- autoplay = autoplay == 1 -- continue = continue == 1 -- continue_autoplay = continue_autoplay == 1 -- listen = listen == 1 -- local = local == 1 -- related_videos = related_videos == 1 -- video_loop = video_loop == 1 -- extend_desc = extend_desc == 1 -- vr_mode = vr_mode == 1 -- save_player_pos = save_player_pos == 1 -- -- if CONFIG.disabled?("dash") && quality == "dash" -- quality = "high" -- end -- -- if CONFIG.disabled?("local") && local -- local = false -- end -- -- if start = query["t"]? || query["time_continue"]? || query["start"]? -- video_start = decode_time(start) -- end -- video_start ||= 0 -- -- if query["end"]? -- video_end = decode_time(query["end"]) -- end -- video_end ||= -1 -- -- raw = query["raw"]?.try &.to_i? -- raw ||= 0 -- raw = raw == 1 -- -- controls = query["controls"]?.try &.to_i? -- controls ||= 1 -- controls = controls >= 1 -- -- params = VideoPreferences.new({ -- annotations: annotations, -- autoplay: autoplay, -- comments: comments, -- continue: continue, -- continue_autoplay: continue_autoplay, -- controls: controls, -- listen: listen, -- local: local, -- player_style: player_style, -- preferred_captions: preferred_captions, -- quality: quality, -- quality_dash: quality_dash, -- raw: raw, -- region: region, -- related_videos: related_videos, -- speed: speed, -- video_end: video_end, -- video_loop: video_loop, -- extend_desc: extend_desc, -- video_start: video_start, -- volume: volume, -- vr_mode: vr_mode, -- save_player_pos: save_player_pos, -- }) -- -- return params --end -- - def build_thumbnails(id) - return { - {host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"}, -diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr -new file mode 100644 -index 000000000..34cf7ff04 ---- /dev/null -+++ b/src/invidious/videos/video_preferences.cr -@@ -0,0 +1,156 @@ -+struct VideoPreferences -+ include JSON::Serializable -+ -+ property annotations : Bool -+ property autoplay : Bool -+ property comments : Array(String) -+ property continue : Bool -+ property continue_autoplay : Bool -+ property controls : Bool -+ property listen : Bool -+ property local : Bool -+ property preferred_captions : Array(String) -+ property player_style : String -+ property quality : String -+ property quality_dash : String -+ property raw : Bool -+ property region : String? -+ property related_videos : Bool -+ property speed : Float32 | Float64 -+ property video_end : Float64 | Int32 -+ property video_loop : Bool -+ property extend_desc : Bool -+ property video_start : Float64 | Int32 -+ property volume : Int32 -+ property vr_mode : Bool -+ property save_player_pos : Bool -+end -+ -+def process_video_params(query, preferences) -+ annotations = query["iv_load_policy"]?.try &.to_i? -+ autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ comments = query["comments"]?.try &.split(",").map(&.downcase) -+ continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ player_style = query["player_style"]? -+ preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) -+ quality = query["quality"]? -+ quality_dash = query["quality_dash"]? -+ region = query["region"]? -+ related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ speed = query["speed"]?.try &.rchop("x").to_f? -+ video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ volume = query["volume"]?.try &.to_i? -+ vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } -+ -+ if preferences -+ # region ||= preferences.region -+ annotations ||= preferences.annotations.to_unsafe -+ autoplay ||= preferences.autoplay.to_unsafe -+ comments ||= preferences.comments -+ continue ||= preferences.continue.to_unsafe -+ continue_autoplay ||= preferences.continue_autoplay.to_unsafe -+ listen ||= preferences.listen.to_unsafe -+ local ||= preferences.local.to_unsafe -+ player_style ||= preferences.player_style -+ preferred_captions ||= preferences.captions -+ quality ||= preferences.quality -+ quality_dash ||= preferences.quality_dash -+ related_videos ||= preferences.related_videos.to_unsafe -+ speed ||= preferences.speed -+ video_loop ||= preferences.video_loop.to_unsafe -+ extend_desc ||= preferences.extend_desc.to_unsafe -+ volume ||= preferences.volume -+ vr_mode ||= preferences.vr_mode.to_unsafe -+ save_player_pos ||= preferences.save_player_pos.to_unsafe -+ end -+ -+ annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe -+ autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe -+ comments ||= CONFIG.default_user_preferences.comments -+ continue ||= CONFIG.default_user_preferences.continue.to_unsafe -+ continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe -+ listen ||= CONFIG.default_user_preferences.listen.to_unsafe -+ local ||= CONFIG.default_user_preferences.local.to_unsafe -+ player_style ||= CONFIG.default_user_preferences.player_style -+ preferred_captions ||= CONFIG.default_user_preferences.captions -+ quality ||= CONFIG.default_user_preferences.quality -+ quality_dash ||= CONFIG.default_user_preferences.quality_dash -+ related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe -+ speed ||= CONFIG.default_user_preferences.speed -+ video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe -+ extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe -+ volume ||= CONFIG.default_user_preferences.volume -+ vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe -+ save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe -+ -+ annotations = annotations == 1 -+ autoplay = autoplay == 1 -+ continue = continue == 1 -+ continue_autoplay = continue_autoplay == 1 -+ listen = listen == 1 -+ local = local == 1 -+ related_videos = related_videos == 1 -+ video_loop = video_loop == 1 -+ extend_desc = extend_desc == 1 -+ vr_mode = vr_mode == 1 -+ save_player_pos = save_player_pos == 1 -+ -+ if CONFIG.disabled?("dash") && quality == "dash" -+ quality = "high" -+ end -+ -+ if CONFIG.disabled?("local") && local -+ local = false -+ end -+ -+ if start = query["t"]? || query["time_continue"]? || query["start"]? -+ video_start = decode_time(start) -+ end -+ video_start ||= 0 -+ -+ if query["end"]? -+ video_end = decode_time(query["end"]) -+ end -+ video_end ||= -1 -+ -+ raw = query["raw"]?.try &.to_i? -+ raw ||= 0 -+ raw = raw == 1 -+ -+ controls = query["controls"]?.try &.to_i? -+ controls ||= 1 -+ controls = controls >= 1 -+ -+ params = VideoPreferences.new({ -+ annotations: annotations, -+ autoplay: autoplay, -+ comments: comments, -+ continue: continue, -+ continue_autoplay: continue_autoplay, -+ controls: controls, -+ listen: listen, -+ local: local, -+ player_style: player_style, -+ preferred_captions: preferred_captions, -+ quality: quality, -+ quality_dash: quality_dash, -+ raw: raw, -+ region: region, -+ related_videos: related_videos, -+ speed: speed, -+ video_end: video_end, -+ video_loop: video_loop, -+ extend_desc: extend_desc, -+ video_start: video_start, -+ volume: volume, -+ vr_mode: vr_mode, -+ save_player_pos: save_player_pos, -+ }) -+ -+ return params -+end - -From cd03fa06aee7596446c0bbe9b77c3832419e8146 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Mon, 23 May 2022 22:37:58 +0200 -Subject: [PATCH 04/16] videos: move 'Caption' and associated global/functions - to a separate file - ---- - spec/spec_helper.cr | 1 + - src/invidious/frontend/watch_page.cr | 2 +- - src/invidious/videos.cr | 168 +---------------------- - src/invidious/videos/caption.cr | 168 +++++++++++++++++++++++ - src/invidious/views/user/preferences.ecr | 2 +- - 5 files changed, 177 insertions(+), 164 deletions(-) - create mode 100644 src/invidious/videos/caption.cr - -diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr -index 6c492e2f8..f8bfa7187 100644 ---- a/spec/spec_helper.cr -+++ b/spec/spec_helper.cr -@@ -5,6 +5,7 @@ require "protodec/utils" - require "yaml" - require "../src/invidious/helpers/*" - require "../src/invidious/channels/*" -+require "../src/invidious/videos/caption" - require "../src/invidious/videos" - require "../src/invidious/comments" - require "../src/invidious/playlists" -diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr -index 9212eb2fd..a9b00860a 100644 ---- a/src/invidious/frontend/watch_page.cr -+++ b/src/invidious/frontend/watch_page.cr -@@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage - getter full_videos : Array(Hash(String, JSON::Any)) - getter video_streams : Array(Hash(String, JSON::Any)) - getter audio_streams : Array(Hash(String, JSON::Any)) -- getter captions : Array(Caption) -+ getter captions : Array(Invidious::Videos::Caption) - - def initialize( - @full_videos, -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index f40126669..45a44c29c 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -1,136 +1,3 @@ --CAPTION_LANGUAGES = { -- "", -- "English", -- "English (auto-generated)", -- "English (United Kingdom)", -- "English (United States)", -- "Afrikaans", -- "Albanian", -- "Amharic", -- "Arabic", -- "Armenian", -- "Azerbaijani", -- "Bangla", -- "Basque", -- "Belarusian", -- "Bosnian", -- "Bulgarian", -- "Burmese", -- "Cantonese (Hong Kong)", -- "Catalan", -- "Cebuano", -- "Chinese", -- "Chinese (China)", -- "Chinese (Hong Kong)", -- "Chinese (Simplified)", -- "Chinese (Taiwan)", -- "Chinese (Traditional)", -- "Corsican", -- "Croatian", -- "Czech", -- "Danish", -- "Dutch", -- "Dutch (auto-generated)", -- "Esperanto", -- "Estonian", -- "Filipino", -- "Finnish", -- "French", -- "French (auto-generated)", -- "Galician", -- "Georgian", -- "German", -- "German (auto-generated)", -- "Greek", -- "Gujarati", -- "Haitian Creole", -- "Hausa", -- "Hawaiian", -- "Hebrew", -- "Hindi", -- "Hmong", -- "Hungarian", -- "Icelandic", -- "Igbo", -- "Indonesian", -- "Indonesian (auto-generated)", -- "Interlingue", -- "Irish", -- "Italian", -- "Italian (auto-generated)", -- "Japanese", -- "Japanese (auto-generated)", -- "Javanese", -- "Kannada", -- "Kazakh", -- "Khmer", -- "Korean", -- "Korean (auto-generated)", -- "Kurdish", -- "Kyrgyz", -- "Lao", -- "Latin", -- "Latvian", -- "Lithuanian", -- "Luxembourgish", -- "Macedonian", -- "Malagasy", -- "Malay", -- "Malayalam", -- "Maltese", -- "Maori", -- "Marathi", -- "Mongolian", -- "Nepali", -- "Norwegian Bokmål", -- "Nyanja", -- "Pashto", -- "Persian", -- "Polish", -- "Portuguese", -- "Portuguese (auto-generated)", -- "Portuguese (Brazil)", -- "Punjabi", -- "Romanian", -- "Russian", -- "Russian (auto-generated)", -- "Samoan", -- "Scottish Gaelic", -- "Serbian", -- "Shona", -- "Sindhi", -- "Sinhala", -- "Slovak", -- "Slovenian", -- "Somali", -- "Southern Sotho", -- "Spanish", -- "Spanish (auto-generated)", -- "Spanish (Latin America)", -- "Spanish (Mexico)", -- "Spanish (Spain)", -- "Sundanese", -- "Swahili", -- "Swedish", -- "Tajik", -- "Tamil", -- "Telugu", -- "Thai", -- "Turkish", -- "Turkish (auto-generated)", -- "Ukrainian", -- "Urdu", -- "Uzbek", -- "Vietnamese", -- "Vietnamese (auto-generated)", -- "Welsh", -- "Western Frisian", -- "Xhosa", -- "Yiddish", -- "Yoruba", -- "Zulu", --} -- - struct Video - include DB::Serializable - -@@ -141,7 +8,7 @@ struct Video - property updated : Time - - @[DB::Field(ignore: true)] -- property captions : Array(Caption)? -+ @captions = [] of Invidious::Videos::Caption - - @[DB::Field(ignore: true)] - property adaptive_fmts : Array(Hash(String, JSON::Any))? -@@ -595,20 +462,12 @@ struct Video - keywords.includes? "YouTube Red" - end - -- def captions : Array(Caption) -- return @captions.as(Array(Caption)) if @captions -- captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| -- name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] -- language_code = caption["languageCode"].to_s -- base_url = caption["baseUrl"].to_s -- -- caption = Caption.new(name.to_s, language_code, base_url) -- caption.name = caption.name.split(" - ")[0] -- caption -+ def captions : Array(Invidious::Videos::Caption) -+ if @captions.empty? && @info.has_key?("captions") -+ @captions = Invidious::Videos::Caption.from_yt_json(info["captions"]) - end -- captions ||= [] of Caption -- @captions = captions -- return @captions.as(Array(Caption)) -+ -+ return @captions - end - - def description -@@ -672,21 +531,6 @@ struct Video - end - end - --struct Caption -- property name -- property language_code -- property base_url -- -- getter name : String -- getter language_code : String -- getter base_url : String -- -- setter name -- -- def initialize(@name, @language_code, @base_url) -- end --end -- - class VideoRedirect < Exception - property video_id : String - -diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr -new file mode 100644 -index 000000000..4642c1a77 ---- /dev/null -+++ b/src/invidious/videos/caption.cr -@@ -0,0 +1,168 @@ -+require "json" -+ -+module Invidious::Videos -+ struct Caption -+ property name : String -+ property language_code : String -+ property base_url : String -+ -+ def initialize(@name, @language_code, @base_url) -+ end -+ -+ # Parse the JSON structure from Youtube -+ def self.from_yt_json(container : JSON::Any) : Array(Caption) -+ caption_tracks = container -+ .dig?("playerCaptionsTracklistRenderer", "captionTracks") -+ .try &.as_a -+ -+ captions_list = [] of Caption -+ return captions_list if caption_tracks.nil? -+ -+ caption_tracks.each do |caption| -+ name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] -+ name = name.to_s.split(" - ")[0] -+ -+ language_code = caption["languageCode"].to_s -+ base_url = caption["baseUrl"].to_s -+ -+ captions_list << Caption.new(name, language_code, base_url) -+ end -+ -+ return captions_list -+ end -+ -+ # List of all caption languages available on Youtube. -+ LANGUAGES = { -+ "", -+ "English", -+ "English (auto-generated)", -+ "English (United Kingdom)", -+ "English (United States)", -+ "Afrikaans", -+ "Albanian", -+ "Amharic", -+ "Arabic", -+ "Armenian", -+ "Azerbaijani", -+ "Bangla", -+ "Basque", -+ "Belarusian", -+ "Bosnian", -+ "Bulgarian", -+ "Burmese", -+ "Cantonese (Hong Kong)", -+ "Catalan", -+ "Cebuano", -+ "Chinese", -+ "Chinese (China)", -+ "Chinese (Hong Kong)", -+ "Chinese (Simplified)", -+ "Chinese (Taiwan)", -+ "Chinese (Traditional)", -+ "Corsican", -+ "Croatian", -+ "Czech", -+ "Danish", -+ "Dutch", -+ "Dutch (auto-generated)", -+ "Esperanto", -+ "Estonian", -+ "Filipino", -+ "Finnish", -+ "French", -+ "French (auto-generated)", -+ "Galician", -+ "Georgian", -+ "German", -+ "German (auto-generated)", -+ "Greek", -+ "Gujarati", -+ "Haitian Creole", -+ "Hausa", -+ "Hawaiian", -+ "Hebrew", -+ "Hindi", -+ "Hmong", -+ "Hungarian", -+ "Icelandic", -+ "Igbo", -+ "Indonesian", -+ "Indonesian (auto-generated)", -+ "Interlingue", -+ "Irish", -+ "Italian", -+ "Italian (auto-generated)", -+ "Japanese", -+ "Japanese (auto-generated)", -+ "Javanese", -+ "Kannada", -+ "Kazakh", -+ "Khmer", -+ "Korean", -+ "Korean (auto-generated)", -+ "Kurdish", -+ "Kyrgyz", -+ "Lao", -+ "Latin", -+ "Latvian", -+ "Lithuanian", -+ "Luxembourgish", -+ "Macedonian", -+ "Malagasy", -+ "Malay", -+ "Malayalam", -+ "Maltese", -+ "Maori", -+ "Marathi", -+ "Mongolian", -+ "Nepali", -+ "Norwegian Bokmål", -+ "Nyanja", -+ "Pashto", -+ "Persian", -+ "Polish", -+ "Portuguese", -+ "Portuguese (auto-generated)", -+ "Portuguese (Brazil)", -+ "Punjabi", -+ "Romanian", -+ "Russian", -+ "Russian (auto-generated)", -+ "Samoan", -+ "Scottish Gaelic", -+ "Serbian", -+ "Shona", -+ "Sindhi", -+ "Sinhala", -+ "Slovak", -+ "Slovenian", -+ "Somali", -+ "Southern Sotho", -+ "Spanish", -+ "Spanish (auto-generated)", -+ "Spanish (Latin America)", -+ "Spanish (Mexico)", -+ "Spanish (Spain)", -+ "Sundanese", -+ "Swahili", -+ "Swedish", -+ "Tajik", -+ "Tamil", -+ "Telugu", -+ "Thai", -+ "Turkish", -+ "Turkish (auto-generated)", -+ "Ukrainian", -+ "Urdu", -+ "Uzbek", -+ "Vietnamese", -+ "Vietnamese (auto-generated)", -+ "Welsh", -+ "Western Frisian", -+ "Xhosa", -+ "Yiddish", -+ "Yoruba", -+ "Zulu", -+ } -+ end -+end -diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr -index dbb5e9db5..d841982ce 100644 ---- a/src/invidious/views/user/preferences.ecr -+++ b/src/invidious/views/user/preferences.ecr -@@ -89,7 +89,7 @@ - - <% preferences.captions.each_with_index do |caption, index| %> - - -From 6aaea7fafa72aecc224089a6b52cad6e6d6daa0f Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Sun, 24 Jul 2022 23:30:58 +0200 -Subject: [PATCH 05/16] Videos: parse data during first fetching - -There will be less data to be stores in the DB cache ---- - .../videos/scheduled_live_extract_spec.cr | 2 - - src/invidious/videos.cr | 300 +++++++++++------- - 2 files changed, 182 insertions(+), 120 deletions(-) - -diff --git a/spec/invidious/videos/scheduled_live_extract_spec.cr b/spec/invidious/videos/scheduled_live_extract_spec.cr -index 6e531bbd9..b80aec0cc 100644 ---- a/spec/invidious/videos/scheduled_live_extract_spec.cr -+++ b/spec/invidious/videos/scheduled_live_extract_spec.cr -@@ -22,7 +22,6 @@ Spectator.describe Invidious::Hashtag do - expect(info["likes"].as_i).to eq(2_283) - - expect(info["genre"].as_s).to eq("Gaming") -- expect(info["genreUrl"].raw).to be_nil - expect(info["genreUcid"].as_s).to be_empty - expect(info["license"].as_s).to be_empty - -@@ -81,7 +80,6 @@ Spectator.describe Invidious::Hashtag do - expect(info["likes"].as_i).to eq(22) - - expect(info["genre"].as_s).to eq("Entertainment") -- expect(info["genreUrl"].raw).to be_nil - expect(info["genreUcid"].as_s).to be_empty - expect(info["license"].as_s).to be_empty - -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index 45a44c29c..6211bcd70 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -1,3 +1,9 @@ -+enum VideoType -+ Video -+ Livestream -+ Scheduled -+end -+ - struct Video - include DB::Serializable - -@@ -27,7 +33,7 @@ struct Video - - def to_json(locale : String?, json : JSON::Builder) - json.object do -- json.field "type", "video" -+ json.field "type", self.video_type - - json.field "title", self.title - json.field "videoId", self.id -@@ -253,61 +259,22 @@ struct Video - to_json(nil, json) - end - -- def title -- info["videoDetails"]["title"]?.try &.as_s || "" -- end -- -- def ucid -- info["videoDetails"]["channelId"]?.try &.as_s || "" -- end -- -- def author -- info["videoDetails"]["author"]?.try &.as_s || "" -- end -- -- def length_seconds : Int32 -- info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i || -- info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0 -- end -- -- def views : Int64 -- info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64 -- end -- -- def likes : Int64 -- info["likes"]?.try &.as_i64 || 0_i64 -- end -- -- def dislikes : Int64 -- info["dislikes"]?.try &.as_i64 || 0_i64 -+ def video_type : VideoType -+ video_type = info["videoType"]?.try &.as_s || "video" -+ return VideoType.parse?(video_type) || VideoType::Video - end - - def published : Time -- info -- .dig?("microformat", "playerMicroformatRenderer", "publishDate") -+ return info["published"]? - .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc - end - - def published=(other : Time) -- info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) -- end -- -- def allow_ratings -- r = info["videoDetails"]["allowRatings"]?.try &.as_bool -- r.nil? ? false : r -+ info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d")) - end - - def live_now -- info["microformat"]?.try &.["playerMicroformatRenderer"]? -- .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false -- end -- -- def is_listed -- info["videoDetails"]["isCrawlable"]?.try &.as_bool || false -- end -- -- def is_upcoming -- info["videoDetails"]["isUpcoming"]?.try &.as_bool || false -+ return (self.video_type == VideoType::Livestream) - end - - def premiere_timestamp : Time? -@@ -316,31 +283,11 @@ struct Video - .try { |t| Time.parse_rfc3339(t.as_s) } - end - -- def keywords -- info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String -- end -- - def related_videos - info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String) - end - -- def allowed_regions -- info -- .dig?("microformat", "playerMicroformatRenderer", "availableCountries") -- .try &.as_a.map &.as_s || [] of String -- end -- -- def author_thumbnail : String -- info["authorThumbnail"]?.try &.as_s || "" -- end -- -- def author_verified : Bool -- info["authorVerified"]?.try &.as_bool || false -- end -- -- def sub_count_text : String -- info["subCountText"]?.try &.as_s || "-" -- end -+ # Methods for parsing streaming data - - def fmt_stream - return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream -@@ -391,6 +338,8 @@ struct Video - adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio") - end - -+ # Misc. methods -+ - def storyboards - storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec") - .try &.as_s.split("|") -@@ -454,8 +403,7 @@ struct Video - end - - def paid -- reason = info.dig?("playabilityStatus", "reason").try &.as_s || "" -- return reason.includes? "requires payment" -+ return (self.reason || "").includes? "requires payment" - end - - def premium -@@ -470,29 +418,6 @@ struct Video - return @captions - end - -- def description -- description = info -- .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText") -- .try &.as_s || "" -- end -- -- # TODO -- def description=(value : String) -- @description = value -- end -- -- def description_html -- info["descriptionHtml"]?.try &.as_s || "

" -- end -- -- def description_html=(value : String) -- info["descriptionHtml"] = JSON::Any.new(value) -- end -- -- def short_description -- info["shortDescription"]?.try &.as_s? || "" -- end -- - def hls_manifest_url : String? - info.dig?("streamingData", "hlsManifestUrl").try &.as_s - end -@@ -501,25 +426,12 @@ struct Video - info.dig?("streamingData", "dashManifestUrl").try &.as_s - end - -- def genre : String -- info["genre"]?.try &.as_s || "" -- end -- - def genre_url : String? - info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil - end - -- def license : String? -- info["license"]?.try &.as_s -- end -- -- def is_family_friendly : Bool -- info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false -- end -- - def is_vr : Bool? -- projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s -- return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type -+ return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type - end - - def projection_type : String? -@@ -529,6 +441,91 @@ struct Video - def reason : String? - info["reason"]?.try &.as_s - end -+ -+ # Macros defining getters/setters for various types of data -+ -+ private macro getset_string(name) -+ # Return {{name.stringify}} from `info` -+ def {{name.id.underscore}} : String -+ return info[{{name.stringify}}]?.try &.as_s || "" -+ end -+ -+ # Update {{name.stringify}} into `info` -+ def {{name.id.underscore}}=(value : String) -+ info[{{name.stringify}}] = JSON::Any.new(value) -+ end -+ -+ {% if flag?(:debug_macros) %} {{debug}} {% end %} -+ end -+ -+ private macro getset_string_array(name) -+ # Return {{name.stringify}} from `info` -+ def {{name.id.underscore}} : Array(String) -+ return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String -+ end -+ -+ # Update {{name.stringify}} into `info` -+ def {{name.id.underscore}}=(value : Array(String)) -+ info[{{name.stringify}}] = JSON::Any.new(value) -+ end -+ -+ {% if flag?(:debug_macros) %} {{debug}} {% end %} -+ end -+ -+ {% for op, type in {i32: Int32, i64: Int64} %} -+ private macro getset_{{op}}(name) -+ def \{{name.id.underscore}} : {{type}} -+ return info[\{{name.stringify}}]?.try &.as_i.to_{{op}} || 0_{{op}} -+ end -+ -+ def \{{name.id.underscore}}=(value : Int) -+ info[\{{name.stringify}}] = JSON::Any.new(value.to_i64) -+ end -+ -+ \{% if flag?(:debug_macros) %} \{{debug}} \{% end %} -+ end -+ {% end %} -+ -+ private macro getset_bool(name) -+ # Return {{name.stringify}} from `info` -+ def {{name.id.underscore}} : Bool -+ return info[{{name.stringify}}]?.try &.as_bool || false -+ end -+ -+ # Update {{name.stringify}} into `info` -+ def {{name.id.underscore}}=(value : Bool) -+ info[{{name.stringify}}] = JSON::Any.new(value) -+ end -+ -+ {% if flag?(:debug_macros) %} {{debug}} {% end %} -+ end -+ -+ # Method definitions, using the macros above -+ -+ getset_string author -+ getset_string authorThumbnail -+ getset_string description -+ getset_string descriptionHtml -+ getset_string genre -+ getset_string genreUcid -+ getset_string license -+ getset_string shortDescription -+ getset_string subCountText -+ getset_string title -+ getset_string ucid -+ -+ getset_string_array allowedRegions -+ getset_string_array keywords -+ -+ getset_i32 lengthSeconds -+ getset_i64 likes -+ getset_i64 views -+ -+ getset_bool allowRatings -+ getset_bool authorVerified -+ getset_bool isFamilyFriendly -+ getset_bool isListed -+ getset_bool isUpcoming - end - - class VideoRedirect < Exception -@@ -684,6 +681,42 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer - raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer - -+ video_details = player_response.dig?("videoDetails") -+ microformat = player_response.dig?("microformat", "playerMicroformatRenderer") -+ -+ raise BrokenTubeException.new("videoDetails") if !video_details -+ raise BrokenTubeException.new("microformat") if !microformat -+ -+ # Basic video infos -+ -+ title = video_details["title"]?.try &.as_s -+ views = video_details["viewCount"]?.try &.as_s.to_i64 -+ -+ length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) -+ .try &.as_s.to_i64 -+ -+ published = microformat["publishDate"]? -+ .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc -+ -+ premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") -+ .try { |t| Time.parse_rfc3339(t.as_s) } -+ -+ live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") -+ .try &.as_bool || false -+ -+ # Extra video infos -+ -+ allowed_regions = microformat["availableCountries"]? -+ .try &.as_a.map &.as_s || [] of String -+ -+ allow_ratings = video_details["allowRatings"]?.try &.as_bool -+ family_friendly = microformat["isFamilySafe"].try &.as_bool -+ is_listed = video_details["isCrawlable"]?.try &.as_bool -+ is_upcoming = video_details["isUpcoming"]?.try &.as_bool -+ -+ keywords = video_details["keywords"]? -+ .try &.as_a.map &.as_s || [] of String -+ - # Related videos - - LOGGER.debug("extract_video_info: parsing related videos...") -@@ -738,6 +771,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - - # Description - -+ description = microformat.dig?("description", "simpleText").try &.as_s || "" - short_description = player_response.dig?("videoDetails", "shortDescription") - - description_html = video_secondary_renderer.try &.dig?("description", "runs") -@@ -749,7 +783,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") - .try &.as_a - -- genre = player_response.dig?("microformat", "playerMicroformatRenderer", "category") -+ genre = microformat["category"]? - genre_ucid = nil - license = nil - -@@ -771,6 +805,9 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - - # Author infos - -+ author = video_details["author"]?.try &.as_s -+ ucid = video_details["channelId"]?.try &.as_s -+ - if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") - author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") - author_verified = has_verified_badge?(author_info["badges"]?) -@@ -782,19 +819,46 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - - # Return data - -+ if live_now -+ video_type = VideoType::Livestream -+ elsif premiere_timestamp.not_nil! -+ video_type = VideoType::Scheduled -+ published = premiere_timestamp || Time.utc -+ else -+ video_type = VideoType::Video -+ end -+ - params = { -- "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), -- "relatedVideos" => JSON::Any.new(related), -- "likes" => JSON::Any.new(likes || 0_i64), -- "dislikes" => JSON::Any.new(0_i64), -+ "videoType" => JSON::Any.new(video_type.to_s), -+ # Basic video infos -+ "title" => JSON::Any.new(title || ""), -+ "views" => JSON::Any.new(views || 0_i64), -+ "likes" => JSON::Any.new(likes || 0_i64), -+ "lengthSeconds" => JSON::Any.new(length_txt || 0_i64), -+ "published" => JSON::Any.new(published.to_rfc3339), -+ # Extra video infos -+ "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }), -+ "allowRatings" => JSON::Any.new(allow_ratings || false), -+ "isFamilyFriendly" => JSON::Any.new(family_friendly || false), -+ "isListed" => JSON::Any.new(is_listed || false), -+ "isUpcoming" => JSON::Any.new(is_upcoming || false), -+ "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), -+ # Related videos -+ "relatedVideos" => JSON::Any.new(related), -+ # Description -+ "description" => JSON::Any.new(description || ""), - "descriptionHtml" => JSON::Any.new(description_html || "

"), -- "genre" => JSON::Any.new(genre.try &.as_s || ""), -- "genreUrl" => JSON::Any.new(nil), -- "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), -- "license" => JSON::Any.new(license.try &.as_s || ""), -- "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), -- "authorVerified" => JSON::Any.new(author_verified), -- "subCountText" => JSON::Any.new(subs_text || "-"), -+ "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), -+ # Video metadata -+ "genre" => JSON::Any.new(genre.try &.as_s || ""), -+ "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), -+ "license" => JSON::Any.new(license.try &.as_s || ""), -+ # Author infos -+ "author" => JSON::Any.new(author || ""), -+ "ucid" => JSON::Any.new(ucid || ""), -+ "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), -+ "authorVerified" => JSON::Any.new(author_verified), -+ "subCountText" => JSON::Any.new(subs_text || "-"), - } - - return params - -From 907ddfa06a08ace8e7f6806e4e8567a18903a790 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Tue, 2 Aug 2022 18:45:43 +0200 -Subject: [PATCH 06/16] spec: Add tests for recent changes - ---- - .../videos/scheduled_live_extract_spec.cr | 185 ++++++++++++++---- - 1 file changed, 151 insertions(+), 34 deletions(-) - -diff --git a/spec/invidious/videos/scheduled_live_extract_spec.cr b/spec/invidious/videos/scheduled_live_extract_spec.cr -index b80aec0cc..ff5aacd52 100644 ---- a/spec/invidious/videos/scheduled_live_extract_spec.cr -+++ b/spec/invidious/videos/scheduled_live_extract_spec.cr -@@ -1,6 +1,6 @@ - require "../../parsers_helper.cr" - --Spectator.describe Invidious::Hashtag do -+Spectator.describe "parse_video_info" do - it "parses scheduled livestreams data (test 1)" do - # Enable mock - _player = load_mock("video/scheduled_live_nintendo.player") -@@ -12,25 +12,50 @@ Spectator.describe Invidious::Hashtag do - # Some basic verifications - expect(typeof(info)).to eq(Hash(String, JSON::Any)) - -- expect(info["shortDescription"].as_s).to eq( -- "Tune in on 6/22 at 7 a.m. PT for a livestreamed Xenoblade Chronicles 3 Direct presentation featuring roughly 20 minutes of information about the upcoming RPG adventure for Nintendo Switch." -- ) -- expect(info["descriptionHtml"].as_s).to eq( -- "Tune in on 6/22 at 7 a.m. PT for a livestreamed Xenoblade Chronicles 3 Direct presentation featuring roughly 20 minutes of information about the upcoming RPG adventure for Nintendo Switch." -- ) -+ expect(info["videoType"].as_s).to eq("Scheduled") -+ -+ # Basic video infos - -+ expect(info["title"].as_s).to eq("Xenoblade Chronicles 3 Nintendo Direct") -+ expect(info["views"].as_i).to eq(160) - expect(info["likes"].as_i).to eq(2_283) -+ expect(info["lengthSeconds"].as_i).to eq(0_i64) -+ expect(info["published"].as_s).to eq("2022-06-22T14:00:00Z") # Unix 1655906400 - -- expect(info["genre"].as_s).to eq("Gaming") -- expect(info["genreUcid"].as_s).to be_empty -- expect(info["license"].as_s).to be_empty -+ # Extra video infos - -- expect(info["authorThumbnail"].as_s).to eq( -- "https://yt3.ggpht.com/ytc/AKedOLTt4vtjREUUNdHlyu9c4gtJjG90M9jQheRlLKy44A=s48-c-k-c0x00ffffff-no-rj" -+ expect(info["allowedRegions"].as_a).to_not be_empty -+ expect(info["allowedRegions"].as_a.size).to eq(249) -+ -+ expect(info["allowedRegions"].as_a).to contain( -+ "AD", "BA", "BB", "BW", "BY", "EG", "GG", "HN", "NP", "NR", "TR", -+ "TT", "TV", "TW", "TZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", -+ "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW" - ) - -- expect(info["authorVerified"].as_bool).to be_true -- expect(info["subCountText"].as_s).to eq("8.5M") -+ expect(info["keywords"].as_a).to_not be_empty -+ expect(info["keywords"].as_a.size).to eq(11) -+ -+ expect(info["keywords"].as_a).to contain_exactly( -+ "nintendo", -+ "game", -+ "gameplay", -+ "fun", -+ "video game", -+ "action", -+ "adventure", -+ "rpg", -+ "play", -+ "switch", -+ "nintendo switch" -+ ).in_any_order -+ -+ expect(info["allowRatings"].as_bool).to be_true -+ expect(info["isFamilyFriendly"].as_bool).to be_true -+ expect(info["isListed"].as_bool).to be_true -+ expect(info["isUpcoming"].as_bool).to be_true -+ -+ # Related videos - - expect(info["relatedVideos"].as_a.size).to eq(20) - -@@ -49,6 +74,32 @@ Spectator.describe Invidious::Hashtag do - expect(info["relatedVideos"][16]["view_count"].as_s).to eq("53510") - expect(info["relatedVideos"][16]["short_view_count"].as_s).to eq("53K") - expect(info["relatedVideos"][16]["author_verified"].as_s).to eq("true") -+ -+ # Description -+ -+ description = "Tune in on 6/22 at 7 a.m. PT for a livestreamed Xenoblade Chronicles 3 Direct presentation featuring roughly 20 minutes of information about the upcoming RPG adventure for Nintendo Switch." -+ -+ expect(info["description"].as_s).to eq(description) -+ expect(info["shortDescription"].as_s).to eq(description) -+ expect(info["descriptionHtml"].as_s).to eq(description) -+ -+ # Video metadata -+ -+ expect(info["genre"].as_s).to eq("Gaming") -+ expect(info["genreUcid"].as_s).to be_empty -+ expect(info["license"].as_s).to be_empty -+ -+ # Author infos -+ -+ expect(info["author"].as_s).to eq("Nintendo") -+ expect(info["ucid"].as_s).to eq("UCGIY_O-8vW4rfX98KlMkvRg") -+ -+ expect(info["authorThumbnail"].as_s).to eq( -+ "https://yt3.ggpht.com/ytc/AKedOLTt4vtjREUUNdHlyu9c4gtJjG90M9jQheRlLKy44A=s48-c-k-c0x00ffffff-no-rj" -+ ) -+ -+ expect(info["authorVerified"].as_bool).to be_true -+ expect(info["subCountText"].as_s).to eq("8.5M") - end - - it "parses scheduled livestreams data (test 2)" do -@@ -62,33 +113,63 @@ Spectator.describe Invidious::Hashtag do - # Some basic verifications - expect(typeof(info)).to eq(Hash(String, JSON::Any)) - -- expect(info["shortDescription"].as_s).to start_with( -- <<-TXT -- PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick. -- -- Join the channel to get exclusive access to perks: https://bit.ly/3Q9rSQL -- TXT -- ) -- expect(info["descriptionHtml"].as_s).to start_with( -- <<-TXT -- PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick. -+ expect(info["videoType"].as_s).to eq("Scheduled") - -- Join the channel to get exclusive access to perks: bit.ly/3Q9rSQL -- TXT -- ) -+ # Basic video infos - -+ expect(info["title"].as_s).to eq("The Truth About Greenpeace w/ Dr. Patrick Moore | PBD Podcast | Ep. 171") -+ expect(info["views"].as_i).to eq(24) - expect(info["likes"].as_i).to eq(22) -+ expect(info["lengthSeconds"].as_i).to eq(0_i64) -+ expect(info["published"].as_s).to eq("2022-07-14T13:00:00Z") # Unix 1657803600 - -- expect(info["genre"].as_s).to eq("Entertainment") -- expect(info["genreUcid"].as_s).to be_empty -- expect(info["license"].as_s).to be_empty -+ # Extra video infos - -- expect(info["authorThumbnail"].as_s).to eq( -- "https://yt3.ggpht.com/61ArDiQshJrvSXcGLhpFfIO3hlMabe2fksitcf6oGob0Mdr5gztdkXxRljICUodL4iuTSrtxW4A=s48-c-k-c0x00ffffff-no-rj" -+ expect(info["allowedRegions"].as_a).to_not be_empty -+ expect(info["allowedRegions"].as_a.size).to eq(249) -+ -+ expect(info["allowedRegions"].as_a).to contain( -+ "AD", "AR", "BA", "BT", "CZ", "FO", "GL", "IO", "KE", "KH", "LS", -+ "LT", "MP", "NO", "PR", "RO", "SE", "SK", "SS", "SX", "SZ", "ZW" - ) - -- expect(info["authorVerified"].as_bool).to be_false -- expect(info["subCountText"].as_s).to eq("227K") -+ expect(info["keywords"].as_a).to_not be_empty -+ expect(info["keywords"].as_a.size).to eq(25) -+ -+ expect(info["keywords"].as_a).to contain_exactly( -+ "Patrick Bet-David", -+ "Valeutainment", -+ "The BetDavid Podcast", -+ "The BetDavid Show", -+ "Betdavid", -+ "PBD", -+ "BetDavid show", -+ "Betdavid podcast", -+ "podcast betdavid", -+ "podcast patrick", -+ "patrick bet david podcast", -+ "Valuetainment podcast", -+ "Entrepreneurs", -+ "Entrepreneurship", -+ "Entrepreneur Motivation", -+ "Entrepreneur Advice", -+ "Startup Entrepreneurs", -+ "valuetainment", -+ "patrick bet david", -+ "PBD podcast", -+ "Betdavid show", -+ "Betdavid Podcast", -+ "Podcast Betdavid", -+ "Show Betdavid", -+ "PBDPodcast" -+ ).in_any_order -+ -+ expect(info["allowRatings"].as_bool).to be_true -+ expect(info["isFamilyFriendly"].as_bool).to be_true -+ expect(info["isListed"].as_bool).to be_true -+ expect(info["isUpcoming"].as_bool).to be_true -+ -+ # Related videos - - expect(info["relatedVideos"].as_a.size).to eq(20) - -@@ -107,5 +188,41 @@ Spectator.describe Invidious::Hashtag do - expect(info["relatedVideos"][9]["view_count"]).to eq("26432") - expect(info["relatedVideos"][9]["short_view_count"]).to eq("26K") - expect(info["relatedVideos"][9]["author_verified"]).to eq("true") -+ -+ # Description -+ -+ description_start_text = <<-TXT -+ PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick. -+ -+ Join the channel to get exclusive access to perks: https://bit.ly/3Q9rSQL -+ TXT -+ -+ expect(info["description"].as_s).to start_with(description_start_text) -+ expect(info["shortDescription"].as_s).to start_with(description_start_text) -+ -+ expect(info["descriptionHtml"].as_s).to start_with( -+ <<-TXT -+ PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick. -+ -+ Join the channel to get exclusive access to perks: bit.ly/3Q9rSQL -+ TXT -+ ) -+ -+ # Video metadata -+ -+ expect(info["genre"].as_s).to eq("Entertainment") -+ expect(info["genreUcid"].as_s).to be_empty -+ expect(info["license"].as_s).to be_empty -+ -+ # Author infos -+ -+ expect(info["author"].as_s).to eq("PBD Podcast") -+ expect(info["ucid"].as_s).to eq("UCGX7nGXpz-CmO_Arg-cgJ7A") -+ -+ expect(info["authorThumbnail"].as_s).to eq( -+ "https://yt3.ggpht.com/61ArDiQshJrvSXcGLhpFfIO3hlMabe2fksitcf6oGob0Mdr5gztdkXxRljICUodL4iuTSrtxW4A=s48-c-k-c0x00ffffff-no-rj" -+ ) -+ expect(info["authorVerified"].as_bool).to be_false -+ expect(info["subCountText"].as_s).to eq("227K") - end - end - -From 7df0cfcbed6100f16b1ccc2bd93aba42cff2b669 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Wed, 3 Aug 2022 21:31:46 +0200 -Subject: [PATCH 07/16] Videos: fix 'views' parsing for livestreams - ---- - src/invidious/videos.cr | 6 +++++- - 1 file changed, 5 insertions(+), 1 deletion(-) - -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index 6211bcd70..b76da8f9f 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -690,7 +690,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - # Basic video infos - - title = video_details["title"]?.try &.as_s -- views = video_details["viewCount"]?.try &.as_s.to_i64 -+ -+ views = video_primary_renderer -+ .dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") -+ .try &.as_s.to_i64 -+ views ||= video_details["viewCount"]?.try &.as_s.to_i64 - - length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) - .try &.as_s.to_i64 - -From 33150f5de388e34cc776a517db099dea39aab392 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Sun, 21 Aug 2022 17:26:30 +0200 -Subject: [PATCH 08/16] spec: Add test cases for regular videos extraction - ---- - mocks | 2 +- - .../videos/regular_videos_extract_spec.cr | 168 ++++++++++++++++++ - 2 files changed, 169 insertions(+), 1 deletion(-) - create mode 100644 spec/invidious/videos/regular_videos_extract_spec.cr - -diff --git a/mocks b/mocks -index c401dd920..dfd53ea6c 160000 ---- a/mocks -+++ b/mocks -@@ -1 +1 @@ --Subproject commit c401dd9203434b561022242c24b0c200d72284c0 -+Subproject commit dfd53ea6ceb3cbcbbce6004f6ce60b330ad0f9b1 -diff --git a/spec/invidious/videos/regular_videos_extract_spec.cr b/spec/invidious/videos/regular_videos_extract_spec.cr -new file mode 100644 -index 000000000..132b37a35 ---- /dev/null -+++ b/spec/invidious/videos/regular_videos_extract_spec.cr -@@ -0,0 +1,168 @@ -+require "../../parsers_helper.cr" -+ -+Spectator.describe "parse_video_info" do -+ it "parses a regular video" do -+ # Enable mock -+ _player = load_mock("video/regular_mrbeast.player") -+ _next = load_mock("video/regular_mrbeast.next") -+ -+ raw_data = _player.merge!(_next) -+ info = parse_video_info("2isYuQZMbdU", raw_data) -+ -+ # Some basic verifications -+ expect(typeof(info)).to eq(Hash(String, JSON::Any)) -+ -+ expect(info["videoType"].as_s).to eq("Video") -+ -+ # Basic video infos -+ -+ expect(info["title"].as_s).to eq("I Gave My 100,000,000th Subscriber An Island") -+ expect(info["views"].as_i).to eq(32_846_329) -+ expect(info["likes"].as_i).to eq(2_611_650) -+ -+ # For some reason the video length from VideoDetails and the -+ # one from microformat differs by 1s... -+ expect(info["lengthSeconds"].as_i).to be_between(930_i64, 931_i64) -+ -+ expect(info["published"].as_s).to eq("2022-08-04T00:00:00Z") -+ -+ # Extra video infos -+ -+ expect(info["allowedRegions"].as_a).to_not be_empty -+ expect(info["allowedRegions"].as_a.size).to eq(249) -+ -+ expect(info["allowedRegions"].as_a).to contain( -+ "AD", "BA", "BB", "BW", "BY", "EG", "GG", "HN", "NP", "NR", "TR", -+ "TT", "TV", "TW", "TZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", -+ "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW" -+ ) -+ -+ expect(info["keywords"].as_a).to be_empty -+ -+ expect(info["allowRatings"].as_bool).to be_true -+ expect(info["isFamilyFriendly"].as_bool).to be_true -+ expect(info["isListed"].as_bool).to be_true -+ expect(info["isUpcoming"].as_bool).to be_false -+ -+ # Related videos -+ -+ expect(info["relatedVideos"].as_a.size).to eq(19) -+ -+ expect(info["relatedVideos"][0]["id"]).to eq("tVWWp1PqDus") -+ expect(info["relatedVideos"][0]["title"]).to eq("100 Girls Vs 100 Boys For $500,000") -+ expect(info["relatedVideos"][0]["author"]).to eq("MrBeast") -+ expect(info["relatedVideos"][0]["ucid"]).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA") -+ expect(info["relatedVideos"][0]["view_count"]).to eq("49702799") -+ expect(info["relatedVideos"][0]["short_view_count"]).to eq("49M") -+ expect(info["relatedVideos"][0]["author_verified"]).to eq("true") -+ -+ # Description -+ -+ description = "🚀Launch a store on Shopify, I’ll buy from 100 random stores that do ▸ " -+ -+ expect(info["description"].as_s).to start_with(description) -+ expect(info["shortDescription"].as_s).to start_with(description) -+ expect(info["descriptionHtml"].as_s).to start_with(description) -+ -+ # Video metadata -+ -+ expect(info["genre"].as_s).to eq("Entertainment") -+ expect(info["genreUcid"].as_s).to be_empty -+ expect(info["license"].as_s).to be_empty -+ -+ # Author infos -+ -+ expect(info["author"].as_s).to eq("MrBeast") -+ expect(info["ucid"].as_s).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA") -+ -+ expect(info["authorThumbnail"].as_s).to eq( -+ "https://yt3.ggpht.com/ytc/AMLnZu84dsnlYtuUFBMC8imQs0IUcTKA9khWAmUOgQZltw=s48-c-k-c0x00ffffff-no-rj" -+ ) -+ -+ expect(info["authorVerified"].as_bool).to be_true -+ expect(info["subCountText"].as_s).to eq("101M") -+ end -+ -+ it "parses a regular video with no descrition/comments" do -+ # Enable mock -+ _player = load_mock("video/regular_no-description.player") -+ _next = load_mock("video/regular_no-description.next") -+ -+ raw_data = _player.merge!(_next) -+ info = parse_video_info("iuevw6218F0", raw_data) -+ -+ # Some basic verifications -+ expect(typeof(info)).to eq(Hash(String, JSON::Any)) -+ -+ expect(info["videoType"].as_s).to eq("Video") -+ -+ # Basic video infos -+ -+ expect(info["title"].as_s).to eq("Chris Rea - Auberge") -+ expect(info["views"].as_i).to eq(10_356_197) -+ expect(info["likes"].as_i).to eq(0) -+ expect(info["lengthSeconds"].as_i).to eq(283_i64) -+ expect(info["published"].as_s).to eq("2012-05-21T00:00:00Z") -+ -+ # Extra video infos -+ -+ expect(info["allowedRegions"].as_a).to_not be_empty -+ expect(info["allowedRegions"].as_a.size).to eq(249) -+ -+ expect(info["allowedRegions"].as_a).to contain( -+ "AD", "BA", "BB", "BW", "BY", "EG", "GG", "HN", "NP", "NR", "TR", -+ "TT", "TV", "TW", "TZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", -+ "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW" -+ ) -+ -+ expect(info["keywords"].as_a).to_not be_empty -+ expect(info["keywords"].as_a.size).to eq(4) -+ -+ expect(info["keywords"].as_a).to contain_exactly( -+ "Chris", -+ "Rea", -+ "Auberge", -+ "1991" -+ ).in_any_order -+ -+ expect(info["allowRatings"].as_bool).to be_true -+ expect(info["isFamilyFriendly"].as_bool).to be_true -+ expect(info["isListed"].as_bool).to be_true -+ expect(info["isUpcoming"].as_bool).to be_false -+ -+ # Related videos -+ -+ expect(info["relatedVideos"].as_a.size).to eq(19) -+ -+ expect(info["relatedVideos"][0]["id"]).to eq("0bkrY_V0yZg") -+ expect(info["relatedVideos"][0]["title"]).to eq( -+ "Chris Rea Best Songs Collection - Chris Rea Greatest Hits Full Album 2022" -+ ) -+ expect(info["relatedVideos"][0]["author"]).to eq("Rock Ultimate") -+ expect(info["relatedVideos"][0]["ucid"]).to eq("UCekSc2A19di9koUIpj8gxlQ") -+ expect(info["relatedVideos"][0]["view_count"]).to eq("1992412") -+ expect(info["relatedVideos"][0]["short_view_count"]).to eq("1.9M") -+ expect(info["relatedVideos"][0]["author_verified"]).to eq("false") -+ -+ # Description -+ -+ expect(info["description"].as_s).to eq(" ") -+ expect(info["shortDescription"].as_s).to be_empty -+ expect(info["descriptionHtml"].as_s).to eq("

") -+ -+ # Video metadata -+ -+ expect(info["genre"].as_s).to eq("Music") -+ expect(info["genreUcid"].as_s).to be_empty -+ expect(info["license"].as_s).to be_empty -+ -+ # Author infos -+ -+ expect(info["author"].as_s).to eq("ChrisReaOfficial") -+ expect(info["ucid"].as_s).to eq("UC_5q6nWPbD30-y6oiWF_oNA") -+ -+ expect(info["authorThumbnail"].as_s).to be_empty -+ expect(info["authorVerified"].as_bool).to be_false -+ expect(info["subCountText"].as_s).to eq("-") -+ end -+end - -From e23ceb6ae92b685152a284f840fa9aee0f1853ab Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Sun, 21 Aug 2022 17:28:27 +0200 -Subject: [PATCH 09/16] videos: Fix extraction code according to tests - ---- - src/invidious/videos.cr | 35 +++++++++++++++++++---------------- - 1 file changed, 19 insertions(+), 16 deletions(-) - -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index b76da8f9f..a01a18b75 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -666,20 +666,20 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - - raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results - -- primary_results = main_results.dig?("results", "results", "contents") -+ # Primary results are not available on Music videos -+ # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725 -+ if primary_results = main_results.dig?("results", "results", "contents") -+ video_primary_renderer = primary_results -+ .as_a.find(&.["videoPrimaryInfoRenderer"]?) -+ .try &.["videoPrimaryInfoRenderer"] - -- raise BrokenTubeException.new("results") if !primary_results -+ video_secondary_renderer = primary_results -+ .as_a.find(&.["videoSecondaryInfoRenderer"]?) -+ .try &.["videoSecondaryInfoRenderer"] - -- video_primary_renderer = primary_results -- .as_a.find(&.["videoPrimaryInfoRenderer"]?) -- .try &.["videoPrimaryInfoRenderer"] -- -- video_secondary_renderer = primary_results -- .as_a.find(&.["videoSecondaryInfoRenderer"]?) -- .try &.["videoSecondaryInfoRenderer"] -- -- raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer -- raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer -+ raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer -+ raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer -+ end - - video_details = player_response.dig?("videoDetails") - microformat = player_response.dig?("microformat", "playerMicroformatRenderer") -@@ -691,9 +691,12 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - - title = video_details["title"]?.try &.as_s - -+ # We have to try to extract viewCount from videoPrimaryInfoRenderer first, -+ # then from videoDetails, as the latter is "0" for livestreams (we want -+ # to get the amount of viewers watching). - views = video_primary_renderer -- .dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") -- .try &.as_s.to_i64 -+ .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") -+ .try &.as_s.to_i64 - views ||= video_details["viewCount"]?.try &.as_s.to_i64 - - length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) -@@ -825,7 +828,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - - if live_now - video_type = VideoType::Livestream -- elsif premiere_timestamp.not_nil! -+ elsif !premiere_timestamp.nil? - video_type = VideoType::Scheduled - published = premiere_timestamp || Time.utc - else -@@ -861,7 +864,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - "author" => JSON::Any.new(author || ""), - "ucid" => JSON::Any.new(ucid || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), -- "authorVerified" => JSON::Any.new(author_verified), -+ "authorVerified" => JSON::Any.new(author_verified || false), - "subCountText" => JSON::Any.new(subs_text || "-"), - } - - -From ae03ed7bf7faeefa3d8e8bf5b6382f56ef154fe8 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Tue, 23 Aug 2022 17:56:09 +0200 -Subject: [PATCH 10/16] videos: move player/next parsing code to a dedicated - file - ---- - spec/parsers_helper.cr | 1 + - src/invidious/videos.cr | 336 -------------------------------- - src/invidious/videos/parser.cr | 337 +++++++++++++++++++++++++++++++++ - 3 files changed, 338 insertions(+), 336 deletions(-) - create mode 100644 src/invidious/videos/parser.cr - -diff --git a/spec/parsers_helper.cr b/spec/parsers_helper.cr -index e91548751..bf05f9eca 100644 ---- a/spec/parsers_helper.cr -+++ b/spec/parsers_helper.cr -@@ -12,6 +12,7 @@ require "../src/invidious/helpers/logger" - require "../src/invidious/helpers/utils" - - require "../src/invidious/videos" -+require "../src/invidious/videos/*" - require "../src/invidious/comments" - - require "../src/invidious/helpers/serialized_yt_data" -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index a01a18b75..9b19bc2a0 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -535,342 +535,6 @@ class VideoRedirect < Exception - end - end - --# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer". --# The former is preferred as it has more videos in it. The second has --# the same 11 first entries as the compact rendered. --# --# TODO: "compactRadioRenderer" (Mix) and --# TODO: Use a proper struct/class instead of a hacky JSON object --def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? -- return nil if !related["videoId"]? -- -- # The compact renderer has video length in seconds, where the end -- # screen rendered has a full text version ("42:40") -- length = related["lengthInSeconds"]?.try &.as_i.to_s -- length ||= related.dig?("lengthText", "simpleText").try do |box| -- decode_length_seconds(box.as_s).to_s -- end -- -- # Both have "short", so the "long" option shouldn't be required -- channel_info = (related["shortBylineText"]? || related["longBylineText"]?) -- .try &.dig?("runs", 0) -- -- author = channel_info.try &.dig?("text") -- author_verified = has_verified_badge?(related["ownerBadges"]?).to_s -- -- ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) } -- -- # "4,088,033 views", only available on compact renderer -- # and when video is not a livestream -- view_count = related.dig?("viewCountText", "simpleText") -- .try &.as_s.gsub(/\D/, "") -- -- short_view_count = related.try do |r| -- HelperExtractors.get_short_view_count(r).to_s -- end -- -- LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") -- -- # TODO: when refactoring video types, make a struct for related videos -- # or reuse an existing type, if that fits. -- return { -- "id" => related["videoId"], -- "title" => related["title"]["simpleText"], -- "author" => author || JSON::Any.new(""), -- "ucid" => JSON::Any.new(ucid || ""), -- "length_seconds" => JSON::Any.new(length || "0"), -- "view_count" => JSON::Any.new(view_count || "0"), -- "short_view_count" => JSON::Any.new(short_view_count || "0"), -- "author_verified" => JSON::Any.new(author_verified), -- } --end -- --def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil) -- # Init client config for the API -- client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) -- if context_screen == "embed" -- client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed -- end -- -- # Fetch data from the player endpoint -- player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) -- -- playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s -- -- if playability_status != "OK" -- subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") -- reason = subreason.try &.[]?("simpleText").try &.as_s -- reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") -- reason ||= player_response.dig("playabilityStatus", "reason").as_s -- -- # Stop here if video is not a scheduled livestream -- if playability_status != "LIVE_STREAM_OFFLINE" -- return { -- "reason" => JSON::Any.new(reason), -- } -- end -- elsif video_id != player_response.dig("videoDetails", "videoId") -- # YouTube may return a different video player response than expected. -- # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 -- raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)") -- else -- reason = nil -- end -- -- # Don't fetch the next endpoint if the video is unavailable. -- if {"OK", "LIVE_STREAM_OFFLINE"}.any?(playability_status) -- next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) -- player_response = player_response.merge(next_response) -- end -- -- params = parse_video_info(video_id, player_response) -- params["reason"] = JSON::Any.new(reason) if reason -- -- # Fetch the video streams using an Android client in order to get the decrypted URLs and -- # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs: -- # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 -- if reason.nil? -- if context_screen == "embed" -- client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed -- else -- client_config.client_type = YoutubeAPI::ClientType::Android -- end -- android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) -- -- # Sometimes, the video is available from the web client, but not on Android, so check -- # that here, and fallback to the streaming data from the web client if needed. -- # See: https://github.com/iv-org/invidious/issues/2549 -- if video_id != android_player.dig("videoDetails", "videoId") -- # YouTube may return a different video player response than expected. -- # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 -- raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (ANDROID client)") -- elsif android_player["playabilityStatus"]["status"] == "OK" -- params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("") -- else -- params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("") -- end -- end -- -- # TODO: clean that up -- {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| -- params[f] = player_response[f] if player_response[f]? -- end -- -- return params --end -- --def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) -- # Top level elements -- -- main_results = player_response.dig?("contents", "twoColumnWatchNextResults") -- -- raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results -- -- # Primary results are not available on Music videos -- # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725 -- if primary_results = main_results.dig?("results", "results", "contents") -- video_primary_renderer = primary_results -- .as_a.find(&.["videoPrimaryInfoRenderer"]?) -- .try &.["videoPrimaryInfoRenderer"] -- -- video_secondary_renderer = primary_results -- .as_a.find(&.["videoSecondaryInfoRenderer"]?) -- .try &.["videoSecondaryInfoRenderer"] -- -- raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer -- raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer -- end -- -- video_details = player_response.dig?("videoDetails") -- microformat = player_response.dig?("microformat", "playerMicroformatRenderer") -- -- raise BrokenTubeException.new("videoDetails") if !video_details -- raise BrokenTubeException.new("microformat") if !microformat -- -- # Basic video infos -- -- title = video_details["title"]?.try &.as_s -- -- # We have to try to extract viewCount from videoPrimaryInfoRenderer first, -- # then from videoDetails, as the latter is "0" for livestreams (we want -- # to get the amount of viewers watching). -- views = video_primary_renderer -- .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") -- .try &.as_s.to_i64 -- views ||= video_details["viewCount"]?.try &.as_s.to_i64 -- -- length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) -- .try &.as_s.to_i64 -- -- published = microformat["publishDate"]? -- .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc -- -- premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") -- .try { |t| Time.parse_rfc3339(t.as_s) } -- -- live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") -- .try &.as_bool || false -- -- # Extra video infos -- -- allowed_regions = microformat["availableCountries"]? -- .try &.as_a.map &.as_s || [] of String -- -- allow_ratings = video_details["allowRatings"]?.try &.as_bool -- family_friendly = microformat["isFamilySafe"].try &.as_bool -- is_listed = video_details["isCrawlable"]?.try &.as_bool -- is_upcoming = video_details["isUpcoming"]?.try &.as_bool -- -- keywords = video_details["keywords"]? -- .try &.as_a.map &.as_s || [] of String -- -- # Related videos -- -- LOGGER.debug("extract_video_info: parsing related videos...") -- -- related = [] of JSON::Any -- -- # Parse "compactVideoRenderer" items (under secondary results) -- secondary_results = main_results -- .dig?("secondaryResults", "secondaryResults", "results") -- secondary_results.try &.as_a.each do |element| -- if item = element["compactVideoRenderer"]? -- related_video = parse_related_video(item) -- related << JSON::Any.new(related_video) if related_video -- end -- end -- -- # If nothing was found previously, fall back to end screen renderer -- if related.empty? -- # Container for "endScreenVideoRenderer" items -- player_overlays = player_response.dig?( -- "playerOverlays", "playerOverlayRenderer", -- "endScreen", "watchNextEndScreenRenderer", "results" -- ) -- -- player_overlays.try &.as_a.each do |element| -- if item = element["endScreenVideoRenderer"]? -- related_video = parse_related_video(item) -- related << JSON::Any.new(related_video) if related_video -- end -- end -- end -- -- # Likes -- -- toplevel_buttons = video_primary_renderer -- .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") -- -- if toplevel_buttons -- likes_button = toplevel_buttons.as_a -- .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") -- .try &.["toggleButtonRenderer"] -- -- if likes_button -- likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) -- .try &.dig?("accessibility", "accessibilityData", "label") -- likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt -- -- LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") -- LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes -- end -- end -- -- # Description -- -- description = microformat.dig?("description", "simpleText").try &.as_s || "" -- short_description = player_response.dig?("videoDetails", "shortDescription") -- -- description_html = video_secondary_renderer.try &.dig?("description", "runs") -- .try &.as_a.try { |t| content_to_comment_html(t, video_id) } -- -- # Video metadata -- -- metadata = video_secondary_renderer -- .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") -- .try &.as_a -- -- genre = microformat["category"]? -- genre_ucid = nil -- license = nil -- -- metadata.try &.each do |row| -- metadata_title = row.dig?("metadataRowRenderer", "title", "simpleText").try &.as_s -- contents = row.dig?("metadataRowRenderer", "contents", 0) -- -- if metadata_title == "Category" -- contents = contents.try &.dig?("runs", 0) -- -- genre = contents.try &.["text"]? -- genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId") -- elsif metadata_title == "License" -- license = contents.try &.dig?("runs", 0, "text") -- elsif metadata_title == "Licensed to YouTube by" -- license = contents.try &.["simpleText"]? -- end -- end -- -- # Author infos -- -- author = video_details["author"]?.try &.as_s -- ucid = video_details["channelId"]?.try &.as_s -- -- if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") -- author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") -- author_verified = has_verified_badge?(author_info["badges"]?) -- -- subs_text = author_info["subscriberCountText"]? -- .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") } -- .try &.as_s.split(" ", 2)[0] -- end -- -- # Return data -- -- if live_now -- video_type = VideoType::Livestream -- elsif !premiere_timestamp.nil? -- video_type = VideoType::Scheduled -- published = premiere_timestamp || Time.utc -- else -- video_type = VideoType::Video -- end -- -- params = { -- "videoType" => JSON::Any.new(video_type.to_s), -- # Basic video infos -- "title" => JSON::Any.new(title || ""), -- "views" => JSON::Any.new(views || 0_i64), -- "likes" => JSON::Any.new(likes || 0_i64), -- "lengthSeconds" => JSON::Any.new(length_txt || 0_i64), -- "published" => JSON::Any.new(published.to_rfc3339), -- # Extra video infos -- "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }), -- "allowRatings" => JSON::Any.new(allow_ratings || false), -- "isFamilyFriendly" => JSON::Any.new(family_friendly || false), -- "isListed" => JSON::Any.new(is_listed || false), -- "isUpcoming" => JSON::Any.new(is_upcoming || false), -- "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), -- # Related videos -- "relatedVideos" => JSON::Any.new(related), -- # Description -- "description" => JSON::Any.new(description || ""), -- "descriptionHtml" => JSON::Any.new(description_html || "

"), -- "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), -- # Video metadata -- "genre" => JSON::Any.new(genre.try &.as_s || ""), -- "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), -- "license" => JSON::Any.new(license.try &.as_s || ""), -- # Author infos -- "author" => JSON::Any.new(author || ""), -- "ucid" => JSON::Any.new(ucid || ""), -- "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), -- "authorVerified" => JSON::Any.new(author_verified || false), -- "subCountText" => JSON::Any.new(subs_text || "-"), -- } -- -- return params --end -- - def get_video(id, refresh = true, region = nil, force_refresh = false) - if (video = Invidious::Database::Videos.select(id)) && !region - # If record was last updated over 10 minutes ago, or video has since premiered, -diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr -new file mode 100644 -index 000000000..ff5d15de0 ---- /dev/null -+++ b/src/invidious/videos/parser.cr -@@ -0,0 +1,337 @@ -+require "json" -+ -+# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer". -+# The former is preferred as it has more videos in it. The second has -+# the same 11 first entries as the compact rendered. -+# -+# TODO: "compactRadioRenderer" (Mix) and -+# TODO: Use a proper struct/class instead of a hacky JSON object -+def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? -+ return nil if !related["videoId"]? -+ -+ # The compact renderer has video length in seconds, where the end -+ # screen rendered has a full text version ("42:40") -+ length = related["lengthInSeconds"]?.try &.as_i.to_s -+ length ||= related.dig?("lengthText", "simpleText").try do |box| -+ decode_length_seconds(box.as_s).to_s -+ end -+ -+ # Both have "short", so the "long" option shouldn't be required -+ channel_info = (related["shortBylineText"]? || related["longBylineText"]?) -+ .try &.dig?("runs", 0) -+ -+ author = channel_info.try &.dig?("text") -+ author_verified = has_verified_badge?(related["ownerBadges"]?).to_s -+ -+ ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) } -+ -+ # "4,088,033 views", only available on compact renderer -+ # and when video is not a livestream -+ view_count = related.dig?("viewCountText", "simpleText") -+ .try &.as_s.gsub(/\D/, "") -+ -+ short_view_count = related.try do |r| -+ HelperExtractors.get_short_view_count(r).to_s -+ end -+ -+ LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") -+ -+ # TODO: when refactoring video types, make a struct for related videos -+ # or reuse an existing type, if that fits. -+ return { -+ "id" => related["videoId"], -+ "title" => related["title"]["simpleText"], -+ "author" => author || JSON::Any.new(""), -+ "ucid" => JSON::Any.new(ucid || ""), -+ "length_seconds" => JSON::Any.new(length || "0"), -+ "view_count" => JSON::Any.new(view_count || "0"), -+ "short_view_count" => JSON::Any.new(short_view_count || "0"), -+ "author_verified" => JSON::Any.new(author_verified), -+ } -+end -+ -+def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil) -+ # Init client config for the API -+ client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) -+ if context_screen == "embed" -+ client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed -+ end -+ -+ # Fetch data from the player endpoint -+ player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) -+ -+ playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s -+ -+ if playability_status != "OK" -+ subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") -+ reason = subreason.try &.[]?("simpleText").try &.as_s -+ reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") -+ reason ||= player_response.dig("playabilityStatus", "reason").as_s -+ -+ # Stop here if video is not a scheduled livestream -+ if playability_status != "LIVE_STREAM_OFFLINE" -+ return { -+ "reason" => JSON::Any.new(reason), -+ } -+ end -+ elsif video_id != player_response.dig("videoDetails", "videoId") -+ # YouTube may return a different video player response than expected. -+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 -+ raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)") -+ else -+ reason = nil -+ end -+ -+ # Don't fetch the next endpoint if the video is unavailable. -+ if {"OK", "LIVE_STREAM_OFFLINE"}.any?(playability_status) -+ next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) -+ player_response = player_response.merge(next_response) -+ end -+ -+ params = parse_video_info(video_id, player_response) -+ params["reason"] = JSON::Any.new(reason) if reason -+ -+ # Fetch the video streams using an Android client in order to get the decrypted URLs and -+ # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs: -+ # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 -+ if reason.nil? -+ if context_screen == "embed" -+ client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed -+ else -+ client_config.client_type = YoutubeAPI::ClientType::Android -+ end -+ android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) -+ -+ # Sometimes, the video is available from the web client, but not on Android, so check -+ # that here, and fallback to the streaming data from the web client if needed. -+ # See: https://github.com/iv-org/invidious/issues/2549 -+ if video_id != android_player.dig("videoDetails", "videoId") -+ # YouTube may return a different video player response than expected. -+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 -+ raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (ANDROID client)") -+ elsif android_player["playabilityStatus"]["status"] == "OK" -+ params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("") -+ else -+ params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("") -+ end -+ end -+ -+ # TODO: clean that up -+ {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| -+ params[f] = player_response[f] if player_response[f]? -+ end -+ -+ return params -+end -+ -+def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) -+ # Top level elements -+ -+ main_results = player_response.dig?("contents", "twoColumnWatchNextResults") -+ -+ raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results -+ -+ # Primary results are not available on Music videos -+ # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725 -+ if primary_results = main_results.dig?("results", "results", "contents") -+ video_primary_renderer = primary_results -+ .as_a.find(&.["videoPrimaryInfoRenderer"]?) -+ .try &.["videoPrimaryInfoRenderer"] -+ -+ video_secondary_renderer = primary_results -+ .as_a.find(&.["videoSecondaryInfoRenderer"]?) -+ .try &.["videoSecondaryInfoRenderer"] -+ -+ raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer -+ raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer -+ end -+ -+ video_details = player_response.dig?("videoDetails") -+ microformat = player_response.dig?("microformat", "playerMicroformatRenderer") -+ -+ raise BrokenTubeException.new("videoDetails") if !video_details -+ raise BrokenTubeException.new("microformat") if !microformat -+ -+ # Basic video infos -+ -+ title = video_details["title"]?.try &.as_s -+ -+ # We have to try to extract viewCount from videoPrimaryInfoRenderer first, -+ # then from videoDetails, as the latter is "0" for livestreams (we want -+ # to get the amount of viewers watching). -+ views = video_primary_renderer -+ .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") -+ .try &.as_s.to_i64 -+ views ||= video_details["viewCount"]?.try &.as_s.to_i64 -+ -+ length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) -+ .try &.as_s.to_i64 -+ -+ published = microformat["publishDate"]? -+ .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc -+ -+ premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") -+ .try { |t| Time.parse_rfc3339(t.as_s) } -+ -+ live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") -+ .try &.as_bool || false -+ -+ # Extra video infos -+ -+ allowed_regions = microformat["availableCountries"]? -+ .try &.as_a.map &.as_s || [] of String -+ -+ allow_ratings = video_details["allowRatings"]?.try &.as_bool -+ family_friendly = microformat["isFamilySafe"].try &.as_bool -+ is_listed = video_details["isCrawlable"]?.try &.as_bool -+ is_upcoming = video_details["isUpcoming"]?.try &.as_bool -+ -+ keywords = video_details["keywords"]? -+ .try &.as_a.map &.as_s || [] of String -+ -+ # Related videos -+ -+ LOGGER.debug("extract_video_info: parsing related videos...") -+ -+ related = [] of JSON::Any -+ -+ # Parse "compactVideoRenderer" items (under secondary results) -+ secondary_results = main_results -+ .dig?("secondaryResults", "secondaryResults", "results") -+ secondary_results.try &.as_a.each do |element| -+ if item = element["compactVideoRenderer"]? -+ related_video = parse_related_video(item) -+ related << JSON::Any.new(related_video) if related_video -+ end -+ end -+ -+ # If nothing was found previously, fall back to end screen renderer -+ if related.empty? -+ # Container for "endScreenVideoRenderer" items -+ player_overlays = player_response.dig?( -+ "playerOverlays", "playerOverlayRenderer", -+ "endScreen", "watchNextEndScreenRenderer", "results" -+ ) -+ -+ player_overlays.try &.as_a.each do |element| -+ if item = element["endScreenVideoRenderer"]? -+ related_video = parse_related_video(item) -+ related << JSON::Any.new(related_video) if related_video -+ end -+ end -+ end -+ -+ # Likes -+ -+ toplevel_buttons = video_primary_renderer -+ .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") -+ -+ if toplevel_buttons -+ likes_button = toplevel_buttons.as_a -+ .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") -+ .try &.["toggleButtonRenderer"] -+ -+ if likes_button -+ likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) -+ .try &.dig?("accessibility", "accessibilityData", "label") -+ likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt -+ -+ LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") -+ LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes -+ end -+ end -+ -+ # Description -+ -+ description = microformat.dig?("description", "simpleText").try &.as_s || "" -+ short_description = player_response.dig?("videoDetails", "shortDescription") -+ -+ description_html = video_secondary_renderer.try &.dig?("description", "runs") -+ .try &.as_a.try { |t| content_to_comment_html(t, video_id) } -+ -+ # Video metadata -+ -+ metadata = video_secondary_renderer -+ .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") -+ .try &.as_a -+ -+ genre = microformat["category"]? -+ genre_ucid = nil -+ license = nil -+ -+ metadata.try &.each do |row| -+ metadata_title = row.dig?("metadataRowRenderer", "title", "simpleText").try &.as_s -+ contents = row.dig?("metadataRowRenderer", "contents", 0) -+ -+ if metadata_title == "Category" -+ contents = contents.try &.dig?("runs", 0) -+ -+ genre = contents.try &.["text"]? -+ genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId") -+ elsif metadata_title == "License" -+ license = contents.try &.dig?("runs", 0, "text") -+ elsif metadata_title == "Licensed to YouTube by" -+ license = contents.try &.["simpleText"]? -+ end -+ end -+ -+ # Author infos -+ -+ author = video_details["author"]?.try &.as_s -+ ucid = video_details["channelId"]?.try &.as_s -+ -+ if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") -+ author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") -+ author_verified = has_verified_badge?(author_info["badges"]?) -+ -+ subs_text = author_info["subscriberCountText"]? -+ .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") } -+ .try &.as_s.split(" ", 2)[0] -+ end -+ -+ # Return data -+ -+ if live_now -+ video_type = VideoType::Livestream -+ elsif !premiere_timestamp.nil? -+ video_type = VideoType::Scheduled -+ published = premiere_timestamp || Time.utc -+ else -+ video_type = VideoType::Video -+ end -+ -+ params = { -+ "videoType" => JSON::Any.new(video_type.to_s), -+ # Basic video infos -+ "title" => JSON::Any.new(title || ""), -+ "views" => JSON::Any.new(views || 0_i64), -+ "likes" => JSON::Any.new(likes || 0_i64), -+ "lengthSeconds" => JSON::Any.new(length_txt || 0_i64), -+ "published" => JSON::Any.new(published.to_rfc3339), -+ # Extra video infos -+ "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }), -+ "allowRatings" => JSON::Any.new(allow_ratings || false), -+ "isFamilyFriendly" => JSON::Any.new(family_friendly || false), -+ "isListed" => JSON::Any.new(is_listed || false), -+ "isUpcoming" => JSON::Any.new(is_upcoming || false), -+ "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), -+ # Related videos -+ "relatedVideos" => JSON::Any.new(related), -+ # Description -+ "description" => JSON::Any.new(description || ""), -+ "descriptionHtml" => JSON::Any.new(description_html || "

"), -+ "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), -+ # Video metadata -+ "genre" => JSON::Any.new(genre.try &.as_s || ""), -+ "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""), -+ "license" => JSON::Any.new(license.try &.as_s || ""), -+ # Author infos -+ "author" => JSON::Any.new(author || ""), -+ "ucid" => JSON::Any.new(ucid || ""), -+ "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), -+ "authorVerified" => JSON::Any.new(author_verified || false), -+ "subCountText" => JSON::Any.new(subs_text || "-"), -+ } -+ -+ return params -+end - -From 87a5d70062b8f4b2b942d027f8c4cf0bb30907eb Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Tue, 23 Aug 2022 19:03:09 +0200 -Subject: [PATCH 11/16] videos: move API's JSON structure to a dedicated module - ---- - src/invidious.cr | 2 + - src/invidious/channels/channels.cr | 2 +- - src/invidious/channels/community.cr | 2 +- - src/invidious/helpers/serialized_yt_data.cr | 4 +- - src/invidious/jsonify/api_v1/video_json.cr | 255 +++++++++++++++++++ - src/invidious/playlists.cr | 2 +- - src/invidious/routes/api/v1/misc.cr | 2 +- - src/invidious/routes/api/v1/videos.cr | 2 +- - src/invidious/videos.cr | 256 +------------------- - 9 files changed, 272 insertions(+), 255 deletions(-) - create mode 100644 src/invidious/jsonify/api_v1/video_json.cr - -diff --git a/src/invidious.cr b/src/invidious.cr -index 8df0c0cdb..2874cc712 100644 ---- a/src/invidious.cr -+++ b/src/invidious.cr -@@ -39,6 +39,8 @@ require "./invidious/yt_backend/*" - require "./invidious/frontend/*" - require "./invidious/videos/*" - -+require "./invidious/jsonify/**" -+ - require "./invidious/*" - require "./invidious/channels/*" - require "./invidious/user/*" -diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr -index e0459cc31..e3d3d9eec 100644 ---- a/src/invidious/channels/channels.cr -+++ b/src/invidious/channels/channels.cr -@@ -29,7 +29,7 @@ struct ChannelVideo - json.field "title", self.title - json.field "videoId", self.id - json.field "videoThumbnails" do -- generate_thumbnails(json, self.id) -+ Invidious::JSONify::APIv1.thumbnails(json, self.id) - end - - json.field "lengthSeconds", self.length_seconds -diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr -index 2a2c74aa6..8e300288f 100644 ---- a/src/invidious/channels/community.cr -+++ b/src/invidious/channels/community.cr -@@ -138,7 +138,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) - json.field "title", video_title - json.field "videoId", video_id - json.field "videoThumbnails" do -- generate_thumbnails(json, video_id) -+ Invidious::JSONify::APIv1.thumbnails(json, video_id) - end - - json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s) -diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr -index 3918bd130..c52e2a0dc 100644 ---- a/src/invidious/helpers/serialized_yt_data.cr -+++ b/src/invidious/helpers/serialized_yt_data.cr -@@ -76,7 +76,7 @@ struct SearchVideo - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoThumbnails" do -- generate_thumbnails(json, self.id) -+ Invidious::JSONify::APIv1.thumbnails(json, self.id) - end - - json.field "description", html_to_content(self.description_html) -@@ -155,7 +155,7 @@ struct SearchPlaylist - json.field "lengthSeconds", video.length_seconds - - json.field "videoThumbnails" do -- generate_thumbnails(json, video.id) -+ Invidious::JSONify::APIv1.thumbnails(json, video.id) - end - end - end -diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr -new file mode 100644 -index 000000000..1082f6d30 ---- /dev/null -+++ b/src/invidious/jsonify/api_v1/video_json.cr -@@ -0,0 +1,255 @@ -+require "json" -+ -+module Invidious::JSONify::APIv1 -+ extend self -+ -+ def video(video : Video, json : JSON::Builder, *, locale : String?) -+ json.object do -+ json.field "type", video.video_type -+ -+ json.field "title", video.title -+ json.field "videoId", video.id -+ -+ json.field "error", video.info["reason"] if video.info["reason"]? -+ -+ json.field "videoThumbnails" do -+ self.thumbnails(json, video.id) -+ end -+ json.field "storyboards" do -+ self.storyboards(json, video.id, video.storyboards) -+ end -+ -+ json.field "description", video.description -+ json.field "descriptionHtml", video.description_html -+ json.field "published", video.published.to_unix -+ json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale)) -+ json.field "keywords", video.keywords -+ -+ json.field "viewCount", video.views -+ json.field "likeCount", video.likes -+ json.field "dislikeCount", 0_i64 -+ -+ json.field "paid", video.paid -+ json.field "premium", video.premium -+ json.field "isFamilyFriendly", video.is_family_friendly -+ json.field "allowedRegions", video.allowed_regions -+ json.field "genre", video.genre -+ json.field "genreUrl", video.genre_url -+ -+ json.field "author", video.author -+ json.field "authorId", video.ucid -+ json.field "authorUrl", "/channel/#{video.ucid}" -+ -+ json.field "authorThumbnails" do -+ json.array do -+ qualities = {32, 48, 76, 100, 176, 512} -+ -+ qualities.each do |quality| -+ json.object do -+ json.field "url", video.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") -+ json.field "width", quality -+ json.field "height", quality -+ end -+ end -+ end -+ end -+ -+ json.field "subCountText", video.sub_count_text -+ -+ json.field "lengthSeconds", video.length_seconds -+ json.field "allowRatings", video.allow_ratings -+ json.field "rating", 0_i64 -+ json.field "isListed", video.is_listed -+ json.field "liveNow", video.live_now -+ json.field "isUpcoming", video.is_upcoming -+ -+ if video.premiere_timestamp -+ json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix -+ end -+ -+ if hlsvp = video.hls_manifest_url -+ hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL) -+ json.field "hlsUrl", hlsvp -+ end -+ -+ json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{video.id}" -+ -+ json.field "adaptiveFormats" do -+ json.array do -+ video.adaptive_fmts.each do |fmt| -+ json.object do -+ # Only available on regular videos, not livestreams/OTF streams -+ if init_range = fmt["initRange"]? -+ json.field "init", "#{init_range["start"]}-#{init_range["end"]}" -+ end -+ if index_range = fmt["indexRange"]? -+ json.field "index", "#{index_range["start"]}-#{index_range["end"]}" -+ end -+ -+ # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only) -+ json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? -+ -+ json.field "url", fmt["url"] -+ json.field "itag", fmt["itag"].as_i.to_s -+ json.field "type", fmt["mimeType"] -+ json.field "clen", fmt["contentLength"]? || "-1" -+ json.field "lmt", fmt["lastModified"] -+ json.field "projectionType", fmt["projectionType"] -+ -+ if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) -+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 -+ json.field "fps", fps -+ json.field "container", fmt_info["ext"] -+ json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] -+ -+ if fmt_info["height"]? -+ json.field "resolution", "#{fmt_info["height"]}p" -+ -+ quality_label = "#{fmt_info["height"]}p" -+ if fps > 30 -+ quality_label += "60" -+ end -+ json.field "qualityLabel", quality_label -+ -+ if fmt_info["width"]? -+ json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" -+ end -+ end -+ end -+ -+ # Livestream chunk infos -+ json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec") -+ json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec") -+ -+ # Audio-related data -+ json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality") -+ json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate") -+ json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels") -+ -+ # Extra misc stuff -+ json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo") -+ json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack") -+ end -+ end -+ end -+ end -+ -+ json.field "formatStreams" do -+ json.array do -+ video.fmt_stream.each do |fmt| -+ json.object do -+ json.field "url", fmt["url"] -+ json.field "itag", fmt["itag"].as_i.to_s -+ json.field "type", fmt["mimeType"] -+ json.field "quality", fmt["quality"] -+ -+ fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) -+ if fmt_info -+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 -+ json.field "fps", fps -+ json.field "container", fmt_info["ext"] -+ json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] -+ -+ if fmt_info["height"]? -+ json.field "resolution", "#{fmt_info["height"]}p" -+ -+ quality_label = "#{fmt_info["height"]}p" -+ if fps > 30 -+ quality_label += "60" -+ end -+ json.field "qualityLabel", quality_label -+ -+ if fmt_info["width"]? -+ json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" -+ end -+ end -+ end -+ end -+ end -+ end -+ end -+ -+ json.field "captions" do -+ json.array do -+ video.captions.each do |caption| -+ json.object do -+ json.field "label", caption.name -+ json.field "language_code", caption.language_code -+ json.field "url", "/api/v1/captions/#{video.id}?label=#{URI.encode_www_form(caption.name)}" -+ end -+ end -+ end -+ end -+ -+ json.field "recommendedVideos" do -+ json.array do -+ video.related_videos.each do |rv| -+ if rv["id"]? -+ json.object do -+ json.field "videoId", rv["id"] -+ json.field "title", rv["title"] -+ json.field "videoThumbnails" do -+ self.thumbnails(json, rv["id"]) -+ end -+ -+ json.field "author", rv["author"] -+ json.field "authorUrl", "/channel/#{rv["ucid"]?}" -+ json.field "authorId", rv["ucid"]? -+ if rv["author_thumbnail"]? -+ json.field "authorThumbnails" do -+ json.array do -+ qualities = {32, 48, 76, 100, 176, 512} -+ -+ qualities.each do |quality| -+ json.object do -+ json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-") -+ json.field "width", quality -+ json.field "height", quality -+ end -+ end -+ end -+ end -+ end -+ -+ json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i -+ json.field "viewCountText", rv["short_view_count"]? -+ json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 -+ end -+ end -+ end -+ end -+ end -+ end -+ end -+ -+ def thumbnails(json, id) -+ json.array do -+ build_thumbnails(id).each do |thumbnail| -+ json.object do -+ json.field "quality", thumbnail[:name] -+ json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" -+ json.field "width", thumbnail[:width] -+ json.field "height", thumbnail[:height] -+ end -+ end -+ end -+ end -+ -+ def storyboards(json, id, storyboards) -+ json.array do -+ storyboards.each do |storyboard| -+ json.object do -+ json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" -+ json.field "templateUrl", storyboard[:url] -+ json.field "width", storyboard[:width] -+ json.field "height", storyboard[:height] -+ json.field "count", storyboard[:count] -+ json.field "interval", storyboard[:interval] -+ json.field "storyboardWidth", storyboard[:storyboard_width] -+ json.field "storyboardHeight", storyboard[:storyboard_height] -+ json.field "storyboardCount", storyboard[:storyboard_count] -+ end -+ end -+ end -+ end -+end -diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr -index c4eb75078..57f1f53e6 100644 ---- a/src/invidious/playlists.cr -+++ b/src/invidious/playlists.cr -@@ -56,7 +56,7 @@ struct PlaylistVideo - json.field "authorUrl", "/channel/#{self.ucid}" - - json.field "videoThumbnails" do -- generate_thumbnails(json, self.id) -+ Invidious::JSONify::APIv1.thumbnails(json, self.id) - end - - if index -diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr -index 844fedb87..43d360e68 100644 ---- a/src/invidious/routes/api/v1/misc.cr -+++ b/src/invidious/routes/api/v1/misc.cr -@@ -124,7 +124,7 @@ module Invidious::Routes::API::V1::Misc - - json.field "videoThumbnails" do - json.array do -- generate_thumbnails(json, video.id) -+ Invidious::JSONify::APIv1.thumbnails(json, video.id) - end - end - -diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr -index 1b7b4fa70..6f1f59164 100644 ---- a/src/invidious/routes/api/v1/videos.cr -+++ b/src/invidious/routes/api/v1/videos.cr -@@ -185,7 +185,7 @@ module Invidious::Routes::API::V1::Videos - response = JSON.build do |json| - json.object do - json.field "storyboards" do -- generate_storyboards(json, id, storyboards) -+ Invidious::JSONify::APIv1.storyboards(json, id, storyboards) - end - end - end -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index 9b19bc2a0..fcc9a8a4b 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -31,234 +31,25 @@ struct Video - end - end - -- def to_json(locale : String?, json : JSON::Builder) -- json.object do -- json.field "type", self.video_type -- -- json.field "title", self.title -- json.field "videoId", self.id -- -- json.field "error", info["reason"] if info["reason"]? -- -- json.field "videoThumbnails" do -- generate_thumbnails(json, self.id) -- end -- json.field "storyboards" do -- generate_storyboards(json, self.id, self.storyboards) -- end -- -- json.field "description", self.description -- json.field "descriptionHtml", self.description_html -- json.field "published", self.published.to_unix -- json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) -- json.field "keywords", self.keywords -- -- json.field "viewCount", self.views -- json.field "likeCount", self.likes -- json.field "dislikeCount", 0_i64 -- -- json.field "paid", self.paid -- json.field "premium", self.premium -- json.field "isFamilyFriendly", self.is_family_friendly -- json.field "allowedRegions", self.allowed_regions -- json.field "genre", self.genre -- json.field "genreUrl", self.genre_url -- -- json.field "author", self.author -- json.field "authorId", self.ucid -- json.field "authorUrl", "/channel/#{self.ucid}" -- -- json.field "authorThumbnails" do -- json.array do -- qualities = {32, 48, 76, 100, 176, 512} -- -- qualities.each do |quality| -- json.object do -- json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") -- json.field "width", quality -- json.field "height", quality -- end -- end -- end -- end -- -- json.field "subCountText", self.sub_count_text -- -- json.field "lengthSeconds", self.length_seconds -- json.field "allowRatings", self.allow_ratings -- json.field "rating", 0_i64 -- json.field "isListed", self.is_listed -- json.field "liveNow", self.live_now -- json.field "isUpcoming", self.is_upcoming -- -- if self.premiere_timestamp -- json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix -- end -- -- if hlsvp = self.hls_manifest_url -- hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL) -- json.field "hlsUrl", hlsvp -- end -- -- json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}" -- -- json.field "adaptiveFormats" do -- json.array do -- self.adaptive_fmts.each do |fmt| -- json.object do -- # Only available on regular videos, not livestreams/OTF streams -- if init_range = fmt["initRange"]? -- json.field "init", "#{init_range["start"]}-#{init_range["end"]}" -- end -- if index_range = fmt["indexRange"]? -- json.field "index", "#{index_range["start"]}-#{index_range["end"]}" -- end -- -- # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only) -- json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]? -- -- json.field "url", fmt["url"] -- json.field "itag", fmt["itag"].as_i.to_s -- json.field "type", fmt["mimeType"] -- json.field "clen", fmt["contentLength"]? || "-1" -- json.field "lmt", fmt["lastModified"] -- json.field "projectionType", fmt["projectionType"] -- -- if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) -- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 -- json.field "fps", fps -- json.field "container", fmt_info["ext"] -- json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] -- -- if fmt_info["height"]? -- json.field "resolution", "#{fmt_info["height"]}p" -- -- quality_label = "#{fmt_info["height"]}p" -- if fps > 30 -- quality_label += "60" -- end -- json.field "qualityLabel", quality_label -- -- if fmt_info["width"]? -- json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" -- end -- end -- end -- -- # Livestream chunk infos -- json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec") -- json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec") -- -- # Audio-related data -- json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality") -- json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate") -- json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels") -- -- # Extra misc stuff -- json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo") -- json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack") -- end -- end -- end -- end -- -- json.field "formatStreams" do -- json.array do -- self.fmt_stream.each do |fmt| -- json.object do -- json.field "url", fmt["url"] -- json.field "itag", fmt["itag"].as_i.to_s -- json.field "type", fmt["mimeType"] -- json.field "quality", fmt["quality"] -- -- fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) -- if fmt_info -- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 -- json.field "fps", fps -- json.field "container", fmt_info["ext"] -- json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] -- -- if fmt_info["height"]? -- json.field "resolution", "#{fmt_info["height"]}p" -- -- quality_label = "#{fmt_info["height"]}p" -- if fps > 30 -- quality_label += "60" -- end -- json.field "qualityLabel", quality_label -- -- if fmt_info["width"]? -- json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" -- end -- end -- end -- end -- end -- end -- end -+ # Methods for API v1 JSON - -- json.field "captions" do -- json.array do -- self.captions.each do |caption| -- json.object do -- json.field "label", caption.name -- json.field "language_code", caption.language_code -- json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" -- end -- end -- end -- end -- -- json.field "recommendedVideos" do -- json.array do -- self.related_videos.each do |rv| -- if rv["id"]? -- json.object do -- json.field "videoId", rv["id"] -- json.field "title", rv["title"] -- json.field "videoThumbnails" do -- generate_thumbnails(json, rv["id"]) -- end -- -- json.field "author", rv["author"] -- json.field "authorUrl", "/channel/#{rv["ucid"]?}" -- json.field "authorId", rv["ucid"]? -- if rv["author_thumbnail"]? -- json.field "authorThumbnails" do -- json.array do -- qualities = {32, 48, 76, 100, 176, 512} -- -- qualities.each do |quality| -- json.object do -- json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-") -- json.field "width", quality -- json.field "height", quality -- end -- end -- end -- end -- end -- -- json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i -- json.field "viewCountText", rv["short_view_count"]? -- json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 -- end -- end -- end -- end -- end -- end -+ def to_json(locale : String?, json : JSON::Builder) -+ Invidious::JSONify::APIv1.video(self, json, locale: locale) - end - - # TODO: remove the locale and follow the crystal convention - def to_json(locale : String?, _json : Nil) -- JSON.build { |json| to_json(locale, json) } -+ JSON.build do |json| -+ Invidious::JSONify::APIv1.video(self, json, locale: locale) -+ end - end - - def to_json(json : JSON::Builder | Nil = nil) - to_json(nil, json) - end - -+ # Misc methods -+ - def video_type : VideoType - video_type = info["videoType"]?.try &.as_s || "video" - return VideoType.parse?(video_type) || VideoType::Video -@@ -631,34 +422,3 @@ def build_thumbnails(id) - {host: HOST_URL, height: 90, width: 120, name: "end", url: "3"}, - } - end -- --def generate_thumbnails(json, id) -- json.array do -- build_thumbnails(id).each do |thumbnail| -- json.object do -- json.field "quality", thumbnail[:name] -- json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" -- json.field "width", thumbnail[:width] -- json.field "height", thumbnail[:height] -- end -- end -- end --end -- --def generate_storyboards(json, id, storyboards) -- json.array do -- storyboards.each do |storyboard| -- json.object do -- json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" -- json.field "templateUrl", storyboard[:url] -- json.field "width", storyboard[:width] -- json.field "height", storyboard[:height] -- json.field "count", storyboard[:count] -- json.field "interval", storyboard[:interval] -- json.field "storyboardWidth", storyboard[:storyboard_width] -- json.field "storyboardHeight", storyboard[:storyboard_height] -- json.field "storyboardCount", storyboard[:storyboard_count] -- end -- end -- end --end - -From d659a451d6dece62dbb091a958083c8a347da5b1 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Tue, 23 Aug 2022 19:04:08 +0200 -Subject: [PATCH 12/16] videos: remove unused 'VideoRedirect' exception - ---- - src/invidious/jsonify/api_v1/common.cr | 18 ++++++++++++++++++ - src/invidious/jsonify/api_v1/video_json.cr | 13 ------------- - src/invidious/routes/api/manifest.cr | 2 -- - src/invidious/routes/api/v1/videos.cr | 9 --------- - src/invidious/routes/embed.cr | 2 -- - src/invidious/routes/watch.cr | 2 -- - src/invidious/videos.cr | 7 ------- - 7 files changed, 18 insertions(+), 35 deletions(-) - create mode 100644 src/invidious/jsonify/api_v1/common.cr - -diff --git a/src/invidious/jsonify/api_v1/common.cr b/src/invidious/jsonify/api_v1/common.cr -new file mode 100644 -index 000000000..64b06465f ---- /dev/null -+++ b/src/invidious/jsonify/api_v1/common.cr -@@ -0,0 +1,18 @@ -+require "json" -+ -+module Invidious::JSONify::APIv1 -+ extend self -+ -+ def thumbnails(json : JSON::Builder, id : String) -+ json.array do -+ build_thumbnails(id).each do |thumbnail| -+ json.object do -+ json.field "quality", thumbnail[:name] -+ json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" -+ json.field "width", thumbnail[:width] -+ json.field "height", thumbnail[:height] -+ end -+ end -+ end -+ end -+end -diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr -index 1082f6d30..0a5173ce6 100644 ---- a/src/invidious/jsonify/api_v1/video_json.cr -+++ b/src/invidious/jsonify/api_v1/video_json.cr -@@ -222,19 +222,6 @@ module Invidious::JSONify::APIv1 - end - end - -- def thumbnails(json, id) -- json.array do -- build_thumbnails(id).each do |thumbnail| -- json.object do -- json.field "quality", thumbnail[:name] -- json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" -- json.field "width", thumbnail[:width] -- json.field "height", thumbnail[:height] -- end -- end -- end -- end -- - def storyboards(json, id, storyboards) - json.array do - storyboards.each do |storyboard| -diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr -index bfb8a3777..ae65f10d2 100644 ---- a/src/invidious/routes/api/manifest.cr -+++ b/src/invidious/routes/api/manifest.cr -@@ -14,8 +14,6 @@ module Invidious::Routes::API::Manifest - - begin - video = get_video(id, region: region) -- rescue ex : VideoRedirect -- return env.redirect env.request.resource.gsub(id, ex.video_id) - rescue ex : NotFoundException - haltf env, status_code: 404 - rescue ex -diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr -index 6f1f59164..a6b2eb4e4 100644 ---- a/src/invidious/routes/api/v1/videos.cr -+++ b/src/invidious/routes/api/v1/videos.cr -@@ -9,9 +9,6 @@ module Invidious::Routes::API::V1::Videos - - begin - video = get_video(id, region: region) -- rescue ex : VideoRedirect -- env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) -- return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) - rescue ex : NotFoundException - return error_json(404, ex) - rescue ex -@@ -41,9 +38,6 @@ module Invidious::Routes::API::V1::Videos - - begin - video = get_video(id, region: region) -- rescue ex : VideoRedirect -- env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) -- return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) - rescue ex : NotFoundException - haltf env, 404 - rescue ex -@@ -168,9 +162,6 @@ module Invidious::Routes::API::V1::Videos - - begin - video = get_video(id, region: region) -- rescue ex : VideoRedirect -- env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) -- return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) - rescue ex : NotFoundException - haltf env, 404 - rescue ex -diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr -index e64865877..289d87c9a 100644 ---- a/src/invidious/routes/embed.cr -+++ b/src/invidious/routes/embed.cr -@@ -131,8 +131,6 @@ module Invidious::Routes::Embed - - begin - video = get_video(id, region: params.region) -- rescue ex : VideoRedirect -- return env.redirect env.request.resource.gsub(id, ex.video_id) - rescue ex : NotFoundException - return error_template(404, ex) - rescue ex -diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr -index fe1d8e548..5f481557a 100644 ---- a/src/invidious/routes/watch.cr -+++ b/src/invidious/routes/watch.cr -@@ -61,8 +61,6 @@ module Invidious::Routes::Watch - - begin - video = get_video(id, region: params.region) -- rescue ex : VideoRedirect -- return env.redirect env.request.resource.gsub(id, ex.video_id) - rescue ex : NotFoundException - LOGGER.error("get_video not found: #{id} : #{ex.message}") - return error_template(404, ex) -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index fcc9a8a4b..bec26de9e 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -319,13 +319,6 @@ struct Video - getset_bool isUpcoming - end - --class VideoRedirect < Exception -- property video_id : String -- -- def initialize(@video_id) -- end --end -- - def get_video(id, refresh = true, region = nil, force_refresh = false) - if (video = Invidious::Database::Videos.select(id)) && !region - # If record was last updated over 10 minutes ago, or video has since premiered, - -From 83795c245aace771fb73936b22d3de7ced0df9df Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Thu, 8 Sep 2022 00:06:58 +0200 -Subject: [PATCH 13/16] videos: Support the new like button's structure - ---- - src/invidious/videos/parser.cr | 12 +++++++++++- - 1 file changed, 11 insertions(+), 1 deletion(-) - -diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr -index ff5d15de0..701c4e77b 100644 ---- a/src/invidious/videos/parser.cr -+++ b/src/invidious/videos/parser.cr -@@ -227,11 +227,21 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") - - if toplevel_buttons -- likes_button = toplevel_buttons.as_a -+ likes_button = toplevel_buttons.try &.as_a - .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") - .try &.["toggleButtonRenderer"] - -+ # New format as of september 2022 -+ likes_button ||= toplevel_buttons.try &.as_a -+ .find(&.["segmentedLikeDislikeButtonRenderer"]?) -+ .try &.dig?( -+ "segmentedLikeDislikeButtonRenderer", -+ "likeButton", "toggleButtonRenderer" -+ ) -+ - if likes_button -+ # Note: The like count from `toggledText` is off by one, as it would -+ # represent the new like count in the event where the user clicks on "like". - likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?) - .try &.dig?("accessibility", "accessibilityData", "label") - likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt - -From db91d3af66b52e8f7127b2b3b826111126027c6d Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Wed, 14 Sep 2022 20:08:36 +0200 -Subject: [PATCH 14/16] videos: Fix some bugs - ---- - src/invidious/jsonify/api_v1/video_json.cr | 11 ++++++++++- - src/invidious/videos/parser.cr | 8 ++++---- - 2 files changed, 14 insertions(+), 5 deletions(-) - -diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr -index 0a5173ce6..642789aa0 100644 ---- a/src/invidious/jsonify/api_v1/video_json.cr -+++ b/src/invidious/jsonify/api_v1/video_json.cr -@@ -93,7 +93,16 @@ module Invidious::JSONify::APIv1 - json.field "itag", fmt["itag"].as_i.to_s - json.field "type", fmt["mimeType"] - json.field "clen", fmt["contentLength"]? || "-1" -- json.field "lmt", fmt["lastModified"] -+ -+ # Last modified is a unix timestamp with µS, with the dot omitted. -+ # E.g: 1638056732(.)141582 -+ # -+ # On livestreams, it's not present, so always fall back to the -+ # current unix timestamp (up to mS precision) for compatibility. -+ last_modified = fmt["lastModified"]? -+ last_modified ||= "#{Time.utc.to_unix_ms.to_s}000" -+ json.field "lmt", last_modified -+ - json.field "projectionType", fmt["projectionType"] - - if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"]) -diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr -index 701c4e77b..53372942b 100644 ---- a/src/invidious/videos/parser.cr -+++ b/src/invidious/videos/parser.cr -@@ -159,10 +159,10 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - # We have to try to extract viewCount from videoPrimaryInfoRenderer first, - # then from videoDetails, as the latter is "0" for livestreams (we want - # to get the amount of viewers watching). -- views = video_primary_renderer -+ views_txt = video_primary_renderer - .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") -- .try &.as_s.to_i64 -- views ||= video_details["viewCount"]?.try &.as_s.to_i64 -+ views_txt ||= video_details["viewCount"]? -+ views = views_txt.try &.as_s.gsub(/\D/, "").to_i64? - - length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) - .try &.as_s.to_i64 -@@ -270,7 +270,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any - license = nil - - metadata.try &.each do |row| -- metadata_title = row.dig?("metadataRowRenderer", "title", "simpleText").try &.as_s -+ metadata_title = extract_text(row.dig?("metadataRowRenderer", "title")) - contents = row.dig?("metadataRowRenderer", "contents", 0) - - if metadata_title == "Category" - -From 2acff70811eeb82d7944b358e03171a775106e86 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Mon, 3 Oct 2022 21:58:52 +0200 -Subject: [PATCH 15/16] videos: handle different JSON structs being present in - cache - ---- - src/invidious/videos.cr | 17 ++++++++++++++++- - src/invidious/videos/parser.cr | 6 +++++- - 2 files changed, 21 insertions(+), 2 deletions(-) - -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index bec26de9e..c055f2a7a 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -7,6 +7,16 @@ end - struct Video - include DB::Serializable - -+ # Version of the JSON structure -+ # It prevents us from loading an incompatible version from cache -+ # (either newer or older, if instances with different versions run -+ # concurrently, e.g during a version upgrade rollout). -+ # -+ # NOTE: don't forget to bump this number if any change is made to -+ # the `params` structure in videos/parser.cr!!! -+ # -+ SCHEMA_VERSION = 2 -+ - property id : String - - @[DB::Field(converter: Video::JSONConverter)] -@@ -55,6 +65,10 @@ struct Video - return VideoType.parse?(video_type) || VideoType::Video - end - -+ def schema_version : Int -+ return info["version"]?.try &.as_i || 1 -+ end -+ - def published : Time - return info["published"]? - .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc -@@ -326,7 +340,8 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) - if (refresh && - (Time.utc - video.updated > 10.minutes) || - (video.premiere_timestamp.try &.< Time.utc)) || -- force_refresh -+ force_refresh || -+ video.schema_version != Video::SCHEMA_VERSION # cache control - begin - video = fetch_video(id, region) - Invidious::Database::Videos.update(video) -diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr -index 53372942b..64c8d21a9 100644 ---- a/src/invidious/videos/parser.cr -+++ b/src/invidious/videos/parser.cr -@@ -71,7 +71,8 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ - # Stop here if video is not a scheduled livestream - if playability_status != "LIVE_STREAM_OFFLINE" - return { -- "reason" => JSON::Any.new(reason), -+ "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), -+ "reason" => JSON::Any.new(reason), - } - end - elsif video_id != player_response.dig("videoDetails", "videoId") -@@ -121,6 +122,9 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ - params[f] = player_response[f] if player_response[f]? - end - -+ # Data structure version, for cache control -+ params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) -+ - return params - end - - -From cc5c83333f2a51dc178b698a548b64f01a2ff453 Mon Sep 17 00:00:00 2001 -From: Samantaz Fox -Date: Sun, 6 Nov 2022 21:23:58 +0100 -Subject: [PATCH 16/16] videos: improve fetching of streaming data - ---- - src/invidious/videos.cr | 6 --- - src/invidious/videos/parser.cr | 78 +++++++++++++++++++++------------- - 2 files changed, 48 insertions(+), 36 deletions(-) - -diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr -index c055f2a7a..786ef4166 100644 ---- a/src/invidious/videos.cr -+++ b/src/invidious/videos.cr -@@ -380,12 +380,6 @@ def fetch_video(id, region) - end - end - -- # Try to fetch video info using an embedded client -- if info["reason"]? -- embed_info = extract_video_info(video_id: id, context_screen: "embed") -- info = embed_info if !embed_info["reason"]? -- end -- - if reason = info["reason"]? - if reason == "Video unavailable" - raise NotFoundException.new(reason.as_s || "") -diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr -index 64c8d21a9..e3f6170d2 100644 ---- a/src/invidious/videos/parser.cr -+++ b/src/invidious/videos/parser.cr -@@ -50,12 +50,9 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? - } - end - --def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil) -+def extract_video_info(video_id : String, proxy_region : String? = nil) - # Init client config for the API - client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) -- if context_screen == "embed" -- client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed -- end - - # Fetch data from the player endpoint - player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) -@@ -69,7 +66,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ - reason ||= player_response.dig("playabilityStatus", "reason").as_s - - # Stop here if video is not a scheduled livestream -- if playability_status != "LIVE_STREAM_OFFLINE" -+ if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) - return { - "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), - "reason" => JSON::Any.new(reason), -@@ -84,7 +81,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ - end - - # Don't fetch the next endpoint if the video is unavailable. -- if {"OK", "LIVE_STREAM_OFFLINE"}.any?(playability_status) -+ if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) - next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) - player_response = player_response.merge(next_response) - end -@@ -92,33 +89,34 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ - params = parse_video_info(video_id, player_response) - params["reason"] = JSON::Any.new(reason) if reason - -- # Fetch the video streams using an Android client in order to get the decrypted URLs and -- # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs: -- # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 -+ new_player_response = nil -+ - if reason.nil? -- if context_screen == "embed" -- client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed -- else -- client_config.client_type = YoutubeAPI::ClientType::Android -- end -- android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) -- -- # Sometimes, the video is available from the web client, but not on Android, so check -- # that here, and fallback to the streaming data from the web client if needed. -- # See: https://github.com/iv-org/invidious/issues/2549 -- if video_id != android_player.dig("videoDetails", "videoId") -- # YouTube may return a different video player response than expected. -- # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 -- raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (ANDROID client)") -- elsif android_player["playabilityStatus"]["status"] == "OK" -- params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("") -- else -- params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("") -- end -+ # Fetch the video streams using an Android client in order to get the -+ # decrypted URLs and maybe fix throttling issues (#2194). See the -+ # following issue for an explanation about decrypted URLs: -+ # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 -+ client_config.client_type = YoutubeAPI::ClientType::Android -+ new_player_response = try_fetch_streaming_data(video_id, client_config) -+ elsif !reason.includes?("your country") # Handled separately -+ # The Android embedded client could help here -+ client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed -+ new_player_response = try_fetch_streaming_data(video_id, client_config) -+ end -+ -+ # Last hope -+ if new_player_response.nil? -+ client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed -+ new_player_response = try_fetch_streaming_data(video_id, client_config) -+ end -+ -+ # Replace player response and reset reason -+ if !new_player_response.nil? -+ player_response = new_player_response -+ params.delete("reason") - end - -- # TODO: clean that up -- {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| -+ {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f| - params[f] = player_response[f] if player_response[f]? - end - -@@ -128,6 +126,26 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ - return params - end - -+def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? -+ LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") -+ response = YoutubeAPI.player(video_id: id, params: "", client_config: client_config) -+ -+ playability_status = response["playabilityStatus"]["status"] -+ LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") -+ -+ if id != response.dig("videoDetails", "videoId") -+ # YouTube may return a different video player response than expected. -+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 -+ raise VideoNotAvailableException.new( -+ "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)" -+ ) -+ elsif playability_status == "OK" -+ return response -+ else -+ return nil -+ end -+end -+ - def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) - # Top level elements -