From b0df3774dbb9494007bd5a7de62ae8cf756f3c7c Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Wed, 1 Nov 2023 21:56:25 +0330 Subject: [PATCH 1/7] Add sort options to streams --- src/invidious/channels/videos.cr | 89 ++++++++++++++++++++++--- src/invidious/routes/api/v1/channels.cr | 3 +- src/invidious/routes/channels.cr | 7 +- 3 files changed, 86 insertions(+), 13 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index beb86e08..87af0caf 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -68,6 +68,76 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end +def produce_channel_livestreams_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) + object_inner_2 = { + "2:0:embedded" => { + "1:0:varint" => 0_i64, + }, + "5:varint" => 50_i64, + "6:varint" => 1_i64, + "7:varint" => (page * 30).to_i64, + "9:varint" => 1_i64, + "10:varint" => 0_i64, + } + + object_inner_2_encoded = object_inner_2 + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + sort_by_numerical = + case sort_by + when "newest" then 1_i64 + when "popular" then 2_i64 + when "oldest" then 4_i64 + else 1_i64 # Fallback to "newest" + end + + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { + "14:embedded" => { + "1:embedded" => { + "1:string" => object_inner_2_encoded, + }, + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "3:varint" => sort_by_numerical, + }, + }, + }, + } + + object_inner_1_encoded = object_inner_1 + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_inner_1_encoded, + "35:string" => "browse-feed#{ucid}videos102", + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end + +# Used in bypass_captcha_job.cr +def produce_channel_livestream_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) + continuation = produce_channel_livestreams_continuation(ucid, page, auto_generated, sort_by, v2) + return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" +end + module Invidious::Channel::Tabs extend self @@ -144,21 +214,24 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil) + def make_initial_livestream_ctoken(ucid, sort_by) : String + return produce_channel_livestreams_continuation(ucid, sort_by: sort_by) + end + + def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") if continuation.nil? - # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) + continuation ||= make_initial_livestream_ctoken(channel.ucid, sort_by) end + initial_data = YoutubeAPI.browse(continuation: continuation) + return extract_items(initial_data, channel.author, channel.ucid) end - def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) + def get_60_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") if continuation.nil? - # Fetch the first "page" of streams - items, next_continuation = get_livestreams(channel) + # Fetch the first "page" of stream + items, next_continuation = get_livestreams(channel, sort_by: sort_by) else # Fetch a "page" of streams using the given continuation token items, next_continuation = get_livestreams(channel, continuation: continuation) diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 67018660..f759f6ab 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -207,11 +207,12 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? begin videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) rescue ex return error_json(500, ex) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index d4d8b1c1..c18644ec 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -81,13 +81,12 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for livestreams - sort_by = "" - sort_options = [] of String + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: (sort_by || "newest") ) selected_tab = Frontend::ChannelPage::TabsAvailable::Streams From 63e5d72466d8b879bfa54f60ed475451e868e691 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Mon, 20 Nov 2023 15:50:59 +0330 Subject: [PATCH 2/7] Remove unused function produce_channel_livestream_url --- src/invidious/channels/videos.cr | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 87af0caf..c389dec3 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -132,12 +132,6 @@ def produce_channel_livestreams_continuation(ucid, page = 1, auto_generated = ni return continuation end -# Used in bypass_captcha_job.cr -def produce_channel_livestream_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_livestreams_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end - module Invidious::Channel::Tabs extend self From 0d63ad5a7f13316652117205b1efe2ff2bda00e6 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Wed, 22 Nov 2023 14:50:08 +0330 Subject: [PATCH 3/7] Use a single function for fetching channel contents --- src/invidious/channels/videos.cr | 93 ++++++-------------------------- 1 file changed, 16 insertions(+), 77 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index c389dec3..368ec2b8 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,4 +1,4 @@ -def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) +def produce_channel_content_continuation(ucid, content, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) object_inner_2 = { "2:0:embedded" => { "1:0:varint" => 0_i64, @@ -16,6 +16,13 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } + content_numerical = + case content + when "videos" then 15 + when "livestreams" then 14 + else 15 # Fallback to "videos" + end + sort_by_numerical = case sort_by when "newest" then 1_i64 @@ -27,7 +34,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so object_inner_1 = { "110:embedded" => { "3:embedded" => { - "15:embedded" => { + "#{content_numerical}:embedded" => { "1:embedded" => { "1:string" => object_inner_2_encoded, }, @@ -62,76 +69,16 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end +def make_initial_content_ctoken(ucid, content, sort_by) : String + return produce_channel_content_continuation(ucid, content, sort_by: sort_by) +end + # Used in bypass_captcha_job.cr def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) + continuation = produce_channel_content_continuation(ucid, "videos", page, auto_generated, sort_by, v2) return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end -def produce_channel_livestreams_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object_inner_2 = { - "2:0:embedded" => { - "1:0:varint" => 0_i64, - }, - "5:varint" => 50_i64, - "6:varint" => 1_i64, - "7:varint" => (page * 30).to_i64, - "9:varint" => 1_i64, - "10:varint" => 0_i64, - } - - object_inner_2_encoded = object_inner_2 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end - - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "14:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, - }, - }, - }, - } - - object_inner_1_encoded = object_inner_1 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => object_inner_1_encoded, - "35:string" => "browse-feed#{ucid}videos102", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - module Invidious::Channel::Tabs extend self @@ -139,10 +86,6 @@ module Invidious::Channel::Tabs # Regular videos # ------------------- - def make_initial_video_ctoken(ucid, sort_by) : String - return produce_channel_videos_continuation(ucid, sort_by: sort_by) - end - # Wrapper for AboutChannel, as we still need to call get_videos with # an author name and ucid directly (e.g in RSS feeds). # TODO: figure out how to get rid of that @@ -164,7 +107,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_video_ctoken(ucid, sort_by) + continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -208,13 +151,9 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def make_initial_livestream_ctoken(ucid, sort_by) : String - return produce_channel_livestreams_continuation(ucid, sort_by: sort_by) - end - def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") if continuation.nil? - continuation ||= make_initial_livestream_ctoken(channel.ucid, sort_by) + continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) end initial_data = YoutubeAPI.browse(continuation: continuation) From 162b89d9427ec4cbeb4661d7a7866847ba582880 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Thu, 23 Nov 2023 14:44:37 +0330 Subject: [PATCH 4/7] Fix format in videos.cr --- src/invidious/channels/videos.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 368ec2b8..80ec498c 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -18,9 +18,9 @@ def produce_channel_content_continuation(ucid, content, page = 1, auto_generated content_numerical = case content - when "videos" then 15 + when "videos" then 15 when "livestreams" then 14 - else 15 # Fallback to "videos" + else 15 # Fallback to "videos" end sort_by_numerical = From 6251d8d43f8b0491e4ff86fa8433b92a91b5fafd Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Sat, 25 Nov 2023 00:46:11 +0330 Subject: [PATCH 5/7] Rename a variable in videos.cr --- src/invidious/channels/videos.cr | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 80ec498c..cc683788 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,4 +1,4 @@ -def produce_channel_content_continuation(ucid, content, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) +def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) object_inner_2 = { "2:0:embedded" => { "1:0:varint" => 0_i64, @@ -16,8 +16,8 @@ def produce_channel_content_continuation(ucid, content, page = 1, auto_generated .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } - content_numerical = - case content + content_type_numerical = + case content_type when "videos" then 15 when "livestreams" then 14 else 15 # Fallback to "videos" @@ -34,7 +34,7 @@ def produce_channel_content_continuation(ucid, content, page = 1, auto_generated object_inner_1 = { "110:embedded" => { "3:embedded" => { - "#{content_numerical}:embedded" => { + "#{content_type_numerical}:embedded" => { "1:embedded" => { "1:string" => object_inner_2_encoded, }, @@ -69,8 +69,8 @@ def produce_channel_content_continuation(ucid, content, page = 1, auto_generated return continuation end -def make_initial_content_ctoken(ucid, content, sort_by) : String - return produce_channel_content_continuation(ucid, content, sort_by: sort_by) +def make_initial_content_ctoken(ucid, content_type, sort_by) : String + return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by) end # Used in bypass_captcha_job.cr From 5f2b43d6534fcbb12d9c66c4e99b16863bd29893 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Sat, 25 Nov 2023 00:48:27 +0330 Subject: [PATCH 6/7] Remove unecessary if condition in videos.cr --- src/invidious/channels/videos.cr | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index cc683788..02fc4d2f 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -152,9 +152,7 @@ module Invidious::Channel::Tabs # ------------------- def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") - if continuation.nil? - continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) - end + continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) From cf61af67abc1bffdeefd080542d11c2eae13d754 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Thu, 30 Nov 2023 14:34:01 +0330 Subject: [PATCH 7/7] Update src/invidious/routes/channels.cr sort_by for consistency --- src/invidious/routes/channels.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index c18644ec..b5b4ad96 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -81,12 +81,12 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation, sort_by: (sort_by || "newest") + channel, continuation: continuation, sort_by: sort_by ) selected_tab = Frontend::ChannelPage::TabsAvailable::Streams