From a82d21ff78787e24c331478c0fc71e6a8f4df980 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Fri, 21 Jan 2022 21:27:50 -0600 Subject: [PATCH 1/2] Cleanup channel helpers code --- src/invidious/channels/channels.cr | 24 ++++++++------------ src/invidious/database/channels.cr | 9 +------- src/invidious/jobs/refresh_channels_job.cr | 2 +- src/invidious/routes/api/v1/authenticated.cr | 2 +- src/invidious/routes/preferences.cr | 10 ++++---- src/invidious/routes/subscriptions.cr | 2 +- src/invidious/users.cr | 2 +- 7 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 155ec559..b9f762e7 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -114,8 +114,9 @@ class ChannelRedirect < Exception end end -def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_threads = 10) +def get_batch_channels(channels) finished_channel = Channel(String | Nil).new + max_threads = 10 spawn do active_threads = 0 @@ -130,7 +131,7 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th active_threads += 1 spawn do begin - get_channel(ucid, refresh, pull_all_videos) + get_channel(ucid) finished_channel.send(ucid) rescue ex finished_channel.send(nil) @@ -151,23 +152,18 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th return final end -def get_channel(id, refresh = true, pull_all_videos = true) - if channel = Invidious::Database::Channels.select(id) - if refresh && Time.utc - channel.updated > 10.minutes - channel = fetch_channel(id, pull_all_videos: pull_all_videos) - Invidious::Database::Channels.insert(channel, update_on_conflict: true) - end - else - channel = fetch_channel(id, pull_all_videos: pull_all_videos) - Invidious::Database::Channels.insert(channel) - end +def get_channel(id) + channel = Invidious::Database::Channels.select(id) + return channel if channel + channel = fetch_channel(id, pull_all_videos: false) + Invidious::Database::Channels.insert(channel) return channel end -def fetch_channel(ucid, pull_all_videos = true, locale = nil) +def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.debug("fetch_channel: #{ucid}") - LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") + LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}") LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr index 134cf59d..e88b4989 100644 --- a/src/invidious/database/channels.cr +++ b/src/invidious/database/channels.cr @@ -10,7 +10,7 @@ module Invidious::Database::Channels # Insert / delete # ------------------- - def insert(channel : InvidiousChannel, update_on_conflict : Bool = false) + def insert(channel : InvidiousChannel) channel_array = channel.to_a request = <<-SQL @@ -18,13 +18,6 @@ module Invidious::Database::Channels VALUES (#{arg_array(channel_array)}) SQL - if update_on_conflict - request += <<-SQL - ON CONFLICT (id) DO UPDATE - SET author = $2, updated = $3 - SQL - end - PG_DB.exec(request, args: channel_array) end diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 941089c1..55fb8154 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -30,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob spawn do begin LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") - channel = fetch_channel(id, CONFIG.full_refresh) + channel = fetch_channel(id, pull_all_videos: CONFIG.full_refresh) lim_fibers = max_fibers diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index fda655ef..4d0fe030 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -92,7 +92,7 @@ module Invidious::Routes::API::V1::Authenticated ucid = env.params.url["ucid"] if !user.subscriptions.includes? ucid - get_channel(ucid, false, false) + get_channel(ucid) Invidious::Database::Users.subscribe_channel(user, ucid) end diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index faae03bc..9c740cf2 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -327,7 +327,7 @@ module Invidious::Routes::PreferencesRoute user.subscriptions += body["subscriptions"].as_a.map(&.as_s) user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, false, false) + user.subscriptions = get_batch_channels(user.subscriptions) Invidious::Database::Users.update_subscriptions(user) end @@ -409,7 +409,7 @@ module Invidious::Routes::PreferencesRoute end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, false, false) + user.subscriptions = get_batch_channels(user.subscriptions) Invidious::Database::Users.update_subscriptions(user) when "import_freetube" @@ -418,7 +418,7 @@ module Invidious::Routes::PreferencesRoute end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, false, false) + user.subscriptions = get_batch_channels(user.subscriptions) Invidious::Database::Users.update_subscriptions(user) when "import_newpipe_subscriptions" @@ -437,7 +437,7 @@ module Invidious::Routes::PreferencesRoute end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, false, false) + user.subscriptions = get_batch_channels(user.subscriptions) Invidious::Database::Users.update_subscriptions(user) when "import_newpipe" @@ -456,7 +456,7 @@ module Invidious::Routes::PreferencesRoute user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, false, false) + user.subscriptions = get_batch_channels(user.subscriptions) Invidious::Database::Users.update_subscriptions(user) diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr index 29152afb..ec8fe67b 100644 --- a/src/invidious/routes/subscriptions.cr +++ b/src/invidious/routes/subscriptions.cr @@ -51,7 +51,7 @@ module Invidious::Routes::Subscriptions case action when "action_create_subscription_to_channel" if !user.subscriptions.includes? channel_id - get_channel(channel_id, false, false) + get_channel(channel_id) Invidious::Database::Users.subscribe_channel(user, channel_id) end when "action_remove_subscriptions" diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 49074994..a7ee72a9 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -74,7 +74,7 @@ def fetch_user(sid, headers) end end - channels = get_batch_channels(channels, false, false) + channels = get_batch_channels(channels) email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"])) if email From e92b3779ad6ac530b4979dfeccb66e96d75d14c9 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Mon, 24 Jan 2022 23:14:13 -0600 Subject: [PATCH 2/2] Add back in refreshing of channels every 2 days --- src/invidious/channels/channels.cr | 10 ++++++---- src/invidious/database/channels.cr | 9 ++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index b9f762e7..6905b6f8 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -152,12 +152,14 @@ def get_batch_channels(channels) return final end -def get_channel(id) +def get_channel(id) : InvidiousChannel channel = Invidious::Database::Channels.select(id) - return channel if channel - channel = fetch_channel(id, pull_all_videos: false) - Invidious::Database::Channels.insert(channel) + if channel.nil? || (Time.utc - channel.updated) > 2.days + channel = fetch_channel(id, pull_all_videos: false) + Invidious::Database::Channels.insert(channel, update_on_conflict: true) + end + return channel end diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr index e88b4989..134cf59d 100644 --- a/src/invidious/database/channels.cr +++ b/src/invidious/database/channels.cr @@ -10,7 +10,7 @@ module Invidious::Database::Channels # Insert / delete # ------------------- - def insert(channel : InvidiousChannel) + def insert(channel : InvidiousChannel, update_on_conflict : Bool = false) channel_array = channel.to_a request = <<-SQL @@ -18,6 +18,13 @@ module Invidious::Database::Channels VALUES (#{arg_array(channel_array)}) SQL + if update_on_conflict + request += <<-SQL + ON CONFLICT (id) DO UPDATE + SET author = $2, updated = $3 + SQL + end + PG_DB.exec(request, args: channel_array) end