From 998edba6f064eb4e09ca286ad33bfd967ef03e66 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 26 Nov 2021 19:36:31 +0100 Subject: [PATCH 01/16] Move DB queries related to 'videos' in a separate module --- src/invidious.cr | 3 ++- src/invidious/database/base.cr | 4 +++ src/invidious/database/videos.cr | 43 ++++++++++++++++++++++++++++++++ src/invidious/videos.cr | 10 +++----- 4 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 src/invidious/database/base.cr create mode 100644 src/invidious/database/videos.cr diff --git a/src/invidious.cr b/src/invidious.cr index ade13608..405fcadf 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -20,12 +20,13 @@ require "kemal" require "athena-negotiation" require "openssl/hmac" require "option_parser" -require "pg" require "sqlite3" require "xml" require "yaml" require "compress/zip" require "protodec/utils" + +require "./invidious/database/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/*" diff --git a/src/invidious/database/base.cr b/src/invidious/database/base.cr new file mode 100644 index 00000000..055a6284 --- /dev/null +++ b/src/invidious/database/base.cr @@ -0,0 +1,4 @@ +require "pg" + +module Invidious::Database +end diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr new file mode 100644 index 00000000..e1fa01c3 --- /dev/null +++ b/src/invidious/database/videos.cr @@ -0,0 +1,43 @@ +require "./base.cr" + +module Invidious::Database::Videos + extend self + + def insert(video : Video) + request = <<-SQL + INSERT INTO videos + VALUES ($1, $2, $3) + ON CONFLICT (id) DO NOTHING + SQL + + PG_DB.exec(request, video.id, video.info.to_json, video.updated) + end + + def delete(id) + request = <<-SQL + DELETE FROM videos * + WHERE id = $1 + SQL + + PG_DB.exec(request, id) + end + + def update(video : Video) + request = <<-SQL + UPDATE videos + SET (id, info, updated) = ($1, $2, $3) + WHERE id = $1 + SQL + + PG_DB.exec(request, video.id, video.info.to_json, video.updated) + end + + def select(id : String) : Video? + request = <<-SQL + SELECT * FROM videos + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: Video) + end +end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index d4ef0900..645d3678 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -994,7 +994,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ end def get_video(id, db, refresh = true, region = nil, force_refresh = false) - if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region + if (video = Invidious::Database::Videos.select(id)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) if (refresh && @@ -1003,17 +1003,15 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false) force_refresh begin video = fetch_video(id, region) - db.exec("UPDATE videos SET (id, info, updated) = ($1, $2, $3) WHERE id = $1", video.id, video.info.to_json, video.updated) + Invidious::Database::Videos.update(video) rescue ex - db.exec("DELETE FROM videos * WHERE id = $1", id) + Invidious::Database::Videos.delete(id) raise ex end end else video = fetch_video(id, region) - if !region - db.exec("INSERT INTO videos VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", video.id, video.info.to_json, video.updated) - end + Invidious::Database::Videos.insert(video) if !region end return video From 3deafe9f8da2805ab19900fbdfb4e90d0a2cea03 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 30 Nov 2021 02:24:24 +0100 Subject: [PATCH 02/16] Move DB queries related to playlists in a separate module (1/3) --- src/invidious.cr | 7 +- src/invidious/database/playlists.cr | 94 ++++++++++++++++++++ src/invidious/playlists.cr | 10 +-- src/invidious/routes/api/v1/authenticated.cr | 14 ++- src/invidious/routes/playlists.cr | 14 ++- 5 files changed, 108 insertions(+), 31 deletions(-) create mode 100644 src/invidious/database/playlists.cr diff --git a/src/invidious.cr b/src/invidious.cr index 405fcadf..28d8ddac 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -816,11 +816,8 @@ post "/data_control" do |env| index: Random::Secure.rand(0_i64..Int64::MAX), }) - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist.id) + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) end end end diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr new file mode 100644 index 00000000..037e25b7 --- /dev/null +++ b/src/invidious/database/playlists.cr @@ -0,0 +1,94 @@ +require "./base.cr" + +# +# This module contains functions related to the "playlists" table. +# +module Invidious::Database::Playlists + extend self + + # ------------------- + # Insert / delete + # ------------------- + + def insert(playlist : InvidiousPlaylist) + playlist_array = playlist.to_a + + request = <<-SQL + INSERT INTO playlists + VALUES (#{arg_array(playlist_array)}) + SQL + + PG_DB.exec(request, args: playlist_array) + end + + # this function is a bit special: it will also remove all videos + # related to the given playlist ID in the "playlist_videos" table, + # in addition to deleting said ID from "playlists". + def delete(id : String) + request = <<-SQL + DELETE FROM playlist_videos * WHERE plid = $1; + DELETE FROM playlists * WHERE id = $1 + SQL + + PG_DB.exec(request, id) + end + + # ------------------- + # Update + # ------------------- + + def update_video_added(id : String, index : String | Int64) + request = <<-SQL + UPDATE playlists + SET index = array_append(index, $1), + video_count = cardinality(index) + 1, + updated = $2 + WHERE id = $3 + SQL + + PG_DB.exec(request, index, Time.utc, id) + end + + def update_video_removed(id : String, index : String | Int64) + request = <<-SQL + UPDATE playlists + SET index = array_remove(index, $1), + video_count = cardinality(index) - 1, + updated = $2 + WHERE id = $3 + SQL + + PG_DB.exec(request, index, Time.utc, id) + end +end + +# +# This module contains functions related to the "playlist_videos" table. +# +module Invidious::Database::PlaylistVideos + extend self + + # ------------------- + # Insert / Delete + # ------------------- + + def insert(video : PlaylistVideo) + video_array = video.to_a + + request = <<-SQL + INSERT INTO playlist_videos + VALUES (#{arg_array(video_array)}) + SQL + + PG_DB.exec(request, args: video_array) + end + + def delete(index) + request = <<-SQL + DELETE FROM playlist_videos * + WHERE index = $1 + SQL + + PG_DB.exec(request, index) + end +end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index f37667b5..685fa1c7 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -261,10 +261,7 @@ def create_playlist(db, title, privacy, user) index: [] of Int64, }) - playlist_array = playlist.to_a - args = arg_array(playlist_array) - - db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array) + Invidious::Database::Playlists.insert(playlist) return playlist end @@ -282,10 +279,7 @@ def subscribe_playlist(db, user, playlist) index: [] of Int64, }) - playlist_array = playlist.to_a - args = arg_array(playlist_array) - - db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array) + Invidious::Database::Playlists.insert(playlist) return playlist end diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index aaf728ff..4fe8cd30 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -216,8 +216,7 @@ module Invidious::Routes::API::V1::Authenticated return error_json(403, "Invalid user") end - PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) - PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + Invidious::Database::Playlists.delete(plid) env.response.status_code = 204 end @@ -266,11 +265,8 @@ module Invidious::Routes::API::V1::Authenticated index: Random::Secure.rand(0_i64..Int64::MAX), }) - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, plid) + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(plid, playlist_video.index) env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}" env.response.status_code = 201 @@ -302,8 +298,8 @@ module Invidious::Routes::API::V1::Authenticated return error_json(404, "Playlist does not contain index") end - PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) - PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, plid) + Invidious::Database::PlaylistVideos.delete(index) + Invidious::Database::Playlists.update_video_removed(plid, index) env.response.status_code = 204 end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 7b7bd03f..d29aef09 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -122,8 +122,7 @@ module Invidious::Routes::Playlists return env.redirect referer end - PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) - PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + Invidious::Database::Playlists.delete(plid) env.redirect "/feed/playlists" end @@ -363,15 +362,12 @@ module Invidious::Routes::Playlists index: Random::Secure.rand(0_i64..Int64::MAX), }) - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id) + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index) when "action_remove_video" index = env.params.query["set_video_id"] - PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) - PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id) + Invidious::Database::PlaylistVideos.delete(index) + Invidious::Database::Playlists.update_video_removed(playlist_id, index) when "action_move_video_before" # TODO: Playlist stub else From 46d08237c6979912275b416a9294a807e5598bc5 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 30 Nov 2021 03:11:21 +0100 Subject: [PATCH 03/16] Move DB queries related to playlists in a separate module (2/3) --- src/invidious.cr | 2 +- src/invidious/database/playlists.cr | 80 ++++++++++++++++++++ src/invidious/playlists.cr | 2 +- src/invidious/routes/api/v1/authenticated.cr | 20 +++-- src/invidious/routes/feeds.cr | 4 +- src/invidious/routes/playlists.cr | 18 +++-- src/invidious/views/playlist.ecr | 2 +- 7 files changed, 108 insertions(+), 20 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 28d8ddac..93b3357a 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -790,7 +790,7 @@ post "/data_control" do |env| next if !privacy playlist = create_playlist(PG_DB, title, privacy, user) - PG_DB.exec("UPDATE playlists SET description = $1 WHERE id = $2", description, playlist.id) + Invidious::Database::Playlists.update_description(playlist.id, description) videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr index 037e25b7..1dba64f3 100644 --- a/src/invidious/database/playlists.cr +++ b/src/invidious/database/playlists.cr @@ -37,6 +37,36 @@ module Invidious::Database::Playlists # Update # ------------------- + def update(id : String, title : String, privacy, description, updated) + request = <<-SQL + UPDATE playlists + SET title = $1, privacy = $2, description = $3, updated = $4 + WHERE id = $5 + SQL + + PG_DB.exec(request, title, privacy, description, updated, id) + end + + def update_description(id : String, description) + request = <<-SQL + UPDATE playlists + SET description = $1 + WHERE id = $2 + SQL + + PG_DB.exec(request, description, id) + end + + def update_subscription_time(id : String) + request = <<-SQL + UPDATE playlists + SET subscribed = $1 + WHERE id = $2 + SQL + + PG_DB.exec(request, Time.utc, id) + end + def update_video_added(id : String, index : String | Int64) request = <<-SQL UPDATE playlists @@ -60,6 +90,56 @@ module Invidious::Database::Playlists PG_DB.exec(request, index, Time.utc, id) end + + # ------------------- + # Salect + # ------------------- + + def select(*, id : String, raise_on_fail : Bool = false) : InvidiousPlaylist? + request = <<-SQL + SELECT * FROM playlists + WHERE id = $1 + SQL + + if raise_on_fail + return PG_DB.query_one(request, id, as: InvidiousPlaylist) + else + return PG_DB.query_one?(request, id, as: InvidiousPlaylist) + end + end + + def select_all(*, author : String) : Array(InvidiousPlaylist) + request = <<-SQL + SELECT * FROM playlists + WHERE author = $1 + SQL + + return PG_DB.query_all(request, author, as: InvidiousPlaylist) + end + + # ------------------- + # Misc checks + # ------------------- + + # Check if given playlist ID exists + def exists?(id : String) : Bool + request = <<-SQL + SELECT id FROM playlists + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: String).nil? + end + + # Count how many playlist a user has created. + def count_owned_by(author : String) : Int64 + request = <<-SQL + SELECT count(*) FROM playlists + WHERE author = $1 + SQL + + return PG_DB.query_one?(request, author, as: Int64) || 0_i64 + end end # diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 685fa1c7..f68dc3b0 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -323,7 +323,7 @@ end def get_playlist(db, plid, locale, refresh = true, force_refresh = false) if plid.starts_with? "IV" - if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if playlist = Invidious::Database::Playlists.select(id: plid) return playlist else raise InfoException.new("Playlist does not exist.") diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 4fe8cd30..d74dca5c 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "application/json" user = env.get("user").as(User) - playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousPlaylist) + playlists = Invidious::Database::Playlists.select_all(author: user.email) JSON.build do |json| json.array do @@ -153,7 +153,7 @@ module Invidious::Routes::API::V1::Authenticated return error_json(400, "Invalid privacy setting.") end - if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 + if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 return error_json(400, "User cannot have more than 100 playlists.") end @@ -172,9 +172,12 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "application/json" user = env.get("user").as(User) - plid = env.params.url["plid"] + plid = env.params.url["plid"]? + if !plid || plid.empty? + return error_json(400, "A playlist ID is required") + end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -195,7 +198,8 @@ module Invidious::Routes::API::V1::Authenticated updated = playlist.updated end - PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + Invidious::Database::Playlists.update(plid, title, privacy, description, updated) + env.response.status_code = 204 end @@ -207,7 +211,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -229,7 +233,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -285,7 +289,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] index = env.params.url["index"].to_i64(16) - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 9650bcf4..6424ab47 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -264,7 +264,7 @@ module Invidious::Routes::Feeds path = env.request.path if plid.starts_with? "IV" - if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if playlist = Invidious::Database::Playlists.select(id: plid) videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) return XML.build(indent: " ", encoding: "UTF-8") do |xml| @@ -364,7 +364,7 @@ module Invidious::Routes::Feeds if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]? PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid) elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]? - PG_DB.exec("UPDATE playlists SET subscribed = $1 WHERE id = $2", Time.utc, ucid) + Invidious::Database::Playlists.update_subscription_time(plid) else haltf env, status_code: 400 end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index d29aef09..b73782d5 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -46,7 +46,7 @@ module Invidious::Routes::Playlists return error_template(400, "Invalid privacy setting.") end - if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 + if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 return error_template(400, "User cannot have more than 100 playlists.") end @@ -85,7 +85,11 @@ module Invidious::Routes::Playlists sid = sid.as(String) plid = env.params.query["list"]? - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !plid || plid.empty? + return error_template(400, "A playlist ID is required") + end + + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email return env.redirect referer end @@ -117,7 +121,7 @@ module Invidious::Routes::Playlists return error_template(400, ex) end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email return env.redirect referer end @@ -148,7 +152,7 @@ module Invidious::Routes::Playlists page ||= 1 begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true) if !playlist || playlist.author != user.email return env.redirect referer end @@ -189,7 +193,7 @@ module Invidious::Routes::Playlists return error_template(400, ex) end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email return env.redirect referer end @@ -206,7 +210,7 @@ module Invidious::Routes::Playlists updated = playlist.updated end - PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + Invidious::Database::Playlists.update(plid, title, privacy, description, updated) env.redirect "/playlist?list=#{plid}" end @@ -232,7 +236,7 @@ module Invidious::Routes::Playlists page ||= 1 begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true) if !playlist || playlist.author != user.email return env.redirect referer end diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index 136981da..7825b1f0 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -61,7 +61,7 @@
<% else %> - <% if PG_DB.query_one?("SELECT id FROM playlists WHERE id = $1", playlist.id, as: String).nil? %> + <% if Invidious::Database::Playlists.exists?(playlist.id) %>
<% else %>
From d94d4c204548eff69ff7c310782c291c89c83bb2 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 1 Dec 2021 22:05:21 +0100 Subject: [PATCH 04/16] Move DB queries related to statistics in a separate module --- src/invidious/database/statistics.cr | 49 ++++++++++++++++++++ src/invidious/jobs/statistics_refresh_job.cr | 10 ++-- 2 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 src/invidious/database/statistics.cr diff --git a/src/invidious/database/statistics.cr b/src/invidious/database/statistics.cr new file mode 100644 index 00000000..1df549e2 --- /dev/null +++ b/src/invidious/database/statistics.cr @@ -0,0 +1,49 @@ +require "./base.cr" + +module Invidious::Database::Statistics + extend self + + # ------------------- + # User stats + # ------------------- + + def count_users_total : Int64 + request = <<-SQL + SELECT count(*) FROM users + SQL + + PG_DB.query_one(request, as: Int64) + end + + def count_users_active_1m : Int64 + request = <<-SQL + SELECT count(*) FROM users + WHERE CURRENT_TIMESTAMP - updated < '6 months' + SQL + + PG_DB.query_one(request, as: Int64) + end + + def count_users_active_6m : Int64 + request = <<-SQL + SELECT count(*) FROM users + WHERE CURRENT_TIMESTAMP - updated < '1 month' + SQL + + PG_DB.query_one(request, as: Int64) + end + + # ------------------- + # Channel stats + # ------------------- + + def channel_last_update : Time? + request = <<-SQL + SELECT updated FROM channels + ORDER BY updated DESC + LIMIT 1 + SQL + + PG_DB.query_one?(request, as: Time) + end +end diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index 6569c0a1..a113bd77 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -47,12 +47,14 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob private def refresh_stats users = STATISTICS.dig("usage", "users").as(Hash(String, Int64)) - users["total"] = db.query_one("SELECT count(*) FROM users", as: Int64) - users["activeHalfyear"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64) - users["activeMonth"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64) + + users["total"] = Invidious::Database::Statistics.count_users_total + users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m + users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m + STATISTICS["metadata"] = { "updatedAt" => Time.utc.to_unix, - "lastChannelRefreshedAt" => db.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64, + "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64, } end end From c021b93b5c7d38504b9cf40307d89b81241adfd9 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 2 Dec 2021 19:16:41 +0100 Subject: [PATCH 05/16] Move DB queries related to channels in a separate module --- src/invidious.cr | 8 +- src/invidious/channels/channels.cr | 23 +-- src/invidious/database/channels.cr | 149 ++++++++++++++++++ src/invidious/helpers/helpers.cr | 5 +- src/invidious/jobs/pull_popular_videos_job.cr | 9 +- src/invidious/jobs/refresh_channels_job.cr | 4 +- src/invidious/routes/api/v1/authenticated.cr | 8 +- src/invidious/routes/feeds.cr | 5 +- src/invidious/users.cr | 5 +- 9 files changed, 164 insertions(+), 52 deletions(-) create mode 100644 src/invidious/database/channels.cr diff --git a/src/invidious.cr b/src/invidious.cr index 93b3357a..97809160 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -649,13 +649,7 @@ get "/subscription_manager" do |env| format = env.params.query["format"]? format ||= "rss" - if user.subscriptions.empty? - values = "'{}'" - else - values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" - end - - subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) + subscriptions = Invidious::Database::Channels.select(user.subscriptions) subscriptions.sort_by!(&.author.downcase) if action_takeout diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 827b6534..5d962ab4 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -152,21 +152,14 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma end def get_channel(id, db, refresh = true, pull_all_videos = true) - if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) + if channel = Invidious::Database::Channels.select(id) if refresh && Time.utc - channel.updated > 10.minutes channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args}) \ - ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array) + Invidious::Database::Channels.insert(channel, update_on_conflict: true) end else channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args})", args: channel_array) + Invidious::Database::Channels.insert(channel) end return channel @@ -241,10 +234,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) # We don't include the 'premiere_timestamp' here because channel pages don't include them, # meaning the above timestamp is always null - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) + was_insert = Invidious::Database::ChannelVideos.insert(video) if was_insert LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") @@ -284,10 +274,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # so since they don't provide a published date here we can safely ignore them. if Time.utc - video.published > 1.minute - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) + was_insert = Invidious::Database::ChannelVideos.insert(video) db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr new file mode 100644 index 00000000..134cf59d --- /dev/null +++ b/src/invidious/database/channels.cr @@ -0,0 +1,149 @@ +require "./base.cr" + +# +# This module contains functions related to the "channels" table. +# +module Invidious::Database::Channels + extend self + + # ------------------- + # Insert / delete + # ------------------- + + def insert(channel : InvidiousChannel, update_on_conflict : Bool = false) + channel_array = channel.to_a + + request = <<-SQL + INSERT INTO channels + VALUES (#{arg_array(channel_array)}) + SQL + + if update_on_conflict + request += <<-SQL + ON CONFLICT (id) DO UPDATE + SET author = $2, updated = $3 + SQL + end + + PG_DB.exec(request, args: channel_array) + end + + # ------------------- + # Update + # ------------------- + + def update_author(id : String, author : String) + request = <<-SQL + UPDATE channels + SET updated = $1, author = $2, deleted = false + WHERE id = $3 + SQL + + PG_DB.exec(request, Time.utc, author, id) + end + + def update_mark_deleted(id : String) + request = <<-SQL + UPDATE channels + SET updated = $1, deleted = true + WHERE id = $2 + SQL + + PG_DB.exec(request, Time.utc, id) + end + + # ------------------- + # Select + # ------------------- + + def select(id : String) : InvidiousChannel? + request = <<-SQL + SELECT * FROM channels + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: InvidiousChannel) + end + + def select(ids : Array(String)) : Array(InvidiousChannel)? + return [] of InvidiousChannel if ids.empty? + values = ids.map { |id| %(('#{id}')) }.join(",") + + request = <<-SQL + SELECT * FROM channels + WHERE id = ANY(VALUES #{values}) + SQL + + return PG_DB.query_all(request, as: InvidiousChannel) + end +end + +# +# This module contains functions related to the "channel_videos" table. +# +module Invidious::Database::ChannelVideos + extend self + + # ------------------- + # Insert + # ------------------- + + # This function returns the status of the query (i.e: success?) + def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool + if with_premiere_timestamp + last_items = "premiere_timestamp = $9, views = $10" + else + last_items = "views = $10" + end + + request = <<-SQL + INSERT INTO channel_videos + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (id) DO UPDATE + SET title = $2, published = $3, updated = $4, ucid = $5, + author = $6, length_seconds = $7, live_now = $8, #{last_items} + RETURNING (xmax=0) AS was_insert + SQL + + return PG_DB.query_one(request, *video.to_tuple, as: Bool) + end + + # ------------------- + # Select + # ------------------- + + def select(ids : Array(String)) : Array(ChannelVideo) + return [] of ChannelVideo if ids.empty? + + request = <<-SQL + SELECT * FROM channel_videos + WHERE id IN (#{arg_array(ids)}) + ORDER BY published DESC + SQL + + return PG_DB.query_all(request, args: ids, as: ChannelVideo) + end + + def select_notfications(ucid : String, since : Time) : Array(ChannelVideo) + request = <<-SQL + SELECT * FROM channel_videos + WHERE ucid = $1 AND published > $2 + ORDER BY published DESC + LIMIT 15 + SQL + + return PG_DB.query_all(request, ucid, since, as: ChannelVideo) + end + + def select_popular_videos : Array(ChannelVideo) + request = <<-SQL + SELECT DISTINCT ON (ucid) * + FROM channel_videos + WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d + GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) + ORDER BY ucid, published DESC + SQL + + PG_DB.query_all(request, as: ChannelVideo) + end +end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 96a78eb9..014c04a8 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -235,11 +235,12 @@ def create_notification_stream(env, topics, connection_channel) spawn do begin if since + since_unix = Time.unix(since.not_nil!) + topics.try &.each do |topic| case topic when .match(/UC[A-Za-z0-9_-]{22}/) - PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", - topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| + Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) if fields_text = env.params.query["fields"]? diff --git a/src/invidious/jobs/pull_popular_videos_job.cr b/src/invidious/jobs/pull_popular_videos_job.cr index 38de816e..dc785bae 100644 --- a/src/invidious/jobs/pull_popular_videos_job.cr +++ b/src/invidious/jobs/pull_popular_videos_job.cr @@ -1,11 +1,4 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob - QUERY = <<-SQL - SELECT DISTINCT ON (ucid) * - FROM channel_videos - WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d - GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) - ORDER BY ucid, published DESC - SQL POPULAR_VIDEOS = Atomic.new([] of ChannelVideo) private getter db : DB::Database @@ -14,7 +7,7 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob def begin loop do - videos = db.query_all(QUERY, as: ChannelVideo) + videos = Invidious::Database::ChannelVideos.select_popular_videos .sort_by!(&.published) .reverse! diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 2321e964..c224c745 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -35,11 +35,11 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob lim_fibers = max_fibers LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB") - db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id) + Invidious::Database::Channels.update_author(id, channel.author) rescue ex LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}") if ex.message == "Deleted or invalid channel" - db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id) + Invidious::Database::Channels.update_mark_deleted(id) else lim_fibers = 1 LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s") diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index d74dca5c..a3ac2add 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -72,13 +72,7 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "application/json" user = env.get("user").as(User) - if user.subscriptions.empty? - values = "'{}'" - else - values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" - end - - subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) + subscriptions = Invidious::Database::Channels.select(user.subscriptions) JSON.build do |json| json.array do diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 6424ab47..78e6bd40 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -416,10 +416,7 @@ module Invidious::Routes::Feeds views: video.views, }) - was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, - updated = $4, ucid = $5, author = $6, length_seconds = $7, - live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) + was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true) PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 584082be..92143437 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -242,10 +242,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) if user.preferences.notifications_only && !notifications.empty? # Only show notifications - - args = arg_array(notifications) - - notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: ChannelVideo) + notifications = Invidious::Database::ChannelVideos.select(notifications) videos = [] of ChannelVideo notifications.sort_by!(&.published).reverse! From 92eea3b18b406e7eb86e1bd95dfaf9078f49ed72 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 2 Dec 2021 23:57:13 +0100 Subject: [PATCH 06/16] Move DB queries related to session tokens in a separate module --- src/invidious.cr | 10 +-- src/invidious/database/nonces.cr | 46 ++++++++++++ src/invidious/database/sessions.cr | 74 ++++++++++++++++++++ src/invidious/helpers/handlers.cr | 4 +- src/invidious/helpers/tokens.cr | 8 +-- src/invidious/routes/api/v1/authenticated.cr | 6 +- src/invidious/routes/login.cr | 6 +- src/invidious/users.cr | 8 +-- 8 files changed, 140 insertions(+), 22 deletions(-) create mode 100644 src/invidious/database/nonces.cr create mode 100644 src/invidious/database/sessions.cr diff --git a/src/invidious.cr b/src/invidious.cr index 97809160..94620a26 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -247,7 +247,7 @@ before_all do |env| # Invidious users only have SID if !env.request.cookies.has_key? "SSID" - if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) + if email = Invidious::Database::SessionIDs.select_email(sid) user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) csrf_token = generate_response(sid, { ":authorize_token", @@ -633,6 +633,7 @@ get "/subscription_manager" do |env| end user = user.as(User) + sid = sid.as(String) if !user.password # Refresh account @@ -1008,7 +1009,7 @@ post "/delete_account" do |env| view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("DELETE FROM users * WHERE email = $1", user.email) - PG_DB.exec("DELETE FROM session_ids * WHERE email = $1", user.email) + Invidious::Database::SessionIDs.delete(email: user.email) PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") env.request.cookies.each do |cookie| @@ -1150,8 +1151,7 @@ get "/token_manager" do |env| end user = user.as(User) - - tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1 ORDER BY issued DESC", user.email, as: {session: String, issued: Time}) + tokens = Invidious::Database::SessionIDs.select_all(user.email) templated "token_manager" end @@ -1200,7 +1200,7 @@ post "/token_ajax" do |env| case action when .starts_with? "action_revoke_token" - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1 AND email = $2", session, user.email) + Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else next error_json(400, "Unsupported action #{action}") end diff --git a/src/invidious/database/nonces.cr b/src/invidious/database/nonces.cr new file mode 100644 index 00000000..469fcbd8 --- /dev/null +++ b/src/invidious/database/nonces.cr @@ -0,0 +1,46 @@ +require "./base.cr" + +module Invidious::Database::Nonces + extend self + + # ------------------- + # Insert + # ------------------- + + def insert(nonce : String, expire : Time) + request = <<-SQL + INSERT INTO nonces + VALUES ($1, $2) + ON CONFLICT DO NOTHING + SQL + + PG_DB.exec(request, nonce, expire) + end + + # ------------------- + # Update + # ------------------- + + def update_set_expired(nonce : String) + request = <<-SQL + UPDATE nonces + SET expire = $1 + WHERE nonce = $2 + SQL + + PG_DB.exec(request, Time.utc(1990, 1, 1), nonce) + end + + # ------------------- + # Select + # ------------------- + + def select(nonce : String) : Tuple(String, Time)? + request = <<-SQL + SELECT * FROM nonces + WHERE nonce = $1 + SQL + + return PG_DB.query_one?(request, nonce, as: {String, Time}) + end +end diff --git a/src/invidious/database/sessions.cr b/src/invidious/database/sessions.cr new file mode 100644 index 00000000..d5f85dd6 --- /dev/null +++ b/src/invidious/database/sessions.cr @@ -0,0 +1,74 @@ +require "./base.cr" + +module Invidious::Database::SessionIDs + extend self + + # ------------------- + # Insert + # ------------------- + + def insert(sid : String, email : String, handle_conflicts : Bool = false) + request = <<-SQL + INSERT INTO session_ids + VALUES ($1, $2, $3) + SQL + + request += " ON CONFLICT (id) DO NOTHING" if handle_conflicts + + PG_DB.exec(request, sid, email, Time.utc) + end + + # ------------------- + # Delete + # ------------------- + + def delete(*, sid : String) + request = <<-SQL + DELETE FROM session_ids * + WHERE id = $1 + SQL + + PG_DB.exec(request, sid) + end + + def delete(*, email : String) + request = <<-SQL + DELETE FROM session_ids * + WHERE email = $1 + SQL + + PG_DB.exec(request, email) + end + + def delete(*, sid : String, email : String) + request = <<-SQL + DELETE FROM session_ids * + WHERE id = $1 AND email = $2 + SQL + + PG_DB.exec(request, sid, email) + end + + # ------------------- + # Select + # ------------------- + + def select_email(sid : String) : String? + request = <<-SQL + SELECT email FROM session_ids + WHERE id = $1 + SQL + + PG_DB.query_one?(request, sid, as: String) + end + + def select_all(email : String) : Array({session: String, issued: Time}) + request = <<-SQL + SELECT id, issued FROM session_ids + WHERE email = $1 + ORDER BY issued DESC + SQL + + PG_DB.query_all(request, email, as: {session: String, issued: Time}) + end +end diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 045b6701..0aa86e64 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -99,7 +99,7 @@ class AuthHandler < Kemal::Handler session = URI.decode_www_form(token["session"].as_s) scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil) - if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", session, as: String) + if email = Invidious::Database::SessionIDs.select_email(session) user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) end elsif sid = env.request.cookies["SID"]?.try &.value @@ -107,7 +107,7 @@ class AuthHandler < Kemal::Handler raise "Cannot use token as SID" end - if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) + if email = Invidious::Database::SessionIDs.select_email(sid) user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) end diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index 3874799a..91405822 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -2,7 +2,7 @@ require "crypto/subtle" def generate_token(email, scopes, expire, key, db) session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}" - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc) + Invidious::Database::SessionIDs.insert(session, email) token = { "session" => session, @@ -30,7 +30,7 @@ def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = fa if use_nonce nonce = Random::Secure.hex(16) - db.exec("INSERT INTO nonces VALUES ($1, $2) ON CONFLICT DO NOTHING", nonce, expire) + Invidious::Database::Nonces.insert(nonce, expire) token["nonce"] = nonce end @@ -92,9 +92,9 @@ def validate_request(token, session, request, key, db, locale = nil) raise InfoException.new("Invalid signature") end - if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time})) + if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s)) if nonce[1] > Time.utc - db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0]) + Invidious::Database::Nonces.update_set_expired(nonce[0]) else raise InfoException.new("Erroneous token") end diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index a3ac2add..c95007c2 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -312,7 +312,7 @@ module Invidious::Routes::API::V1::Authenticated user = env.get("user").as(User) scopes = env.get("scopes").as(Array(String)) - tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1", user.email, as: {session: String, issued: Time}) + tokens = Invidious::Database::SessionIDs.select_all(user.email) JSON.build do |json| json.array do @@ -400,9 +400,9 @@ module Invidious::Routes::API::V1::Authenticated # Allow tokens to revoke other tokens with correct scope if session == env.get("session").as(String) - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) + Invidious::Database::SessionIDs.delete(sid: session) elsif scopes_include_scope(scopes, "GET:tokens") - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) + Invidious::Database::SessionIDs.delete(sid: session) else return error_json(400, "Cannot revoke session #{session}") end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 2a50561d..e70206cc 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -336,7 +336,7 @@ module Invidious::Routes::Login if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + Invidious::Database::SessionIDs.insert(sid, email) if Kemal.config.ssl || CONFIG.https_only secure = true @@ -455,7 +455,7 @@ module Invidious::Routes::Login args = arg_array(user_array) PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + Invidious::Database::SessionIDs.insert(sid, email) view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") @@ -511,7 +511,7 @@ module Invidious::Routes::Login return error_template(400, ex) end - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid) + Invidious::Database::SessionIDs.delete(sid: sid) env.request.cookies.each do |cookie| cookie.expires = Time.utc(1990, 1, 1) diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 92143437..3e9a9e68 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -30,7 +30,7 @@ struct User end def get_user(sid, headers, db, refresh = true) - if email = db.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) + if email = Invidious::Database::SessionIDs.select_email(sid) user = db.query_one("SELECT * FROM users WHERE email = $1", email, as: User) if refresh && Time.utc - user.updated > 1.minute @@ -42,8 +42,7 @@ def get_user(sid, headers, db, refresh = true) db.exec("INSERT INTO users VALUES (#{args}) \ ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) - db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ - ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) + Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin view_name = "subscriptions_#{sha256(user.email)}" @@ -60,8 +59,7 @@ def get_user(sid, headers, db, refresh = true) db.exec("INSERT INTO users VALUES (#{args}) \ ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) - db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ - ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) + Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin view_name = "subscriptions_#{sha256(user.email)}" From 094f83564297257a956d2e42d3b70adfa78b3185 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 3 Dec 2021 02:27:51 +0100 Subject: [PATCH 07/16] Move DB queries related to 'users' in a separate module (1/2) --- src/invidious.cr | 15 +-- src/invidious/database/users.cr | 129 +++++++++++++++++++ src/invidious/helpers/handlers.cr | 4 +- src/invidious/routes/api/v1/authenticated.cr | 4 +- src/invidious/routes/feeds.cr | 2 +- src/invidious/routes/login.cr | 9 +- src/invidious/routes/watch.cr | 2 +- src/invidious/users.cr | 16 +-- 8 files changed, 147 insertions(+), 34 deletions(-) create mode 100644 src/invidious/database/users.cr diff --git a/src/invidious.cr b/src/invidious.cr index 94620a26..91f19d69 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -248,7 +248,7 @@ before_all do |env| # Invidious users only have SID if !env.request.cookies.has_key? "SSID" if email = Invidious::Database::SessionIDs.select_email(sid) - user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + user = Invidious::Database::Users.select!(email: email) csrf_token = generate_response(sid, { ":authorize_token", ":playlist_ajax", @@ -458,10 +458,10 @@ post "/watch_ajax" do |env| case action when "action_mark_watched" if !user.watched.includes? id - PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.email) + Invidious::Database::Users.mark_watched(user, id) end when "action_mark_unwatched" - PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email) + Invidious::Database::Users.mark_unwatched(user, id) else next error_json(400, "Unsupported action #{action}") end @@ -599,16 +599,15 @@ post "/subscription_ajax" do |env| # Sync subscriptions with YouTube subscribe_ajax(channel_id, action, env.request.headers) end - email = user.email case action when "action_create_subscription_to_channel" if !user.subscriptions.includes? channel_id get_channel(channel_id, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email) + Invidious::Database::Users.subscribe_channel(user, channel_id) end when "action_remove_subscriptions" - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) + Invidious::Database::Users.unsubscribe_channel(user, channel_id) else next error_json(400, "Unsupported action #{action}") end @@ -1008,7 +1007,7 @@ post "/delete_account" do |env| end view_name = "subscriptions_#{sha256(user.email)}" - PG_DB.exec("DELETE FROM users * WHERE email = $1", user.email) + Invidious::Database::Users.delete(user) Invidious::Database::SessionIDs.delete(email: user.email) PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") @@ -1059,7 +1058,7 @@ post "/clear_watch_history" do |env| next error_template(400, ex) end - PG_DB.exec("UPDATE users SET watched = '{}' WHERE email = $1", user.email) + Invidious::Database::Users.clear_watch_history(user) env.redirect referer end diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr new file mode 100644 index 00000000..aa3b9f85 --- /dev/null +++ b/src/invidious/database/users.cr @@ -0,0 +1,129 @@ +require "./base.cr" + +module Invidious::Database::Users + extend self + + # ------------------- + # Insert / delete + # ------------------- + + def insert(user : User, update_on_conflict : Bool = false) + user_array = user.to_a + user_array[4] = user_array[4].to_json # User preferences + + request = <<-SQL + INSERT INTO users + VALUES (#{arg_array(user_array)}) + SQL + + if update_on_conflict + request += <<-SQL + ON CONFLICT (email) DO UPDATE + SET updated = $1, subscriptions = $3 + SQL + end + + PG_DB.exec(request, args: user_array) + end + + def delete(user : User) + request = <<-SQL + DELETE FROM users * + WHERE email = $1 + SQL + + PG_DB.exec(request, user.email) + end + + # ------------------- + # Update (history) + # ------------------- + + def mark_watched(user : User, vid : String) + request = <<-SQL + UPDATE users + SET watched = array_append(watched, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, vid, user.email) + end + + def mark_unwatched(user : User, vid : String) + request = <<-SQL + UPDATE users + SET watched = array_remove(watched, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, vid, user.email) + end + + def clear_watch_history(user : User) + request = <<-SQL + UPDATE users + SET watched = '{}' + WHERE email = $1 + SQL + + PG_DB.exec(request, user.email) + end + + # ------------------- + # Update (channels) + # ------------------- + + def subscribe_channel(user : User, ucid : String) + request = <<-SQL + UPDATE users + SET feed_needs_update = true, + subscriptions = array_append(subscriptions,$1) + WHERE email = $2 + SQL + + PG_DB.exec(request, ucid, user.email) + end + + def unsubscribe_channel(user : User, ucid : String) + request = <<-SQL + UPDATE users + SET feed_needs_update = true, + subscriptions = array_remove(subscriptions, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, ucid, user.email) + end + + # ------------------- + # Select + # ------------------- + + def select(*, email : String) : User? + request = <<-SQL + SELECT * FROM users + WHERE email = $1 + SQL + + return PG_DB.query_one?(request, email, as: User) + end + + # Same as select, but can raise an exception + def select!(*, email : String) : User + request = <<-SQL + SELECT * FROM users + WHERE email = $1 + SQL + + return PG_DB.query_one(request, email, as: User) + end + + def select(*, token : String) : User? + request = <<-SQL + SELECT * FROM users + WHERE token = $1 + SQL + + return PG_DB.query_one?(request, token, as: User) + end +end diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 0aa86e64..d52035c7 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -100,7 +100,7 @@ class AuthHandler < Kemal::Handler scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil) if email = Invidious::Database::SessionIDs.select_email(session) - user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + user = Invidious::Database::Users.select!(email: email) end elsif sid = env.request.cookies["SID"]?.try &.value if sid.starts_with? "v1:" @@ -108,7 +108,7 @@ class AuthHandler < Kemal::Handler end if email = Invidious::Database::SessionIDs.select_email(sid) - user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + user = Invidious::Database::Users.select!(email: email) end scopes = [":*"] diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index c95007c2..d9b58ebf 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -94,7 +94,7 @@ module Invidious::Routes::API::V1::Authenticated if !user.subscriptions.includes? ucid get_channel(ucid, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) + Invidious::Database::Users.subscribe_channel(user, ucid) end # For Google accounts, access tokens don't have enough information to @@ -110,7 +110,7 @@ module Invidious::Routes::API::V1::Authenticated ucid = env.params.url["ucid"] - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", ucid, user.email) + Invidious::Database::Users.unsubscribe_channel(user, ucid) env.response.status_code = 204 end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 78e6bd40..4e7ec9ad 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -220,7 +220,7 @@ module Invidious::Routes::Feeds haltf env, status_code: 403 end - user = PG_DB.query_one?("SELECT * FROM users WHERE token = $1", token.strip, as: User) + user = Invidious::Database::Users.select(token: token.strip) if !user haltf env, status_code: 403 end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index e70206cc..8f703464 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -327,7 +327,7 @@ module Invidious::Routes::Login return error_template(401, "Password is a required field") end - user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1", email, as: User) + user = Invidious::Database::Users.select(email: email) if user if !user.password @@ -449,12 +449,7 @@ module Invidious::Routes::Login end end - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - - args = arg_array(user_array) - - PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) + Invidious::Database::Users.insert(user) Invidious::Database::SessionIDs.insert(sid, email) view_name = "subscriptions_#{sha256(user.email)}" diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index b24222ff..c1ec0bc6 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -76,7 +76,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("iv_load_policy") if watched && !watched.includes? id - PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email) + Invidious::Database::Users.mark_watched(user.as(User), id) end if notifications && notifications.includes? id diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 3e9a9e68..933c451d 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -31,17 +31,12 @@ end def get_user(sid, headers, db, refresh = true) if email = Invidious::Database::SessionIDs.select_email(sid) - user = db.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + user = Invidious::Database::Users.select!(email: email) if refresh && Time.utc - user.updated > 1.minute user, sid = fetch_user(sid, headers, db) - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - args = arg_array(user_array) - - db.exec("INSERT INTO users VALUES (#{args}) \ - ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) + Invidious::Database::Users.insert(user, update_on_conflict: true) Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin @@ -52,13 +47,8 @@ def get_user(sid, headers, db, refresh = true) end else user, sid = fetch_user(sid, headers, db) - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - args = arg_array(user.to_a) - - db.exec("INSERT INTO users VALUES (#{args}) \ - ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) + Invidious::Database::Users.insert(user, update_on_conflict: true) Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin From 7691f5352025d7b2158ebae73417dd8619baea32 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 3 Dec 2021 03:29:52 +0100 Subject: [PATCH 08/16] Move DB queries related to 'users' in a separate module (2/2) --- src/invidious.cr | 18 ++-- src/invidious/channels/channels.cr | 7 +- src/invidious/database/users.cr | 89 ++++++++++++++++++++ src/invidious/routes/api/v1/authenticated.cr | 5 +- src/invidious/routes/embed.cr | 2 +- src/invidious/routes/feeds.cr | 7 +- src/invidious/routes/login.cr | 8 +- src/invidious/routes/preferences.cr | 20 ++--- src/invidious/routes/watch.cr | 2 +- src/invidious/users.cr | 6 +- 10 files changed, 121 insertions(+), 43 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 91f19d69..0149be11 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -759,18 +759,18 @@ post "/data_control" do |env| user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) end if body["watch_history"]? user.watched += body["watch_history"].as_a.map(&.as_s) user.watched.uniq! - PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email) + Invidious::Database::Users.update_watch_history(user) end if body["preferences"]? user.preferences = Preferences.from_json(body["preferences"].to_json) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email) + Invidious::Database::Users.update_preferences(user) end if playlists = body["playlists"]?.try &.as_a? @@ -831,7 +831,7 @@ post "/data_control" do |env| user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) when "import_freetube" user.subscriptions += body.scan(/"channelId":"(?[a-zA-Z0-9_-]{24})"/).map do |md| md["channel_id"] @@ -840,7 +840,7 @@ post "/data_control" do |env| user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) when "import_newpipe_subscriptions" body = JSON.parse(body) user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| @@ -859,7 +859,7 @@ post "/data_control" do |env| user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) when "import_newpipe" Compress::Zip::Reader.open(IO::Memory.new(body)) do |file| file.each_entry do |entry| @@ -871,14 +871,14 @@ post "/data_control" do |env| user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v=")) user.watched.uniq! - PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email) + Invidious::Database::Users.update_watch_history(user) user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) user.subscriptions.uniq! user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) db.close tempfile.delete @@ -962,7 +962,7 @@ post "/change_password" do |env| end new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) - PG_DB.exec("UPDATE users SET password = $1 WHERE email = $2", new_password.to_s, user.email) + Invidious::Database::Users.update_password(user, new_password.to_s) env.redirect referer end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 5d962ab4..2ec510f0 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -238,8 +238,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) if was_insert LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) + Invidious::Database::Users.add_notification(video) else LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") end @@ -275,9 +274,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) # so since they don't provide a published date here we can safely ignore them. if Time.utc - video.published > 1.minute was_insert = Invidious::Database::ChannelVideos.insert(video) - - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + Invidious::Database::Users.add_notification(video) if was_insert end end diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr index aa3b9f85..71650918 100644 --- a/src/invidious/database/users.cr +++ b/src/invidious/database/users.cr @@ -39,6 +39,16 @@ module Invidious::Database::Users # Update (history) # ------------------- + def update_watch_history(user : User) + request = <<-SQL + UPDATE users + SET watched = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.watched, user.email) + end + def mark_watched(user : User, vid : String) request = <<-SQL UPDATE users @@ -73,6 +83,16 @@ module Invidious::Database::Users # Update (channels) # ------------------- + def update_subscriptions(user : User) + request = <<-SQL + UPDATE users + SET feed_needs_update = true, subscriptions = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.subscriptions, user.email) + end + def subscribe_channel(user : User, ucid : String) request = <<-SQL UPDATE users @@ -95,6 +115,65 @@ module Invidious::Database::Users PG_DB.exec(request, ucid, user.email) end + # ------------------- + # Update (notifs) + # ------------------- + + def add_notification(video : ChannelVideo) + request = <<-SQL + UPDATE users + SET notifications = array_append(notifications, $1), + feed_needs_update = true + WHERE $2 = ANY(subscriptions) + SQL + + PG_DB.exec(request, video.id, video.ucid) + end + + def remove_notification(user : User, vid : String) + request = <<-SQL + UPDATE users + SET notifications = array_remove(notifications, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, vid, user.email) + end + + def clear_notifications(user : User) + request = <<-SQL + UPDATE users + SET notifications = $1, updated = $2 + WHERE email = $3 + SQL + + PG_DB.exec(request, [] of String, Time.utc, user) + end + + # ------------------- + # Update (misc) + # ------------------- + + def update_preferences(user : User) + request = <<-SQL + UPDATE users + SET preferences = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.preferences.to_json, user.email) + end + + def update_password(user : User, pass : String) + request = <<-SQL + UPDATE users + SET password = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.email, pass) + end + # ------------------- # Select # ------------------- @@ -126,4 +205,14 @@ module Invidious::Database::Users return PG_DB.query_one?(request, token, as: User) end + + def select_notifications(user : User) : Array(String) + request = <<-SQL + SELECT notifications + FROM users + WHERE email = $1 + SQL + + return PG_DB.query_one(request, user.email, as: Array(String)) + end end diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index d9b58ebf..62b09f79 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -22,12 +22,11 @@ module Invidious::Routes::API::V1::Authenticated user = env.get("user").as(User) begin - preferences = Preferences.from_json(env.request.body || "{}") + user.preferences = Preferences.from_json(env.request.body || "{}") rescue - preferences = user.preferences end - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + Invidious::Database::Users.update_preferences(user) env.response.status_code = 204 end diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 049ee344..2c648b5a 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -137,7 +137,7 @@ module Invidious::Routes::Embed # end if notifications && notifications.includes? id - PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email) + Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 4e7ec9ad..be58dd8d 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -99,8 +99,7 @@ module Invidious::Routes::Feeds # we know a user has looked at their feed e.g. in the past 10 minutes, # they've already seen a video posted 20 minutes ago, and don't need # to be notified. - PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.utc, - user.email) + Invidious::Database::Users.clear_notifications(user) user.notifications = [] of String env.set "user", user @@ -417,9 +416,7 @@ module Invidious::Routes::Feeds }) was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true) - - PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + Invidious::Database::Users.add_notification(video) if was_insert end end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 8f703464..c94fd09b 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -303,8 +303,8 @@ module Invidious::Routes::Login end if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + user.preferences = env.get("preferences").as(Preferences) + Invidious::Database::Users.update_preferences(user) cookie = env.request.cookies["PREFS"] cookie.expires = Time.utc(1990, 1, 1) @@ -470,8 +470,8 @@ module Invidious::Routes::Login end if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + user.preferences = env.get("preferences").as(Preferences) + Invidious::Database::Users.update_preferences(user) cookie = env.request.cookies["PREFS"] cookie.expires = Time.utc(1990, 1, 1) diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 15c00700..a832076c 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -170,11 +170,12 @@ module Invidious::Routes::PreferencesRoute vr_mode: vr_mode, show_nick: show_nick, save_player_pos: save_player_pos, - }.to_json).to_json + }.to_json) if user = env.get? "user" user = user.as(User) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + user.preferences = preferences + Invidious::Database::Users.update_preferences(user) if CONFIG.admins.includes? user.email CONFIG.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home @@ -220,10 +221,10 @@ module Invidious::Routes::PreferencesRoute end if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, secure: secure, http_only: true) end end @@ -241,18 +242,15 @@ module Invidious::Routes::PreferencesRoute if user = env.get? "user" user = user.as(User) - preferences = user.preferences - case preferences.dark_mode + case user.preferences.dark_mode when "dark" - preferences.dark_mode = "light" + user.preferences.dark_mode = "light" else - preferences.dark_mode = "dark" + user.preferences.dark_mode = "dark" end - preferences = preferences.to_json - - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + Invidious::Database::Users.update_preferences(user) else preferences = env.get("preferences").as(Preferences) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index c1ec0bc6..f7bd7d81 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -80,7 +80,7 @@ module Invidious::Routes::Watch end if notifications && notifications.includes? id - PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email) + Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 933c451d..efc4dd52 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -224,8 +224,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE) offset = (page - 1) * limit - notifications = db.query_one("SELECT notifications FROM users WHERE email = $1", user.email, - as: Array(String)) + notifications = Invidious::Database::Users.select_notifications(user) view_name = "subscriptions_#{sha256(user.email)}" if user.preferences.notifications_only && !notifications.empty? @@ -296,8 +295,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) else nil # Ignore end - notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, as: Array(String)) - + notifications = Invidious::Database::Users.select_notifications(user) notifications = videos.select { |v| notifications.includes? v.id } videos = videos - notifications end From 85cf27119cb230259550bfb795dffcb724ebebf3 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 6 Dec 2021 17:02:15 +0100 Subject: [PATCH 09/16] Move DB queries related to playlists in a separate module (3/3) --- src/invidious.cr | 4 +- src/invidious/database/playlists.cr | 83 +++++++++++++++++++++++++++++ src/invidious/playlists.cr | 10 ++-- src/invidious/routes/feeds.cr | 5 +- src/invidious/views/watch.ecr | 2 +- 5 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 0149be11..561fc9cf 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -656,7 +656,7 @@ get "/subscription_manager" do |env| if format == "json" env.response.content_type = "application/json" env.response.headers["content-disposition"] = "attachment" - playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + playlists = Invidious::Database::Playlists.select_like_iv(user.email) next JSON.build do |json| json.object do @@ -672,7 +672,7 @@ get "/subscription_manager" do |env| json.field "privacy", playlist.privacy.to_s json.field "videos" do json.array do - PG_DB.query_all("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 500", playlist.id, playlist.index, as: String).each do |video_id| + Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id| json.string video_id end end diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr index 1dba64f3..950d5f4b 100644 --- a/src/invidious/database/playlists.cr +++ b/src/invidious/database/playlists.cr @@ -117,6 +117,39 @@ module Invidious::Database::Playlists return PG_DB.query_all(request, author, as: InvidiousPlaylist) end + # ------------------- + # Salect (filtered) + # ------------------- + + def select_like_iv(email : String) : Array(InvidiousPlaylist) + request = <<-SQL + SELECT * FROM playlists + WHERE author = $1 AND id LIKE 'IV%' + ORDER BY created + SQL + + PG_DB.query_all(request, email, as: InvidiousPlaylist) + end + + def select_not_like_iv(email : String) : Array(InvidiousPlaylist) + request = <<-SQL + SELECT * FROM playlists + WHERE author = $1 AND id NOT LIKE 'IV%' + ORDER BY created + SQL + + PG_DB.query_all(request, email, as: InvidiousPlaylist) + end + + def select_user_created_playlists(email : String) : Array({String, String}) + request = <<-SQL + SELECT id,title FROM playlists + WHERE author = $1 AND id LIKE 'IV%' + SQL + + PG_DB.query_all(request, email, as: {String, String}) + end + # ------------------- # Misc checks # ------------------- @@ -148,6 +181,8 @@ end module Invidious::Database::PlaylistVideos extend self + private alias VideoIndex = Int64 | Array(Int64) + # ------------------- # Insert / Delete # ------------------- @@ -171,4 +206,52 @@ module Invidious::Database::PlaylistVideos PG_DB.exec(request, index) end + + # ------------------- + # Salect + # ------------------- + + def select(plid : String, index : VideoIndex, offset, limit = 100) : Array(PlaylistVideo) + request = <<-SQL + SELECT * FROM playlist_videos + WHERE plid = $1 + ORDER BY array_position($2, index) + LIMIT $3 + OFFSET $4 + SQL + + return PG_DB.query_all(request, plid, index, limit, offset, as: PlaylistVideo) + end + + def select_index(plid : String, vid : String) : Int64? + request = <<-SQL + SELECT index FROM playlist_videos + WHERE plid = $1 AND id = $2 + LIMIT 1 + SQL + + return PG_DB.query_one?(request, plid, vid, as: Int64) + end + + def select_one_id(plid : String, index : VideoIndex) : String? + request = <<-SQL + SELECT id FROM playlist_videos + WHERE plid = $1 + ORDER BY array_position($2, index) + LIMIT 1 + SQL + + return PG_DB.query_one?(request, plid, index, as: String) + end + + def select_ids(plid : String, index : VideoIndex, limit = 500) : Array(String) + request = <<-SQL + SELECT id FROM playlist_videos + WHERE plid = $1 + ORDER BY array_position($2, index) + LIMIT $3 + SQL + + return PG_DB.query_all(request, plid, index, limit, as: String) + end end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index f68dc3b0..9128f7db 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -200,8 +200,8 @@ struct InvidiousPlaylist json.field "videos" do json.array do - if !offset || offset == 0 - index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, video_id, as: Int64) + if (!offset || offset == 0) && !video_id.nil? + index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id) offset = self.index.index(index) || 0 end @@ -225,7 +225,8 @@ struct InvidiousPlaylist end def thumbnail - @thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------" + # TODO: Get playlist thumbnail from playlist data rather than first video + @thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------" "/vi/#{@thumbnail_id}/mqdefault.jpg" end @@ -411,8 +412,7 @@ def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil) end if playlist.is_a? InvidiousPlaylist - db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", - playlist.id, playlist.index, offset, as: PlaylistVideo) + Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100) else if video_id initial_data = YoutubeAPI.next({ diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index be58dd8d..b58a988f 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -15,13 +15,14 @@ module Invidious::Routes::Feeds user = user.as(User) - items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + # TODO: make a single DB call and separate the items here? + items_created = Invidious::Database::Playlists.select_like_iv(user.email) items_created.map! do |item| item.author = "" item end - items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_saved = Invidious::Database::Playlists.select_not_like_iv(user.email) items_saved.map! do |item| item.author = "" item diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index b85ea59d..fa4fe083 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -138,7 +138,7 @@ we're going to need to do it here in order to allow for translations.

<% if user %> - <% playlists = PG_DB.query_all("SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%'", user.email, as: {String, String}) %> + <% playlists = Invidious::Database::Playlists.select_user_created_playlists(user.email) %> <% if !playlists.empty? %>
From 914cfbd953d5a2c3c6f8ae98f350b60bfb38b9a2 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 6 Dec 2021 17:24:49 +0100 Subject: [PATCH 10/16] Move DB queries related to 'annotations' in a separate module --- src/invidious/database/annotations.cr | 24 ++++++++++++++++++++++++ src/invidious/helpers/helpers.cr | 2 +- src/invidious/routes/api/v1/videos.cr | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 src/invidious/database/annotations.cr diff --git a/src/invidious/database/annotations.cr b/src/invidious/database/annotations.cr new file mode 100644 index 00000000..03749473 --- /dev/null +++ b/src/invidious/database/annotations.cr @@ -0,0 +1,24 @@ +require "./base.cr" + +module Invidious::Database::Annotations + extend self + + def insert(id : String, annotations : String) + request = <<-SQL + INSERT INTO annotations + VALUES ($1, $2) + ON CONFLICT DO NOTHING + SQL + + PG_DB.exec(request, id, annotations) + end + + def select(id : String) : Annotation? + request = <<-SQL + SELECT * FROM annotations + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: Annotation) + end +end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 014c04a8..c5f6c6c5 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -183,7 +183,7 @@ def cache_annotation(db, id, annotations) end end - db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) if has_legacy_annotations + Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations end def create_notification_stream(env, topics, connection_channel) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 4c7179ce..f982329c 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -239,7 +239,7 @@ module Invidious::Routes::API::V1::Videos case source when "archive" - if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation)) + if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id)) annotations = cached_annotation.annotations else index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') From 6704ce3214ed8c9a3acb69e0fc53824324a610fc Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 6 Dec 2021 18:03:37 +0100 Subject: [PATCH 11/16] Move DB utility functions to the proper module --- src/invidious.cr | 20 ++--- src/invidious/database/base.cr | 106 ++++++++++++++++++++++++ src/invidious/helpers/helpers.cr | 105 ----------------------- src/invidious/jobs/refresh_feeds_job.cr | 2 +- 4 files changed, 117 insertions(+), 116 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 561fc9cf..ef02f143 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -113,19 +113,19 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) # Check table integrity if CONFIG.check_tables - check_enum(PG_DB, "privacy", PlaylistPrivacy) + Invidious::Database.check_enum(PG_DB, "privacy", PlaylistPrivacy) - check_table(PG_DB, "channels", InvidiousChannel) - check_table(PG_DB, "channel_videos", ChannelVideo) - check_table(PG_DB, "playlists", InvidiousPlaylist) - check_table(PG_DB, "playlist_videos", PlaylistVideo) - check_table(PG_DB, "nonces", Nonce) - check_table(PG_DB, "session_ids", SessionId) - check_table(PG_DB, "users", User) - check_table(PG_DB, "videos", Video) + Invidious::Database.check_table(PG_DB, "channels", InvidiousChannel) + Invidious::Database.check_table(PG_DB, "channel_videos", ChannelVideo) + Invidious::Database.check_table(PG_DB, "playlists", InvidiousPlaylist) + Invidious::Database.check_table(PG_DB, "playlist_videos", PlaylistVideo) + Invidious::Database.check_table(PG_DB, "nonces", Nonce) + Invidious::Database.check_table(PG_DB, "session_ids", SessionId) + Invidious::Database.check_table(PG_DB, "users", User) + Invidious::Database.check_table(PG_DB, "videos", Video) if CONFIG.cache_annotations - check_table(PG_DB, "annotations", Annotation) + Invidious::Database.check_table(PG_DB, "annotations", Annotation) end end diff --git a/src/invidious/database/base.cr b/src/invidious/database/base.cr index 055a6284..6e49ea1a 100644 --- a/src/invidious/database/base.cr +++ b/src/invidious/database/base.cr @@ -1,4 +1,110 @@ require "pg" module Invidious::Database + extend self + + def check_enum(db, enum_name, struct_type = nil) + return # TODO + + if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) + LOGGER.info("check_enum: CREATE TYPE #{enum_name}") + + db.using_connection do |conn| + conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql")) + end + end + end + + def check_table(db, table_name, struct_type = nil) + # Create table if it doesn't exist + begin + db.exec("SELECT * FROM #{table_name} LIMIT 0") + rescue ex + LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}") + + db.using_connection do |conn| + conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql")) + end + end + + return if !struct_type + + struct_array = struct_type.type_array + column_array = get_column_array(db, table_name) + column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?[\d\D]*?)\);/) + .try &.["types"].split(",").map(&.strip).reject &.starts_with?("CONSTRAINT") + + return if !column_types + + struct_array.each_with_index do |name, i| + if name != column_array[i]? + if !column_array[i]? + new_column = column_types.select(&.starts_with?(name))[0] + LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + next + end + + # Column doesn't exist + if !column_array.includes? name + new_column = column_types.select(&.starts_with?(name))[0] + db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + end + + # Column exists but in the wrong position, rotate + if struct_array.includes? column_array[i] + until name == column_array[i] + new_column = column_types.select(&.starts_with?(column_array[i]))[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new") + + # There's a column we didn't expect + if !new_column + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + + column_array = get_column_array(db, table_name) + next + end + + LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + + LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") + db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") + + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + + LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") + db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") + + column_array = get_column_array(db, table_name) + end + else + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + end + end + end + + return if column_array.size <= struct_array.size + + column_array.each do |column| + if !struct_array.includes? column + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + end + end + end + + def get_column_array(db, table_name) + column_array = [] of String + db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs| + rs.column_count.times do |i| + column = rs.as(PG::ResultSet).field(i) + column_array << column.name + end + end + + return column_array + end end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index c5f6c6c5..982bcda1 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -60,111 +60,6 @@ def html_to_content(description_html : String) return description end -def check_enum(db, enum_name, struct_type = nil) - return # TODO - - if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) - LOGGER.info("check_enum: CREATE TYPE #{enum_name}") - - db.using_connection do |conn| - conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql")) - end - end -end - -def check_table(db, table_name, struct_type = nil) - # Create table if it doesn't exist - begin - db.exec("SELECT * FROM #{table_name} LIMIT 0") - rescue ex - LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}") - - db.using_connection do |conn| - conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql")) - end - end - - return if !struct_type - - struct_array = struct_type.type_array - column_array = get_column_array(db, table_name) - column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?[\d\D]*?)\);/) - .try &.["types"].split(",").map(&.strip).reject &.starts_with?("CONSTRAINT") - - return if !column_types - - struct_array.each_with_index do |name, i| - if name != column_array[i]? - if !column_array[i]? - new_column = column_types.select(&.starts_with?(name))[0] - LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - next - end - - # Column doesn't exist - if !column_array.includes? name - new_column = column_types.select(&.starts_with?(name))[0] - db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - end - - # Column exists but in the wrong position, rotate - if struct_array.includes? column_array[i] - until name == column_array[i] - new_column = column_types.select(&.starts_with?(column_array[i]))[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new") - - # There's a column we didn't expect - if !new_column - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - - column_array = get_column_array(db, table_name) - next - end - - LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - - LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") - db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") - - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - - LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") - db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") - - column_array = get_column_array(db, table_name) - end - else - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - end - end - end - - return if column_array.size <= struct_array.size - - column_array.each do |column| - if !struct_array.includes? column - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") - end - end -end - -def get_column_array(db, table_name) - column_array = [] of String - db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs| - rs.column_count.times do |i| - column = rs.as(PG::ResultSet).field(i) - column_array << column.name - end - end - - return column_array -end - def cache_annotation(db, id, annotations) if !CONFIG.cache_annotations return diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index 926c27fa..4b52c959 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -25,7 +25,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob spawn do begin # Drop outdated views - column_array = get_column_array(db, view_name) + column_array = Invidious::Database.get_column_array(db, view_name) ChannelVideo.type_array.each_with_index do |name, i| if name != column_array[i]? LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}") From 9bad7e29405536abfe35dcd1c4315918659a5d3c Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 7 Dec 2021 03:40:15 +0100 Subject: [PATCH 12/16] Clean useless database arguments (1/5) --- src/invidious.cr | 12 ++++++------ src/invidious/channels/channels.cr | 12 ++++++------ src/invidious/routes/api/v1/authenticated.cr | 2 +- src/invidious/users.cr | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index ef02f143..e1d3e37b 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -603,7 +603,7 @@ post "/subscription_ajax" do |env| case action when "action_create_subscription_to_channel" if !user.subscriptions.includes? channel_id - get_channel(channel_id, PG_DB, false, false) + get_channel(channel_id, false, false) Invidious::Database::Users.subscribe_channel(user, channel_id) end when "action_remove_subscriptions" @@ -757,7 +757,7 @@ post "/data_control" do |env| user.subscriptions += body["subscriptions"].as_a.map(&.as_s) user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) Invidious::Database::Users.update_subscriptions(user) end @@ -829,7 +829,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) Invidious::Database::Users.update_subscriptions(user) when "import_freetube" @@ -838,7 +838,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) Invidious::Database::Users.update_subscriptions(user) when "import_newpipe_subscriptions" @@ -857,7 +857,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) Invidious::Database::Users.update_subscriptions(user) when "import_newpipe" @@ -876,7 +876,7 @@ post "/data_control" do |env| user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) Invidious::Database::Users.update_subscriptions(user) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 2ec510f0..155ec559 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -114,7 +114,7 @@ class ChannelRedirect < Exception end end -def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) +def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_threads = 10) finished_channel = Channel(String | Nil).new spawn do @@ -130,7 +130,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma active_threads += 1 spawn do begin - get_channel(ucid, db, refresh, pull_all_videos) + get_channel(ucid, refresh, pull_all_videos) finished_channel.send(ucid) rescue ex finished_channel.send(nil) @@ -151,21 +151,21 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma return final end -def get_channel(id, db, refresh = true, pull_all_videos = true) +def get_channel(id, refresh = true, pull_all_videos = true) if channel = Invidious::Database::Channels.select(id) if refresh && Time.utc - channel.updated > 10.minutes - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) + channel = fetch_channel(id, pull_all_videos: pull_all_videos) Invidious::Database::Channels.insert(channel, update_on_conflict: true) end else - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) + channel = fetch_channel(id, pull_all_videos: pull_all_videos) Invidious::Database::Channels.insert(channel) end return channel end -def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) +def fetch_channel(ucid, pull_all_videos = true, locale = nil) LOGGER.debug("fetch_channel: #{ucid}") LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 62b09f79..c4c06420 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -92,7 +92,7 @@ module Invidious::Routes::API::V1::Authenticated ucid = env.params.url["ucid"] if !user.subscriptions.includes? ucid - get_channel(ucid, PG_DB, false, false) + get_channel(ucid, false, false) Invidious::Database::Users.subscribe_channel(user, ucid) end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index efc4dd52..ef97d3d6 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -74,7 +74,7 @@ def fetch_user(sid, headers, db) end end - channels = get_batch_channels(channels, db, false, false) + channels = get_batch_channels(channels, false, false) email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"])) if email From c25d664edcd300c920cb22a419b5cd98d31ce3a2 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 6 Dec 2021 22:28:16 +0100 Subject: [PATCH 13/16] Clean useless database arguments (2/5) --- src/invidious.cr | 32 ++++++++++---------- src/invidious/helpers/handlers.cr | 2 +- src/invidious/helpers/tokens.cr | 6 ++-- src/invidious/routes/api/v1/authenticated.cr | 4 +-- src/invidious/routes/feeds.cr | 2 +- src/invidious/routes/login.cr | 12 ++++---- src/invidious/routes/playlists.cr | 14 ++++----- src/invidious/users.cr | 20 ++++++------ 8 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index e1d3e37b..149fdef3 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -256,7 +256,7 @@ before_all do |env| ":subscription_ajax", ":token_ajax", ":watch_ajax", - }, HMAC_KEY, PG_DB, 1.week) + }, HMAC_KEY, 1.week) preferences = user.preferences env.set "preferences", preferences @@ -270,7 +270,7 @@ before_all do |env| headers["Cookie"] = env.request.headers["Cookie"] begin - user, sid = get_user(sid, headers, PG_DB, false) + user, sid = get_user(sid, headers, false) csrf_token = generate_response(sid, { ":authorize_token", ":playlist_ajax", @@ -278,7 +278,7 @@ before_all do |env| ":subscription_ajax", ":token_ajax", ":watch_ajax", - }, HMAC_KEY, PG_DB, 1.week) + }, HMAC_KEY, 1.week) preferences = user.preferences env.set "preferences", preferences @@ -438,7 +438,7 @@ post "/watch_ajax" do |env| end begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect next error_template(400, ex) @@ -575,7 +575,7 @@ post "/subscription_ajax" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect next error_template(400, ex) @@ -639,7 +639,7 @@ get "/subscription_manager" do |env| headers = HTTP::Headers.new headers["Cookie"] = env.request.headers["Cookie"] - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers) end action_takeout = env.params.query["action_takeout"]?.try &.to_i? @@ -906,7 +906,7 @@ get "/change_password" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) templated "change_password" end @@ -932,7 +932,7 @@ post "/change_password" do |env| end begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end @@ -980,7 +980,7 @@ get "/delete_account" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) templated "delete_account" end @@ -1001,7 +1001,7 @@ post "/delete_account" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end @@ -1032,7 +1032,7 @@ get "/clear_watch_history" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) templated "clear_watch_history" end @@ -1053,7 +1053,7 @@ post "/clear_watch_history" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end @@ -1075,7 +1075,7 @@ get "/authorize_token" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY) scopes = env.params.query["scopes"]?.try &.split(",") scopes ||= [] of String @@ -1106,7 +1106,7 @@ post "/authorize_token" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end @@ -1115,7 +1115,7 @@ post "/authorize_token" do |env| callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? - access_token = generate_token(user.email, scopes, expire, HMAC_KEY, PG_DB) + access_token = generate_token(user.email, scopes, expire, HMAC_KEY) if callback_url access_token = URI.encode_www_form(access_token) @@ -1179,7 +1179,7 @@ post "/token_ajax" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect next error_template(400, ex) diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index d52035c7..d140a858 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -97,7 +97,7 @@ class AuthHandler < Kemal::Handler if token = env.request.headers["Authorization"]? token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) session = URI.decode_www_form(token["session"].as_s) - scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil) + scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, nil) if email = Invidious::Database::SessionIDs.select_email(session) user = Invidious::Database::Users.select!(email: email) diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index 91405822..8b076e39 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -1,6 +1,6 @@ require "crypto/subtle" -def generate_token(email, scopes, expire, key, db) +def generate_token(email, scopes, expire, key) session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}" Invidious::Database::SessionIDs.insert(session, email) @@ -19,7 +19,7 @@ def generate_token(email, scopes, expire, key, db) return token.to_json end -def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = false) +def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false) expire = Time.utc + expire token = { @@ -63,7 +63,7 @@ def sign_token(key, hash) return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip end -def validate_request(token, session, request, key, db, locale = nil) +def validate_request(token, session, request, key, locale = nil) case token when String token = JSON.parse(URI.decode_www_form(token)).as_h diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index c4c06420..7ebc71fe 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -353,7 +353,7 @@ module Invidious::Routes::API::V1::Authenticated if sid = env.get?("sid").try &.as(String) env.response.content_type = "text/html" - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB, use_nonce: true) + csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true) return templated "authorize_token" else env.response.content_type = "application/json" @@ -367,7 +367,7 @@ module Invidious::Routes::API::V1::Authenticated end end - access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB) + access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY) if callback_url access_token = URI.encode_www_form(access_token) diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index b58a988f..187dd247 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -84,7 +84,7 @@ module Invidious::Routes::Feeds headers["Cookie"] = env.request.headers["Cookie"] if !user.password - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers) end max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index c94fd09b..64da3e4e 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -275,7 +275,7 @@ module Invidious::Routes::Login raise "Couldn't get SID." end - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers) # We are now logged in traceback << "done.
" @@ -393,9 +393,9 @@ module Invidious::Routes::Login prompt = "" if captcha_type == "image" - captcha = generate_captcha(HMAC_KEY, PG_DB) + captcha = generate_captcha(HMAC_KEY) else - captcha = generate_text_captcha(HMAC_KEY, PG_DB) + captcha = generate_text_captcha(HMAC_KEY) end return templated "login" @@ -412,7 +412,7 @@ module Invidious::Routes::Login answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) begin - validate_request(tokens[0], answer, env.request, HMAC_KEY, PG_DB, locale) + validate_request(tokens[0], answer, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end @@ -427,7 +427,7 @@ module Invidious::Routes::Login error_exception = Exception.new tokens.each do |token| begin - validate_request(token, answer, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, answer, env.request, HMAC_KEY, locale) found_valid_captcha = true rescue ex error_exception = ex @@ -501,7 +501,7 @@ module Invidious::Routes::Login token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index b73782d5..a8097ab7 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -12,7 +12,7 @@ module Invidious::Routes::Playlists user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY) templated "create_playlist" end @@ -31,7 +31,7 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end @@ -94,7 +94,7 @@ module Invidious::Routes::Playlists return env.redirect referer end - csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY) templated "delete_playlist" end @@ -116,7 +116,7 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end @@ -166,7 +166,7 @@ module Invidious::Routes::Playlists videos = [] of PlaylistVideo end - csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY) templated "edit_playlist" end @@ -188,7 +188,7 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end @@ -286,7 +286,7 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect return error_template(400, ex) diff --git a/src/invidious/users.cr b/src/invidious/users.cr index ef97d3d6..ad836d61 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -29,31 +29,31 @@ struct User end end -def get_user(sid, headers, db, refresh = true) +def get_user(sid, headers, refresh = true) if email = Invidious::Database::SessionIDs.select_email(sid) user = Invidious::Database::Users.select!(email: email) if refresh && Time.utc - user.updated > 1.minute - user, sid = fetch_user(sid, headers, db) + user, sid = fetch_user(sid, headers) Invidious::Database::Users.insert(user, update_on_conflict: true) Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin view_name = "subscriptions_#{sha256(user.email)}" - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") rescue ex end end else - user, sid = fetch_user(sid, headers, db) + user, sid = fetch_user(sid, headers) Invidious::Database::Users.insert(user, update_on_conflict: true) Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin view_name = "subscriptions_#{sha256(user.email)}" - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") rescue ex end end @@ -61,7 +61,7 @@ def get_user(sid, headers, db, refresh = true) return user, sid end -def fetch_user(sid, headers, db) +def fetch_user(sid, headers) feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) feed = XML.parse_html(feed.body) @@ -118,7 +118,7 @@ def create_user(sid, email, password) return user, sid end -def generate_captcha(key, db) +def generate_captcha(key) second = Random::Secure.rand(12) second_angle = second * 30 second = second * 5 @@ -170,16 +170,16 @@ def generate_captcha(key, db) return { question: image, - tokens: {generate_response(answer, {":login"}, key, db, use_nonce: true)}, + tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, } end -def generate_text_captcha(key, db) +def generate_text_captcha(key) response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body) response = JSON.parse(response) tokens = response["a"].as_a.map do |answer| - generate_response(answer.as_s, {":login"}, key, db, use_nonce: true) + generate_response(answer.as_s, {":login"}, key, use_nonce: true) end return { From 40ed4a0506277f62d66c7f7ed8cca54c62a78a02 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 7 Dec 2021 00:42:55 +0100 Subject: [PATCH 14/16] Clean useless database arguments (3/5) --- src/invidious.cr | 2 +- src/invidious/playlists.cr | 12 ++++++------ src/invidious/routes/api/v1/authenticated.cr | 2 +- src/invidious/routes/api/v1/misc.cr | 2 +- src/invidious/routes/embed.cr | 8 ++++---- src/invidious/routes/feeds.cr | 2 +- src/invidious/routes/playlists.cr | 14 +++++++------- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 149fdef3..d400c0ba 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -783,7 +783,7 @@ post "/data_control" do |env| next if !description next if !privacy - playlist = create_playlist(PG_DB, title, privacy, user) + playlist = create_playlist(title, privacy, user) Invidious::Database::Playlists.update_description(playlist.id, description) videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 9128f7db..ee7e12aa 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -125,7 +125,7 @@ struct Playlist json.field "videos" do json.array do - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) + videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id) videos.each do |video| video.to_json(json) end @@ -205,7 +205,7 @@ struct InvidiousPlaylist offset = self.index.index(index) || 0 end - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) + videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id) videos.each_with_index do |video, index| video.to_json(json, offset + index) end @@ -247,7 +247,7 @@ struct InvidiousPlaylist end end -def create_playlist(db, title, privacy, user) +def create_playlist(title, privacy, user) plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" playlist = InvidiousPlaylist.new({ @@ -267,7 +267,7 @@ def create_playlist(db, title, privacy, user) return playlist end -def subscribe_playlist(db, user, playlist) +def subscribe_playlist(user, playlist) playlist = InvidiousPlaylist.new({ title: playlist.title.byte_slice(0, 150), id: playlist.id, @@ -322,7 +322,7 @@ def produce_playlist_continuation(id, index) return continuation end -def get_playlist(db, plid, locale, refresh = true, force_refresh = false) +def get_playlist(plid, locale, refresh = true, force_refresh = false) if plid.starts_with? "IV" if playlist = Invidious::Database::Playlists.select(id: plid) return playlist @@ -404,7 +404,7 @@ def fetch_playlist(plid, locale) }) end -def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil) +def get_playlist_videos(playlist, offset, locale = nil, video_id = nil) # Show empy playlist if requested page is out of range # (e.g, when a new playlist has been created, offset will be negative) if offset >= playlist.video_count || offset < 0 diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 7ebc71fe..c3f751f7 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -150,7 +150,7 @@ module Invidious::Routes::API::V1::Authenticated return error_json(400, "User cannot have more than 100 playlists.") end - playlist = create_playlist(PG_DB, title, privacy, user) + playlist = create_playlist(title, privacy, user) env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}" env.response.status_code = 201 { diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 1621c9ef..ac0576a0 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -34,7 +34,7 @@ module Invidious::Routes::API::V1::Misc end begin - playlist = get_playlist(PG_DB, plid, locale) + playlist = get_playlist(plid, locale) rescue ex : InfoException return error_json(404, ex) rescue ex diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 2c648b5a..13422993 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -6,9 +6,9 @@ module Invidious::Routes::Embed if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") begin - playlist = get_playlist(PG_DB, plid, locale: locale) + playlist = get_playlist(plid, locale: locale) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) + videos = get_playlist_videos(playlist, offset: offset, locale: locale) rescue ex return error_template(500, ex) end @@ -60,9 +60,9 @@ module Invidious::Routes::Embed if plid begin - playlist = get_playlist(PG_DB, plid, locale: locale) + playlist = get_playlist(plid, locale: locale) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) + videos = get_playlist_videos(playlist, offset: offset, locale: locale) rescue ex return error_template(500, ex) end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 187dd247..458519b8 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -265,7 +265,7 @@ module Invidious::Routes::Feeds if plid.starts_with? "IV" if playlist = Invidious::Database::Playlists.select(id: plid) - videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) + videos = get_playlist_videos(playlist, offset: 0, locale: locale) return XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index a8097ab7..d33c699b 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -50,7 +50,7 @@ module Invidious::Routes::Playlists return error_template(400, "User cannot have more than 100 playlists.") end - playlist = create_playlist(PG_DB, title, privacy, user) + playlist = create_playlist(title, privacy, user) env.redirect "/playlist?list=#{playlist.id}" end @@ -66,8 +66,8 @@ module Invidious::Routes::Playlists user = user.as(User) playlist_id = env.params.query["list"] - playlist = get_playlist(PG_DB, playlist_id, locale) - subscribe_playlist(PG_DB, user, playlist) + playlist = get_playlist(playlist_id, locale) + subscribe_playlist(user, playlist) env.redirect "/playlist?list=#{playlist.id}" end @@ -161,7 +161,7 @@ module Invidious::Routes::Playlists end begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale) rescue ex videos = [] of PlaylistVideo end @@ -314,7 +314,7 @@ module Invidious::Routes::Playlists begin playlist_id = env.params.query["playlist_id"] - playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist) + playlist = get_playlist(playlist_id, locale).as(InvidiousPlaylist) raise "Invalid user" if playlist.author != user.email rescue ex if redirect @@ -405,7 +405,7 @@ module Invidious::Routes::Playlists end begin - playlist = get_playlist(PG_DB, plid, locale) + playlist = get_playlist(plid, locale) rescue ex return error_template(500, ex) end @@ -422,7 +422,7 @@ module Invidious::Routes::Playlists end begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale) rescue ex return error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}") end From d74873fed1a4da2c2eb51a47932207b65ca473e5 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 7 Dec 2021 02:55:43 +0100 Subject: [PATCH 15/16] Clean useless database arguments (4/5) --- src/invidious.cr | 2 +- src/invidious/helpers/helpers.cr | 6 +++--- src/invidious/routes/api/manifest.cr | 2 +- src/invidious/routes/api/v1/authenticated.cr | 2 +- src/invidious/routes/api/v1/videos.cr | 8 ++++---- src/invidious/routes/embed.cr | 4 ++-- src/invidious/routes/feeds.cr | 2 +- src/invidious/routes/playlists.cr | 2 +- src/invidious/routes/video_playback.cr | 2 +- src/invidious/routes/watch.cr | 4 ++-- src/invidious/videos.cr | 4 ++-- 11 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index d400c0ba..fb67af87 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -793,7 +793,7 @@ post "/data_control" do |env| next if !video_id begin - video = get_video(video_id, PG_DB) + video = get_video(video_id) rescue ex next end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 982bcda1..c3b53339 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -60,7 +60,7 @@ def html_to_content(description_html : String) return description end -def cache_annotation(db, id, annotations) +def cache_annotation(id, annotations) if !CONFIG.cache_annotations return end @@ -99,7 +99,7 @@ def create_notification_stream(env, topics, connection_channel) published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3]) video_id = TEST_IDS[rand(TEST_IDS.size)] - video = get_video(video_id, PG_DB) + video = get_video(video_id) video.published = published response = JSON.parse(video.to_json(locale, nil)) @@ -176,7 +176,7 @@ def create_notification_stream(env, topics, connection_channel) next end - video = get_video(video_id, PG_DB) + video = get_video(video_id) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 12687ec6..b6183001 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -13,7 +13,7 @@ module Invidious::Routes::API::Manifest unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index c3f751f7..44603c9a 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -245,7 +245,7 @@ module Invidious::Routes::API::V1::Authenticated end begin - video = get_video(video_id, PG_DB) + video = get_video(video_id) rescue ex return error_json(500, ex) end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index f982329c..4d244e7f 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Videos region = env.params.query["region"]? begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) @@ -36,7 +36,7 @@ module Invidious::Routes::API::V1::Videos # getting video info. begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) @@ -157,7 +157,7 @@ module Invidious::Routes::API::V1::Videos region = env.params.query["region"]? begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) @@ -271,7 +271,7 @@ module Invidious::Routes::API::V1::Videos annotations = response.body - cache_annotation(PG_DB, id, annotations) + cache_annotation(id, annotations) end else # "youtube" response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}") diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 13422993..ab722ae2 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -30,7 +30,7 @@ module Invidious::Routes::Embed id = env.params.url["id"] plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") - continuation = process_continuation(PG_DB, env.params.query, plid, id) + continuation = process_continuation(env.params.query, plid, id) if md = env.params.query["playlist"]? .try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/) @@ -119,7 +119,7 @@ module Invidious::Routes::Embed subscriptions ||= [] of String begin - video = get_video(id, PG_DB, region: params.region) + video = get_video(id, region: params.region) rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 458519b8..5dcef351 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -393,7 +393,7 @@ module Invidious::Routes::Feeds published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - video = get_video(id, PG_DB, force_refresh: true) + video = get_video(id, force_refresh: true) # Deliver notifications to `/api/v1/auth/notifications` payload = { diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index d33c699b..d437b79c 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -345,7 +345,7 @@ module Invidious::Routes::Playlists video_id = env.params.query["video_id"] begin - video = get_video(video_id, PG_DB) + video = get_video(video_id) rescue ex if redirect return error_template(500, ex) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 06ba6b8c..8a58b034 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -263,7 +263,7 @@ module Invidious::Routes::VideoPlayback haltf env, status_code: 400, response: "TESTING" end - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } url = fmt.try &.["url"]?.try &.as_s diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index f7bd7d81..1198f48f 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -39,7 +39,7 @@ module Invidious::Routes::Watch end plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") - continuation = process_continuation(PG_DB, env.params.query, plid, id) + continuation = process_continuation(env.params.query, plid, id) nojs = env.params.query["nojs"]? @@ -60,7 +60,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - video = get_video(id, PG_DB, region: params.region) + video = get_video(id, region: params.region) rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 645d3678..6c89b445 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -993,7 +993,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ return params end -def get_video(id, db, refresh = true, region = nil, force_refresh = false) +def get_video(id, refresh = true, region = nil, force_refresh = false) if (video = Invidious::Database::Videos.select(id)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) @@ -1056,7 +1056,7 @@ def itag_to_metadata?(itag : JSON::Any) return VIDEO_FORMATS[itag.to_s]? end -def process_continuation(db, query, plid, id) +def process_continuation(query, plid, id) continuation = nil if plid if index = query["index"]?.try &.to_i? From 302fecbdcb8924b2f3a7cf8905f901c044cec728 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 7 Dec 2021 03:57:18 +0100 Subject: [PATCH 16/16] Clean useless database arguments (5/5) --- src/invidious/helpers/utils.cr | 2 -- src/invidious/routes/api/v1/authenticated.cr | 2 +- src/invidious/routes/feeds.cr | 4 ++-- src/invidious/users.cr | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 7bbbcb92..8453d605 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,5 +1,3 @@ -require "db" - # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html def ci_lower_bound(pos, n) if n == 0 diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 44603c9a..fda655ef 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -44,7 +44,7 @@ module Invidious::Routes::API::V1::Authenticated page = env.params.query["page"]?.try &.to_i? page ||= 1 - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + videos, notifications = get_subscription_feed(user, max_results, page) JSON.build do |json| json.object do diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 5dcef351..fd8c25ce 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -94,7 +94,7 @@ module Invidious::Routes::Feeds page = env.params.query["page"]?.try &.to_i? page ||= 1 - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + videos, notifications = get_subscription_feed(user, max_results, page) # "updated" here is used for delivering new notifications, so if # we know a user has looked at their feed e.g. in the past 10 minutes, @@ -234,7 +234,7 @@ module Invidious::Routes::Feeds params = HTTP::Params.parse(env.params.query["params"]? || "") - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + videos, notifications = get_subscription_feed(user, max_results, page) XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", diff --git a/src/invidious/users.cr b/src/invidious/users.cr index ad836d61..49074994 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -220,7 +220,7 @@ def subscribe_ajax(channel_id, action, env_headers) end end -def get_subscription_feed(db, user, max_results = 40, page = 1) +def get_subscription_feed(user, max_results = 40, page = 1) limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE) offset = (page - 1) * limit