diff --git a/src/invidious.cr b/src/invidious.cr index ade13608..fb67af87 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -20,12 +20,13 @@ require "kemal" require "athena-negotiation" require "openssl/hmac" require "option_parser" -require "pg" require "sqlite3" require "xml" require "yaml" require "compress/zip" require "protodec/utils" + +require "./invidious/database/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/*" @@ -112,19 +113,19 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) # Check table integrity if CONFIG.check_tables - check_enum(PG_DB, "privacy", PlaylistPrivacy) + Invidious::Database.check_enum(PG_DB, "privacy", PlaylistPrivacy) - check_table(PG_DB, "channels", InvidiousChannel) - check_table(PG_DB, "channel_videos", ChannelVideo) - check_table(PG_DB, "playlists", InvidiousPlaylist) - check_table(PG_DB, "playlist_videos", PlaylistVideo) - check_table(PG_DB, "nonces", Nonce) - check_table(PG_DB, "session_ids", SessionId) - check_table(PG_DB, "users", User) - check_table(PG_DB, "videos", Video) + Invidious::Database.check_table(PG_DB, "channels", InvidiousChannel) + Invidious::Database.check_table(PG_DB, "channel_videos", ChannelVideo) + Invidious::Database.check_table(PG_DB, "playlists", InvidiousPlaylist) + Invidious::Database.check_table(PG_DB, "playlist_videos", PlaylistVideo) + Invidious::Database.check_table(PG_DB, "nonces", Nonce) + Invidious::Database.check_table(PG_DB, "session_ids", SessionId) + Invidious::Database.check_table(PG_DB, "users", User) + Invidious::Database.check_table(PG_DB, "videos", Video) if CONFIG.cache_annotations - check_table(PG_DB, "annotations", Annotation) + Invidious::Database.check_table(PG_DB, "annotations", Annotation) end end @@ -246,8 +247,8 @@ before_all do |env| # Invidious users only have SID if !env.request.cookies.has_key? "SSID" - if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) - user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + if email = Invidious::Database::SessionIDs.select_email(sid) + user = Invidious::Database::Users.select!(email: email) csrf_token = generate_response(sid, { ":authorize_token", ":playlist_ajax", @@ -255,7 +256,7 @@ before_all do |env| ":subscription_ajax", ":token_ajax", ":watch_ajax", - }, HMAC_KEY, PG_DB, 1.week) + }, HMAC_KEY, 1.week) preferences = user.preferences env.set "preferences", preferences @@ -269,7 +270,7 @@ before_all do |env| headers["Cookie"] = env.request.headers["Cookie"] begin - user, sid = get_user(sid, headers, PG_DB, false) + user, sid = get_user(sid, headers, false) csrf_token = generate_response(sid, { ":authorize_token", ":playlist_ajax", @@ -277,7 +278,7 @@ before_all do |env| ":subscription_ajax", ":token_ajax", ":watch_ajax", - }, HMAC_KEY, PG_DB, 1.week) + }, HMAC_KEY, 1.week) preferences = user.preferences env.set "preferences", preferences @@ -437,7 +438,7 @@ post "/watch_ajax" do |env| end begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect next error_template(400, ex) @@ -457,10 +458,10 @@ post "/watch_ajax" do |env| case action when "action_mark_watched" if !user.watched.includes? id - PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.email) + Invidious::Database::Users.mark_watched(user, id) end when "action_mark_unwatched" - PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email) + Invidious::Database::Users.mark_unwatched(user, id) else next error_json(400, "Unsupported action #{action}") end @@ -574,7 +575,7 @@ post "/subscription_ajax" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect next error_template(400, ex) @@ -598,16 +599,15 @@ post "/subscription_ajax" do |env| # Sync subscriptions with YouTube subscribe_ajax(channel_id, action, env.request.headers) end - email = user.email case action when "action_create_subscription_to_channel" if !user.subscriptions.includes? channel_id - get_channel(channel_id, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email) + get_channel(channel_id, false, false) + Invidious::Database::Users.subscribe_channel(user, channel_id) end when "action_remove_subscriptions" - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) + Invidious::Database::Users.unsubscribe_channel(user, channel_id) else next error_json(400, "Unsupported action #{action}") end @@ -632,13 +632,14 @@ get "/subscription_manager" do |env| end user = user.as(User) + sid = sid.as(String) if !user.password # Refresh account headers = HTTP::Headers.new headers["Cookie"] = env.request.headers["Cookie"] - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers) end action_takeout = env.params.query["action_takeout"]?.try &.to_i? @@ -648,20 +649,14 @@ get "/subscription_manager" do |env| format = env.params.query["format"]? format ||= "rss" - if user.subscriptions.empty? - values = "'{}'" - else - values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" - end - - subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) + subscriptions = Invidious::Database::Channels.select(user.subscriptions) subscriptions.sort_by!(&.author.downcase) if action_takeout if format == "json" env.response.content_type = "application/json" env.response.headers["content-disposition"] = "attachment" - playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + playlists = Invidious::Database::Playlists.select_like_iv(user.email) next JSON.build do |json| json.object do @@ -677,7 +672,7 @@ get "/subscription_manager" do |env| json.field "privacy", playlist.privacy.to_s json.field "videos" do json.array do - PG_DB.query_all("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 500", playlist.id, playlist.index, as: String).each do |video_id| + Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id| json.string video_id end end @@ -762,20 +757,20 @@ post "/data_control" do |env| user.subscriptions += body["subscriptions"].as_a.map(&.as_s) user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) end if body["watch_history"]? user.watched += body["watch_history"].as_a.map(&.as_s) user.watched.uniq! - PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email) + Invidious::Database::Users.update_watch_history(user) end if body["preferences"]? user.preferences = Preferences.from_json(body["preferences"].to_json) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email) + Invidious::Database::Users.update_preferences(user) end if playlists = body["playlists"]?.try &.as_a? @@ -788,8 +783,8 @@ post "/data_control" do |env| next if !description next if !privacy - playlist = create_playlist(PG_DB, title, privacy, user) - PG_DB.exec("UPDATE playlists SET description = $1 WHERE id = $2", description, playlist.id) + playlist = create_playlist(title, privacy, user) + Invidious::Database::Playlists.update_description(playlist.id, description) videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 @@ -798,7 +793,7 @@ post "/data_control" do |env| next if !video_id begin - video = get_video(video_id, PG_DB) + video = get_video(video_id) rescue ex next end @@ -815,11 +810,8 @@ post "/data_control" do |env| index: Random::Secure.rand(0_i64..Int64::MAX), }) - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist.id) + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) end end end @@ -837,18 +829,18 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) when "import_freetube" user.subscriptions += body.scan(/"channelId":"(?[a-zA-Z0-9_-]{24})"/).map do |md| md["channel_id"] end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) when "import_newpipe_subscriptions" body = JSON.parse(body) user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| @@ -865,9 +857,9 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) when "import_newpipe" Compress::Zip::Reader.open(IO::Memory.new(body)) do |file| file.each_entry do |entry| @@ -879,14 +871,14 @@ post "/data_control" do |env| user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v=")) user.watched.uniq! - PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email) + Invidious::Database::Users.update_watch_history(user) user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) + Invidious::Database::Users.update_subscriptions(user) db.close tempfile.delete @@ -914,7 +906,7 @@ get "/change_password" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) templated "change_password" end @@ -940,7 +932,7 @@ post "/change_password" do |env| end begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end @@ -970,7 +962,7 @@ post "/change_password" do |env| end new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) - PG_DB.exec("UPDATE users SET password = $1 WHERE email = $2", new_password.to_s, user.email) + Invidious::Database::Users.update_password(user, new_password.to_s) env.redirect referer end @@ -988,7 +980,7 @@ get "/delete_account" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) templated "delete_account" end @@ -1009,14 +1001,14 @@ post "/delete_account" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end view_name = "subscriptions_#{sha256(user.email)}" - PG_DB.exec("DELETE FROM users * WHERE email = $1", user.email) - PG_DB.exec("DELETE FROM session_ids * WHERE email = $1", user.email) + Invidious::Database::Users.delete(user) + Invidious::Database::SessionIDs.delete(email: user.email) PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") env.request.cookies.each do |cookie| @@ -1040,7 +1032,7 @@ get "/clear_watch_history" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) templated "clear_watch_history" end @@ -1061,12 +1053,12 @@ post "/clear_watch_history" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end - PG_DB.exec("UPDATE users SET watched = '{}' WHERE email = $1", user.email) + Invidious::Database::Users.clear_watch_history(user) env.redirect referer end @@ -1083,7 +1075,7 @@ get "/authorize_token" do |env| user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY) scopes = env.params.query["scopes"]?.try &.split(",") scopes ||= [] of String @@ -1114,7 +1106,7 @@ post "/authorize_token" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex next error_template(400, ex) end @@ -1123,7 +1115,7 @@ post "/authorize_token" do |env| callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? - access_token = generate_token(user.email, scopes, expire, HMAC_KEY, PG_DB) + access_token = generate_token(user.email, scopes, expire, HMAC_KEY) if callback_url access_token = URI.encode_www_form(access_token) @@ -1158,8 +1150,7 @@ get "/token_manager" do |env| end user = user.as(User) - - tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1 ORDER BY issued DESC", user.email, as: {session: String, issued: Time}) + tokens = Invidious::Database::SessionIDs.select_all(user.email) templated "token_manager" end @@ -1188,7 +1179,7 @@ post "/token_ajax" do |env| token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect next error_template(400, ex) @@ -1208,7 +1199,7 @@ post "/token_ajax" do |env| case action when .starts_with? "action_revoke_token" - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1 AND email = $2", session, user.email) + Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else next error_json(400, "Unsupported action #{action}") end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 827b6534..155ec559 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -114,7 +114,7 @@ class ChannelRedirect < Exception end end -def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) +def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_threads = 10) finished_channel = Channel(String | Nil).new spawn do @@ -130,7 +130,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma active_threads += 1 spawn do begin - get_channel(ucid, db, refresh, pull_all_videos) + get_channel(ucid, refresh, pull_all_videos) finished_channel.send(ucid) rescue ex finished_channel.send(nil) @@ -151,28 +151,21 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma return final end -def get_channel(id, db, refresh = true, pull_all_videos = true) - if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) +def get_channel(id, refresh = true, pull_all_videos = true) + if channel = Invidious::Database::Channels.select(id) if refresh && Time.utc - channel.updated > 10.minutes - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args}) \ - ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array) + channel = fetch_channel(id, pull_all_videos: pull_all_videos) + Invidious::Database::Channels.insert(channel, update_on_conflict: true) end else - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) - channel_array = channel.to_a - args = arg_array(channel_array) - - db.exec("INSERT INTO channels VALUES (#{args})", args: channel_array) + channel = fetch_channel(id, pull_all_videos: pull_all_videos) + Invidious::Database::Channels.insert(channel) end return channel end -def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) +def fetch_channel(ucid, pull_all_videos = true, locale = nil) LOGGER.debug("fetch_channel: #{ucid}") LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") @@ -241,15 +234,11 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) # We don't include the 'premiere_timestamp' here because channel pages don't include them, # meaning the above timestamp is always null - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) + was_insert = Invidious::Database::ChannelVideos.insert(video) if was_insert LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) + Invidious::Database::Users.add_notification(video) else LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") end @@ -284,13 +273,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # so since they don't provide a published date here we can safely ignore them. if Time.utc - video.published > 1.minute - was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + was_insert = Invidious::Database::ChannelVideos.insert(video) + Invidious::Database::Users.add_notification(video) if was_insert end end diff --git a/src/invidious/database/annotations.cr b/src/invidious/database/annotations.cr new file mode 100644 index 00000000..03749473 --- /dev/null +++ b/src/invidious/database/annotations.cr @@ -0,0 +1,24 @@ +require "./base.cr" + +module Invidious::Database::Annotations + extend self + + def insert(id : String, annotations : String) + request = <<-SQL + INSERT INTO annotations + VALUES ($1, $2) + ON CONFLICT DO NOTHING + SQL + + PG_DB.exec(request, id, annotations) + end + + def select(id : String) : Annotation? + request = <<-SQL + SELECT * FROM annotations + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: Annotation) + end +end diff --git a/src/invidious/database/base.cr b/src/invidious/database/base.cr new file mode 100644 index 00000000..6e49ea1a --- /dev/null +++ b/src/invidious/database/base.cr @@ -0,0 +1,110 @@ +require "pg" + +module Invidious::Database + extend self + + def check_enum(db, enum_name, struct_type = nil) + return # TODO + + if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) + LOGGER.info("check_enum: CREATE TYPE #{enum_name}") + + db.using_connection do |conn| + conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql")) + end + end + end + + def check_table(db, table_name, struct_type = nil) + # Create table if it doesn't exist + begin + db.exec("SELECT * FROM #{table_name} LIMIT 0") + rescue ex + LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}") + + db.using_connection do |conn| + conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql")) + end + end + + return if !struct_type + + struct_array = struct_type.type_array + column_array = get_column_array(db, table_name) + column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?[\d\D]*?)\);/) + .try &.["types"].split(",").map(&.strip).reject &.starts_with?("CONSTRAINT") + + return if !column_types + + struct_array.each_with_index do |name, i| + if name != column_array[i]? + if !column_array[i]? + new_column = column_types.select(&.starts_with?(name))[0] + LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + next + end + + # Column doesn't exist + if !column_array.includes? name + new_column = column_types.select(&.starts_with?(name))[0] + db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + end + + # Column exists but in the wrong position, rotate + if struct_array.includes? column_array[i] + until name == column_array[i] + new_column = column_types.select(&.starts_with?(column_array[i]))[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new") + + # There's a column we didn't expect + if !new_column + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + + column_array = get_column_array(db, table_name) + next + end + + LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + + LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") + db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") + + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + + LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") + db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") + + column_array = get_column_array(db, table_name) + end + else + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + end + end + end + + return if column_array.size <= struct_array.size + + column_array.each do |column| + if !struct_array.includes? column + LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + end + end + end + + def get_column_array(db, table_name) + column_array = [] of String + db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs| + rs.column_count.times do |i| + column = rs.as(PG::ResultSet).field(i) + column_array << column.name + end + end + + return column_array + end +end diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr new file mode 100644 index 00000000..134cf59d --- /dev/null +++ b/src/invidious/database/channels.cr @@ -0,0 +1,149 @@ +require "./base.cr" + +# +# This module contains functions related to the "channels" table. +# +module Invidious::Database::Channels + extend self + + # ------------------- + # Insert / delete + # ------------------- + + def insert(channel : InvidiousChannel, update_on_conflict : Bool = false) + channel_array = channel.to_a + + request = <<-SQL + INSERT INTO channels + VALUES (#{arg_array(channel_array)}) + SQL + + if update_on_conflict + request += <<-SQL + ON CONFLICT (id) DO UPDATE + SET author = $2, updated = $3 + SQL + end + + PG_DB.exec(request, args: channel_array) + end + + # ------------------- + # Update + # ------------------- + + def update_author(id : String, author : String) + request = <<-SQL + UPDATE channels + SET updated = $1, author = $2, deleted = false + WHERE id = $3 + SQL + + PG_DB.exec(request, Time.utc, author, id) + end + + def update_mark_deleted(id : String) + request = <<-SQL + UPDATE channels + SET updated = $1, deleted = true + WHERE id = $2 + SQL + + PG_DB.exec(request, Time.utc, id) + end + + # ------------------- + # Select + # ------------------- + + def select(id : String) : InvidiousChannel? + request = <<-SQL + SELECT * FROM channels + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: InvidiousChannel) + end + + def select(ids : Array(String)) : Array(InvidiousChannel)? + return [] of InvidiousChannel if ids.empty? + values = ids.map { |id| %(('#{id}')) }.join(",") + + request = <<-SQL + SELECT * FROM channels + WHERE id = ANY(VALUES #{values}) + SQL + + return PG_DB.query_all(request, as: InvidiousChannel) + end +end + +# +# This module contains functions related to the "channel_videos" table. +# +module Invidious::Database::ChannelVideos + extend self + + # ------------------- + # Insert + # ------------------- + + # This function returns the status of the query (i.e: success?) + def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool + if with_premiere_timestamp + last_items = "premiere_timestamp = $9, views = $10" + else + last_items = "views = $10" + end + + request = <<-SQL + INSERT INTO channel_videos + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (id) DO UPDATE + SET title = $2, published = $3, updated = $4, ucid = $5, + author = $6, length_seconds = $7, live_now = $8, #{last_items} + RETURNING (xmax=0) AS was_insert + SQL + + return PG_DB.query_one(request, *video.to_tuple, as: Bool) + end + + # ------------------- + # Select + # ------------------- + + def select(ids : Array(String)) : Array(ChannelVideo) + return [] of ChannelVideo if ids.empty? + + request = <<-SQL + SELECT * FROM channel_videos + WHERE id IN (#{arg_array(ids)}) + ORDER BY published DESC + SQL + + return PG_DB.query_all(request, args: ids, as: ChannelVideo) + end + + def select_notfications(ucid : String, since : Time) : Array(ChannelVideo) + request = <<-SQL + SELECT * FROM channel_videos + WHERE ucid = $1 AND published > $2 + ORDER BY published DESC + LIMIT 15 + SQL + + return PG_DB.query_all(request, ucid, since, as: ChannelVideo) + end + + def select_popular_videos : Array(ChannelVideo) + request = <<-SQL + SELECT DISTINCT ON (ucid) * + FROM channel_videos + WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d + GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) + ORDER BY ucid, published DESC + SQL + + PG_DB.query_all(request, as: ChannelVideo) + end +end diff --git a/src/invidious/database/nonces.cr b/src/invidious/database/nonces.cr new file mode 100644 index 00000000..469fcbd8 --- /dev/null +++ b/src/invidious/database/nonces.cr @@ -0,0 +1,46 @@ +require "./base.cr" + +module Invidious::Database::Nonces + extend self + + # ------------------- + # Insert + # ------------------- + + def insert(nonce : String, expire : Time) + request = <<-SQL + INSERT INTO nonces + VALUES ($1, $2) + ON CONFLICT DO NOTHING + SQL + + PG_DB.exec(request, nonce, expire) + end + + # ------------------- + # Update + # ------------------- + + def update_set_expired(nonce : String) + request = <<-SQL + UPDATE nonces + SET expire = $1 + WHERE nonce = $2 + SQL + + PG_DB.exec(request, Time.utc(1990, 1, 1), nonce) + end + + # ------------------- + # Select + # ------------------- + + def select(nonce : String) : Tuple(String, Time)? + request = <<-SQL + SELECT * FROM nonces + WHERE nonce = $1 + SQL + + return PG_DB.query_one?(request, nonce, as: {String, Time}) + end +end diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr new file mode 100644 index 00000000..950d5f4b --- /dev/null +++ b/src/invidious/database/playlists.cr @@ -0,0 +1,257 @@ +require "./base.cr" + +# +# This module contains functions related to the "playlists" table. +# +module Invidious::Database::Playlists + extend self + + # ------------------- + # Insert / delete + # ------------------- + + def insert(playlist : InvidiousPlaylist) + playlist_array = playlist.to_a + + request = <<-SQL + INSERT INTO playlists + VALUES (#{arg_array(playlist_array)}) + SQL + + PG_DB.exec(request, args: playlist_array) + end + + # this function is a bit special: it will also remove all videos + # related to the given playlist ID in the "playlist_videos" table, + # in addition to deleting said ID from "playlists". + def delete(id : String) + request = <<-SQL + DELETE FROM playlist_videos * WHERE plid = $1; + DELETE FROM playlists * WHERE id = $1 + SQL + + PG_DB.exec(request, id) + end + + # ------------------- + # Update + # ------------------- + + def update(id : String, title : String, privacy, description, updated) + request = <<-SQL + UPDATE playlists + SET title = $1, privacy = $2, description = $3, updated = $4 + WHERE id = $5 + SQL + + PG_DB.exec(request, title, privacy, description, updated, id) + end + + def update_description(id : String, description) + request = <<-SQL + UPDATE playlists + SET description = $1 + WHERE id = $2 + SQL + + PG_DB.exec(request, description, id) + end + + def update_subscription_time(id : String) + request = <<-SQL + UPDATE playlists + SET subscribed = $1 + WHERE id = $2 + SQL + + PG_DB.exec(request, Time.utc, id) + end + + def update_video_added(id : String, index : String | Int64) + request = <<-SQL + UPDATE playlists + SET index = array_append(index, $1), + video_count = cardinality(index) + 1, + updated = $2 + WHERE id = $3 + SQL + + PG_DB.exec(request, index, Time.utc, id) + end + + def update_video_removed(id : String, index : String | Int64) + request = <<-SQL + UPDATE playlists + SET index = array_remove(index, $1), + video_count = cardinality(index) - 1, + updated = $2 + WHERE id = $3 + SQL + + PG_DB.exec(request, index, Time.utc, id) + end + + # ------------------- + # Salect + # ------------------- + + def select(*, id : String, raise_on_fail : Bool = false) : InvidiousPlaylist? + request = <<-SQL + SELECT * FROM playlists + WHERE id = $1 + SQL + + if raise_on_fail + return PG_DB.query_one(request, id, as: InvidiousPlaylist) + else + return PG_DB.query_one?(request, id, as: InvidiousPlaylist) + end + end + + def select_all(*, author : String) : Array(InvidiousPlaylist) + request = <<-SQL + SELECT * FROM playlists + WHERE author = $1 + SQL + + return PG_DB.query_all(request, author, as: InvidiousPlaylist) + end + + # ------------------- + # Salect (filtered) + # ------------------- + + def select_like_iv(email : String) : Array(InvidiousPlaylist) + request = <<-SQL + SELECT * FROM playlists + WHERE author = $1 AND id LIKE 'IV%' + ORDER BY created + SQL + + PG_DB.query_all(request, email, as: InvidiousPlaylist) + end + + def select_not_like_iv(email : String) : Array(InvidiousPlaylist) + request = <<-SQL + SELECT * FROM playlists + WHERE author = $1 AND id NOT LIKE 'IV%' + ORDER BY created + SQL + + PG_DB.query_all(request, email, as: InvidiousPlaylist) + end + + def select_user_created_playlists(email : String) : Array({String, String}) + request = <<-SQL + SELECT id,title FROM playlists + WHERE author = $1 AND id LIKE 'IV%' + SQL + + PG_DB.query_all(request, email, as: {String, String}) + end + + # ------------------- + # Misc checks + # ------------------- + + # Check if given playlist ID exists + def exists?(id : String) : Bool + request = <<-SQL + SELECT id FROM playlists + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: String).nil? + end + + # Count how many playlist a user has created. + def count_owned_by(author : String) : Int64 + request = <<-SQL + SELECT count(*) FROM playlists + WHERE author = $1 + SQL + + return PG_DB.query_one?(request, author, as: Int64) || 0_i64 + end +end + +# +# This module contains functions related to the "playlist_videos" table. +# +module Invidious::Database::PlaylistVideos + extend self + + private alias VideoIndex = Int64 | Array(Int64) + + # ------------------- + # Insert / Delete + # ------------------- + + def insert(video : PlaylistVideo) + video_array = video.to_a + + request = <<-SQL + INSERT INTO playlist_videos + VALUES (#{arg_array(video_array)}) + SQL + + PG_DB.exec(request, args: video_array) + end + + def delete(index) + request = <<-SQL + DELETE FROM playlist_videos * + WHERE index = $1 + SQL + + PG_DB.exec(request, index) + end + + # ------------------- + # Salect + # ------------------- + + def select(plid : String, index : VideoIndex, offset, limit = 100) : Array(PlaylistVideo) + request = <<-SQL + SELECT * FROM playlist_videos + WHERE plid = $1 + ORDER BY array_position($2, index) + LIMIT $3 + OFFSET $4 + SQL + + return PG_DB.query_all(request, plid, index, limit, offset, as: PlaylistVideo) + end + + def select_index(plid : String, vid : String) : Int64? + request = <<-SQL + SELECT index FROM playlist_videos + WHERE plid = $1 AND id = $2 + LIMIT 1 + SQL + + return PG_DB.query_one?(request, plid, vid, as: Int64) + end + + def select_one_id(plid : String, index : VideoIndex) : String? + request = <<-SQL + SELECT id FROM playlist_videos + WHERE plid = $1 + ORDER BY array_position($2, index) + LIMIT 1 + SQL + + return PG_DB.query_one?(request, plid, index, as: String) + end + + def select_ids(plid : String, index : VideoIndex, limit = 500) : Array(String) + request = <<-SQL + SELECT id FROM playlist_videos + WHERE plid = $1 + ORDER BY array_position($2, index) + LIMIT $3 + SQL + + return PG_DB.query_all(request, plid, index, limit, as: String) + end +end diff --git a/src/invidious/database/sessions.cr b/src/invidious/database/sessions.cr new file mode 100644 index 00000000..d5f85dd6 --- /dev/null +++ b/src/invidious/database/sessions.cr @@ -0,0 +1,74 @@ +require "./base.cr" + +module Invidious::Database::SessionIDs + extend self + + # ------------------- + # Insert + # ------------------- + + def insert(sid : String, email : String, handle_conflicts : Bool = false) + request = <<-SQL + INSERT INTO session_ids + VALUES ($1, $2, $3) + SQL + + request += " ON CONFLICT (id) DO NOTHING" if handle_conflicts + + PG_DB.exec(request, sid, email, Time.utc) + end + + # ------------------- + # Delete + # ------------------- + + def delete(*, sid : String) + request = <<-SQL + DELETE FROM session_ids * + WHERE id = $1 + SQL + + PG_DB.exec(request, sid) + end + + def delete(*, email : String) + request = <<-SQL + DELETE FROM session_ids * + WHERE email = $1 + SQL + + PG_DB.exec(request, email) + end + + def delete(*, sid : String, email : String) + request = <<-SQL + DELETE FROM session_ids * + WHERE id = $1 AND email = $2 + SQL + + PG_DB.exec(request, sid, email) + end + + # ------------------- + # Select + # ------------------- + + def select_email(sid : String) : String? + request = <<-SQL + SELECT email FROM session_ids + WHERE id = $1 + SQL + + PG_DB.query_one?(request, sid, as: String) + end + + def select_all(email : String) : Array({session: String, issued: Time}) + request = <<-SQL + SELECT id, issued FROM session_ids + WHERE email = $1 + ORDER BY issued DESC + SQL + + PG_DB.query_all(request, email, as: {session: String, issued: Time}) + end +end diff --git a/src/invidious/database/statistics.cr b/src/invidious/database/statistics.cr new file mode 100644 index 00000000..1df549e2 --- /dev/null +++ b/src/invidious/database/statistics.cr @@ -0,0 +1,49 @@ +require "./base.cr" + +module Invidious::Database::Statistics + extend self + + # ------------------- + # User stats + # ------------------- + + def count_users_total : Int64 + request = <<-SQL + SELECT count(*) FROM users + SQL + + PG_DB.query_one(request, as: Int64) + end + + def count_users_active_1m : Int64 + request = <<-SQL + SELECT count(*) FROM users + WHERE CURRENT_TIMESTAMP - updated < '6 months' + SQL + + PG_DB.query_one(request, as: Int64) + end + + def count_users_active_6m : Int64 + request = <<-SQL + SELECT count(*) FROM users + WHERE CURRENT_TIMESTAMP - updated < '1 month' + SQL + + PG_DB.query_one(request, as: Int64) + end + + # ------------------- + # Channel stats + # ------------------- + + def channel_last_update : Time? + request = <<-SQL + SELECT updated FROM channels + ORDER BY updated DESC + LIMIT 1 + SQL + + PG_DB.query_one?(request, as: Time) + end +end diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr new file mode 100644 index 00000000..71650918 --- /dev/null +++ b/src/invidious/database/users.cr @@ -0,0 +1,218 @@ +require "./base.cr" + +module Invidious::Database::Users + extend self + + # ------------------- + # Insert / delete + # ------------------- + + def insert(user : User, update_on_conflict : Bool = false) + user_array = user.to_a + user_array[4] = user_array[4].to_json # User preferences + + request = <<-SQL + INSERT INTO users + VALUES (#{arg_array(user_array)}) + SQL + + if update_on_conflict + request += <<-SQL + ON CONFLICT (email) DO UPDATE + SET updated = $1, subscriptions = $3 + SQL + end + + PG_DB.exec(request, args: user_array) + end + + def delete(user : User) + request = <<-SQL + DELETE FROM users * + WHERE email = $1 + SQL + + PG_DB.exec(request, user.email) + end + + # ------------------- + # Update (history) + # ------------------- + + def update_watch_history(user : User) + request = <<-SQL + UPDATE users + SET watched = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.watched, user.email) + end + + def mark_watched(user : User, vid : String) + request = <<-SQL + UPDATE users + SET watched = array_append(watched, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, vid, user.email) + end + + def mark_unwatched(user : User, vid : String) + request = <<-SQL + UPDATE users + SET watched = array_remove(watched, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, vid, user.email) + end + + def clear_watch_history(user : User) + request = <<-SQL + UPDATE users + SET watched = '{}' + WHERE email = $1 + SQL + + PG_DB.exec(request, user.email) + end + + # ------------------- + # Update (channels) + # ------------------- + + def update_subscriptions(user : User) + request = <<-SQL + UPDATE users + SET feed_needs_update = true, subscriptions = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.subscriptions, user.email) + end + + def subscribe_channel(user : User, ucid : String) + request = <<-SQL + UPDATE users + SET feed_needs_update = true, + subscriptions = array_append(subscriptions,$1) + WHERE email = $2 + SQL + + PG_DB.exec(request, ucid, user.email) + end + + def unsubscribe_channel(user : User, ucid : String) + request = <<-SQL + UPDATE users + SET feed_needs_update = true, + subscriptions = array_remove(subscriptions, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, ucid, user.email) + end + + # ------------------- + # Update (notifs) + # ------------------- + + def add_notification(video : ChannelVideo) + request = <<-SQL + UPDATE users + SET notifications = array_append(notifications, $1), + feed_needs_update = true + WHERE $2 = ANY(subscriptions) + SQL + + PG_DB.exec(request, video.id, video.ucid) + end + + def remove_notification(user : User, vid : String) + request = <<-SQL + UPDATE users + SET notifications = array_remove(notifications, $1) + WHERE email = $2 + SQL + + PG_DB.exec(request, vid, user.email) + end + + def clear_notifications(user : User) + request = <<-SQL + UPDATE users + SET notifications = $1, updated = $2 + WHERE email = $3 + SQL + + PG_DB.exec(request, [] of String, Time.utc, user) + end + + # ------------------- + # Update (misc) + # ------------------- + + def update_preferences(user : User) + request = <<-SQL + UPDATE users + SET preferences = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.preferences.to_json, user.email) + end + + def update_password(user : User, pass : String) + request = <<-SQL + UPDATE users + SET password = $1 + WHERE email = $2 + SQL + + PG_DB.exec(request, user.email, pass) + end + + # ------------------- + # Select + # ------------------- + + def select(*, email : String) : User? + request = <<-SQL + SELECT * FROM users + WHERE email = $1 + SQL + + return PG_DB.query_one?(request, email, as: User) + end + + # Same as select, but can raise an exception + def select!(*, email : String) : User + request = <<-SQL + SELECT * FROM users + WHERE email = $1 + SQL + + return PG_DB.query_one(request, email, as: User) + end + + def select(*, token : String) : User? + request = <<-SQL + SELECT * FROM users + WHERE token = $1 + SQL + + return PG_DB.query_one?(request, token, as: User) + end + + def select_notifications(user : User) : Array(String) + request = <<-SQL + SELECT notifications + FROM users + WHERE email = $1 + SQL + + return PG_DB.query_one(request, user.email, as: Array(String)) + end +end diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr new file mode 100644 index 00000000..e1fa01c3 --- /dev/null +++ b/src/invidious/database/videos.cr @@ -0,0 +1,43 @@ +require "./base.cr" + +module Invidious::Database::Videos + extend self + + def insert(video : Video) + request = <<-SQL + INSERT INTO videos + VALUES ($1, $2, $3) + ON CONFLICT (id) DO NOTHING + SQL + + PG_DB.exec(request, video.id, video.info.to_json, video.updated) + end + + def delete(id) + request = <<-SQL + DELETE FROM videos * + WHERE id = $1 + SQL + + PG_DB.exec(request, id) + end + + def update(video : Video) + request = <<-SQL + UPDATE videos + SET (id, info, updated) = ($1, $2, $3) + WHERE id = $1 + SQL + + PG_DB.exec(request, video.id, video.info.to_json, video.updated) + end + + def select(id : String) : Video? + request = <<-SQL + SELECT * FROM videos + WHERE id = $1 + SQL + + return PG_DB.query_one?(request, id, as: Video) + end +end diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 045b6701..d140a858 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -97,18 +97,18 @@ class AuthHandler < Kemal::Handler if token = env.request.headers["Authorization"]? token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) session = URI.decode_www_form(token["session"].as_s) - scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil) + scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, nil) - if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", session, as: String) - user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + if email = Invidious::Database::SessionIDs.select_email(session) + user = Invidious::Database::Users.select!(email: email) end elsif sid = env.request.cookies["SID"]?.try &.value if sid.starts_with? "v1:" raise "Cannot use token as SID" end - if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) - user = PG_DB.query_one("SELECT * FROM users WHERE email = $1", email, as: User) + if email = Invidious::Database::SessionIDs.select_email(sid) + user = Invidious::Database::Users.select!(email: email) end scopes = [":*"] diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 96a78eb9..c3b53339 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -60,112 +60,7 @@ def html_to_content(description_html : String) return description end -def check_enum(db, enum_name, struct_type = nil) - return # TODO - - if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) - LOGGER.info("check_enum: CREATE TYPE #{enum_name}") - - db.using_connection do |conn| - conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql")) - end - end -end - -def check_table(db, table_name, struct_type = nil) - # Create table if it doesn't exist - begin - db.exec("SELECT * FROM #{table_name} LIMIT 0") - rescue ex - LOGGER.info("check_table: check_table: CREATE TABLE #{table_name}") - - db.using_connection do |conn| - conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql")) - end - end - - return if !struct_type - - struct_array = struct_type.type_array - column_array = get_column_array(db, table_name) - column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?[\d\D]*?)\);/) - .try &.["types"].split(",").map(&.strip).reject &.starts_with?("CONSTRAINT") - - return if !column_types - - struct_array.each_with_index do |name, i| - if name != column_array[i]? - if !column_array[i]? - new_column = column_types.select(&.starts_with?(name))[0] - LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - next - end - - # Column doesn't exist - if !column_array.includes? name - new_column = column_types.select(&.starts_with?(name))[0] - db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - end - - # Column exists but in the wrong position, rotate - if struct_array.includes? column_array[i] - until name == column_array[i] - new_column = column_types.select(&.starts_with?(column_array[i]))[0]?.try &.gsub("#{column_array[i]}", "#{column_array[i]}_new") - - # There's a column we didn't expect - if !new_column - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - - column_array = get_column_array(db, table_name) - next - end - - LOGGER.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - - LOGGER.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") - db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") - - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - - LOGGER.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") - db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") - - column_array = get_column_array(db, table_name) - end - else - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - end - end - end - - return if column_array.size <= struct_array.size - - column_array.each do |column| - if !struct_array.includes? column - LOGGER.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") - db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") - end - end -end - -def get_column_array(db, table_name) - column_array = [] of String - db.query("SELECT * FROM #{table_name} LIMIT 0") do |rs| - rs.column_count.times do |i| - column = rs.as(PG::ResultSet).field(i) - column_array << column.name - end - end - - return column_array -end - -def cache_annotation(db, id, annotations) +def cache_annotation(id, annotations) if !CONFIG.cache_annotations return end @@ -183,7 +78,7 @@ def cache_annotation(db, id, annotations) end end - db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) if has_legacy_annotations + Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations end def create_notification_stream(env, topics, connection_channel) @@ -204,7 +99,7 @@ def create_notification_stream(env, topics, connection_channel) published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3]) video_id = TEST_IDS[rand(TEST_IDS.size)] - video = get_video(video_id, PG_DB) + video = get_video(video_id) video.published = published response = JSON.parse(video.to_json(locale, nil)) @@ -235,11 +130,12 @@ def create_notification_stream(env, topics, connection_channel) spawn do begin if since + since_unix = Time.unix(since.not_nil!) + topics.try &.each do |topic| case topic when .match(/UC[A-Za-z0-9_-]{22}/) - PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", - topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| + Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) if fields_text = env.params.query["fields"]? @@ -280,7 +176,7 @@ def create_notification_stream(env, topics, connection_channel) next end - video = get_video(video_id, PG_DB) + video = get_video(video_id) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index 3874799a..8b076e39 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -1,8 +1,8 @@ require "crypto/subtle" -def generate_token(email, scopes, expire, key, db) +def generate_token(email, scopes, expire, key) session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}" - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc) + Invidious::Database::SessionIDs.insert(session, email) token = { "session" => session, @@ -19,7 +19,7 @@ def generate_token(email, scopes, expire, key, db) return token.to_json end -def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = false) +def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false) expire = Time.utc + expire token = { @@ -30,7 +30,7 @@ def generate_response(session, scopes, key, db, expire = 6.hours, use_nonce = fa if use_nonce nonce = Random::Secure.hex(16) - db.exec("INSERT INTO nonces VALUES ($1, $2) ON CONFLICT DO NOTHING", nonce, expire) + Invidious::Database::Nonces.insert(nonce, expire) token["nonce"] = nonce end @@ -63,7 +63,7 @@ def sign_token(key, hash) return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip end -def validate_request(token, session, request, key, db, locale = nil) +def validate_request(token, session, request, key, locale = nil) case token when String token = JSON.parse(URI.decode_www_form(token)).as_h @@ -92,9 +92,9 @@ def validate_request(token, session, request, key, db, locale = nil) raise InfoException.new("Invalid signature") end - if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time})) + if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s)) if nonce[1] > Time.utc - db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0]) + Invidious::Database::Nonces.update_set_expired(nonce[0]) else raise InfoException.new("Erroneous token") end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 7bbbcb92..8453d605 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,5 +1,3 @@ -require "db" - # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html def ci_lower_bound(pos, n) if n == 0 diff --git a/src/invidious/jobs/pull_popular_videos_job.cr b/src/invidious/jobs/pull_popular_videos_job.cr index 38de816e..dc785bae 100644 --- a/src/invidious/jobs/pull_popular_videos_job.cr +++ b/src/invidious/jobs/pull_popular_videos_job.cr @@ -1,11 +1,4 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob - QUERY = <<-SQL - SELECT DISTINCT ON (ucid) * - FROM channel_videos - WHERE ucid IN (SELECT channel FROM (SELECT UNNEST(subscriptions) AS channel FROM users) AS d - GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) - ORDER BY ucid, published DESC - SQL POPULAR_VIDEOS = Atomic.new([] of ChannelVideo) private getter db : DB::Database @@ -14,7 +7,7 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob def begin loop do - videos = db.query_all(QUERY, as: ChannelVideo) + videos = Invidious::Database::ChannelVideos.select_popular_videos .sort_by!(&.published) .reverse! diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 2321e964..c224c745 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -35,11 +35,11 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob lim_fibers = max_fibers LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB") - db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id) + Invidious::Database::Channels.update_author(id, channel.author) rescue ex LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}") if ex.message == "Deleted or invalid channel" - db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id) + Invidious::Database::Channels.update_mark_deleted(id) else lim_fibers = 1 LOGGER.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s") diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index 926c27fa..4b52c959 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -25,7 +25,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob spawn do begin # Drop outdated views - column_array = get_column_array(db, view_name) + column_array = Invidious::Database.get_column_array(db, view_name) ChannelVideo.type_array.each_with_index do |name, i| if name != column_array[i]? LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}") diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index 6569c0a1..a113bd77 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -47,12 +47,14 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob private def refresh_stats users = STATISTICS.dig("usage", "users").as(Hash(String, Int64)) - users["total"] = db.query_one("SELECT count(*) FROM users", as: Int64) - users["activeHalfyear"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '6 months'", as: Int64) - users["activeMonth"] = db.query_one("SELECT count(*) FROM users WHERE CURRENT_TIMESTAMP - updated < '1 month'", as: Int64) + + users["total"] = Invidious::Database::Statistics.count_users_total + users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m + users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m + STATISTICS["metadata"] = { "updatedAt" => Time.utc.to_unix, - "lastChannelRefreshedAt" => db.query_one?("SELECT updated FROM channels ORDER BY updated DESC LIMIT 1", as: Time).try &.to_unix || 0_i64, + "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64, } end end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index f37667b5..ee7e12aa 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -125,7 +125,7 @@ struct Playlist json.field "videos" do json.array do - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) + videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id) videos.each do |video| video.to_json(json) end @@ -200,12 +200,12 @@ struct InvidiousPlaylist json.field "videos" do json.array do - if !offset || offset == 0 - index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, video_id, as: Int64) + if (!offset || offset == 0) && !video_id.nil? + index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id) offset = self.index.index(index) || 0 end - videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, video_id: video_id) + videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id) videos.each_with_index do |video, index| video.to_json(json, offset + index) end @@ -225,7 +225,8 @@ struct InvidiousPlaylist end def thumbnail - @thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------" + # TODO: Get playlist thumbnail from playlist data rather than first video + @thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------" "/vi/#{@thumbnail_id}/mqdefault.jpg" end @@ -246,7 +247,7 @@ struct InvidiousPlaylist end end -def create_playlist(db, title, privacy, user) +def create_playlist(title, privacy, user) plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" playlist = InvidiousPlaylist.new({ @@ -261,15 +262,12 @@ def create_playlist(db, title, privacy, user) index: [] of Int64, }) - playlist_array = playlist.to_a - args = arg_array(playlist_array) - - db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array) + Invidious::Database::Playlists.insert(playlist) return playlist end -def subscribe_playlist(db, user, playlist) +def subscribe_playlist(user, playlist) playlist = InvidiousPlaylist.new({ title: playlist.title.byte_slice(0, 150), id: playlist.id, @@ -282,10 +280,7 @@ def subscribe_playlist(db, user, playlist) index: [] of Int64, }) - playlist_array = playlist.to_a - args = arg_array(playlist_array) - - db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array) + Invidious::Database::Playlists.insert(playlist) return playlist end @@ -327,9 +322,9 @@ def produce_playlist_continuation(id, index) return continuation end -def get_playlist(db, plid, locale, refresh = true, force_refresh = false) +def get_playlist(plid, locale, refresh = true, force_refresh = false) if plid.starts_with? "IV" - if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if playlist = Invidious::Database::Playlists.select(id: plid) return playlist else raise InfoException.new("Playlist does not exist.") @@ -409,7 +404,7 @@ def fetch_playlist(plid, locale) }) end -def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil) +def get_playlist_videos(playlist, offset, locale = nil, video_id = nil) # Show empy playlist if requested page is out of range # (e.g, when a new playlist has been created, offset will be negative) if offset >= playlist.video_count || offset < 0 @@ -417,8 +412,7 @@ def get_playlist_videos(db, playlist, offset, locale = nil, video_id = nil) end if playlist.is_a? InvidiousPlaylist - db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", - playlist.id, playlist.index, offset, as: PlaylistVideo) + Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100) else if video_id initial_data = YoutubeAPI.next({ diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 12687ec6..b6183001 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -13,7 +13,7 @@ module Invidious::Routes::API::Manifest unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index aaf728ff..fda655ef 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -22,12 +22,11 @@ module Invidious::Routes::API::V1::Authenticated user = env.get("user").as(User) begin - preferences = Preferences.from_json(env.request.body || "{}") + user.preferences = Preferences.from_json(env.request.body || "{}") rescue - preferences = user.preferences end - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + Invidious::Database::Users.update_preferences(user) env.response.status_code = 204 end @@ -45,7 +44,7 @@ module Invidious::Routes::API::V1::Authenticated page = env.params.query["page"]?.try &.to_i? page ||= 1 - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + videos, notifications = get_subscription_feed(user, max_results, page) JSON.build do |json| json.object do @@ -72,13 +71,7 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "application/json" user = env.get("user").as(User) - if user.subscriptions.empty? - values = "'{}'" - else - values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" - end - - subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) + subscriptions = Invidious::Database::Channels.select(user.subscriptions) JSON.build do |json| json.array do @@ -99,8 +92,8 @@ module Invidious::Routes::API::V1::Authenticated ucid = env.params.url["ucid"] if !user.subscriptions.includes? ucid - get_channel(ucid, PG_DB, false, false) - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) + get_channel(ucid, false, false) + Invidious::Database::Users.subscribe_channel(user, ucid) end # For Google accounts, access tokens don't have enough information to @@ -116,7 +109,7 @@ module Invidious::Routes::API::V1::Authenticated ucid = env.params.url["ucid"] - PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", ucid, user.email) + Invidious::Database::Users.unsubscribe_channel(user, ucid) env.response.status_code = 204 end @@ -127,7 +120,7 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "application/json" user = env.get("user").as(User) - playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousPlaylist) + playlists = Invidious::Database::Playlists.select_all(author: user.email) JSON.build do |json| json.array do @@ -153,11 +146,11 @@ module Invidious::Routes::API::V1::Authenticated return error_json(400, "Invalid privacy setting.") end - if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 + if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 return error_json(400, "User cannot have more than 100 playlists.") end - playlist = create_playlist(PG_DB, title, privacy, user) + playlist = create_playlist(title, privacy, user) env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}" env.response.status_code = 201 { @@ -172,9 +165,12 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "application/json" user = env.get("user").as(User) - plid = env.params.url["plid"] + plid = env.params.url["plid"]? + if !plid || plid.empty? + return error_json(400, "A playlist ID is required") + end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -195,7 +191,8 @@ module Invidious::Routes::API::V1::Authenticated updated = playlist.updated end - PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + Invidious::Database::Playlists.update(plid, title, privacy, description, updated) + env.response.status_code = 204 end @@ -207,7 +204,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -216,8 +213,7 @@ module Invidious::Routes::API::V1::Authenticated return error_json(403, "Invalid user") end - PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) - PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + Invidious::Database::Playlists.delete(plid) env.response.status_code = 204 end @@ -230,7 +226,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -249,7 +245,7 @@ module Invidious::Routes::API::V1::Authenticated end begin - video = get_video(video_id, PG_DB) + video = get_video(video_id) rescue ex return error_json(500, ex) end @@ -266,11 +262,8 @@ module Invidious::Routes::API::V1::Authenticated index: Random::Secure.rand(0_i64..Int64::MAX), }) - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, plid) + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(plid, playlist_video.index) env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}" env.response.status_code = 201 @@ -289,7 +282,7 @@ module Invidious::Routes::API::V1::Authenticated plid = env.params.url["plid"] index = env.params.url["index"].to_i64(16) - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email && playlist.privacy.private? return error_json(404, "Playlist does not exist.") end @@ -302,8 +295,8 @@ module Invidious::Routes::API::V1::Authenticated return error_json(404, "Playlist does not contain index") end - PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) - PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, plid) + Invidious::Database::PlaylistVideos.delete(index) + Invidious::Database::Playlists.update_video_removed(plid, index) env.response.status_code = 204 end @@ -318,7 +311,7 @@ module Invidious::Routes::API::V1::Authenticated user = env.get("user").as(User) scopes = env.get("scopes").as(Array(String)) - tokens = PG_DB.query_all("SELECT id, issued FROM session_ids WHERE email = $1", user.email, as: {session: String, issued: Time}) + tokens = Invidious::Database::SessionIDs.select_all(user.email) JSON.build do |json| json.array do @@ -360,7 +353,7 @@ module Invidious::Routes::API::V1::Authenticated if sid = env.get?("sid").try &.as(String) env.response.content_type = "text/html" - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, PG_DB, use_nonce: true) + csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true) return templated "authorize_token" else env.response.content_type = "application/json" @@ -374,7 +367,7 @@ module Invidious::Routes::API::V1::Authenticated end end - access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB) + access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY) if callback_url access_token = URI.encode_www_form(access_token) @@ -406,9 +399,9 @@ module Invidious::Routes::API::V1::Authenticated # Allow tokens to revoke other tokens with correct scope if session == env.get("session").as(String) - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) + Invidious::Database::SessionIDs.delete(sid: session) elsif scopes_include_scope(scopes, "GET:tokens") - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) + Invidious::Database::SessionIDs.delete(sid: session) else return error_json(400, "Cannot revoke session #{session}") end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 1621c9ef..ac0576a0 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -34,7 +34,7 @@ module Invidious::Routes::API::V1::Misc end begin - playlist = get_playlist(PG_DB, plid, locale) + playlist = get_playlist(plid, locale) rescue ex : InfoException return error_json(404, ex) rescue ex diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 4c7179ce..4d244e7f 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Videos region = env.params.query["region"]? begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) @@ -36,7 +36,7 @@ module Invidious::Routes::API::V1::Videos # getting video info. begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) @@ -157,7 +157,7 @@ module Invidious::Routes::API::V1::Videos region = env.params.query["region"]? begin - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) rescue ex : VideoRedirect env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) return error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) @@ -239,7 +239,7 @@ module Invidious::Routes::API::V1::Videos case source when "archive" - if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation)) + if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id)) annotations = cached_annotation.annotations else index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') @@ -271,7 +271,7 @@ module Invidious::Routes::API::V1::Videos annotations = response.body - cache_annotation(PG_DB, id, annotations) + cache_annotation(id, annotations) end else # "youtube" response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}") diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 049ee344..ab722ae2 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -6,9 +6,9 @@ module Invidious::Routes::Embed if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") begin - playlist = get_playlist(PG_DB, plid, locale: locale) + playlist = get_playlist(plid, locale: locale) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) + videos = get_playlist_videos(playlist, offset: offset, locale: locale) rescue ex return error_template(500, ex) end @@ -30,7 +30,7 @@ module Invidious::Routes::Embed id = env.params.url["id"] plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") - continuation = process_continuation(PG_DB, env.params.query, plid, id) + continuation = process_continuation(env.params.query, plid, id) if md = env.params.query["playlist"]? .try &.match(/[a-zA-Z0-9_-]{11}(,[a-zA-Z0-9_-]{11})*/) @@ -60,9 +60,9 @@ module Invidious::Routes::Embed if plid begin - playlist = get_playlist(PG_DB, plid, locale: locale) + playlist = get_playlist(plid, locale: locale) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) + videos = get_playlist_videos(playlist, offset: offset, locale: locale) rescue ex return error_template(500, ex) end @@ -119,7 +119,7 @@ module Invidious::Routes::Embed subscriptions ||= [] of String begin - video = get_video(id, PG_DB, region: params.region) + video = get_video(id, region: params.region) rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex @@ -137,7 +137,7 @@ module Invidious::Routes::Embed # end if notifications && notifications.includes? id - PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email) + Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 9650bcf4..fd8c25ce 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -15,13 +15,14 @@ module Invidious::Routes::Feeds user = user.as(User) - items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + # TODO: make a single DB call and separate the items here? + items_created = Invidious::Database::Playlists.select_like_iv(user.email) items_created.map! do |item| item.author = "" item end - items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_saved = Invidious::Database::Playlists.select_not_like_iv(user.email) items_saved.map! do |item| item.author = "" item @@ -83,7 +84,7 @@ module Invidious::Routes::Feeds headers["Cookie"] = env.request.headers["Cookie"] if !user.password - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers) end max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) @@ -93,14 +94,13 @@ module Invidious::Routes::Feeds page = env.params.query["page"]?.try &.to_i? page ||= 1 - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + videos, notifications = get_subscription_feed(user, max_results, page) # "updated" here is used for delivering new notifications, so if # we know a user has looked at their feed e.g. in the past 10 minutes, # they've already seen a video posted 20 minutes ago, and don't need # to be notified. - PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.utc, - user.email) + Invidious::Database::Users.clear_notifications(user) user.notifications = [] of String env.set "user", user @@ -220,7 +220,7 @@ module Invidious::Routes::Feeds haltf env, status_code: 403 end - user = PG_DB.query_one?("SELECT * FROM users WHERE token = $1", token.strip, as: User) + user = Invidious::Database::Users.select(token: token.strip) if !user haltf env, status_code: 403 end @@ -234,7 +234,7 @@ module Invidious::Routes::Feeds params = HTTP::Params.parse(env.params.query["params"]? || "") - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + videos, notifications = get_subscription_feed(user, max_results, page) XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", @@ -264,8 +264,8 @@ module Invidious::Routes::Feeds path = env.request.path if plid.starts_with? "IV" - if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) + if playlist = Invidious::Database::Playlists.select(id: plid) + videos = get_playlist_videos(playlist, offset: 0, locale: locale) return XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", @@ -364,7 +364,7 @@ module Invidious::Routes::Feeds if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]? PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid) elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]? - PG_DB.exec("UPDATE playlists SET subscribed = $1 WHERE id = $2", Time.utc, ucid) + Invidious::Database::Playlists.update_subscription_time(plid) else haltf env, status_code: 400 end @@ -393,7 +393,7 @@ module Invidious::Routes::Feeds published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - video = get_video(id, PG_DB, force_refresh: true) + video = get_video(id, force_refresh: true) # Deliver notifications to `/api/v1/auth/notifications` payload = { @@ -416,13 +416,8 @@ module Invidious::Routes::Feeds views: video.views, }) - was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, - updated = $4, ucid = $5, author = $6, length_seconds = $7, - live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true) + Invidious::Database::Users.add_notification(video) if was_insert end end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 2a50561d..64da3e4e 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -275,7 +275,7 @@ module Invidious::Routes::Login raise "Couldn't get SID." end - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers) # We are now logged in traceback << "done.
" @@ -303,8 +303,8 @@ module Invidious::Routes::Login end if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + user.preferences = env.get("preferences").as(Preferences) + Invidious::Database::Users.update_preferences(user) cookie = env.request.cookies["PREFS"] cookie.expires = Time.utc(1990, 1, 1) @@ -327,7 +327,7 @@ module Invidious::Routes::Login return error_template(401, "Password is a required field") end - user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1", email, as: User) + user = Invidious::Database::Users.select(email: email) if user if !user.password @@ -336,7 +336,7 @@ module Invidious::Routes::Login if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + Invidious::Database::SessionIDs.insert(sid, email) if Kemal.config.ssl || CONFIG.https_only secure = true @@ -393,9 +393,9 @@ module Invidious::Routes::Login prompt = "" if captcha_type == "image" - captcha = generate_captcha(HMAC_KEY, PG_DB) + captcha = generate_captcha(HMAC_KEY) else - captcha = generate_text_captcha(HMAC_KEY, PG_DB) + captcha = generate_text_captcha(HMAC_KEY) end return templated "login" @@ -412,7 +412,7 @@ module Invidious::Routes::Login answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) begin - validate_request(tokens[0], answer, env.request, HMAC_KEY, PG_DB, locale) + validate_request(tokens[0], answer, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end @@ -427,7 +427,7 @@ module Invidious::Routes::Login error_exception = Exception.new tokens.each do |token| begin - validate_request(token, answer, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, answer, env.request, HMAC_KEY, locale) found_valid_captcha = true rescue ex error_exception = ex @@ -449,13 +449,8 @@ module Invidious::Routes::Login end end - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - - args = arg_array(user_array) - - PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + Invidious::Database::Users.insert(user) + Invidious::Database::SessionIDs.insert(sid, email) view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") @@ -475,8 +470,8 @@ module Invidious::Routes::Login end if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + user.preferences = env.get("preferences").as(Preferences) + Invidious::Database::Users.update_preferences(user) cookie = env.request.cookies["PREFS"] cookie.expires = Time.utc(1990, 1, 1) @@ -506,12 +501,12 @@ module Invidious::Routes::Login token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid) + Invidious::Database::SessionIDs.delete(sid: sid) env.request.cookies.each do |cookie| cookie.expires = Time.utc(1990, 1, 1) diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 7b7bd03f..d437b79c 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -12,7 +12,7 @@ module Invidious::Routes::Playlists user = user.as(User) sid = sid.as(String) - csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY) templated "create_playlist" end @@ -31,7 +31,7 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end @@ -46,11 +46,11 @@ module Invidious::Routes::Playlists return error_template(400, "Invalid privacy setting.") end - if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 + if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 return error_template(400, "User cannot have more than 100 playlists.") end - playlist = create_playlist(PG_DB, title, privacy, user) + playlist = create_playlist(title, privacy, user) env.redirect "/playlist?list=#{playlist.id}" end @@ -66,8 +66,8 @@ module Invidious::Routes::Playlists user = user.as(User) playlist_id = env.params.query["list"] - playlist = get_playlist(PG_DB, playlist_id, locale) - subscribe_playlist(PG_DB, user, playlist) + playlist = get_playlist(playlist_id, locale) + subscribe_playlist(user, playlist) env.redirect "/playlist?list=#{playlist.id}" end @@ -85,12 +85,16 @@ module Invidious::Routes::Playlists sid = sid.as(String) plid = env.params.query["list"]? - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !plid || plid.empty? + return error_template(400, "A playlist ID is required") + end + + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email return env.redirect referer end - csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY) templated "delete_playlist" end @@ -112,18 +116,17 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email return env.redirect referer end - PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) - PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + Invidious::Database::Playlists.delete(plid) env.redirect "/feed/playlists" end @@ -149,7 +152,7 @@ module Invidious::Routes::Playlists page ||= 1 begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true) if !playlist || playlist.author != user.email return env.redirect referer end @@ -158,12 +161,12 @@ module Invidious::Routes::Playlists end begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale) rescue ex videos = [] of PlaylistVideo end - csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB) + csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY) templated "edit_playlist" end @@ -185,12 +188,12 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex return error_template(400, ex) end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid) if !playlist || playlist.author != user.email return env.redirect referer end @@ -207,7 +210,7 @@ module Invidious::Routes::Playlists updated = playlist.updated end - PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + Invidious::Database::Playlists.update(plid, title, privacy, description, updated) env.redirect "/playlist?list=#{plid}" end @@ -233,7 +236,7 @@ module Invidious::Routes::Playlists page ||= 1 begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + playlist = Invidious::Database::Playlists.select(id: plid, raise_on_fail: true) if !playlist || playlist.author != user.email return env.redirect referer end @@ -283,7 +286,7 @@ module Invidious::Routes::Playlists token = env.params.body["csrf_token"]? begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex if redirect return error_template(400, ex) @@ -311,7 +314,7 @@ module Invidious::Routes::Playlists begin playlist_id = env.params.query["playlist_id"] - playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist) + playlist = get_playlist(playlist_id, locale).as(InvidiousPlaylist) raise "Invalid user" if playlist.author != user.email rescue ex if redirect @@ -342,7 +345,7 @@ module Invidious::Routes::Playlists video_id = env.params.query["video_id"] begin - video = get_video(video_id, PG_DB) + video = get_video(video_id) rescue ex if redirect return error_template(500, ex) @@ -363,15 +366,12 @@ module Invidious::Routes::Playlists index: Random::Secure.rand(0_i64..Int64::MAX), }) - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id) + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index) when "action_remove_video" index = env.params.query["set_video_id"] - PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) - PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id) + Invidious::Database::PlaylistVideos.delete(index) + Invidious::Database::Playlists.update_video_removed(playlist_id, index) when "action_move_video_before" # TODO: Playlist stub else @@ -405,7 +405,7 @@ module Invidious::Routes::Playlists end begin - playlist = get_playlist(PG_DB, plid, locale) + playlist = get_playlist(plid, locale) rescue ex return error_template(500, ex) end @@ -422,7 +422,7 @@ module Invidious::Routes::Playlists end begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale) rescue ex return error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}") end diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 15c00700..a832076c 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -170,11 +170,12 @@ module Invidious::Routes::PreferencesRoute vr_mode: vr_mode, show_nick: show_nick, save_player_pos: save_player_pos, - }.to_json).to_json + }.to_json) if user = env.get? "user" user = user.as(User) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + user.preferences = preferences + Invidious::Database::Users.update_preferences(user) if CONFIG.admins.includes? user.email CONFIG.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home @@ -220,10 +221,10 @@ module Invidious::Routes::PreferencesRoute end if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, secure: secure, http_only: true) end end @@ -241,18 +242,15 @@ module Invidious::Routes::PreferencesRoute if user = env.get? "user" user = user.as(User) - preferences = user.preferences - case preferences.dark_mode + case user.preferences.dark_mode when "dark" - preferences.dark_mode = "light" + user.preferences.dark_mode = "light" else - preferences.dark_mode = "dark" + user.preferences.dark_mode = "dark" end - preferences = preferences.to_json - - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + Invidious::Database::Users.update_preferences(user) else preferences = env.get("preferences").as(Preferences) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 06ba6b8c..8a58b034 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -263,7 +263,7 @@ module Invidious::Routes::VideoPlayback haltf env, status_code: 400, response: "TESTING" end - video = get_video(id, PG_DB, region: region) + video = get_video(id, region: region) fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } url = fmt.try &.["url"]?.try &.as_s diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index b24222ff..1198f48f 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -39,7 +39,7 @@ module Invidious::Routes::Watch end plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") - continuation = process_continuation(PG_DB, env.params.query, plid, id) + continuation = process_continuation(env.params.query, plid, id) nojs = env.params.query["nojs"]? @@ -60,7 +60,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - video = get_video(id, PG_DB, region: params.region) + video = get_video(id, region: params.region) rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex @@ -76,11 +76,11 @@ module Invidious::Routes::Watch env.params.query.delete_all("iv_load_policy") if watched && !watched.includes? id - PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email) + Invidious::Database::Users.mark_watched(user.as(User), id) end if notifications && notifications.includes? id - PG_DB.exec("UPDATE users SET notifications = array_remove(notifications, $1) WHERE email = $2", id, user.as(User).email) + Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 584082be..49074994 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -29,43 +29,31 @@ struct User end end -def get_user(sid, headers, db, refresh = true) - if email = db.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) - user = db.query_one("SELECT * FROM users WHERE email = $1", email, as: User) +def get_user(sid, headers, refresh = true) + if email = Invidious::Database::SessionIDs.select_email(sid) + user = Invidious::Database::Users.select!(email: email) if refresh && Time.utc - user.updated > 1.minute - user, sid = fetch_user(sid, headers, db) - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - args = arg_array(user_array) + user, sid = fetch_user(sid, headers) - db.exec("INSERT INTO users VALUES (#{args}) \ - ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) - - db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ - ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) + Invidious::Database::Users.insert(user, update_on_conflict: true) + Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin view_name = "subscriptions_#{sha256(user.email)}" - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") rescue ex end end else - user, sid = fetch_user(sid, headers, db) - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - args = arg_array(user.to_a) + user, sid = fetch_user(sid, headers) - db.exec("INSERT INTO users VALUES (#{args}) \ - ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) - - db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ - ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) + Invidious::Database::Users.insert(user, update_on_conflict: true) + Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true) begin view_name = "subscriptions_#{sha256(user.email)}" - db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") rescue ex end end @@ -73,7 +61,7 @@ def get_user(sid, headers, db, refresh = true) return user, sid end -def fetch_user(sid, headers, db) +def fetch_user(sid, headers) feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) feed = XML.parse_html(feed.body) @@ -86,7 +74,7 @@ def fetch_user(sid, headers, db) end end - channels = get_batch_channels(channels, db, false, false) + channels = get_batch_channels(channels, false, false) email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"])) if email @@ -130,7 +118,7 @@ def create_user(sid, email, password) return user, sid end -def generate_captcha(key, db) +def generate_captcha(key) second = Random::Secure.rand(12) second_angle = second * 30 second = second * 5 @@ -182,16 +170,16 @@ def generate_captcha(key, db) return { question: image, - tokens: {generate_response(answer, {":login"}, key, db, use_nonce: true)}, + tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, } end -def generate_text_captcha(key, db) +def generate_text_captcha(key) response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body) response = JSON.parse(response) tokens = response["a"].as_a.map do |answer| - generate_response(answer.as_s, {":login"}, key, db, use_nonce: true) + generate_response(answer.as_s, {":login"}, key, use_nonce: true) end return { @@ -232,20 +220,16 @@ def subscribe_ajax(channel_id, action, env_headers) end end -def get_subscription_feed(db, user, max_results = 40, page = 1) +def get_subscription_feed(user, max_results = 40, page = 1) limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE) offset = (page - 1) * limit - notifications = db.query_one("SELECT notifications FROM users WHERE email = $1", user.email, - as: Array(String)) + notifications = Invidious::Database::Users.select_notifications(user) view_name = "subscriptions_#{sha256(user.email)}" if user.preferences.notifications_only && !notifications.empty? # Only show notifications - - args = arg_array(notifications) - - notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: ChannelVideo) + notifications = Invidious::Database::ChannelVideos.select(notifications) videos = [] of ChannelVideo notifications.sort_by!(&.published).reverse! @@ -311,8 +295,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) else nil # Ignore end - notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, as: Array(String)) - + notifications = Invidious::Database::Users.select_notifications(user) notifications = videos.select { |v| notifications.includes? v.id } videos = videos - notifications end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index d4ef0900..6c89b445 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -993,8 +993,8 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ return params end -def get_video(id, db, refresh = true, region = nil, force_refresh = false) - if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region +def get_video(id, refresh = true, region = nil, force_refresh = false) + if (video = Invidious::Database::Videos.select(id)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) if (refresh && @@ -1003,17 +1003,15 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false) force_refresh begin video = fetch_video(id, region) - db.exec("UPDATE videos SET (id, info, updated) = ($1, $2, $3) WHERE id = $1", video.id, video.info.to_json, video.updated) + Invidious::Database::Videos.update(video) rescue ex - db.exec("DELETE FROM videos * WHERE id = $1", id) + Invidious::Database::Videos.delete(id) raise ex end end else video = fetch_video(id, region) - if !region - db.exec("INSERT INTO videos VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", video.id, video.info.to_json, video.updated) - end + Invidious::Database::Videos.insert(video) if !region end return video @@ -1058,7 +1056,7 @@ def itag_to_metadata?(itag : JSON::Any) return VIDEO_FORMATS[itag.to_s]? end -def process_continuation(db, query, plid, id) +def process_continuation(query, plid, id) continuation = nil if plid if index = query["index"]?.try &.to_i? diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index 136981da..7825b1f0 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -61,7 +61,7 @@
<% else %> - <% if PG_DB.query_one?("SELECT id FROM playlists WHERE id = $1", playlist.id, as: String).nil? %> + <% if Invidious::Database::Playlists.exists?(playlist.id) %>
<% else %>
diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index b85ea59d..fa4fe083 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -138,7 +138,7 @@ we're going to need to do it here in order to allow for translations.

<% if user %> - <% playlists = PG_DB.query_all("SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%'", user.email, as: {String, String}) %> + <% playlists = Invidious::Database::Playlists.select_user_created_playlists(user.email) %> <% if !playlists.empty? %>