Make use of Search::Query/Filters and associated HTML generator

This commit is contained in:
Samantaz Fox 2022-03-26 20:15:02 +01:00
parent a813955ad3
commit d93a7b315d
No known key found for this signature in database
GPG Key ID: F42821059186176E
10 changed files with 87 additions and 331 deletions

View File

@ -251,18 +251,22 @@ module Invidious::Routes::API::V1::Channels
def self.search(env) def self.search(env)
locale = env.get("preferences").as(Preferences).locale locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
env.response.content_type = "application/json" env.response.content_type = "application/json"
ucid = env.params.url["ucid"] query = Invidious::Search::Query.new(env.params.query, :channel, region)
query = env.params.query["q"]? # Required because we can't (yet) pass multiple parameter to the
query ||= "" # `Search::Query` initializer (in this case, an URL segment)
query.channel = env.params.url["ucid"]
page = env.params.query["page"]?.try &.to_i? begin
page ||= 1 search_results = query.process
rescue ex
return error_json(400, ex)
end
search_results = Invidious::Search::Processors.channel(query, page, ucid)
JSON.build do |json| JSON.build do |json|
json.array do json.array do
search_results.each do |item| search_results.each do |item|

View File

@ -5,34 +5,14 @@ module Invidious::Routes::API::V1::Search
env.response.content_type = "application/json" env.response.content_type = "application/json"
query = env.params.query["q"]? query = Invidious::Search::Query.new(env.params.query, :regular, region)
query ||= ""
page = env.params.query["page"]?.try &.to_i?
page ||= 1
sort_by = env.params.query["sort_by"]?.try &.downcase
sort_by ||= "relevance"
date = env.params.query["date"]?.try &.downcase
date ||= ""
duration = env.params.query["duration"]?.try &.downcase
duration ||= ""
features = env.params.query["features"]?.try &.split(",").map(&.downcase)
features ||= [] of String
content_type = env.params.query["type"]?.try &.downcase
content_type ||= "video"
begin begin
search_params = produce_search_params(page, sort_by, date, content_type, duration, features) search_results = query.process
rescue ex rescue ex
return error_json(400, ex) return error_json(400, ex)
end end
search_results = search(query, search_params, region)
JSON.build do |json| JSON.build do |json|
json.array do json.array do
search_results.each do |item| search_results.each do |item|

View File

@ -212,7 +212,10 @@ module Invidious::Routes::Playlists
end end
def self.add_playlist_items_page(env) def self.add_playlist_items_page(env)
locale = env.get("preferences").as(Preferences).locale prefs = env.get("preferences").as(Preferences)
locale = prefs.locale
region = env.params.query["region"]? || prefs.region
user = env.get? "user" user = env.get? "user"
sid = env.get? "sid" sid = env.get? "sid"
@ -236,17 +239,12 @@ module Invidious::Routes::Playlists
return env.redirect referer return env.redirect referer
end end
query = env.params.query["q"]?
if query
begin begin
search_query, items, operators = process_search_query(query, page, user, region: nil) query = Invidious::Search::Query.new(env.params.query, :playlist, region)
videos = items.select(SearchVideo).map(&.as(SearchVideo)) videos = query.process.select(SearchVideo).map(&.as(SearchVideo))
rescue ex rescue ex
videos = [] of SearchVideo videos = [] of SearchVideo
end end
else
videos = [] of SearchVideo
end
env.set "add_playlist_items", plid env.set "add_playlist_items", plid
templated "add_playlist_items" templated "add_playlist_items"

View File

@ -37,37 +37,29 @@ module Invidious::Routes::Search
end end
def self.search(env) def self.search(env)
locale = env.get("preferences").as(Preferences).locale prefs = env.get("preferences").as(Preferences)
region = env.params.query["region"]? locale = prefs.locale
query = env.params.query["search_query"]? region = env.params.query["region"]? || prefs.region
query ||= env.params.query["q"]?
if !query || query.empty? query = Invidious::Search::Query.new(env.params.query, :regular, region)
if query.empty?
# Display the full page search box implemented in #1977 # Display the full page search box implemented in #1977
env.set "search", "" env.set "search", ""
templated "search_homepage", navbar_search: false templated "search_homepage", navbar_search: false
else else
page = env.params.query["page"]?.try &.to_i?
page ||= 1
user = env.get? "user" user = env.get? "user"
begin begin
search_query, videos, operators = process_search_query(query, page, user, region: region) videos = query.process
rescue ex : ChannelSearchException rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex rescue ex
return error_template(500, ex) return error_template(500, ex)
end end
operator_hash = {} of String => String env.set "search", query.text
operators.each do |operator|
key, value = operator.downcase.split(":")
operator_hash[key] = value
end
env.set "search", query
templated "search" templated "search"
end end
end end

View File

@ -5,113 +5,6 @@ class ChannelSearchException < InfoException
end end
end end
def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem)
return [] of SearchItem if query.empty?
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.search(query, search_params, client_config: client_config)
return extract_items(initial_data)
end
def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "",
duration : String = "", features : Array(String) = [] of String)
object = {
"1:varint" => 0_i64,
"2:embedded" => {} of String => Int64,
"9:varint" => ((page - 1) * 20).to_i64,
}
case sort
when "relevance"
object["1:varint"] = 0_i64
when "rating"
object["1:varint"] = 1_i64
when "upload_date", "date"
object["1:varint"] = 2_i64
when "view_count", "views"
object["1:varint"] = 3_i64
else
raise "No sort #{sort}"
end
case date
when "hour"
object["2:embedded"].as(Hash)["1:varint"] = 1_i64
when "today"
object["2:embedded"].as(Hash)["1:varint"] = 2_i64
when "week"
object["2:embedded"].as(Hash)["1:varint"] = 3_i64
when "month"
object["2:embedded"].as(Hash)["1:varint"] = 4_i64
when "year"
object["2:embedded"].as(Hash)["1:varint"] = 5_i64
else nil # Ignore
end
case content_type
when "video"
object["2:embedded"].as(Hash)["2:varint"] = 1_i64
when "channel"
object["2:embedded"].as(Hash)["2:varint"] = 2_i64
when "playlist"
object["2:embedded"].as(Hash)["2:varint"] = 3_i64
when "movie"
object["2:embedded"].as(Hash)["2:varint"] = 4_i64
when "show"
object["2:embedded"].as(Hash)["2:varint"] = 5_i64
when "all"
#
else
object["2:embedded"].as(Hash)["2:varint"] = 1_i64
end
case duration
when "short"
object["2:embedded"].as(Hash)["3:varint"] = 1_i64
when "long"
object["2:embedded"].as(Hash)["3:varint"] = 2_i64
else nil # Ignore
end
features.each do |feature|
case feature
when "hd"
object["2:embedded"].as(Hash)["4:varint"] = 1_i64
when "subtitles"
object["2:embedded"].as(Hash)["5:varint"] = 1_i64
when "creative_commons", "cc"
object["2:embedded"].as(Hash)["6:varint"] = 1_i64
when "3d"
object["2:embedded"].as(Hash)["7:varint"] = 1_i64
when "live", "livestream"
object["2:embedded"].as(Hash)["8:varint"] = 1_i64
when "purchased"
object["2:embedded"].as(Hash)["9:varint"] = 1_i64
when "4k"
object["2:embedded"].as(Hash)["14:varint"] = 1_i64
when "360"
object["2:embedded"].as(Hash)["15:varint"] = 1_i64
when "location"
object["2:embedded"].as(Hash)["23:varint"] = 1_i64
when "hdr"
object["2:embedded"].as(Hash)["25:varint"] = 1_i64
else nil # Ignore
end
end
if object["2:embedded"].as(Hash).empty?
object.delete("2:embedded")
end
params = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return params
end
def produce_channel_search_continuation(ucid, query, page) def produce_channel_search_continuation(ucid, query, page)
if page <= 1 if page <= 1
idx = 0_i64 idx = 0_i64
@ -146,41 +39,10 @@ def produce_channel_search_continuation(ucid, query, page)
end end
def process_search_query(query, page, user, region) def process_search_query(query, page, user, region)
channel = nil # Parse legacy query
content_type = "all" filters, channel, search_query, subscriptions = Invidious::Search::Filters.from_legacy_filters(query)
date = ""
duration = ""
features = [] of String
sort = "relevance"
subscriptions = nil
operators = query.split(" ").select(&.match(/\w+:[\w,]+/)) if !channel.nil? && !channel.empty?
operators.each do |operator|
key, value = operator.downcase.split(":")
case key
when "channel", "user"
channel = operator.split(":")[-1]
when "content_type", "type"
content_type = value
when "date"
date = value
when "duration"
duration = value
when "feature", "features"
features = value.split(",")
when "sort"
sort = value
when "subscriptions"
subscriptions = value == "true"
else
operators.delete(operator)
end
end
search_query = (query.split(" ") - operators).join(" ")
if channel
items = Invidious::Search::Processors.channel(search_query, page, channel) items = Invidious::Search::Processors.channel(search_query, page, channel)
elsif subscriptions elsif subscriptions
if user if user
@ -190,9 +52,7 @@ def process_search_query(query, page, user, region)
items = [] of ChannelVideo items = [] of ChannelVideo
end end
else else
search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, search_params = filters.to_yt_params(page: page)
duration: duration, features: features)
items = search(search_query, search_params, region) items = search(search_query, search_params, region)
end end
@ -211,5 +71,5 @@ def process_search_query(query, page, user, region)
end end
end end
{search_query, items_without_category, operators} {search_query, items_without_category, filters}
end end

View File

@ -79,7 +79,7 @@ module Invidious::Search
) )
end end
def is_default? : Bool def default? : Bool
return @date.none? && @type.all? && @duration.none? && \ return @date.none? && @type.all? && @duration.none? && \
@features.none? && @sort.relevance? @features.none? && @sort.relevance?
end end

View File

@ -2,22 +2,32 @@ module Invidious::Search
module Processors module Processors
extend self extend self
# Search a youtube channel # Regular search (`/search` endpoint)
# TODO: clean code, and rely more on YoutubeAPI def regular(query : Query) : Array(SearchItem)
def channel(query, page, channel) : Array(SearchItem) search_params = query.filters.to_yt_params(page: query.page)
response = YT_POOL.client &.get("/channel/#{channel}")
if response.status_code == 404 client_config = YoutubeAPI::ClientConfig.new(region: query.region)
response = YT_POOL.client &.get("/user/#{channel}") initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404
initial_data = extract_initial_data(response.body) return extract_items(initial_data)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(channel) if !ucid
else
ucid = channel
end end
continuation = produce_channel_search_continuation(ucid, query, page) # Search a youtube channel
# TODO: clean code, and rely more on YoutubeAPI
def channel(query : Query) : Array(SearchItem)
response = YT_POOL.client &.get("/channel/#{query.channel}")
if response.status_code == 404
response = YT_POOL.client &.get("/user/#{query.channel}")
response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
initial_data = extract_initial_data(response.body)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(query.channel) if !ucid
else
ucid = query.channel
end
continuation = produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation) response_json = YoutubeAPI.browse(continuation)
continuation_items = response_json["onResponseReceivedActions"]? continuation_items = response_json["onResponseReceivedActions"]?
@ -34,7 +44,7 @@ module Invidious::Search
end end
# Search inside of user subscriptions # Search inside of user subscriptions
def subscriptions(query, page, user : Invidious::User) : Array(ChannelVideo) def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
view_name = "subscriptions_#{sha256(user.email)}" view_name = "subscriptions_#{sha256(user.email)}"
return PG_DB.query_all(" return PG_DB.query_all("
@ -46,7 +56,7 @@ module Invidious::Search
as document as document
FROM #{view_name} FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
query, (page - 1) * 20, query.text, (query.page - 1) * 20,
as: ChannelVideo as: ChannelVideo
) )
end end

View File

@ -110,11 +110,10 @@ module Invidious::Search
case @type case @type
when .regular?, .playlist? when .regular?, .playlist?
all_items = search(@query, @filters, @page, @region) items = unnest_items(Processors.regular(self))
items = unnest_items(all_items)
# #
when .channel? when .channel?
items = Processors.channel(@query, @page, @channel) items = Processors.channel(self)
# #
when .subscriptions? when .subscriptions?
if user if user

View File

@ -11,7 +11,9 @@
<legend><a href="/playlist?list=<%= playlist.id %>"><%= translate(locale, "Editing playlist `x`", %|"#{HTML.escape(playlist.title)}"|) %></a></legend> <legend><a href="/playlist?list=<%= playlist.id %>"><%= translate(locale, "Editing playlist `x`", %|"#{HTML.escape(playlist.title)}"|) %></a></legend>
<fieldset> <fieldset>
<input class="pure-input-1" type="search" name="q" <% if query %>value="<%= HTML.escape(query) %>"<% else %>placeholder="<%= translate(locale, "Search for videos") %>"<% end %>> <input class="pure-input-1" type="search" name="q"
<% if query %>value="<%= HTML.escape(query.text) %>"<% end %>
placeholder="<%= translate(locale, "Search for videos") %>">
<input type="hidden" name="list" value="<%= plid %>"> <input type="hidden" name="list" value="<%= plid %>">
</fieldset> </fieldset>
</form> </form>
@ -38,10 +40,11 @@
</div> </div>
<% if query %> <% if query %>
<%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%>
<div class="pure-g h-box"> <div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5"> <div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %> <% if query.page > 1 %>
<a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page - 1 %>"> <a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page - 1 %>">
<%= translate(locale, "Previous page") %> <%= translate(locale, "Previous page") %>
</a> </a>
<% end %> <% end %>
@ -49,7 +52,7 @@
<div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if videos.size >= 20 %> <% if videos.size >= 20 %>
<a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page + 1 %>"> <a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page + 1 %>">
<%= translate(locale, "Next page") %> <%= translate(locale, "Next page") %>
</a> </a>
<% end %> <% end %>

View File

@ -1,124 +1,38 @@
<% content_for "header" do %> <% content_for "header" do %>
<title><%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious</title> <title><%= query.text.size > 30 ? HTML.escape(query.text[0,30].rstrip(".")) + "&hellip;" : HTML.escape(query.text) %> - Invidious</title>
<link rel="stylesheet" href="/css/search.css?v=<%= ASSET_COMMIT %>">
<% end %> <% end %>
<% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %> <%-
search_query_encoded = URI.encode_www_form(query.text, space_to_plus: true)
filter_params = query.filters.to_iv_params
url_prev_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page - 1}"
url_next_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page + 1}"
-%>
<!-- Search redirection and filtering UI --> <!-- Search redirection and filtering UI -->
<% if videos.size == 0 %> <% if videos.size == 0 %>
<h3 style="text-align: center"> <h3 style="text-align: center">
<a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Broken? Try another Invidious Instance!") %></a> <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Broken? Try another Invidious Instance!") %></a>
</h3> </h3>
<% else %> <%- else -%>
<details id="filters"> <%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %>
<summary> <%- end -%>
<h3 style="display:inline"> <%= translate(locale, "filter") %> </h3>
</summary>
<div id="filters" class="pure-g h-box">
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "date") %></b>
<hr/>
<% ["hour", "today", "week", "month", "year"].each do |date| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("date", "all") == date %>
<b><%= translate(locale, date) %></b>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?date:[a-z]+/, "") + " date:" + date) %>&page=<%= page %>">
<%= translate(locale, date) %>
</a>
<% end %>
</div>
<% end %>
</div>
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "content_type") %></b>
<hr/>
<% ["video", "channel", "playlist", "movie", "show"].each do |content_type| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("content_type", "all") == content_type %>
<b><%= translate(locale, content_type) %></b>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?content_type:[a-z]+/, "") + " content_type:" + content_type) %>&page=<%= page %>">
<%= translate(locale, content_type) %>
</a>
<% end %>
</div>
<% end %>
</div>
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "duration") %></b>
<hr/>
<% ["short", "long"].each do |duration| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("duration", "all") == duration %>
<b><%= translate(locale, duration) %></b>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?duration:[a-z]+/, "") + " duration:" + duration) %>&page=<%= page %>">
<%= translate(locale, duration) %>
</a>
<% end %>
</div>
<% end %>
</div>
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "features") %></b>
<hr/>
<% ["hd", "subtitles", "creative_commons", "3d", "live", "purchased", "4k", "360", "location", "hdr"].each do |feature| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("features", "all").includes?(feature) %>
<b><%= translate(locale, feature) %></b>
<% elsif operator_hash.has_key?("features") %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/features:/, "features:" + feature + ",")) %>&page=<%= page %>">
<%= translate(locale, feature) %>
</a>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil! + " features:" + feature) %>&page=<%= page %>">
<%= translate(locale, feature) %>
</a>
<% end %>
</div>
<% end %>
</div>
<div class="pure-u-1-3 pure-u-md-1-5">
<b><%= translate(locale, "sort") %></b>
<hr/>
<% ["relevance", "rating", "date", "views"].each do |sort| %>
<div class="pure-u-1 pure-md-1-5">
<% if operator_hash.fetch("sort", "relevance") == sort %>
<b><%= translate(locale, sort) %></b>
<% else %>
<a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?sort:[a-z]+/, "") + " sort:" + sort) %>&page=<%= page %>">
<%= translate(locale, sort) %>
</a>
<% end %>
</div>
<% end %>
</div>
</div>
</details>
<% end %>
<% if videos.size == 0 %> <% if videos.size == 0 %><hr style="margin: 0;"/><% else %><hr/><% end %>
<hr style="margin: 0;"/>
<% else %>
<hr/>
<% end %>
<div class="pure-g h-box v-box"> <div class="pure-g h-box v-box">
<div class="pure-u-1 pure-u-lg-1-5"> <div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %> <%- if query.page > 1 -%>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>"> <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a>
<%= translate(locale, "Previous page") %> <%- end -%>
</a>
<% end %>
</div> </div>
<div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if videos.size >= 20 %> <%- if videos.size >= 20 -%>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>"> <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a>
<%= translate(locale, "Next page") %> <%- end -%>
</a>
<% end %>
</div> </div>
</div> </div>
@ -130,18 +44,14 @@
<div class="pure-g h-box"> <div class="pure-g h-box">
<div class="pure-u-1 pure-u-lg-1-5"> <div class="pure-u-1 pure-u-lg-1-5">
<% if page > 1 %> <%- if query.page > 1 -%>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>"> <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a>
<%= translate(locale, "Previous page") %> <%- end -%>
</a>
<% end %>
</div> </div>
<div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if videos.size >= 20 %> <%- if videos.size >= 20 -%>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>"> <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a>
<%= translate(locale, "Next page") %> <%- end -%>
</a>
<% end %>
</div> </div>
</div> </div>