Merge pull request #634 from kvch/advanced-search

support time range search
This commit is contained in:
Adam Tauber 2016-07-26 00:06:16 +02:00 committed by GitHub
commit 7d9c898170
25 changed files with 291 additions and 95 deletions

View File

@ -42,7 +42,8 @@ engine_default_args = {'paging': False,
'shortcut': '-',
'disabled': False,
'suspend_end_time': 0,
'continuous_errors': 0}
'continuous_errors': 0,
'time_range_support': False}
def load_module(filename):

View File

@ -21,10 +21,16 @@ from searx.engines.xpath import extract_text
# engine dependent config
categories = ['images']
paging = True
time_range_support = True
# search-url
base_url = 'https://www.deviantart.com/'
search_url = base_url + 'browse/all/?offset={offset}&{query}'
time_range_url = '&order={range}'
time_range_dict = {'day': 11,
'week': 14,
'month': 15}
# do search-request
@ -33,6 +39,8 @@ def request(query, params):
params['url'] = search_url.format(offset=offset,
query=urlencode({'q': query}))
if params['time_range']:
params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])
return params

View File

@ -22,9 +22,15 @@ from searx.languages import language_codes
categories = ['general']
paging = True
language_support = True
time_range_support = True
# search-url
url = 'https://duckduckgo.com/html?{query}&s={offset}'
time_range_url = '&df={range}'
time_range_dict = {'day': 'd',
'week': 'w',
'month': 'm'}
# specific xpath variables
result_xpath = '//div[@class="result results_links results_links_deep web-result "]' # noqa
@ -61,6 +67,9 @@ def request(query, params):
params['url'] = url.format(
query=urlencode({'q': query}), offset=offset)
if params['time_range']:
params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])
return params

View File

@ -24,6 +24,7 @@ categories = ['general']
paging = True
language_support = True
use_locale_domain = True
time_range_support = True
# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests
default_hostname = 'www.google.com'
@ -92,6 +93,11 @@ search_url = ('https://{hostname}' +
search_path +
'?{query}&start={offset}&gws_rd=cr&gbv=1&lr={lang}&ei=x')
time_range_search = "&tbs=qdr:{range}"
time_range_dict = {'day': 'd',
'week': 'w',
'month': 'm'}
# other URLs
map_hostname_start = 'maps.google.'
maps_path = '/maps'
@ -179,6 +185,8 @@ def request(query, params):
query=urlencode({'q': query}),
hostname=google_hostname,
lang=url_lang)
if params['time_range']:
params['url'] += time_range_search.format(range=time_range_dict[params['time_range']])
params['headers']['Accept-Language'] = language
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'

View File

@ -19,12 +19,17 @@ from lxml import html
categories = ['images']
paging = True
safesearch = True
time_range_support = True
search_url = 'https://www.google.com/search'\
'?{query}'\
'&tbm=isch'\
'&ijn=1'\
'&start={offset}'
time_range_search = "&tbs=qdr:{range}"
time_range_dict = {'day': 'd',
'week': 'w',
'month': 'm'}
# do search-request
@ -34,6 +39,8 @@ def request(query, params):
params['url'] = search_url.format(query=urlencode({'q': query}),
offset=offset,
safesearch=safesearch)
if params['time_range']:
params['url'] += time_range_search.format(range=time_range_dict[params['time_range']])
if safesearch and params['safesearch']:
params['url'] += '&' + urlencode({'safe': 'active'})

View File

@ -20,10 +20,12 @@ from searx.engines.xpath import extract_text, extract_url
categories = ['general']
paging = True
language_support = True
time_range_support = True
# search-url
base_url = 'https://search.yahoo.com/'
search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}'
search_url_with_time = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}&age={age}&btf={btf}&fr2=time'
# specific xpath variables
results_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]"
@ -32,6 +34,10 @@ title_xpath = './/h3/a'
content_xpath = './/div[@class="compText aAbs"]'
suggestion_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' AlsoTry ')]//a"
time_range_dict = {'day': ['1d', 'd'],
'week': ['1w', 'w'],
'month': ['1m', 'm']}
# remove yahoo-specific tracking-url
def parse_url(url_string):
@ -51,18 +57,30 @@ def parse_url(url_string):
return unquote(url_string[start:end])
def _get_url(query, offset, language, time_range):
if time_range:
return base_url + search_url_with_time.format(offset=offset,
query=urlencode({'p': query}),
lang=language,
age=time_range_dict[time_range][0],
btf=time_range_dict[time_range][1])
return base_url + search_url.format(offset=offset,
query=urlencode({'p': query}),
lang=language)
def _get_language(params):
if params['language'] == 'all':
return 'en'
return params['language'].split('_')[0]
# do search-request
def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1
language = _get_language(params)
if params['language'] == 'all':
language = 'en'
else:
language = params['language'].split('_')[0]
params['url'] = base_url + search_url.format(offset=offset,
query=urlencode({'p': query}),
lang=language)
params['url'] = _get_url(query, offset, language, params['time_range'])
# TODO required?
params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\

View File

@ -138,6 +138,8 @@ class Search(object):
self.paging = False
self.pageno = 1
self.lang = 'all'
self.time_range = None
self.is_advanced = None
# set blocked engines
self.disabled_engines = request.preferences.engines.get_disabled()
@ -178,9 +180,10 @@ class Search(object):
if len(query_obj.languages):
self.lang = query_obj.languages[-1]
self.engines = query_obj.engines
self.time_range = self.request_data.get('time_range')
self.is_advanced = self.request_data.get('advanced_search')
self.categories = []
self.engines = query_obj.engines
# if engines are calculated from query,
# set categories by using that informations
@ -279,6 +282,9 @@ class Search(object):
if self.lang != 'all' and not engine.language_support:
continue
if self.time_range and not engine.time_range_support:
continue
# set default request parameters
request_params = default_request_params()
request_params['headers']['User-Agent'] = user_agent
@ -293,6 +299,8 @@ class Search(object):
# 0 = None, 1 = Moderate, 2 = Strict
request_params['safesearch'] = request.preferences.get_value('safesearch')
request_params['time_range'] = self.time_range
request_params['advanced_search'] = self.is_advanced
# update request parameters dependent on
# search-engine (contained in engines folder)

View File

@ -4,13 +4,16 @@ $(document).ready(function() {
$('#categories input[type="checkbox"]').each(function(i, checkbox) {
$(checkbox).prop('checked', false);
});
$('#categories label').removeClass('btn-primary').removeClass('active').addClass('btn-default');
$(this).removeClass('btn-default').addClass('btn-primary').addClass('active');
$($(this).children()[0]).prop('checked', 'checked');
$(document.getElementById($(this).attr("for"))).prop('checked', true);
if($('#q').val()) {
$('#search_form').submit();
}
return false;
});
$('#time-range > option').click(function(e) {
if($('#q').val()) {
$('#search_form').submit();
}
});
}
});

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,68 @@
#advanced-search-container {
display:none;
text-align:left;
margin-bottom:1rem;
label, .input-group-addon {
font-size: 1.2rem;
font-weight:normal;
background-color: white;
border: @mild-gray 1px solid;
border-right: none;
color: @dark-gray;
padding-bottom: 0.4rem;
padding-top: 0.4rem;
padding-left: 0.5rem;
padding-right: 0.5rem;
}
label:last-child, .input-group-addon:last-child {
border-right: @mild-gray 1px solid;
}
input[type="radio"] {
display: none;
}
input[type="radio"]:checked + label{
color: @black;
font-weight:bold;
border-bottom: @light-green 5px solid;
}
select {
appearance: none;
-webkit-appearance: none;
-moz-appearance: none;
font-size: 1.2rem;
font-weight:normal;
background-color: white;
border: @mild-gray 1px solid;
color: @dark-gray;
padding-bottom: 0.4rem;
padding-top: 0.4rem;
padding-left: 1rem;
padding-right: 5rem;
margin-right: 0.5rem;
background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA8AAAAPCAQAAACR313BAAAABGdBTUEAALGPC/xhBQAAACBjSFJN
AAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAAmJLR0QA/4ePzL8AAAAJcEhZ
cwAABFkAAARZAVnbJUkAAAAHdElNRQfgBxgLDwB20OFsAAAAbElEQVQY073OsQ3CMAAEwJMYwJGn
sAehpoXJItltBkmcdZBYgIIiQoLglnz3ui+eP+bk5uneteTMZJa6OJuIqvYzSJoqwqBq8gdmTTW8
6/dghxAUq4xsVYT9laBYXCw93Aajh7GPEF23t4fkBYevGFTANkPRAAAAJXRFWHRkYXRlOmNyZWF0
ZQAyMDE2LTA3LTI0VDExOjU1OjU4KzAyOjAwRFqFOQAAACV0RVh0ZGF0ZTptb2RpZnkAMjAxNi0w
Ny0yNFQxMToxNTowMCswMjowMP7RDgQAAAAZdEVYdFNvZnR3YXJlAHd3dy5pbmtzY2FwZS5vcmeb
7jwaAAAAAElFTkSuQmCC) 96% no-repeat;
}
}
#check-advanced {
display: none;
}
#check-advanced:checked ~ #advanced-search-container {
display:block;
}
.advanced {
margin-top:1rem;
text-align:right;
}

View File

@ -12,6 +12,8 @@
@import "search.less";
@import "advanced.less";
@import "cursor.less";
@import "code.less";

View File

@ -1,36 +1,28 @@
.search_categories, #categories {
margin: 10px 0 4px 0;
text-transform: capitalize;
margin-bottom: 0.5rem;
label{
border: none;
box-shadow: none;
font-size: 13px;
padding-bottom: 2px;
color: @gray;
margin-bottom: 5px;
label, .input-group-addon {
font-size: 1.2rem;
font-weight:normal;
background-color: white;
border: @mild-gray 1px solid;
border-right: none;
color: @dark-gray;
padding-bottom: 0.4rem;
padding-top: 0.4rem;
padding-left: 0.5rem;
padding-right: 0.5rem;
}
label:last-child, .input-group-addon:last-child {
border-right: @mild-gray 1px solid;
}
&:hover{
input[type="checkbox"]:checked + label {
color: @black;
background-color: transparent;
font-weight:bold;
border-bottom: @light-green 5px solid;
}
&:active{
box-shadow: none;
}
}
.active, .btn-primary{
color: @black;
font-weight: 700;
border-bottom: 5px solid @light-green;
background-color: transparent;
}
}
#categories{
margin: 0;
}
#main-logo{

View File

@ -2,6 +2,8 @@
@gray: #A4A4A4;
@dim-gray: #F6F9FA;
@dark-gray: #666;
@middle-gray: #F5F5F5;
@mild-gray: #DDD;
@blue: #0088CC;
@red: #F35E77;
@violet: #684898;

View File

@ -0,0 +1,45 @@
#advanced-search-container {
display:none;
text-align:center;
margin-bottom:1rem;
label, .input-group-addon {
font-size: 1.3rem;
font-weight:normal;
background-color: white;
border: #DDD 1px solid;
border-right: none;
color: #333;
padding-bottom: 0.8rem;
padding-top: 0.8rem;
padding-left: 1.2rem;
padding-right: 1.2rem;
}
label:last-child, .input-group-addon:last-child {
border-right: #DDD 1px solid;
}
input[type="radio"] {
display: none;
}
input[type="radio"]:checked + label{
color: black;
font-weight:bold;
background-color: #EEE;
}
}
#check-advanced {
display: none;
}
#check-advanced:checked ~ #advanced-search-container {
display:block;
}
.advanced {
margin-top:1rem;
text-align:right;
}

View File

@ -8,6 +8,8 @@
@import "search.less";
@import "advanced.less";
@import "cursor.less";
@import "code.less";

View File

@ -1,4 +1,28 @@
.search_categories {
margin:10px 0;
.search_categories, #categories {
text-transform: capitalize;
margin-bottom:1.5rem;
margin-top:1.5rem;
label, .input-group-addon {
font-size: 1.3rem;
font-weight:normal;
background-color: white;
border: #DDD 1px solid;
border-right: none;
color: #333;
padding-bottom: 0.8rem;
padding-top: 0.8rem;
padding-left: 1.2rem;
padding-right: 1.2rem;
}
label:last-child, .input-group-addon:last-child {
border-right: #DDD 1px solid;
}
input[type="checkbox"]:checked + label{
color: black;
font-weight:bold;
background-color: #EEE;
}
}

View File

@ -0,0 +1,11 @@
<div class="input-group col-sm-12 advanced">
<input type="checkbox" name="advanced_search" id="check-advanced" {% if advanced_search %} checked="checked"{% endif %}>
<label for="check-advanced">
<span class="glyphicon glyphicon-cog"></span>
{{ _('Advanced settings') }}
</label>
<div id="advanced-search-container">
{% include 'oscar/categories.html' %}
{% include 'oscar/time-range.html' %}
</div>
</div>

View File

@ -1,42 +1,14 @@
<!-- used if scripts are disabled -->
<noscript>
<div id="categories" class="btn-group btn-toggle">
<div id="categories">
{% if rtl %}
{% for category in categories | reverse %}
<!--<div class="checkbox">-->
<input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}_nojs" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />
<label class="btn btn-sm btn-primary active label_hide_if_not_checked" for="checkbox_{{ category|replace(' ', '_') }}_nojs">{{ _(category) }}</label>
<label class="btn btn-sm btn-default label_hide_if_checked" for="checkbox_{{ category|replace(' ', '_') }}_nojs">{{ _(category) }}</label>
<!--</div>-->
{% if category in selected_categories %}<input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}_dis_activation" name="category_{{ category }}" value="off" checked="checked"/>{% endif %}
{% endfor %}
{% else %}
{% for category in categories %}
<!--<div class="checkbox">-->
<input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}_nojs" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />
<label class="btn btn-sm btn-primary active label_hide_if_not_checked" for="checkbox_{{ category|replace(' ', '_') }}_nojs">{{ _(category) }}</label>
<label class="btn btn-sm btn-default label_hide_if_checked" for="checkbox_{{ category|replace(' ', '_') }}_nojs">{{ _(category) }}</label>
<!--</div>-->
{% if category in selected_categories %}<input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}_dis_activation" name="category_{{ category }}" value="off" checked="checked"/>{% endif %}
{% endfor %}
{% endif %}
</div>
</noscript>
<div id="categories" class="btn-group btn-toggle hide_if_nojs" data-toggle="buttons">
{% if rtl %}
{% for category in categories | reverse %}
<label class="btn btn-sm {% if category in selected_categories %}btn-primary active{% else %}btn-default{% endif %}" data-btn-class="primary">
<input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{ _(category) }}
<input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />
<label class="input-group-addon" for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
</label>
{% endfor %}
{% else %}
{% for category in categories %}
<label class="btn btn-sm {% if category in selected_categories %}btn-primary active{% else %}btn-default{% endif %}" data-btn-class="primary">
<input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />{{ _(category) }}
</label>
<input class="hidden" type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} />
<label class="input-group-addon" for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
{% endfor %}
{% endif %}
</div>

View File

@ -36,7 +36,7 @@
<label class="col-sm-3 col-md-2">{{ _('Default categories') }}</label>
{% else %}
<label class="col-sm-3 col-md-2">{{ _('Default categories') }}</label>
<div class="col-sm-11 col-md-10">
<div class="col-sm-11 col-md-10 search-categories">
{% include 'oscar/categories.html' %}
</div>
{% endif %}

View File

@ -1,6 +1,6 @@
{% extends "oscar/base.html" %}
{% block title %}{{ q }} - {% endblock %}
{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}&amp;time_range={{ time_range }}">{% endblock %}
{% block content %}
<div class="row">
<div class="col-sm-8" id="main_results">
@ -41,6 +41,7 @@
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
<input type="hidden" name="q" value="{{ q }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>
</form>
</div>
@ -48,6 +49,7 @@
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
<input type="hidden" name="pageno" value="{{ pageno-1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-forward"></span> {{ _('previous page') }}</button>
</form>
</div>
@ -60,6 +62,7 @@
<input type="hidden" name="q" value="{{ q }}" />
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
<input type="hidden" name="pageno" value="{{ pageno-1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-backward"></span> {{ _('previous page') }}</button>
</form>
</div>
@ -68,6 +71,7 @@
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
<input type="hidden" name="q" value="{{ q }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>
</form>
</div>
@ -118,7 +122,7 @@
<form role="form">
<div class="form-group">
<label for="search_url">{{ _('Search URL') }}</label>
<input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ base_url }}?q={{ q|urlencode }}{% if selected_categories %}&amp;categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}{% if pageno > 1 %}&amp;pageno={{ pageno }}{% endif %}" readonly>
<input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ base_url }}?q={{ q|urlencode }}{% if selected_categories %}&amp;categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}{% if pageno > 1 %}&amp;pageno={{ pageno }}{% endif %}{% if time_range %}&amp;time_range={{ time_range }}{% endif %}" readonly>
</div>
</form>
@ -130,6 +134,7 @@
<input type="hidden" name="format" value="{{ output_type }}">
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1">{% endfor %}
<input type="hidden" name="pageno" value="{{ pageno }}">
<input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default">{{ output_type }}</button>
</form>
{% endfor %}

View File

@ -6,7 +6,5 @@
<button type="submit" class="btn btn-default"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button>
</span>
</div>
<div class="search_categories">
{% include 'oscar/categories.html' %}
</div><!-- / #search_categories -->
{% include 'oscar/advanced.html' %}
</form><!-- / #search_form_full -->

View File

@ -11,11 +11,8 @@
<button type="submit" class="btn btn-default input-lg"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button>
</span>
</div>
<button type="button" class="btn btn-link btn-collapse center-block collapsed hide_if_nojs" data-toggle="collapse" data-target="#search_categories" data-btn-text-collapsed="{{ _('Show search filters') }}" data-btn-text-not-collapsed="{{ _('Hide search filters') }}">{{ _('Show search filters') }}</button>
<div class="row collapse active_if_nojs margin_top_if_nojs" id="search_categories">
<div class="col-md-12 text-center">
{% include 'oscar/categories.html' %}
<div class="input-group col-md-8 col-md-offset-2">
{% include 'oscar/advanced.html' %}
</div>
</div><!-- / #search_categories -->
</form><!-- / #search_form_full -->

View File

@ -0,0 +1,14 @@
<select name="time_range" id="time-range">
<option id="time-range-anytime" value="" {{ "selected" if time_range=="" or not time_range else ""}}>
{{ _('Anytime') }}
</option>
<option id="time-range-day" value="day" {{ "selected" if time_range=="day" else ""}}>
{{ _('Last day') }}
</option>
<option id="time-range-week" value="week" {{ "selected" if time_range=="week" else ""}}>
{{ _('Last week') }}
</option>
<option id="time-range-month" value="month" {{ "selected" if time_range=="month" else ""}}>
{{ _('Last month') }}
</option>
</select>

View File

@ -459,6 +459,8 @@ def index():
paging=search.paging,
number_of_results=format_decimal(number_of_results),
pageno=search.pageno,
advanced_search=search.is_advanced,
time_range=search.time_range,
base_url=get_base_url(),
suggestions=search.result_container.suggestions,
answers=search.result_container.answers,