From 7fdfeca3a43e0e2bd8ef2dcb27cca7745edf596a Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 13 Jan 2017 22:15:11 +0100 Subject: [PATCH 1/2] [mod] add a __common__ template that can't be selected but that provides a common place for shared templates. What has been moved into this template : * opensearch*.xml is always the same whatever the themes. * the text inside */about.html --- searx/templates/__common__/about.html | 62 ++++++++++++++++++ .../{courgette => __common__}/opensearch.xml | 0 .../opensearch_response_rss.xml | 0 searx/templates/courgette/about.html | 63 +------------------ searx/templates/legacy/about.html | 63 +------------------ searx/templates/legacy/opensearch.xml | 28 --------- .../legacy/opensearch_response_rss.xml | 23 ------- searx/templates/oscar/about.html | 63 +------------------ searx/templates/oscar/opensearch.xml | 28 --------- .../oscar/opensearch_response_rss.xml | 23 ------- searx/templates/pix-art/about.html | 63 +------------------ searx/utils.py | 2 + searx/webapp.py | 8 ++- tests/unit/test_webapp.py | 2 + 14 files changed, 75 insertions(+), 353 deletions(-) create mode 100644 searx/templates/__common__/about.html rename searx/templates/{courgette => __common__}/opensearch.xml (100%) rename searx/templates/{courgette => __common__}/opensearch_response_rss.xml (100%) delete mode 100644 searx/templates/legacy/opensearch.xml delete mode 100644 searx/templates/legacy/opensearch_response_rss.xml delete mode 100644 searx/templates/oscar/opensearch.xml delete mode 100644 searx/templates/oscar/opensearch_response_rss.xml diff --git a/searx/templates/__common__/about.html b/searx/templates/__common__/about.html new file mode 100644 index 00000000..d8afab73 --- /dev/null +++ b/searx/templates/__common__/about.html @@ -0,0 +1,62 @@ + +

About searx

+ +

Searx is a metasearch engine, aggregating the results of other search engines while not storing information about its users. +

+

Why use searx?

+ +

If you do care about privacy, want to be a conscious user, or otherwise believe + in digital freedom, make searx your default search engine or run it on your own server

+ +

Technical details - How does it work?

+ +

Searx is a metasearch engine, +inspired by the seeks project.
+It provides basic privacy by mixing your queries with searches on other platforms without storing search data. Queries are made using a POST request on every browser (except chrome*). Therefore they show up in neither our logs, nor your url history. In case of Chrome* users there is an exception, searx uses the search bar to perform GET requests.
+Searx can be added to your browser's search bar; moreover, it can be set as the default search engine. +

+ +

How can I make it my own?

+ +

Searx appreciates your concern regarding logs, so take the code and run it yourself!
Add your Searx to this list to help other people reclaim their privacy and make the Internet freer! +
The more decentralized the Internet is, the more freedom we have!

+ + +

More about searx

+ + + + +
+ +

FAQ

+ +

How to add to firefox?

+

Install searx as a search engine on any version of Firefox! (javascript required)

+ +

Developer FAQ

+ +

New engines?

+ +

Don't forget to restart searx after config edit!

+ +

Installation/WSGI support?

+

See the installation and setup wiki page

+ +

How to debug engines?

+

Stats page contains some useful data about the engines used.

+ + diff --git a/searx/templates/courgette/opensearch.xml b/searx/templates/__common__/opensearch.xml similarity index 100% rename from searx/templates/courgette/opensearch.xml rename to searx/templates/__common__/opensearch.xml diff --git a/searx/templates/courgette/opensearch_response_rss.xml b/searx/templates/__common__/opensearch_response_rss.xml similarity index 100% rename from searx/templates/courgette/opensearch_response_rss.xml rename to searx/templates/__common__/opensearch_response_rss.xml diff --git a/searx/templates/courgette/about.html b/searx/templates/courgette/about.html index faa7b613..08948ee9 100644 --- a/searx/templates/courgette/about.html +++ b/searx/templates/courgette/about.html @@ -1,66 +1,5 @@ {% extends 'courgette/base.html' %} {% block content %} {% include 'courgette/github_ribbon.html' %} -
-

About searx

- -

Searx is a metasearch engine, aggregating the results of other search engines while not storing information about its users. -

-

Why use searx?

-
    -
  • searx may not offer you as personalised results as Google, but it doesn't generate a profile about you
    • -
  • searx doesn't care about what you search for, never shares anything with a third party, and it can't be used to compromise you
    • -
  • searx is free software, the code is 100% open and you can help to make it better. See more on github
    • -
-

If you do care about privacy, want to be a conscious user, or otherwise believe - in digital freedom, make searx your default search engine or run it on your own server

- -

Technical details - How does it work?

- -

Searx is a metasearch engine, -inspired by the seeks project.
-It provides basic privacy by mixing your queries with searches on other platforms without storing search data. Queries are made using a POST request on every browser (except chrome*). Therefore they show up in neither our logs, nor your url history. In case of Chrome* users there is an exception, searx uses the search bar to perform GET requests.
-Searx can be added to your browser's search bar; moreover, it can be set as the default search engine. -

- -

How can I make it my own?

- -

Searx appreciates your concern regarding logs, so take the code and run it yourself!
Add your Searx to this list to help other people reclaim their privacy and make the Internet freer! -
The more decentralized the Internet, is the more freedom we have!

- - -

More about searx

- - - - -
- -

FAQ

- -

How to add to firefox?

-

Install searx as a search engine on any version of Firefox! (javascript required)

- -

Developer FAQ

- -

New engines?

- -

Don't forget to restart searx after config edit!

- -

Installation/WSGI support?

-

See the installation and setup wiki page

- -

How to debug engines?

-

Stats page contains some useful data about the engines used.

- -
+{% include '__common__/about.html' %} {% endblock %} diff --git a/searx/templates/legacy/about.html b/searx/templates/legacy/about.html index 580321e4..f773e3a7 100644 --- a/searx/templates/legacy/about.html +++ b/searx/templates/legacy/about.html @@ -1,66 +1,5 @@ {% extends 'legacy/base.html' %} {% block content %} {% include 'legacy/github_ribbon.html' %} -
-

About searx

- -

Searx is a metasearch engine, aggregating the results of other search engines while not storing information about its users. -

-

Why use searx?

-
    -
  • searx may not offer you as personalised results as Google, but it doesn't generate a profile about you
    • -
  • searx doesn't care about what you search for, never shares anything with a third party, and it can't be used to compromise you
    • -
  • searx is free software, the code is 100% open and you can help to make it better. See more on github
    • -
-

If you do care about privacy, want to be a conscious user, or otherwise believe - in digital freedom, make searx your default search engine or run it on your own server

- -

Technical details - How does it work?

- -

Searx is a metasearch engine, -inspired by the seeks project.
-It provides basic privacy by mixing your queries with searches on other platforms without storing search data. Queries are made using a POST request on every browser (except chrome*). Therefore they show up in neither our logs, nor your url history. In case of Chrome* users there is an exception, if searx used from the search bar it performs GET requests.
-Searx can be added to your browser's search bar; moreover, it can be set as the default search engine. -

- -

How can I make it my own?

- -

Searx appreciates your concern regarding logs, so take the code and run it yourself!
Add your Searx to this list to help other people reclaim their privacy and make the Internet freer! -
The more decentralized Internet is the more freedom we have!

- - -

More about searx

- - - - -
- -

FAQ

- -

How to add to firefox?

-

Install searx as a search engine on any version of Firefox! (javascript required)

- -

Developer FAQ

- -

New engines?

- -

Don't forget to restart searx after config edit!

- -

Installation/WSGI support?

-

See the installation and setup wiki page

- -

How to debug engines?

-

Stats page contains some useful data about the engines used.

- -
+{% include '__common__/about.html' %} {% endblock %} diff --git a/searx/templates/legacy/opensearch.xml b/searx/templates/legacy/opensearch.xml deleted file mode 100644 index 15d3eb79..00000000 --- a/searx/templates/legacy/opensearch.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - {{ instance_name }} - a privacy-respecting, hackable metasearch engine - UTF-8 - {{ urljoin(host, url_for('static', filename='img/favicon.png')) }} - searx metasearch - {% if opensearch_method == 'get' %} - - {% if autocomplete %} - - - - - {% endif %} - {% else %} - - - - {% if autocomplete %} - - - - - - {% endif %} - {% endif %} - diff --git a/searx/templates/legacy/opensearch_response_rss.xml b/searx/templates/legacy/opensearch_response_rss.xml deleted file mode 100644 index ddb60fa5..00000000 --- a/searx/templates/legacy/opensearch_response_rss.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - Searx search: {{ q|e }} - {{ base_url }}?q={{ q|e }} - Search results for "{{ q|e }}" - searx - {{ number_of_results }} - 1 - {{ number_of_results }} - - diff --git a/searx/templates/oscar/about.html b/searx/templates/oscar/about.html index 67373817..bc7fed8e 100644 --- a/searx/templates/oscar/about.html +++ b/searx/templates/oscar/about.html @@ -1,66 +1,5 @@ {% extends "oscar/base.html" %} {% block title %}{{ _('about') }} - {% endblock %} {% block content %} - -

About searx

- -

Searx is a metasearch engine, aggregating the results of other search engines while not storing information about its users. -

-

Why use searx?

-
    -
  • searx may not offer you as personalised results as Google, but it doesn't generate a profile about you
    • -
  • searx doesn't care about what you search for, never shares anything with a third party, and it can't be used to compromise you
    • -
  • searx is free software, the code is 100% open and you can help to make it better. See more on github
    • -
-

If you do care about privacy, want to be a conscious user, or otherwise believe - in digital freedom, make searx your default search engine or run it on your own server

- -

Technical details - How does it work?

- -

Searx is a metasearch engine, -inspired by the seeks project.
-It provides basic privacy by mixing your queries with searches on other platforms without storing search data. Queries are made using a POST request on every browser (except chrome*). Therefore they show up in neither our logs, nor your url history. In case of Chrome* users there is an exception, searx uses the search bar to perform GET requests.
-Searx can be added to your browser's search bar; moreover, it can be set as the default search engine. -

- -

How can I make it my own?

- -

Searx appreciates your concern regarding logs, so take the code and run it yourself!
Add your Searx to this list to help other people reclaim their privacy and make the Internet freer! -
The more decentralized the Internet is, the more freedom we have!

- - -

More about searx

- - - - -
- -

FAQ

- -

How to add to firefox?

-

Install searx as a search engine on any version of Firefox! (javascript required)

- -

Developer FAQ

- -

New engines?

- -

Don't forget to restart searx after config edit!

- -

Installation/WSGI support?

-

See the installation and setup wiki page

- -

How to debug engines?

-

Stats page contains some useful data about the engines used.

- - +{% include '__common__/about.html' %} {% endblock %} diff --git a/searx/templates/oscar/opensearch.xml b/searx/templates/oscar/opensearch.xml deleted file mode 100644 index 15d3eb79..00000000 --- a/searx/templates/oscar/opensearch.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - {{ instance_name }} - a privacy-respecting, hackable metasearch engine - UTF-8 - {{ urljoin(host, url_for('static', filename='img/favicon.png')) }} - searx metasearch - {% if opensearch_method == 'get' %} - - {% if autocomplete %} - - - - - {% endif %} - {% else %} - - - - {% if autocomplete %} - - - - - - {% endif %} - {% endif %} - diff --git a/searx/templates/oscar/opensearch_response_rss.xml b/searx/templates/oscar/opensearch_response_rss.xml deleted file mode 100644 index ddb60fa5..00000000 --- a/searx/templates/oscar/opensearch_response_rss.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - Searx search: {{ q|e }} - {{ base_url }}?q={{ q|e }} - Search results for "{{ q|e }}" - searx - {{ number_of_results }} - 1 - {{ number_of_results }} - - diff --git a/searx/templates/pix-art/about.html b/searx/templates/pix-art/about.html index 041b036f..f76a6893 100644 --- a/searx/templates/pix-art/about.html +++ b/searx/templates/pix-art/about.html @@ -1,65 +1,4 @@ {% extends 'pix-art/base.html' %} {% block content %} -
-

About searx

- -

Searx is a metasearch engine, aggregating the results of other search engines while not storing information about its users. -

-

Why use searx?

-
    -
  • searx may not offer you as personalised results as Google, but it doesn't generate a profile about you
    • -
  • searx doesn't care about what you search for, never shares anything with a third party, and it can't be used to compromise you
    • -
  • searx is free software, the code is 100% open and you can help to make it better. See more on github
    • -
-

If you do care about privacy, want to be a conscious user, or otherwise believe - in digital freedom, make searx your default search engine or run it on your own server

- -

Technical details - How does it work?

- -

Searx is a metasearch engine, -inspired by the seeks project.
-It provides basic privacy by mixing your queries with searches on other platforms without storing search data. Queries are made using a POST request on every browser (except chrome*). Therefore they show up in neither our logs, nor your url history. In case of Chrome* users there is an exception, if searx used from the search bar it performs GET requests.
-Searx can be added to your browser's search bar; moreover, it can be set as the default search engine. -

- -

How can I make it my own?

- -

Searx appreciates your concern regarding logs, so take the code and run it yourself!
Add your Searx to this list to help other people reclaim their privacy and make the Internet freer! -
The more decentralized Internet is the more freedom we have!

- - -

More about searx

- - - - -
- -

FAQ

- -

How to add to firefox?

-

Install searx as a search engine on any version of Firefox! (javascript required)

- -

Developer FAQ

- -

New engines?

- -

Don't forget to restart searx after config edit!

- -

Installation/WSGI support?

-

See the installation and setup wiki page

- -

How to debug engines?

-

Stats page contains some useful data about the engines used.

- -
+{% include '__common__/about.html' %} {% endblock %} diff --git a/searx/utils.py b/searx/utils.py index faa63485..35cb6f8a 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -175,6 +175,8 @@ def get_themes(root): templates_path = os.path.join(root, 'templates') themes = os.listdir(os.path.join(static_path, 'themes')) + if '__common__' in themes: + themes.remove('__common__') return static_path, templates_path, themes diff --git a/searx/webapp.py b/searx/webapp.py index 0b716931..31d85ff5 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -226,7 +226,7 @@ def get_current_theme_name(override=None): 2. cookies 3. settings""" - if override and override in themes: + if override and (override in themes or override == '__common__'): return override theme_name = request.args.get('theme', request.preferences.get_value('theme')) if theme_name not in themes: @@ -501,7 +501,8 @@ def index(): results=results, q=request.form['q'], number_of_results=number_of_results, - base_url=get_base_url() + base_url=get_base_url(), + override_theme='__common__', ) return Response(response_rss, mimetype='text/xml') @@ -722,7 +723,8 @@ def opensearch(): ret = render('opensearch.xml', opensearch_method=method, host=get_base_url(), - urljoin=urljoin) + urljoin=urljoin, + override_theme='__common__') resp = Response(response=ret, status=200, diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index ac5bf8c9..5e5f0b4b 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -46,6 +46,8 @@ class ViewsTestCase(SearxTestCase): Search.search = search_mock def get_current_theme_name_mock(override=None): + if override: + return override return 'legacy' webapp.get_current_theme_name = get_current_theme_name_mock From 15eef0ebdb15af80c026302bef250dc7f4417951 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 20 Jan 2017 18:52:47 +0100 Subject: [PATCH 2/2] [enh] validate input and raise an exception inside search.py. The exception message is output in json and rss format. --- searx/exceptions.py | 32 +++++++++ searx/search.py | 66 +++++++++++------- .../__common__/opensearch_response_rss.xml | 6 ++ searx/webapp.py | 67 ++++++++++++++----- 4 files changed, 133 insertions(+), 38 deletions(-) create mode 100644 searx/exceptions.py diff --git a/searx/exceptions.py b/searx/exceptions.py new file mode 100644 index 00000000..c605ddca --- /dev/null +++ b/searx/exceptions.py @@ -0,0 +1,32 @@ +''' +searx is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +searx is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with searx. If not, see < http://www.gnu.org/licenses/ >. + +(C) 2017- by Alexandre Flament, +''' + + +class SearxException(Exception): + pass + + +class SearxParameterException(SearxException): + + def __init__(self, name, value): + if value == '' or value is None: + message = 'Empty ' + name + ' parameter' + else: + message = 'Invalid value "' + value + '" for parameter ' + name + super(SearxParameterException, self).__init__(message) + self.parameter_name = name + self.parameter_value = value diff --git a/searx/search.py b/searx/search.py index e0f0cfd6..0bb77447 100644 --- a/searx/search.py +++ b/searx/search.py @@ -31,11 +31,16 @@ from searx.query import RawTextQuery, SearchQuery from searx.results import ResultContainer from searx import logger from searx.plugins import plugins +from searx.languages import language_codes +from searx.exceptions import SearxParameterException logger = logger.getChild('search') number_of_searches = 0 +language_code_set = set(l[0].lower() for l in language_codes) +language_code_set.add('all') + def send_http_request(engine, request_params, start_time, timeout_limit): # for page_load_time stats @@ -182,33 +187,13 @@ def default_request_params(): def get_search_query_from_webapp(preferences, form): - query = None - query_engines = [] - query_categories = [] - query_pageno = 1 - query_lang = 'all' - query_time_range = None + # no text for the query ? + if not form.get('q'): + raise SearxParameterException('q', '') # set blocked engines disabled_engines = preferences.engines.get_disabled() - # set specific language if set - query_lang = preferences.get_value('language') - - # safesearch - query_safesearch = preferences.get_value('safesearch') - - # TODO better exceptions - if not form.get('q'): - raise Exception('noquery') - - # set pagenumber - pageno_param = form.get('pageno', '1') - if not pageno_param.isdigit() or int(pageno_param) < 1: - pageno_param = 1 - - query_pageno = int(pageno_param) - # parse query, if tags are set, which change # the serch engine or search-language raw_text_query = RawTextQuery(form['q'], disabled_engines) @@ -217,6 +202,13 @@ def get_search_query_from_webapp(preferences, form): # set query query = raw_text_query.getSearchQuery() + # get and check page number + pageno_param = form.get('pageno', '1') + if not pageno_param.isdigit() or int(pageno_param) < 1: + raise SearxParameterException('pageno', pageno_param) + query_pageno = int(pageno_param) + + # get language # set specific language if set on request, query or preferences # TODO support search with multible languages if len(raw_text_query.languages): @@ -226,10 +218,38 @@ def get_search_query_from_webapp(preferences, form): else: query_lang = preferences.get_value('language') + # check language + if query_lang not in language_code_set: + raise SearxParameterException('language', query_lang) + + # get safesearch + if 'safesearch' in form: + query_safesearch = form.get('safesearch') + # first check safesearch + if not query_safesearch.isdigit(): + raise SearxParameterException('safesearch', query_safesearch) + query_safesearch = int(query_safesearch) + else: + query_safesearch = preferences.get_value('safesearch') + + # safesearch : second check + if query_safesearch < 0 or query_safesearch > 2: + raise SearxParameterException('safesearch', query_safesearch) + + # get time_range query_time_range = form.get('time_range') + # check time_range + if not(query_time_range is None)\ + and not (query_time_range in ['', 'day', 'week', 'month', 'year']): + raise SearxParameterException('time_range', query_time_range) + + # query_engines query_engines = raw_text_query.engines + # query_categories + query_categories = [] + # if engines are calculated from query, # set categories by using that informations if query_engines and raw_text_query.specific: diff --git a/searx/templates/__common__/opensearch_response_rss.xml b/searx/templates/__common__/opensearch_response_rss.xml index ddb60fa5..32c42e7c 100644 --- a/searx/templates/__common__/opensearch_response_rss.xml +++ b/searx/templates/__common__/opensearch_response_rss.xml @@ -11,6 +11,12 @@ {{ number_of_results }}