Merge branch 'master' of https://github.com/asciimoo/searx into filtron

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

docs/conf.py

@@ -12,7 +12,7 @@ DOCS_URL = os.environ.get("DOCS_URL", "https://asciimoo.github.io/searx/")
 # Project --------------------------------------------------------------
 
 project = u'searx'
-copyright = u'2015-2019, Adam Tauber, Noémi Ványi'
+copyright = u'2015-2020, Adam Tauber, Noémi Ványi'
 author = u'Adam Tauber'
 release, version = VERSION_STRING, VERSION_STRING
 highlight_language = 'none'

docs/dev/reST.rst

@@ -285,7 +285,7 @@ content becomes smart.
    :rst:role:`pep`            :pep:`8`                           ``:pep:`8```
    sphinx.ext.extlinks_
    --------------------------------------------------------------------------------------------------
-   project's wiki article     :wiki:`Searx-instances`            ``:wiki:`Searx-instances```
+   project's wiki article     :wiki:`Offline-engines`            ``:wiki:`Offline-engines```
    to docs public URL         :docs:`dev/reST.html`              ``:docs:`dev/reST.html```
    files & folders origin     :origin:`docs/dev/reST.rst`        ``:origin:`docs/dev/reST.rst```
    pull request               :pull:`1756`                       ``:pull:`1756```

docs/index.rst

@@ -8,8 +8,8 @@ Searx is a free internet metasearch engine which aggregates results from more
 than 70 search services.  Users are neither tracked nor profiled.  Additionally,
 searx can be used over Tor for online anonymity.
 
-Get started with searx by using one of the :wiki:`Searx-instances`.  If you
+Get started with searx by using one of the Searx-instances_.  If you don't trust
-don't trust anyone, you can set up your own, see :ref:`installation`.
+anyone, you can set up your own, see :ref:`installation`.
 
 .. sidebar::  Features
 
@@ -32,3 +32,5 @@ don't trust anyone, you can set up your own, see :ref:`installation`.
    dev/index
    utils/index
    blog/index
+
+.. _Searx-instances: https://searx.space

searx/engines/etools.py

@@ -0,0 +1,54 @@
+"""
+ eTools (Web)
+
+ @website      https://www.etools.ch
+ @provide-api  no
+ @using-api    no
+ @results      HTML
+ @stable       no (HTML can change)
+ @parse        url, title, content
+"""
+
+from lxml import html
+from searx.engines.xpath import extract_text
+from searx.url_utils import quote
+from searx.utils import eval_xpath
+
+categories = ['general']
+paging = False
+language_support = False
+safesearch = True
+
+base_url = 'https://www.etools.ch'
+search_path = '/searchAdvancedSubmit.do'\
+    '?query={search_term}'\
+    '&pageResults=20'\
+    '&safeSearch={safesearch}'
+
+
+def request(query, params):
+    if params['safesearch']:
+        safesearch = 'true'
+    else:
+        safesearch = 'false'
+
+    params['url'] = base_url + search_path.format(search_term=quote(query), safesearch=safesearch)
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    for result in eval_xpath(dom, '//table[@class="result"]//td[@class="record"]'):
+        url = eval_xpath(result, './a/@href')[0]
+        title = extract_text(eval_xpath(result, './a//text()'))
+        content = extract_text(eval_xpath(result, './/div[@class="text"]//text()'))
+
+        results.append({'url': url,
+                        'title': title,
+                        'content': content})
+
+    return results

searx/preferences.py

@@ -4,6 +4,7 @@ from sys import version
 
 from searx import settings, autocomplete
 from searx.languages import language_codes as languages
+from searx.utils import match_language
 from searx.url_utils import parse_qs, urlencode
 
 if version[0] == '3':
@@ -11,7 +12,7 @@ if version[0] == '3':
 
 
 COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5  # 5 years
-LANGUAGE_CODES = [l[0].split('-')[0] for l in languages]
+LANGUAGE_CODES = [l[0] for l in languages]
 LANGUAGE_CODES.append('all')
 DISABLED = 0
 ENABLED = 1
@@ -132,6 +133,10 @@ class SetSetting(Setting):
 class SearchLanguageSetting(EnumStringSetting):
     """Available choices may change, so user's value may not be in choices anymore"""
 
+    def _validate_selection(self, selection):
+        if not match_language(selection, self.choices, fallback=None) and selection != "":
+            raise ValidationException('Invalid language code: "{0}"'.format(selection))
+
     def parse(self, data):
         if data not in self.choices and data != self.value:
             # hack to give some backwards compatibility with old language cookies
@@ -268,7 +273,7 @@ class Preferences(object):
         super(Preferences, self).__init__()
 
         self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories + ['none']),
-                                   'language': SearchLanguageSetting(settings['ui']['default_locale'],
+                                   'language': SearchLanguageSetting(settings['search']['default_lang'],
                                                                      choices=list(LANGUAGE_CODES) + ['']),
                                    'locale': EnumStringSetting(settings['ui']['default_locale'],
                                                                choices=list(settings['locales'].keys()) + ['']),

searx/settings.yml

@@ -5,6 +5,7 @@ general:
 search:
     safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
     autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
+    default_lang : "" # Default search language - leave blank to detect from browser information or use codes from 'languages.py'
     ban_time_on_fail : 5 # ban time in seconds after engine errors
     max_ban_time_on_fail : 120 # max ban time in seconds after engine errors
 
@@ -202,6 +203,11 @@ engines:
     timeout: 3.0
     disabled : True
 
+  - name : etools
+    engine : etools
+    shortcut : eto
+    disabled : True
+
   - name : etymonline
     engine : xpath
     paging : True

searx/webapp.py

@@ -157,14 +157,16 @@ _category_names = (gettext('files'),
 outgoing_proxies = settings['outgoing'].get('proxies') or None
 
 
+def _get_browser_language(request, lang_list):
+    for lang in request.headers.get("Accept-Language", "en").split(","):
+        locale = match_language(lang, lang_list, fallback=None)
+        if locale is not None:
+            return locale
+
+
 @babel.localeselector
 def get_locale():
-    locale = "en-US"
+    locale = _get_browser_language(request, settings['locales'].keys())
-
-    for lang in request.headers.get("Accept-Language", locale).split(","):
-        locale = match_language(lang, settings['locales'].keys(), fallback=None)
-        if locale is not None:
-            break
 
     logger.debug("default locale from browser info is `%s`", locale)
 
@@ -372,8 +374,7 @@ def render(template_name, override_theme=None, **kwargs):
     kwargs['language_codes'] = languages
     if 'current_language' not in kwargs:
         kwargs['current_language'] = match_language(request.preferences.get_value('language'),
-                                                    LANGUAGE_CODES,
+                                                    LANGUAGE_CODES)
-                                                    fallback=locale)
 
     # override url_for function in templates
     kwargs['url_for'] = url_for_theme
@@ -444,11 +445,10 @@ def pre_request():
             request.errors.append(gettext('Invalid settings'))
 
     # init search language and locale
-    locale = get_locale()
     if not preferences.get_value("language"):
-        preferences.parse_dict({"language": locale})
+        preferences.parse_dict({"language": _get_browser_language(request, LANGUAGE_CODES)})
     if not preferences.get_value("locale"):
-        preferences.parse_dict({"locale": locale})
+        preferences.parse_dict({"locale": get_locale()})
 
     # request.user_plugins
     request.user_plugins = []