[fix] engine - yahoo: rewrite and fix issues

Languages are supported by mapping the language to a domain.  If domain is not
found in :py:obj:`lang2domain` URL ``<lang>.search.yahoo.com`` is used.

Closes #3020

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/data/engines_languages.json

@@ -28582,7 +28582,7 @@
     "sv",
     "th",
     "tr",
-    "zh-CHS",
+    "zh_chs",
-    "zh-CHT"
+    "zh_cht"
   ]
 }

searx/engines/yahoo.py

@@ -3,9 +3,18 @@
  Yahoo (Web)
 """
 
-from urllib.parse import unquote, urlencode
+from urllib.parse import (
+    unquote,
+    urlencode,
+)
 from lxml import html
-from searx.utils import extract_text, extract_url, match_language, eval_xpath
+
+from searx.utils import (
+    eval_xpath_getindex,
+    eval_xpath_list,
+    extract_text,
+    match_language,
+)
 
 # about
 about = {
@@ -21,29 +30,75 @@ about = {
 categories = ['general']
 paging = True
 time_range_support = True
+supported_languages_url = 'https://search.yahoo.com/preferences/languages'
+"""Supported languages are read from Yahoo preference page."""
 
-# search-url
+time_range_dict = {
-base_url = 'https://search.yahoo.com/'
+    'day': ('1d', 'd'),
-search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}'
+    'week': ('1w', 'w'),
-search_url_with_time = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}&age={age}&btf={btf}&fr2=time'
+    'month': ('1m', 'm'),
+}
 
-supported_languages_url = 'https://search.yahoo.com/web/advanced'
+language_aliases = {
+    'zh-HK': 'zh_chs',
+    'zh-CN': 'zh_chs',  # dead since 2015 / routed to hk.search.yahoo.com
+    'zh-TW': 'zh_cht',
+}
 
-# specific xpath variables
+lang2domain = {
-results_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]"
+    'zh_chs': 'hk.search.yahoo.com',
-url_xpath = './/h3/a/@href'
+    'zh_cht': 'tw.search.yahoo.com',
-title_xpath = './/h3/a'
+    'en': 'search.yahoo.com',
-content_xpath = './/div[contains(@class, "compText")]'
-suggestion_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' AlsoTry ')]//a"
 
-time_range_dict = {'day': ['1d', 'd'],
+    'bg': 'search.yahoo.com',
-                   'week': ['1w', 'w'],
+    'cs': 'search.yahoo.com',
-                   'month': ['1m', 'm']}
+    'da': 'search.yahoo.com',
+    'el': 'search.yahoo.com',
+    'et': 'search.yahoo.com',
+    'he': 'search.yahoo.com',
+    'hr': 'search.yahoo.com',
+    'ja': 'search.yahoo.com',
+    'ko': 'search.yahoo.com',
+    'sk': 'search.yahoo.com',
+    'sl': 'search.yahoo.com',
 
-language_aliases = {'zh-CN': 'zh-CHS', 'zh-TW': 'zh-CHT', 'zh-HK': 'zh-CHT'}
+}
+
+
+def _get_language(params):
+
+    lang = language_aliases.get(params['language'])
+    if lang is None:
+        lang = match_language(
+            params['language'], supported_languages, language_aliases
+        )
+    lang = lang.split('-')[0]
+    return lang
+
+
+def request(query, params):
+    """build request"""
+    offset = (params['pageno'] - 1) * 7 + 1
+    lang = _get_language(params)
+    age, btf = time_range_dict.get(params['time_range'], ('', ''))
+
+    args = urlencode({
+        'p': query,
+        'ei': 'UTF-8',
+        'fl': 1,
+        'vl': 'lang_' + lang,
+        'btf': btf,
+        'fr2': 'time',
+        'age': age,
+        'b': offset,
+        'xargs': 0,
+    })
+
+    domain = lang2domain.get(lang, '%s.search.yahoo.com' % lang)
+    params['url'] = 'https://%s/search?%s' % (domain, args)
+    return params
 
 
-# remove yahoo-specific tracking-url
 def parse_url(url_string):
     endings = ['/RS', '/RK']
     endpositions = []
@@ -61,86 +116,37 @@ def parse_url(url_string):
         return unquote(url_string[start:end])
 
 
-def _get_url(query, offset, language, time_range):
-    if time_range in time_range_dict:
-        return base_url + search_url_with_time.format(offset=offset,
-                                                      query=urlencode({'p': query}),
-                                                      lang=language,
-                                                      age=time_range_dict[time_range][0],
-                                                      btf=time_range_dict[time_range][1])
-    return base_url + search_url.format(offset=offset,
-                                        query=urlencode({'p': query}),
-                                        lang=language)
-
-
-def _get_language(params):
-    if params['language'] == 'all':
-        return 'en'
-
-    language = match_language(params['language'], supported_languages, language_aliases)
-    if language not in language_aliases.values():
-        language = language.split('-')[0]
-    language = language.replace('-', '_').lower()
-
-    return language
-
-
-# do search-request
-def request(query, params):
-    if params['time_range'] and params['time_range'] not in time_range_dict:
-        return params
-
-    offset = (params['pageno'] - 1) * 10 + 1
-    language = _get_language(params)
-
-    params['url'] = _get_url(query, offset, language, params['time_range'])
-
-    # TODO required?
-    params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\
-        .format(lang=language)
-
-    return params
-
-
-# get response from search-request
 def response(resp):
     results = []
-
     dom = html.fromstring(resp.text)
 
-    try:
+    for result in eval_xpath_list(dom, '//div[contains(@class,"algo-sr")]'):
-        results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0]
+        url = eval_xpath_getindex(result, './/h3/a/@href', 0, default=None)
-                          .split()[0].replace(',', ''))
+        if url is None:
-        results.append({'number_of_results': results_num})
-    except:
-        pass
-
-    # parse results
-    for result in eval_xpath(dom, results_xpath):
-        try:
-            url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url))
-            title = extract_text(eval_xpath(result, title_xpath)[0])
-        except:
             continue
+        url = parse_url(url)
 
-        content = extract_text(eval_xpath(result, content_xpath)[0])
+        title = eval_xpath_getindex(result, './/h3/a', 0, default=None)
+        if title is None:
+            continue
+        offset = len(extract_text(title.xpath('span')))
+        title = extract_text(title)[offset:]
+
+        content = eval_xpath_getindex(
+            result, './/div[contains(@class, "compText")]', 0, default=''
+        )
+        if content:
+            content = extract_text(content)
 
         # append result
         results.append({'url': url,
                         'title': title,
                         'content': content})
 
-    # if no suggestion found, return results
+    for suggestion in eval_xpath_list(dom, '//div[contains(@class, "AlsoTry")]'):
-    suggestions = eval_xpath(dom, suggestion_xpath)
-    if not suggestions:
-        return results
-
-    # parse suggestion
-    for suggestion in suggestions:
         # append suggestion
         results.append({'suggestion': extract_text(suggestion)})
 
-    # return results
     return results
 
 
@@ -148,13 +154,9 @@ def response(resp):
 def _fetch_supported_languages(resp):
     supported_languages = []
     dom = html.fromstring(resp.text)
-    options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input')
+    offset = len('lang_')
-    for option in options:
+
-        code_parts = eval_xpath(option, './@value')[0][5:].split('_')
+    for val in eval_xpath_list(dom, '//div[contains(@class, "lang-item")]/input/@value'):
-        if len(code_parts) == 2:
+        supported_languages.append(val[offset:])
-            code = code_parts[0] + '-' + code_parts[1].upper()
-        else:
-            code = code_parts[0]
-        supported_languages.append(code)
 
     return supported_languages