devol
/
searx
mirror of https://github.com/searx/searx
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #2359 from dalf/update-duden 

[mod] duden engine
This commit is contained in:
Alexandre Flament 2020-12-08 20:33:38 +01:00 committed by GitHub
parent bd4869ecd0 9bf594cbcf
commit a77d8c8227
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 19 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
searx/engines/duden.py
View File

@ -8,11 +8,10 @@
@parse url, title, content @parse url, title, content
""" """
from lxml import html, etree
import re import re
from urllib.parse import quote, urljoin from urllib.parse import quote, urljoin
from searx.utils import extract_text, eval_xpath from lxml import html
from searx import logger from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
categories = ['general'] categories = ['general']
paging = True paging = True
@ -40,6 +39,9 @@ def request(query, params):
params['url'] = search_url_fmt.format(query=quote(query)) params['url'] = search_url_fmt.format(query=quote(query))
else: else:
params['url'] = search_url.format(offset=offset, query=quote(query)) params['url'] = search_url.format(offset=offset, query=quote(query))
# after the last page of results, spelling corrections are returned after a HTTP redirect
# whatever the page number is
params['soft_max_redirects'] = 1
return params return params
@ -51,28 +53,21 @@ def response(resp):
dom = html.fromstring(resp.text) dom = html.fromstring(resp.text)
try: number_of_results_element =\
number_of_results_string =\ eval_xpath_getindex(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()',
re.sub('[^0-9]', '', 0, default=None)
eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]) if number_of_results_element is not None:
number_of_results_string = re.sub('[^0-9]', '', number_of_results_element)
results.append({'number_of_results': int(number_of_results_string)}) results.append({'number_of_results': int(number_of_results_string)})
except: for result in eval_xpath_list(dom, '//section[not(contains(@class, "essay"))]'):
logger.debug("Couldn't read number of results.") url = eval_xpath_getindex(result, './/h2/a', 0).get('href')
url = urljoin(base_url, url)
for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'): title = eval_xpath(result, 'string(.//h2/a)').strip()
try: content = extract_text(eval_xpath(result, './/p'))
url = eval_xpath(result, './/h2/a')[0].get('href') # append result
url = urljoin(base_url, url) results.append({'url': url,
title = eval_xpath(result, 'string(.//h2/a)').strip() 'title': title,
content = extract_text(eval_xpath(result, './/p')) 'content': content})
# append result
results.append({'url': url,
'title': title,
'content': content})
except:
logger.debug('result parse error in:\n%s', etree.tostring(result, pretty_print=True))
continue
return results return results