devol
/
searx
mirror of https://github.com/searx/searx
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' into dsgvo

This commit is contained in:
cy8aer 2018-07-09 15:34:43 +02:00 committed by GitHub
parent b786ac9a63 d51732c0e5
commit 78be030500
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 29 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
searx/engines/bing.py
View File

@ -16,7 +16,7 @@
from lxml import html from lxml import html
from searx.engines.xpath import extract_text from searx.engines.xpath import extract_text
from searx.url_utils import urlencode from searx.url_utils import urlencode
from searx.utils import match_language from searx.utils import match_language, gen_useragent
# engine dependent config # engine dependent config
categories = ['general'] categories = ['general']
@ -43,6 +43,9 @@ def request(query, params):
offset=offset) offset=offset)
params['url'] = base_url + search_path params['url'] = base_url + search_path
params['headers']['User-Agent'] = gen_useragent('Windows NT 6.3; WOW64')
return params return params

3
searx/engines/wikidata.py
View File

@ -54,6 +54,7 @@ value_xpath = './/div[contains(@class,"wikibase-statementview-mainsnak")]'\
+ '/*/div[contains(@class,"wikibase-snakview-value")]' + '/*/div[contains(@class,"wikibase-snakview-value")]'
language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator")]' language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator")]'
calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]' calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
def request(query, params): def request(query, params):
@ -313,7 +314,7 @@ def add_image(result):
for property_id in property_ids: for property_id in property_ids:
image = result.xpath(property_xpath.replace('{propertyid}', property_id)) image = result.xpath(property_xpath.replace('{propertyid}', property_id))
if image: if image:
image_name = image[0].xpath(value_xpath) image_name = image[0].xpath(media_xpath)
image_src = url_image.replace('{filename}', extract_text(image_name[0])) image_src = url_image.replace('{filename}', extract_text(image_name[0]))
return image_src return image_src

21
searx/utils.py
View File

@ -39,14 +39,15 @@ else:
logger = logger.getChild('utils') logger = logger.getChild('utils')
ua_versions = ('40.0', ua_versions = ('52.8.1',
'41.0', '53.0',
'42.0', '54.0',
'43.0', '55.0',
'44.0', '56.0',
'45.0', '57.0',
'46.0', '58.0',
'47.0') '59.0',
'60.0.2')
ua_os = ('Windows NT 6.3; WOW64', ua_os = ('Windows NT 6.3; WOW64',
'X11; Linux x86_64', 'X11; Linux x86_64',
@ -58,9 +59,9 @@ blocked_tags = ('script',
'style') 'style')
def gen_useragent(): def gen_useragent(os=None):
# TODO # TODO
return ua.format(os=choice(ua_os), version=choice(ua_versions)) return ua.format(os=os or choice(ua_os), version=choice(ua_versions))
def searx_useragent(): def searx_useragent():

21
tests/unit/engines/test_wikidata.py
View File

@ -123,9 +123,10 @@ class TestWikidataEngine(SearxTestCase):
<div class="wikibase-statementview-mainsnak"> <div class="wikibase-statementview-mainsnak">
<div> <div>
<div class="wikibase-snakview-value"> <div class="wikibase-snakview-value">
<a href="https://commons.wikimedia.org/wiki/File:image.png"> <div class="commons-media-caption">
image.png <a href="https://commons.wikimedia.org/wiki/File:image.png">image.png</a>
</a> <br/>2,687 &#215; 3,356; 1.22 MB
</div>
</div> </div>
</div> </div>
</div> </div>
@ -156,9 +157,10 @@ class TestWikidataEngine(SearxTestCase):
<div class="wikibase-statementview-mainsnak"> <div class="wikibase-statementview-mainsnak">
<div> <div>
<div class="wikibase-snakview-value"> <div class="wikibase-snakview-value">
<a href="https://commons.wikimedia.org/wiki/File:icon.png"> <div class="commons-media-caption">
icon.png <a href="https://commons.wikimedia.org/wiki/File:icon.png">icon.png</a>
</a> <br/>671 &#215; 671; 18 KB</div>
</div>
</div> </div>
</div> </div>
</div> </div>
@ -179,9 +181,10 @@ class TestWikidataEngine(SearxTestCase):
<div class="wikibase-statementview-mainsnak"> <div class="wikibase-statementview-mainsnak">
<div> <div>
<div class="wikibase-snakview-value"> <div class="wikibase-snakview-value">
<a href="https://commons.wikimedia.org/wiki/File:logo.png"> <div class="commons-media-caption">
logo.png <a href="https://commons.wikimedia.org/wiki/File:logo.png">logo.png</a>
</a> <br/>170 &#215; 170; 1 KB
</div>
</div> </div>
</div> </div>
</div> </div>