devol
/
searx
mirror of https://github.com/searx/searx
1
0
Fork 0
Code Issues Releases Wiki Activity

[enh] implement image support for swisscows engine

This commit is contained in:
Thomas Pointhuber 2015-06-01 12:30:07 +02:00
parent d923b2a420
commit 2d81079384
2 changed files with 55 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
searx/engines/swisscows.py
View File

@ -1,5 +1,5 @@
""" """
Swisscows (Web) Swisscows (Web, Images)
@website https://swisscows.ch @website https://swisscows.ch
@provide-api no @provide-api no
@ -15,7 +15,7 @@ from urllib import urlencode, unquote
import re import re
# engine dependent config # engine dependent config
categories = ['general'] categories = ['general', 'images']
paging = True paging = True
language_support = True language_support = True
@ -24,7 +24,7 @@ base_url = 'https://swisscows.ch/'
search_string = '?{query}&page={page}' search_string = '?{query}&page={page}'
# regex # regex
regex_json = re.compile('initialData: {"Request":(.|\n)*}\]},\s*environment') regex_json = re.compile('initialData: {"Request":(.|\n)*},\s*environment')
regex_json_remove_start = re.compile('^initialData:\s*') regex_json_remove_start = re.compile('^initialData:\s*')
regex_json_remove_end = re.compile(',\s*environment$') regex_json_remove_end = re.compile(',\s*environment$')
regex_img_url_remove_start = re.compile('^https?://i\.swisscows\.ch/\?link=') regex_img_url_remove_start = re.compile('^https?://i\.swisscows\.ch/\?link=')
@ -45,6 +45,10 @@ def request(query, params):
'region': region}), 'region': region}),
page=params['pageno']) page=params['pageno'])
# image search query is something like 'image?{query}&page={page}'
if params['category'] == 'images':
search_path = 'image' + search_path
params['url'] = base_url + search_path params['url'] = base_url + search_path
return params return params
@ -63,12 +67,30 @@ def response(resp):
json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group())) json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))
json = loads(json_raw) json = loads(json_raw)
# parse normal results # parse results
for result in json['Results'].get('items', []): for result in json['Results'].get('items', []):
result_title = result['Title'].replace(u'\uE000', '').replace(u'\uE001', '')
# parse image results
if result.get('ContentType', '').startswith('image'):
img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
# append result # append result
results.append({'url': result['Url'].replace(u'\uE000', '').replace(u'\uE001', ''), results.append({'url': result['SourceUrl'],
'title': result['Title'].replace(u'\uE000', '').replace(u'\uE001', ''), 'title': result['Title'],
'content': result['Description'].replace(u'\uE000', '').replace(u'\uE001', '')}) 'content': '',
'img_src': img_url,
'template': 'images.html'})
# parse general results
else:
result_url = result['Url'].replace(u'\uE000', '').replace(u'\uE001', '')
result_content = result['Description'].replace(u'\uE000', '').replace(u'\uE001', '')
# append result
results.append({'url': result_url,
'title': result_title,
'content': result_content})
# parse images # parse images
for result in json.get('Images', []): for result in json.get('Images', []):

30
searx/tests/engines/test_swisscows.py
View File

@ -51,7 +51,23 @@ class TestSwisscowsEngine(SearxTestCase):
"Description":"\uE000This should\uE001 be the content.", "Description":"\uE000This should\uE001 be the content.",
"Url":"http://this.should.be.the.link/", "Url":"http://this.should.be.the.link/",
"DisplayUrl":"www.\uE000this.should.be.the\uE001.link", "DisplayUrl":"www.\uE000this.should.be.the\uE001.link",
"Id":"782ef287-e439-451c-b380-6ebc14ba033d"} "Id":"782ef287-e439-451c-b380-6ebc14ba033d"},
{"Title":"Datei:This should1.svg",
"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png",
"SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should1.svg",
"DisplayUrl":"de.wikipedia.org/wiki/Datei:This should1.svg",
"Width":950,
"Height":534,
"FileSize":92100,
"ContentType":"image/jpeg",
"Thumbnail":{
"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png",
"ContentType":"image/jpeg",
"Width":300,
"Height":168,
"FileSize":9134},
"Id":"6a97a542-8f65-425f-b7f6-1178c3aba7be"
}
],"TotalCount":55300, ],"TotalCount":55300,
"Query":"This should " "Query":"This should "
}, },
@ -94,11 +110,15 @@ class TestSwisscowsEngine(SearxTestCase):
response = mock.Mock(content=html) response = mock.Mock(content=html)
results = swisscows.response(response) results = swisscows.response(response)
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
self.assertEqual(len(results), 2) self.assertEqual(len(results), 3)
self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['title'], 'This should be the title')
self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
self.assertEqual(results[0]['content'], 'This should be the content.') self.assertEqual(results[0]['content'], 'This should be the content.')
self.assertEqual(results[1]['title'], 'Datei:This should.svg') self.assertEqual(results[1]['title'], 'Datei:This should1.svg')
self.assertEqual(results[1]['url'], 'http://de.wikipedia.org/wiki/Datei:This should.svg') self.assertEqual(results[1]['url'], 'http://de.wikipedia.org/wiki/Datei:This should1.svg')
self.assertEqual(results[1]['img_src'], 'http://ts2.mm.This/should.png') self.assertEqual(results[1]['img_src'], 'http://ts2.mm.This/should1.png')
self.assertEqual(results[1]['template'], 'images.html') self.assertEqual(results[1]['template'], 'images.html')
self.assertEqual(results[2]['title'], 'Datei:This should.svg')
self.assertEqual(results[2]['url'], 'http://de.wikipedia.org/wiki/Datei:This should.svg')
self.assertEqual(results[2]['img_src'], 'http://ts2.mm.This/should.png')
self.assertEqual(results[2]['template'], 'images.html')