diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py index 17e2a7aa..3d26c9cc 100644 --- a/searx/engines/yacy.py +++ b/searx/engines/yacy.py @@ -25,10 +25,10 @@ number_of_results = 5 # search-url base_url = 'http://localhost:8090' search_url = '/yacysearch.json?{query}'\ - '&startRecord={offset}'\ - '&maximumRecords={limit}'\ - '&contentdom={search_type}'\ - '&resource=global' # noqa + '&startRecord={offset}'\ + '&maximumRecords={limit}'\ + '&contentdom={search_type}'\ + '&resource=global' # yacy specific type-definitions search_types = {'general': 'text', @@ -41,7 +41,7 @@ search_types = {'general': 'text', # do search-request def request(query, params): offset = (params['pageno'] - 1) * number_of_results - search_type = search_types.get(params['category'], '0') + search_type = search_types.get(params.get('category'), '0') params['url'] = base_url +\ search_url.format(query=urlencode({'query': query}), @@ -66,9 +66,12 @@ def response(resp): if not raw_search_results: return [] - search_results = raw_search_results.get('channels', {})[0].get('items', []) + search_results = raw_search_results.get('channels', []) - for result in search_results: + if len(search_results) == 0: + return [] + + for result in search_results[0].get('items', []): # parse image results if result.get('image'): # append result @@ -88,7 +91,7 @@ def response(resp): 'content': result['description'], 'publishedDate': publishedDate}) - #TODO parse video, audio and file results + # TODO parse video, audio and file results # return results return results diff --git a/searx/tests/engines/test_yacy.py b/searx/tests/engines/test_yacy.py new file mode 100644 index 00000000..f49532cf --- /dev/null +++ b/searx/tests/engines/test_yacy.py @@ -0,0 +1,96 @@ +from collections import defaultdict +import mock +from searx.engines import yacy +from searx.testing import SearxTestCase + + +class TestYacyEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr_FR' + params = yacy.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('localhost', params['url']) + self.assertIn('fr', params['url']) + + dicto['language'] = 'all' + params = yacy.request(query, dicto) + self.assertIn('url', params) + self.assertNotIn('lr=lang_', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, yacy.response, None) + self.assertRaises(AttributeError, yacy.response, []) + self.assertRaises(AttributeError, yacy.response, '') + self.assertRaises(AttributeError, yacy.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(yacy.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(yacy.response(response), []) + + json = """ + { + "channels": [ + { + "title": "YaCy P2P-Search for test", + "description": "Search for test", + "link": "http://search.yacy.de:7001/yacysearch.html?query=test&resource=global&contentdom=0", + "image": { + "url": "http://search.yacy.de:7001/env/grafics/yacy.png", + "title": "Search for test", + "link": "http://search.yacy.de:7001/yacysearch.html?query=test&resource=global&contentdom=0" + }, + "totalResults": "249", + "startIndex": "0", + "itemsPerPage": "5", + "searchTerms": "test", + "items": [ + { + "title": "This is the title", + "link": "http://this.is.the.url", + "code": "", + "description": "This should be the content", + "pubDate": "Sat, 08 Jun 2013 02:00:00 +0200", + "size": "44213", + "sizename": "43 kbyte", + "guid": "lzh_1T_5FP-A", + "faviconCode": "XTS4uQ_5FP-A", + "host": "www.gamestar.de", + "path": "/spiele/city-of-heroes-freedom/47019.html", + "file": "47019.html", + "urlhash": "lzh_1T_5FP-A", + "ranking": "0.20106804" + }, + { + "title": "This is the title2", + "icon": "/ViewImage.png?maxwidth=96&maxheight=96&code=7EbAbW6BpPOA", + "image": "http://image.url/image.png", + "cache": "/ViewImage.png?quadratic=&url=http://golem.ivwbox.de/cgi-bin/ivw/CP/G_INET?d=14071378", + "url": "http://this.is.the.url", + "urlhash": "7EbAbW6BpPOA", + "host": "www.golem.de", + "width": "-1", + "height": "-1" + } + ] + } + ] + } + """ + response = mock.Mock(text=json) + results = yacy.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'http://this.is.the.url') + self.assertEqual(results[0]['content'], 'This should be the content') + self.assertEqual(results[1]['img_src'], 'http://image.url/image.png') + self.assertEqual(results[1]['content'], '') + self.assertEqual(results[1]['url'], 'http://this.is.the.url') + self.assertEqual(results[1]['title'], 'This is the title2') diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py index e3e0938c..4a27f5ad 100644 --- a/searx/tests/test_engines.py +++ b/searx/tests/test_engines.py @@ -30,6 +30,7 @@ from searx.tests.engines.test_subtitleseeker import * # noqa from searx.tests.engines.test_twitter import * # noqa from searx.tests.engines.test_vimeo import * # noqa from searx.tests.engines.test_www500px import * # noqa -from searx.tests.engines.test_youtube import * # noqa +from searx.tests.engines.test_yacy import * # noqa from searx.tests.engines.test_yahoo import * # noqa +from searx.tests.engines.test_youtube import * # noqa from searx.tests.engines.test_yahoo_news import * # noqa