diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index c6c5b0d0..161f7513 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -35,7 +35,7 @@ suggestion_xpath = '//div[@id="satat"]//a' def parse_url(url_string): endings = ['/RS', '/RK'] endpositions = [] - start = url_string.find('http', url_string.find('/RU=')+1) + start = url_string.find('http', url_string.find('/RU=') + 1) for ending in endings: endpos = url_string.rfind(ending) @@ -91,7 +91,7 @@ def response(resp): 'content': content}) # if no suggestion found, return results - if not suggestion_xpath: + if not dom.xpath(suggestion_xpath): return results # parse suggestion diff --git a/searx/tests/engines/test_yahoo.py b/searx/tests/engines/test_yahoo.py new file mode 100644 index 00000000..e5c78701 --- /dev/null +++ b/searx/tests/engines/test_yahoo.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import yahoo +from searx.testing import SearxTestCase + + +class TestYahooEngine(SearxTestCase): + + def test_parse_url(self): + test_url = 'http://r.search.yahoo.com/_ylt=A0LEb9JUSKcAEGRXNyoA;_ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb' +\ + '2xvA2Jm2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10/RU=https%3a%2f%2fthis.is.the.url%2f/RK=0/RS=' +\ + 'dtcJsfP4mEeBOjnVfUQ-' + url = yahoo.parse_url(test_url) + self.assertEqual('https://this.is.the.url/', url) + + test_url = 'http://r.search.yahoo.com/_ylt=A0LElb9JUSKcAEGRXNyoA;_ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb' +\ + '2xvA2Jm2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10/RU=https%3a%2f%2fthis.is.the.url%2f/RS=' +\ + 'dtcJsfP4mEeBOjnVfUQ-' + url = yahoo.parse_url(test_url) + self.assertEqual('https://this.is.the.url/', url) + + test_url = 'https://this.is.the.url/' + url = yahoo.parse_url(test_url) + self.assertEqual('https://this.is.the.url/', url) + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr_FR' + params = yahoo.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('search.yahoo.com', params['url']) + self.assertIn('fr', params['url']) + self.assertIn('cookies', params) + self.assertIn('sB', params['cookies']) + self.assertIn('fr', params['cookies']['sB']) + + dicto['language'] = 'all' + params = yahoo.request(query, dicto) + self.assertIn('cookies', params) + self.assertIn('sB', params['cookies']) + self.assertIn('en', params['cookies']['sB']) + self.assertIn('en', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, yahoo.response, None) + self.assertRaises(AttributeError, yahoo.response, []) + self.assertRaises(AttributeError, yahoo.response, '') + self.assertRaises(AttributeError, yahoo.response, '[]') + + response = mock.Mock(text='') + self.assertEqual(yahoo.response(response), []) + + html = """ +
+
+

+ + This is the title + +

+
+ www.test.com +
+ This is the content +
+
+
+

Also Try

+ + + + + + +
+ + + This is the suggestion + + +
+
+ """ + response = mock.Mock(text=html) + results = yahoo.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'https://this.is.the.url/') + self.assertEqual(results[0]['content'], 'This is the content') + self.assertEqual(results[1]['suggestion'], 'This is the suggestion') + + html = """ +
+
+

+ + This is the title + +

+
+ www.test.com +
+ This is the content +
+
+
+
+

+ + This is the title + +

+
+ www.test.com +
+ This is the content +
+
+
+
+

+

+
+ www.test.com +
+ This is the content +
+
+ """ + response = mock.Mock(text=html) + results = yahoo.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This is the title') + self.assertEqual(results[0]['url'], 'https://this.is.the.url/') + self.assertEqual(results[0]['content'], 'This is the content') + + html = """ +
  • +
    • + """ + response = mock.Mock(text=html) + results = yahoo.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py index 65f182e8..259ebcf3 100644 --- a/searx/tests/test_engines.py +++ b/searx/tests/test_engines.py @@ -27,4 +27,5 @@ from searx.tests.engines.test_twitter import * # noqa from searx.tests.engines.test_vimeo import * # noqa from searx.tests.engines.test_www500px import * # noqa from searx.tests.engines.test_youtube import * # noqa +from searx.tests.engines.test_yahoo import * # noqa from searx.tests.engines.test_yahoo_news import * # noqa