diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py index 973ede9a..94425062 100644 --- a/searx/engines/btdigg.py +++ b/searx/engines/btdigg.py @@ -23,11 +23,6 @@ paging = True url = 'https://btdigg.org' search_url = url + '/search?q={search_term}&p={pageno}' -# specific xpath variables -magnet_xpath = './/a[@title="Torrent magnet link"]' -torrent_xpath = './/a[@title="Download torrent file"]' -content_xpath = './/span[@class="font11px lightgrey block"]' - # do search-request def request(query, params): @@ -52,8 +47,8 @@ def response(resp): # parse results for result in search_res: link = result.xpath('.//td[@class="torrent_name"]//a')[0] - href = urljoin(url, link.attrib['href']) - title = escape(extract_text(link.xpath('.//text()'))) + href = urljoin(url, link.attrib.get('href')) + title = escape(extract_text(link)) content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0])) content = "
".join(content.split("\n")) @@ -81,7 +76,7 @@ def response(resp): filesize = int(filesize * 1024 * 1024 * 1024) elif filesize_multiplier == 'MB': filesize = int(filesize * 1024 * 1024) - elif filesize_multiplier == 'kb': + elif filesize_multiplier == 'KB': filesize = int(filesize * 1024) except: filesize = None diff --git a/searx/tests/engines/test_btdigg.py b/searx/tests/engines/test_btdigg.py new file mode 100644 index 00000000..4947b71d --- /dev/null +++ b/searx/tests/engines/test_btdigg.py @@ -0,0 +1,384 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import btdigg +from searx.testing import SearxTestCase + + +class TestBtdiggEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = btdigg.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('btdigg.org', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, btdigg.response, None) + self.assertRaises(AttributeError, btdigg.response, []) + self.assertRaises(AttributeError, btdigg.response, '') + self.assertRaises(AttributeError, btdigg.response, '[]') + + response = mock.Mock(text='') + self.assertEqual(btdigg.response(response), []) + + html = """ +
+ + + + + +
1 + + + + +
+ Should be the title +
+ + + + + + + + + + + +
+ [magnet] + + [cloud] + + Taille: + 8 B + + Fichiers: + 710 + + Téléchargements: + 5 + + Temps: + 417.8 jours + + Dernière mise à jour: + 5.3 jours + + Faux: + Aucun +
+ 
+                            Content
+
+
+
+ """ + response = mock.Mock(text=html) + results = btdigg.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Should be the title') + self.assertEqual(results[0]['url'], 'https://btdigg.org/url') + self.assertEqual(results[0]['content'], 'Content') + self.assertEqual(results[0]['seed'], 5) + self.assertEqual(results[0]['leech'], 0) + self.assertEqual(results[0]['filesize'], 8) + self.assertEqual(results[0]['files'], 710) + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:magnet&dn=Test') + + html = """ +
+ +
+
+ """ + response = mock.Mock(text=html) + results = btdigg.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + html = """ +
+ + + + + + + + + + + + + + + + + + + + + +
1 + + + + +
+ Should be the title +
+ + + + + + + + + + + +
+ [magnet] + + [cloud] + + Taille: + 1 KB + + Fichiers: + 710 + + Téléchargements: + 5 + + Temps: + 417.8 jours + + Dernière mise à jour: + 5.3 jours + + Faux: + Aucun +
+ 
+                            Content
+
+
1 + + + + +
+ Should be the title +
+ + + + + + + + + + + +
+ [magnet] + + [cloud] + + Taille: + 1 MB + + Fichiers: + a + + Téléchargements: + 4 + + Temps: + 417.8 jours + + Dernière mise à jour: + 5.3 jours + + Faux: + Aucun +
+ 
+                            Content
+
+
1 + + + + +
+ Should be the title +
+ + + + + + + + + + + +
+ [magnet] + + [cloud] + + Taille: + 1 GB + + Fichiers: + 710 + + Téléchargements: + 3 + + Temps: + 417.8 jours + + Dernière mise à jour: + 5.3 jours + + Faux: + Aucun +
+ 
+                            Content
+
+
1 + + + + +
+ Should be the title +
+ + + + + + + + + + + +
+ [magnet] + + [cloud] + + Taille: + 1 TB + + Fichiers: + 710 + + Téléchargements: + 2 + + Temps: + 417.8 jours + + Dernière mise à jour: + 5.3 jours + + Faux: + Aucun +
+ 
+                            Content
+
+
1 + + + + +
+ Should be the title +
+ + + + + + + + + + + +
+ [magnet] + + [cloud] + + Taille: + a TB + + Fichiers: + 710 + + Téléchargements: + z + + Temps: + 417.8 jours + + Dernière mise à jour: + 5.3 jours + + Faux: + Aucun +
+ 
+                            Content
+
+
+
+ """ + response = mock.Mock(text=html) + results = btdigg.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 5) + self.assertEqual(results[0]['title'], 'Should be the title') + self.assertEqual(results[0]['url'], 'https://btdigg.org/url') + self.assertEqual(results[0]['content'], 'Content') + self.assertEqual(results[0]['seed'], 5) + self.assertEqual(results[0]['leech'], 0) + self.assertEqual(results[0]['files'], 710) + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:magnet&dn=Test') + self.assertEqual(results[0]['filesize'], 1024) + self.assertEqual(results[1]['filesize'], 1048576) + self.assertEqual(results[2]['filesize'], 1073741824) + self.assertEqual(results[3]['filesize'], 1099511627776) diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py index bfdd1de4..b07444e4 100644 --- a/searx/tests/test_engines.py +++ b/searx/tests/test_engines.py @@ -1,6 +1,7 @@ from searx.tests.engines.test_bing import * # noqa from searx.tests.engines.test_bing_images import * # noqa from searx.tests.engines.test_bing_news import * # noqa +from searx.tests.engines.test_btdigg import * # noqa from searx.tests.engines.test_dailymotion import * # noqa from searx.tests.engines.test_deezer import * # noqa from searx.tests.engines.test_deviantart import * # noqa