From 1e2b60f3805a1cdc87cff0c5753bd355ad00cc98 Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 21:31:40 +0200 Subject: [PATCH 1/3] [mod] tokyotoshokan dependecy to other engine removed --- searx/engines/tokyotoshokan.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/searx/engines/tokyotoshokan.py b/searx/engines/tokyotoshokan.py index 9a6b5e57..dfd2e779 100644 --- a/searx/engines/tokyotoshokan.py +++ b/searx/engines/tokyotoshokan.py @@ -14,8 +14,8 @@ import re from lxml import html from searx.engines.xpath import extract_text from datetime import datetime -from searx.engines.nyaa import int_or_zero, get_filesize_mul from searx.url_utils import urlencode +from searx.utils import get_torrent_size # engine dependent config categories = ['files', 'videos', 'music'] @@ -26,6 +26,17 @@ base_url = 'https://www.tokyotosho.info/' search_url = base_url + 'search.php?{query}' +# convert a variable to integer or return 0 if it's not a number +def int_or_zero(num): + if isinstance(num, list): + if len(num) < 1: + return 0 + num = num[0] + if num.isdigit(): + return int(num) + return 0 + + # do search-request def request(query, params): query = urlencode({'page': params['pageno'], 'terms': query}) @@ -76,8 +87,7 @@ def response(resp): try: # ('1.228', 'GB') groups = size_re.match(item).groups() - multiplier = get_filesize_mul(groups[1]) - params['filesize'] = int(multiplier * float(groups[0])) + params['filesize'] = get_torrent_size(groups[0], groups[1]) except: pass elif item.startswith('Date:'): From 01330f71cd60cab6c1ce9a19c28c2729f2d02344 Mon Sep 17 00:00:00 2001 From: misnyo Date: Thu, 31 Aug 2017 21:32:30 +0200 Subject: [PATCH 2/3] [fix] nyaa.si fixed --- searx/engines/nyaa.py | 97 ++++++++++++------------- tests/unit/engines/test_nyaa.py | 124 +++++++++++++++++++++++--------- 2 files changed, 137 insertions(+), 84 deletions(-) diff --git a/searx/engines/nyaa.py b/searx/engines/nyaa.py index 272c712c..0d5bfc91 100644 --- a/searx/engines/nyaa.py +++ b/searx/engines/nyaa.py @@ -1,7 +1,7 @@ """ - Nyaa.se (Anime Bittorrent tracker) + Nyaa.si (Anime Bittorrent tracker) - @website http://www.nyaa.se/ + @website http://www.nyaa.si/ @provide-api no @using-api no @results HTML @@ -12,50 +12,25 @@ from lxml import html from searx.engines.xpath import extract_text from searx.url_utils import urlencode +from searx.utils import get_torrent_size # engine dependent config categories = ['files', 'images', 'videos', 'music'] paging = True # search-url -base_url = 'http://www.nyaa.se/' +base_url = 'http://www.nyaa.si/' search_url = base_url + '?page=search&{query}&offset={offset}' # xpath queries -xpath_results = '//table[@class="tlist"]//tr[contains(@class, "tlistrow")]' -xpath_category = './/td[@class="tlisticon"]/a' -xpath_title = './/td[@class="tlistname"]/a' -xpath_torrent_file = './/td[@class="tlistdownload"]/a' -xpath_filesize = './/td[@class="tlistsize"]/text()' -xpath_seeds = './/td[@class="tlistsn"]/text()' -xpath_leeches = './/td[@class="tlistln"]/text()' -xpath_downloads = './/td[@class="tlistdn"]/text()' - - -# convert a variable to integer or return 0 if it's not a number -def int_or_zero(num): - if isinstance(num, list): - if len(num) < 1: - return 0 - num = num[0] - if num.isdigit(): - return int(num) - return 0 - - -# get multiplier to convert torrent size to bytes -def get_filesize_mul(suffix): - return { - 'KB': 1024, - 'MB': 1024 ** 2, - 'GB': 1024 ** 3, - 'TB': 1024 ** 4, - - 'KIB': 1024, - 'MIB': 1024 ** 2, - 'GIB': 1024 ** 3, - 'TIB': 1024 ** 4 - }[str(suffix).upper()] +xpath_results = '//table[contains(@class, "torrent-list")]//tr[not(th)]' +xpath_category = './/td[1]/a[1]' +xpath_title = './/td[2]/a[last()]' +xpath_torrent_links = './/td[3]/a' +xpath_filesize = './/td[4]/text()' +xpath_seeds = './/td[6]/text()' +xpath_leeches = './/td[7]/text()' +xpath_downloads = './/td[8]/text()' # do search-request @@ -72,6 +47,14 @@ def response(resp): dom = html.fromstring(resp.text) for result in dom.xpath(xpath_results): + # defaults + filesize = 0 + seed = 0 + leech = 0 + downloads = 0 + magnet_link = "" + torrent_link = "" + # category in which our torrent belongs category = result.xpath(xpath_category)[0].attrib.get('title') @@ -80,26 +63,37 @@ def response(resp): title = extract_text(page_a) # link to the page - href = page_a.attrib.get('href') + href = base_url + page_a.attrib.get('href') - # link to the torrent file - torrent_link = result.xpath(xpath_torrent_file)[0].attrib.get('href') + for link in result.xpath(xpath_torrent_links): + url = link.attrib.get('href') + if 'magnet' in url: + # link to the magnet + magnet_link = url + else: + # link to the torrent file + torrent_link = url - # torrent size + # get seeders and leechers try: - file_size, suffix = result.xpath(xpath_filesize)[0].split(' ') - file_size = int(float(file_size) * get_filesize_mul(suffix)) + seed = int(result.xpath(xpath_seeds)[0]) + leech = int(result.xpath(xpath_leeches)[0]) except: - file_size = None + pass - # seed count - seed = int_or_zero(result.xpath(xpath_seeds)) - - # leech count - leech = int_or_zero(result.xpath(xpath_leeches)) + # let's try to calculate the torrent size + try: + filesize_info = result.xpath(xpath_filesize)[0] + filesize, filesize_multiplier = filesize_info.split() + filesize = get_torrent_size(filesize, filesize_multiplier) + except: + pass # torrent downloads count - downloads = int_or_zero(result.xpath(xpath_downloads)) + try: + downloads = result.xpath(xpath_downloads)[0] + except: + pass # content string contains all information not included into template content = 'Category: "{category}". Downloaded {downloads} times.' @@ -110,8 +104,9 @@ def response(resp): 'content': content, 'seed': seed, 'leech': leech, - 'filesize': file_size, + 'filesize': filesize, 'torrentfile': torrent_link, + 'magnetlink': magnet_link, 'template': 'torrent.html'}) return results diff --git a/tests/unit/engines/test_nyaa.py b/tests/unit/engines/test_nyaa.py index db412e1c..6dcafc6b 100644 --- a/tests/unit/engines/test_nyaa.py +++ b/tests/unit/engines/test_nyaa.py @@ -13,38 +13,92 @@ class TestNyaaEngine(SearxTestCase): params = nyaa.request(query, dic) self.assertTrue('url' in params) self.assertTrue(query in params['url']) - self.assertTrue('nyaa.se' in params['url']) + self.assertTrue('nyaa.si' in params['url']) def test_response(self): resp = mock.Mock(text='') self.assertEqual(nyaa.response(resp), []) html = """ - - - - - - - - - - - - - +
- - English-translated Anime - - - - Sample torrent title - - - - DL - - 10 MiB136660
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
Category
+ 		+
Name
+ 		+ + + +
Link
+ 		+ +
Size
+ 		+ +
Date
+ 		+ + + + + + + + +
+ + Anime - English-translated + + + Sample title 1 + + + + 723.7 MiB2017-08-21 11:241312
+ + Anime - English-translated + + + Sample title 2 + + + 8.2 GiB2017-04-08 01:40101206
""" @@ -52,15 +106,19 @@ class TestNyaaEngine(SearxTestCase): results = nyaa.response(resp) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) r = results[0] - self.assertTrue(r['url'].find('www.nyaa.se/?page3') >= 0) - self.assertTrue(r['torrentfile'].find('www.nyaa.se/?page_dl') >= 0) - self.assertTrue(r['content'].find('English-translated Anime') >= 0) - self.assertTrue(r['content'].find('Downloaded 666 times.') >= 0) + self.assertTrue(r['url'].find('1') >= 0) + self.assertTrue(r['torrentfile'].find('1.torrent') >= 0) + self.assertTrue(r['content'].find('Anime - English-translated') >= 0) + self.assertTrue(r['content'].find('Downloaded 12 times.') >= 0) - self.assertEqual(r['title'], 'Sample torrent title') + self.assertEqual(r['title'], 'Sample title 1') self.assertEqual(r['seed'], 1) self.assertEqual(r['leech'], 3) - self.assertEqual(r['filesize'], 10 * 1024 * 1024) + self.assertEqual(r['filesize'], 723700000) + + r = results[1] + self.assertTrue(r['url'].find('2') >= 0) + self.assertTrue(r['magnetlink'].find('magnet:') >= 0) From 33fd938016def4496876d89d0ccaa53f47705005 Mon Sep 17 00:00:00 2001 From: misnyo Date: Mon, 4 Sep 2017 20:05:04 +0200 Subject: [PATCH 3/3] [mod] int_or_zero refactored to searx_utils --- searx/engines/nyaa.py | 30 +++++++++++++----------------- searx/engines/tokyotoshokan.py | 13 +------------ searx/utils.py | 9 +++++++++ 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/searx/engines/nyaa.py b/searx/engines/nyaa.py index 0d5bfc91..6a8e598c 100644 --- a/searx/engines/nyaa.py +++ b/searx/engines/nyaa.py @@ -12,7 +12,7 @@ from lxml import html from searx.engines.xpath import extract_text from searx.url_utils import urlencode -from searx.utils import get_torrent_size +from searx.utils import get_torrent_size, int_or_zero # engine dependent config categories = ['files', 'images', 'videos', 'music'] @@ -49,14 +49,14 @@ def response(resp): for result in dom.xpath(xpath_results): # defaults filesize = 0 - seed = 0 - leech = 0 - downloads = 0 magnet_link = "" torrent_link = "" # category in which our torrent belongs - category = result.xpath(xpath_category)[0].attrib.get('title') + try: + category = result.xpath(xpath_category)[0].attrib.get('title') + except: + pass # torrent title page_a = result.xpath(xpath_title)[0] @@ -74,12 +74,14 @@ def response(resp): # link to the torrent file torrent_link = url - # get seeders and leechers - try: - seed = int(result.xpath(xpath_seeds)[0]) - leech = int(result.xpath(xpath_leeches)[0]) - except: - pass + # seed count + seed = int_or_zero(result.xpath(xpath_seeds)) + + # leech count + leech = int_or_zero(result.xpath(xpath_leeches)) + + # torrent downloads count + downloads = int_or_zero(result.xpath(xpath_downloads)) # let's try to calculate the torrent size try: @@ -89,12 +91,6 @@ def response(resp): except: pass - # torrent downloads count - try: - downloads = result.xpath(xpath_downloads)[0] - except: - pass - # content string contains all information not included into template content = 'Category: "{category}". Downloaded {downloads} times.' content = content.format(category=category, downloads=downloads) diff --git a/searx/engines/tokyotoshokan.py b/searx/engines/tokyotoshokan.py index dfd2e779..77321204 100644 --- a/searx/engines/tokyotoshokan.py +++ b/searx/engines/tokyotoshokan.py @@ -15,7 +15,7 @@ from lxml import html from searx.engines.xpath import extract_text from datetime import datetime from searx.url_utils import urlencode -from searx.utils import get_torrent_size +from searx.utils import get_torrent_size, int_or_zero # engine dependent config categories = ['files', 'videos', 'music'] @@ -26,17 +26,6 @@ base_url = 'https://www.tokyotosho.info/' search_url = base_url + 'search.php?{query}' -# convert a variable to integer or return 0 if it's not a number -def int_or_zero(num): - if isinstance(num, list): - if len(num) < 1: - return 0 - num = num[0] - if num.isdigit(): - return int(num) - return 0 - - # do search-request def request(query, params): query = urlencode({'page': params['pageno'], 'terms': query}) diff --git a/searx/utils.py b/searx/utils.py index 3df57116..35b20ad8 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -290,6 +290,15 @@ def convert_str_to_int(number_str): return 0 +# convert a variable to integer or return 0 if it's not a number +def int_or_zero(num): + if isinstance(num, list): + if len(num) < 1: + return 0 + num = num[0] + return convert_str_to_int(num) + + def is_valid_lang(lang): is_abbr = (len(lang) == 2) if is_abbr: