Merge pull request #2133 from unixfox/fix/piratebay-engine

[fix] piratebay engine

searx/engines/piratebay.py

@@ -1,44 +1,50 @@
 #  Piratebay (Videos, Music, Files)
 #
-# @website     https://thepiratebay.se
+# @website     https://thepiratebay.org
-# @provide-api no (nothing found)
+# @provide-api yes (https://apibay.org/)
 #
-# @using-api   no
+# @using-api   yes
-# @results     HTML (using search portal)
+# @results     JSON
-# @stable      yes (HTML can change)
+# @stable      no (the API is not documented nor versioned)
-# @parse       url, title, content, seed, leech, magnetlink
+# @parse       url, title, seed, leech, magnetlink, filesize, publishedDate
 
-from lxml import html
+from json import loads
+from datetime import datetime
 from operator import itemgetter
-from searx.engines.xpath import extract_text
+from searx.url_utils import quote
-from searx.url_utils import quote, urljoin
+from searx.utils import get_torrent_size
 
 # engine dependent config
-categories = ['videos', 'music', 'files']
+categories = ["videos", "music", "files"]
-paging = True
 
 # search-url
-url = 'https://thepiratebay.org/'
+url = "https://thepiratebay.org/"
-search_url = url + 'search/{search_term}/{pageno}/99/{search_type}'
+search_url = "https://apibay.org/q.php?q={search_term}&cat={search_type}"
+
+# default trackers provided by thepiratebay
+trackers = [
+    "udp://tracker.coppersurfer.tk:6969/announce",
+    "udp://9.rarbg.to:2920/announce",
+    "udp://tracker.opentrackr.org:1337",
+    "udp://tracker.internetwarriors.net:1337/announce",
+    "udp://tracker.leechers-paradise.org:6969/announce",
+    "udp://tracker.coppersurfer.tk:6969/announce",
+    "udp://tracker.pirateparty.gr:6969/announce",
+    "udp://tracker.cyberia.is:6969/announce",
+]
 
 # piratebay specific type-definitions
-search_types = {'files': '0',
+search_types = {"files": "0",
-                'music': '100',
+                "music": "100",
-                'videos': '200'}
+                "videos": "200"}
-
-# specific xpath variables
-magnet_xpath = './/a[@title="Download this torrent using magnet"]'
-torrent_xpath = './/a[@title="Download this torrent"]'
-content_xpath = './/font[@class="detDesc"]'
 
 
 # do search-request
 def request(query, params):
-    search_type = search_types.get(params['category'], '0')
+    search_type = search_types.get(params["category"], "0")
 
-    params['url'] = search_url.format(search_term=quote(query),
+    params["url"] = search_url.format(search_term=quote(query),
-                                      search_type=search_type,
+                                      search_type=search_type)
-                                      pageno=params['pageno'] - 1)
 
     return params
 
@@ -47,50 +53,43 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
+    search_res = loads(resp.text)
-
-    search_res = dom.xpath('//table[@id="searchResult"]//tr')
 
     # return empty array if nothing is found
-    if not search_res:
+    if search_res[0]["name"] == "No results returned":
         return []
 
     # parse results
-    for result in search_res[1:]:
+    for result in search_res:
-        link = result.xpath('.//div[@class="detName"]//a')[0]
+        link = url + "description.php?id=" + result["id"]
-        href = urljoin(url, link.attrib.get('href'))
+        magnetlink = "magnet:?xt=urn:btih:" + result["info_hash"] + "&dn=" + result["name"]
-        title = extract_text(link)
+        + "&tr=" + "&tr=".join(trackers)
-        content = extract_text(result.xpath(content_xpath))
-        seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
 
-        # convert seed to int if possible
+        params = {
-        if seed.isdigit():
+            "url": link,
-            seed = int(seed)
+            "title": result["name"],
-        else:
+            "seed": result["seeders"],
-            seed = 0
+            "leech": result["leechers"],
+            "magnetlink": magnetlink,
+            "template": "torrent.html"
+        }
 
-        # convert leech to int if possible
+        # extract and convert creation date
-        if leech.isdigit():
+        try:
-            leech = int(leech)
+            date = datetime.fromtimestamp(result.added)
-        else:
+            params['publishedDate'] = date
-            leech = 0
+        except:
+            pass
 
-        magnetlink = result.xpath(magnet_xpath)[0]
+        # let's try to calculate the torrent size
-        torrentfile_links = result.xpath(torrent_xpath)
+        try:
-        if torrentfile_links:
+            filesize = get_torrent_size(result["size"], "B")
-            torrentfile_link = torrentfile_links[0].attrib.get('href')
+            params['filesize'] = filesize
-        else:
+        except:
-            torrentfile_link = None
+            pass
 
         # append result
-        results.append({'url': href,
+        results.append(params)
-                        'title': title,
-                        'content': content,
-                        'seed': seed,
-                        'leech': leech,
-                        'magnetlink': magnetlink.attrib.get('href'),
-                        'torrentfile': torrentfile_link,
-                        'template': 'torrent.html'})
 
     # return results sorted by seeder
-    return sorted(results, key=itemgetter('seed'), reverse=True)
+    return sorted(results, key=itemgetter("seed"), reverse=True)

searx/settings.yml

@@ -544,7 +544,8 @@ engines:
   - name : piratebay
     engine : piratebay
     shortcut : tpb
-    url: https://pirateproxy.red/
+    # You may need to change this URL to a proxy if piratebay is blocked in your country
+    url: https://thepiratebay.org/
     timeout : 3.0
 
   - name : pubmed