Improved cloudflare detection

src/instances/cloudflare.json

@@ -1,14 +1,13 @@
 [
   "https://invidious.kavin.rocks",
-  "https://invidious-us.kavin.rocks",
   "https://invidious.lunar.icu",
+  "https://invidious-us.kavin.rocks",
   "https://send.silkky.cloud",
   "https://nhanh.cloud",
   "https://nitter.domain.glass",
   "https://nitter.actionsack.com",
   "https://birdsite.xanny.family",
   "https://nitter.moomoo.me",
-  "https://nittereu.moomoo.me",
   "https://nitter.alefvanoon.xyz",
   "https://notabird.site",
   "https://nitter.silkky.cloud",

src/instances/cloudflare_ips.py

@@ -1,30 +0,0 @@
-# Note: Run this script from the root of the repo
-
-import requests
-import json
-import subprocess
-from colorama import Fore, Back, Style
-
-def init_cloudflare():
-    r = requests.get('https://www.cloudflare.com/ips-v4')
-    myList = []
-    for i in r.text.split('\n'):
-        out = subprocess.run(
-            ["sh", "./src/instances/get_possible_ips.sh", i],
-            capture_output=True,
-            text=True
-        )
-        myList += out.stdout.splitlines()
-    print(Fore.GREEN + 'Fetched ' +
-          Fore.RED + 'Cloudflare IPs' +
-          Style.RESET_ALL)
-
-    return myList
-
-cloudflare_ips = init_cloudflare()
-print(cloudflare_ips)
-
-json_object = json.dumps(cloudflare_ips, ensure_ascii=False, indent=2)
-with open('./src/instances/cloudflare_ips.json', 'w') as outfile:
-    outfile.write(json_object)
-print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/cloudflare_ips.json')

src/instances/data.json

@@ -5,13 +5,12 @@
       "https://vid.puffyan.us",
       "https://invidious.snopyta.org",
       "https://invidious.kavin.rocks",
-      "https://invidio.xamh.de",
       "https://inv.riverside.rocks",
-      "https://invidious-us.kavin.rocks",
       "https://invidious.osi.kr",
-      "https://tube.cthd.icu",
       "https://y.com.sb",
+      "https://tube.cthd.icu",
       "https://invidious.flokinet.to",
+      "https://yt.artemislena.eu",
       "https://invidious.lunar.icu",
       "https://invidious.mutahar.rocks",
       "https://youtube.076.ne.jp",
@@ -19,8 +18,9 @@
       "https://inv.bp.mutahar.rocks",
       "https://invidious.weblibre.org",
       "https://invidious.privacy.gd",
-      "https://yt.artemislena.eu",
-      "https://invidious.namazso.eu"
+      "https://invidio.xamh.de",
+      "https://invidious.namazso.eu",
+      "https://invidious-us.kavin.rocks"
     ],
     "tor": [
       "http://c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid.onion",
@@ -79,7 +79,6 @@
       "https://nitter.hu",
       "https://twitr.gq",
       "https://nitter.moomoo.me",
-      "https://nittereu.moomoo.me",
       "https://bird.trom.tf",
       "https://nitter.it",
       "https://twitter.censors.us",

src/instances/get_instances.py

@@ -12,6 +12,7 @@ import subprocess
 
 mightyList = {}
 
+
 def filterLastSlash(urlList):
     tmp = []
     for i in urlList:
@@ -22,22 +23,48 @@ def filterLastSlash(urlList):
             tmp.append(i)
     return tmp
 
-cloudflare_ips = []
-with open('./src/instances/cloudflare_ips.json', 'r') as file:
-    cloudflare_ips = json.load(file)
+
+def ip2bin(ip): return "".join(
+    map(
+        str,
+        [
+            "{0:08b}".format(int(x)) for x in ip.split(".")
+        ]
+    )
+)
+
+
+def get_cloudflare_ips():
+    r = requests.get('https://www.cloudflare.com/ips-v4')
+    return r.text.split('\n')
+
+
+cloudflare_ips = get_cloudflare_ips()
+
 
 def is_cloudflare(url):
-    href = urlparse(url)
-    ip = ''
+    instance_ip = None
     try:
-        ip = socket.gethostbyname(href.hostname)
+        instance_ip = socket.gethostbyname(urlparse(url).hostname)
+        if instance_ip is None:
+            return False
     except:
         return False
+    instance_bin = ip2bin(instance_ip)
 
-    if ip in cloudflare_ips:
-        return True
-    else:
-        return False
+    for cloudflare_ip_mask in cloudflare_ips:
+        cloudflare_ip = cloudflare_ip_mask.split('/')[0]
+        cloudflare_bin = ip2bin(cloudflare_ip)
+
+        mask = int(cloudflare_ip_mask.split('/')[1])
+
+        cloudflare_bin_masked = cloudflare_bin[:mask]
+        instance_bin_masked = instance_bin[:mask]
+
+        if cloudflare_bin_masked == instance_bin_masked:
+            print(url + ' is ' + Fore.RED + 'cloudflare' + Style.RESET_ALL)
+            return True
+    return False
 
 
 # Invidious
@@ -313,9 +340,8 @@ for k1, v1 in mightyList.items():
                     mightyList[k1][k2].remove(instance)
                     print("removed " + instance)
                 else:
-                    if (is_cloudflare(instance)):
+                    if not instance.endswith('.onion') and not instance.endswith('.i2p') and is_cloudflare(instance):
                         cloudflareMightyList.append(instance)
-                        
 
 
 # Writing to file
@@ -330,4 +356,3 @@ with open('./src/instances/cloudflare.json', 'w') as outfile:
 print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/cloudflare.json')
 
 # print(json_object)
-

src/instances/get_possible_ips.sh

@@ -1 +0,0 @@
-nmap -sL -n $1 | awk '/Nmap scan report/{print $NF}'

src/instances/piped.json

@@ -16,3 +16,4 @@
       "http://piped2bbch4xslbl2ckr6k62q56kon56ffowxaqzy42ai22a4sash3ad.onion"
     ]
   }
+}