From d308102880553b955f74e785032cec5c2b009116 Mon Sep 17 00:00:00 2001 From: ManeraKai Date: Sat, 16 Apr 2022 16:06:32 +0300 Subject: [PATCH] Added cloudflare detection #99 --- src/assets/javascripts/helpers/common.js | 20 ++++++++- src/instances/cloudflare.json | 53 +++++++++++++++++++++++ src/instances/data.json | 9 ++-- src/instances/get_instances.py | 55 +++++++++++++++++++++--- src/instances/get_possible_ips.sh | 1 + 5 files changed, 124 insertions(+), 14 deletions(-) create mode 100644 src/instances/cloudflare.json create mode 100644 src/instances/get_possible_ips.sh diff --git a/src/assets/javascripts/helpers/common.js b/src/assets/javascripts/helpers/common.js index 6dd0e7f..287604d 100644 --- a/src/assets/javascripts/helpers/common.js +++ b/src/assets/javascripts/helpers/common.js @@ -15,6 +15,16 @@ function getRandomInstance(instances) { return instances[~~(instances.length * Math.random())]; } +let cloudflareList = []; +async function initCloudflareList() { + return new Promise(resolve => { + fetch('/instances/cloudflare.json').then(response => response.text()).then(data => { + console.log(data); + cloudflareList = data; + resolve(); + }) + }); +} async function wholeInit() { await youtubeHelper.init(); await twitterHelper.init(); @@ -26,6 +36,7 @@ async function wholeInit() { await mediumHelper.init(); await sendTargetsHelper.init(); await tikTokHelper.init(); + await initCloudflareList(); } async function updateInstances() { @@ -79,7 +90,9 @@ function protocolHost(url) { return `${url.protocol}//${url.host}`; } -function processDefaultCustomInstances( + + +async function processDefaultCustomInstances( name, protocol, nameHelper, @@ -95,6 +108,8 @@ function processDefaultCustomInstances( let nameCheckListElement = nameProtocolElement.getElementsByClassName('checklist')[0]; let nameDefaultRedirects; + await initCloudflareList(); + function calcNameCheckBoxes() { let isTrue = true; for (const item of nameHelper.getRedirects()[name][protocol]) @@ -109,10 +124,11 @@ function processDefaultCustomInstances( nameDefaultRedirects = getNameRedirectsChecks(); + console.log('cloudflareList', cloudflareList) nameCheckListElement.innerHTML = [ `
Toggle All
`, - ...nameHelper.getRedirects()[name][protocol].map((x) => `
${x}
`), + ...nameHelper.getRedirects()[name][protocol].map(x => `
${x}${cloudflareList.includes(x) ? ' cloudflare' : ''}
`), ].join('\n
\n'); localise.localisePage(); diff --git a/src/instances/cloudflare.json b/src/instances/cloudflare.json new file mode 100644 index 0000000..9918a9e --- /dev/null +++ b/src/instances/cloudflare.json @@ -0,0 +1,53 @@ +[ + "https://invidious.kavin.rocks", + "https://invidious-us.kavin.rocks", + "https://invidious.lunar.icu", + "https://send.silkky.cloud", + "https://nhanh.cloud", + "https://nitter.domain.glass", + "https://nitter.actionsack.com", + "https://birdsite.xanny.family", + "https://nitter.moomoo.me", + "https://nittereu.moomoo.me", + "https://nitter.alefvanoon.xyz", + "https://notabird.site", + "https://nitter.silkky.cloud", + "https://fuckthesacklers.network", + "https://nitter.govt.land", + "https://nitter.winscloud.net", + "https://twtr.bch.bar", + "https://nitter.lunar.icu", + "https://bib.actionsack.com", + "https://biblio.alefvanoon.xyz", + "https://libreddit.silkky.cloud", + "https://libreddit.domain.glass", + "https://libreddit.jamiethalacker.dev", + "https://r.nf", + "https://libreddit.alefvanoon.xyz", + "https://libreddit.flux.industries", + "https://libreddit.hu", + "https://lr.stilic.ml", + "https://reddi.tk", + "https://r.walkx.org", + "https://libreddit.yonalee.eu", + "https://libreddit.winscloud.net", + "https://libreddit.lunar.icu", + "https://teddit.domain.glass", + "https://teddit.alefvanoon.xyz", + "https://wikiless.alefvanoon.xyz", + "https://wiki.604kph.xyz", + "https://wikiless.lunar.icu", + "https://st.alefvanoon.xyz", + "https://translate.syncpundit.com", + "https://searx.josie.lol", + "https://searx.kujonello.cf", + "https://searx.org", + "https://searx.roughs.ru", + "https://searx.run", + "https://searx.tk", + "https://s.alefvanoon.xyz", + "https://search.albony.xyz", + "https://whoogle.lunar.icu", + "https://rimgo.lunar.icu", + "https://i.actionsack.com" +] \ No newline at end of file diff --git a/src/instances/data.json b/src/instances/data.json index d1212ea..3ee84a5 100644 --- a/src/instances/data.json +++ b/src/instances/data.json @@ -341,7 +341,6 @@ "https://search.st8.at", "https://search.stinpriza.org", "https://search.trom.tf", - "https://search.vojkovic.xyz", "https://search.zdechov.net", "https://searx.bissisoft.com", "https://searx.divided-by-zero.eu", @@ -405,6 +404,7 @@ "i2p": [], "normal": [ "https://darmarit.org/searx", + "https://etsi.me", "https://northboot.xyz", "https://paulgo.io", "https://s.zhaocloud.net", @@ -412,12 +412,14 @@ "https://search.mdosch.de", "https://search.neet.works", "https://search.ononoki.org", + "https://search.vojkovic.xyz", "https://search.zzls.xyz", "https://searx.be", "https://searx.ebnar.xyz", "https://searx.esmailelbob.xyz", "https://searx.fmac.xyz", "https://searx.gnous.eu", + "https://searx.jaska.cc", "https://searx.mha.fi", "https://searx.namejeff.xyz", "https://searx.prvcy.eu", @@ -468,6 +470,7 @@ ] }, "peertube": [ + "https://videos.rabbit-company.com", "https://video.paradigmthreat.net", "https://video.lincolncyber.com", "https://vid.twhtv.club", @@ -925,7 +928,6 @@ "https://tube.toontoet.nl", "https://video.gyt.is", "https://peertube.0x5e.eu", - "https://turkum.me", "https://peertube.jensdiemer.de", "https://tube.futuretic.fr", "https://libra.syntazia.org", @@ -1067,7 +1069,6 @@ "https://videos.pzelawski.xyz", "https://peertube.zoz-serv.org", "https://videos.stadtfabrikanten.org", - "https://archive.vidicon.org", "https://peertube.gargantia.fr", "https://tube.melonbread.xyz", "https://tube.grap.coop", @@ -1404,7 +1405,6 @@ "https://peertube.uno", "https://tube.ksl-bmx.de", "https://tube.tchncs.de", - "https://peertube.kosebamse.com", "https://yunopeertube.myddns.me", "https://peertube.anon-kenkai.com", "https://tube.maiti.info", @@ -1448,7 +1448,6 @@ "https://peertube.we-keys.fr", "https://artitube.artifaille.fr", "https://peertube.fr", - "https://peertube.nayya.org", "https://peertube.amicale.net", "https://aperi.tube", "https://tube.ac-lyon.fr", diff --git a/src/instances/get_instances.py b/src/instances/get_instances.py index 8ecec16..6194c6b 100644 --- a/src/instances/get_instances.py +++ b/src/instances/get_instances.py @@ -7,6 +7,8 @@ from bs4 import BeautifulSoup import re from colorama import Fore, Back, Style from urllib.parse import urlparse +import socket +import subprocess mightyList = {} @@ -22,6 +24,39 @@ def filterLastSlash(urlList): return tmp +def init_cloudflare(): + r = requests.get('https://www.cloudflare.com/ips-v4') + myList = [] + for i in r.text.split('\n'): + out = subprocess.run( + ["sh", "./src/instances/get_possible_ips.sh", i], + capture_output=True, + text=True + ) + myList += out.stdout.splitlines() + print(Fore.GREEN + 'Fetched ' + + Fore.RED + 'Cloudflare IPs' + + Style.RESET_ALL) + return myList + + +cloudflare_ips = init_cloudflare() + + +def is_cloudflare(url): + href = urlparse(url) + ip = '' + try: + ip = socket.gethostbyname(href.hostname) + except: + return False + + if ip in cloudflare_ips: + return True + else: + return False + + # Invidious r = requests.get('https://api.invidious.io/instances.json') rJson = json.loads(r.text) @@ -281,11 +316,12 @@ print(Fore.GREEN + 'Fetched ' + Style.RESET_ALL + 'Peertube') def isValid(url): # This code is contributed by avanitrachhadiya2155 try: result = urlparse(url) - return all([result.scheme, result.netloc, result.path]) + return all([result.scheme, result.netloc]) except: return False +cloudflareMightyList = [] for k1, v1 in mightyList.items(): if type(mightyList[k1]) is dict: for k2, v2 in mightyList[k1].items(): @@ -293,17 +329,22 @@ for k1, v1 in mightyList.items(): if (not isValid(instance)): mightyList[k1][k2].remove(instance) print("removed " + instance) + else: + if (is_cloudflare(instance)): + cloudflareMightyList.append(instance) + - elif type(mightyList[k1]) is list: - for instance in mightyList[k1]: - if (not isValid(instance)): - mightyList[k1].remove(instance) - print("removed " + instance) # Writing to file json_object = json.dumps(mightyList, ensure_ascii=False, indent=2) with open('./src/instances/data.json', 'w') as outfile: outfile.write(json_object) +print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json') + +json_object = json.dumps(cloudflareMightyList, ensure_ascii=False, indent=2) +with open('./src/instances/cloudflare.json', 'w') as outfile: + outfile.write(json_object) +print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/cloudflare.json') + # print(json_object) -print(Fore.BLUE + 'wrote ' + Style.RESET_ALL + 'instances/data.json') diff --git a/src/instances/get_possible_ips.sh b/src/instances/get_possible_ips.sh new file mode 100644 index 0000000..81347c5 --- /dev/null +++ b/src/instances/get_possible_ips.sh @@ -0,0 +1 @@ +nmap -sL -n $1 | awk '/Nmap scan report/{print $NF}' \ No newline at end of file