miracle091/dnscrypt-proxy
miracle091
/
dnscrypt-proxy
mirror of https://github.com/DNSCrypt/dnscrypt-proxy.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Format generate-domains-blocklist.py with Black

This commit is contained in:
Frank Denis 2020-09-12 23:34:39 +02:00
parent 5a1b87130d
commit 018d8412be
1 changed files with 33 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
utils/generate-domains-blocklists/generate-domains-blocklist.py
View File

@ -57,7 +57,8 @@ def parse_list(content, trusted=False):
rx_comment = re.compile(r"^(#|$)") rx_comment = re.compile(r"^(#|$)")
rx_inline_comment = re.compile(r"\s*#\s*[a-z0-9-].*$") rx_inline_comment = re.compile(r"\s*#\s*[a-z0-9-].*$")
rx_u = re.compile( rx_u = re.compile(
r"^@*\|\|([a-z0-9][a-z0-9.-]*[.][a-z]{2,})\^?(\$(popup|third-party))?$") r"^@*\|\|([a-z0-9][a-z0-9.-]*[.][a-z]{2,})\^?(\$(popup|third-party))?$"
)
rx_l = re.compile(r"^([a-z0-9][a-z0-9.-]*[.][a-z]{2,})$") rx_l = re.compile(r"^([a-z0-9][a-z0-9.-]*[.][a-z]{2,})$")
rx_h = re.compile( rx_h = re.compile(
r"^[0-9]{1,3}[.][0-9]{1,3}[.][0-9]{1,3}[.][0-9]{1,3}\s+([a-z0-9][a-z0-9.-]*[.][a-z]{2,})$" r"^[0-9]{1,3}[.][0-9]{1,3}[.][0-9]{1,3}[.][0-9]{1,3}\s+([a-z0-9][a-z0-9.-]*[.][a-z]{2,})$"
@ -86,12 +87,13 @@ def parse_list(content, trusted=False):
def print_restricted_name(output_fd, name, time_restrictions): def print_restricted_name(output_fd, name, time_restrictions):
if name in time_restrictions: if name in time_restrictions:
print("{}\t{}".format( print("{}\t{}".format(name, time_restrictions[name]), file=output_fd, end="\n")
name, time_restrictions[name]), file=output_fd, end='\n')
else: else:
print( print(
"# ignored: [{}] was in the time-restricted list, " "# ignored: [{}] was in the time-restricted list, "
"but without a time restriction label".format(name), file=output_fd, end='\n' "but without a time restriction label".format(name),
file=output_fd,
end="\n",
) )
@ -113,8 +115,7 @@ def load_from_url(url):
except urllib.URLError as err: except urllib.URLError as err:
raise Exception("[{}] could not be loaded: {}\n".format(url, err)) raise Exception("[{}] could not be loaded: {}\n".format(url, err))
if trusted is False and response.getcode() != 200: if trusted is False and response.getcode() != 200:
raise Exception("[{}] returned HTTP code {}\n".format( raise Exception("[{}] returned HTTP code {}\n".format(url, response.getcode()))
url, response.getcode()))
content = response.read() content = response.read()
if URLLIB_NEW: if URLLIB_NEW:
content = content.decode("utf-8", errors="replace") content = content.decode("utf-8", errors="replace")
@ -135,7 +136,7 @@ def is_glob(pattern):
if c == "?" or c == "[": if c == "?" or c == "[":
maybe_glob = True maybe_glob = True
elif c == "*" and i != 0: elif c == "*" and i != 0:
if i < len(pattern) - 1 or pattern[i - 1] == '.': if i < len(pattern) - 1 or pattern[i - 1] == ".":
maybe_glob = True maybe_glob = True
if maybe_glob: if maybe_glob:
try: try:
@ -215,11 +216,13 @@ def blocklists_from_config_file(
if time_restricted_url: if time_restricted_url:
time_restricted_content, _trusted = load_from_url(time_restricted_url) time_restricted_content, _trusted = load_from_url(time_restricted_url)
time_restricted_names, time_restrictions, _globs = parse_trusted_list( time_restricted_names, time_restrictions, _globs = parse_trusted_list(
time_restricted_content) time_restricted_content
)
if time_restricted_names: if time_restricted_names:
print("########## Time-based blocklist ##########\n", print(
file=output_fd, end='\n') "########## Time-based blocklist ##########\n", file=output_fd, end="\n"
)
for name in time_restricted_names: for name in time_restricted_names:
print_restricted_name(output_fd, name, time_restrictions) print_restricted_name(output_fd, name, time_restrictions)
@ -234,8 +237,11 @@ def blocklists_from_config_file(
# Process blocklists # Process blocklists
for url, names in blocklists.items(): for url, names in blocklists.items():
print("\n\n########## Blocklist from {} ##########\n".format( print(
url), file=output_fd, end='\n') "\n\n########## Blocklist from {} ##########\n".format(url),
file=output_fd,
end="\n",
)
ignored, glob_ignored, allowed = 0, 0, 0 ignored, glob_ignored, allowed = 0, 0, 0
list_names = list() list_names = list()
for name in names: for name in names:
@ -251,18 +257,23 @@ def blocklists_from_config_file(
list_names.sort(key=name_cmp) list_names.sort(key=name_cmp)
if ignored: if ignored:
print("# Ignored duplicates: {}".format( print("# Ignored duplicates: {}".format(ignored), file=output_fd, end="\n")
ignored), file=output_fd, end='\n')
if glob_ignored: if glob_ignored:
print("# Ignored due to overlapping local patterns: {}".format( print(
glob_ignored), file=output_fd, end='\n') "# Ignored due to overlapping local patterns: {}".format(glob_ignored),
file=output_fd,
end="\n",
)
if allowed: if allowed:
print( print(
"# Ignored entries due to the allowlist: {}".format(allowed), file=output_fd, end='\n') "# Ignored entries due to the allowlist: {}".format(allowed),
file=output_fd,
end="\n",
)
if ignored or glob_ignored or allowed: if ignored or glob_ignored or allowed:
print(file=output_fd, end='\n') print(file=output_fd, end="\n")
for name in list_names: for name in list_names:
print(name, file=output_fd, end='\n') print(name, file=output_fd, end="\n")
output_fd.close() output_fd.close()
@ -311,7 +322,7 @@ args = argp.parse_args()
whitelist = args.whitelist whitelist = args.whitelist
if whitelist: if whitelist:
print('Use of -w WHITELIST has been removed. Please use -a ALLOWLIST instead.') print("Use of -w WHITELIST has been removed. Please use -a ALLOWLIST instead.")
exit(1) exit(1)
conf = args.config conf = args.config
@ -321,4 +332,5 @@ ignore_retrieval_failure = args.ignore_retrieval_failure
output_file = args.output_file output_file = args.output_file
blocklists_from_config_file( blocklists_from_config_file(
conf, allowlist, time_restricted, ignore_retrieval_failure, output_file) conf, allowlist, time_restricted, ignore_retrieval_failure, output_file
)