Further block/allow updates (#1406)
* ConfigFile change to allowlist and blocklist * revised names and warnings * consistent file naming in kebab case, and generic use of blocklist and allowlist in cmoments for clarity * update ci files * further allow/blocklist updates * improve language in comments Co-authored-by: Ian Bashford <ianbashford@gmail.com>
This commit is contained in:
Ian Bashford
GitHub
parent
10710def50
commit
af564522ec
|
|
@ -1,16 +1,16 @@
|
|||
|
||||
###########################
|
||||
# Whitelist #
|
||||
# Allowlist #
|
||||
###########################
|
||||
|
||||
## Rules for name-based query whitelisting, one per line
|
||||
## Rules for allowing queries based on name, one per line
|
||||
##
|
||||
## Example of valid patterns:
|
||||
##
|
||||
## ads.* | matches anything with an "ads." prefix
|
||||
## *.example.com | matches example.com and all names within that zone such as www.example.com
|
||||
## example.com | identical to the above
|
||||
## =example.com | whitelists example.com but not *.example.com
|
||||
## =example.com | allows example.com but not *.example.com
|
||||
## *sex* | matches any name containing that substring
|
||||
## ads[0-9]* | matches "ads" followed by one or more digits
|
||||
## ads*.example* | *, ? and [] can be used anywhere, but prefixes/suffixes are faster
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
##############################
|
||||
# IP blacklist #
|
||||
# IP blocklist #
|
||||
##############################
|
||||
|
||||
## Response containing blacklisted IP addresses will be blocked
|
||||
## Rules for IP-based response blocking
|
||||
##
|
||||
## Sample feeds of suspect IP addresses:
|
||||
## - https://github.com/stamparm/ipsum
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
|
||||
###########################
|
||||
# Blacklist #
|
||||
# Blocklist #
|
||||
###########################
|
||||
|
||||
## Rules for name-based query blocking, one per line
|
||||
|
|
|
|||
|
|
@ -507,23 +507,23 @@ cache_neg_max_ttl = 600
|
|||
|
||||
|
||||
######################################################
|
||||
# Pattern-based allowlisting (blocklists bypass) #
|
||||
# Pattern-based allow lists (blocklists bypass) #
|
||||
######################################################
|
||||
|
||||
## Allowlists support the same patterns as blocklists
|
||||
## If a name matches a allowlist entry, the corresponding session
|
||||
## If a name matches an allowlist entry, the corresponding session
|
||||
## will bypass names and IP filters.
|
||||
##
|
||||
## Time-based rules are also supported to make some websites only accessible at specific times of the day.
|
||||
|
||||
[allowed_names]
|
||||
|
||||
## Path to the file of allowlisting rules (absolute, or relative to the same directory as the config file)
|
||||
## Path to the file of allow list rules (absolute, or relative to the same directory as the config file)
|
||||
|
||||
# allowed_names_file = 'allowed-names.txt'
|
||||
|
||||
|
||||
## Optional path to a file logging allowlisted queries
|
||||
## Optional path to a file logging allowed queries
|
||||
|
||||
# log_file = 'allowed-names.log'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,30 +1,30 @@
|
|||
##################################################################################
|
||||
# #
|
||||
# Generate a black list of domains using public data sources, and the local #
|
||||
# domains-blacklist-local-additions.txt file. #
|
||||
# Generate a block list of domains using public data sources, and the local #
|
||||
# domains-blocklist-local-additions.txt file. #
|
||||
# #
|
||||
# The default configuration is just indicative, and corresponds to the one #
|
||||
# used to produce the public "mybase" set. #
|
||||
# #
|
||||
# Comment out the URLs of the sources you wish to disable, leave the ones #
|
||||
# you would like enabled uncommented. Then run the script to build the #
|
||||
# dnscrypt-blacklist-domains.txt file: #
|
||||
# dnscrypt-blocklist-domains.txt file: #
|
||||
# #
|
||||
# $ generate-domains-blacklist.py > dnscrypt-blacklist-domains.txt #
|
||||
# $ generate-domains-blocklist.py > dnscrypt-blacklist-domains.txt #
|
||||
# #
|
||||
# Domains that should never be blocked can be put into a file named #
|
||||
# domains-whitelist.txt. #
|
||||
# domains-allowlist.txt. #
|
||||
# #
|
||||
# That blacklist file can then be used in the dnscrypt-proxy.toml file: #
|
||||
# That blocklist file can then be used in the dnscrypt-proxy.toml file: #
|
||||
# #
|
||||
# [blacklist] #
|
||||
# [blocklist] #
|
||||
# #
|
||||
# blacklist_file = 'dnscrypt-blacklist-domains.txt' #
|
||||
# blocklist_file = 'dnscrypt-blocklist-domains.txt' #
|
||||
# #
|
||||
##################################################################################
|
||||
|
||||
# Local additions
|
||||
file:domains-blacklist-local-additions.txt
|
||||
file:domains-blocklist-local-additions.txt
|
||||
|
||||
# AdAway is an open source ad blocker for Android using the hosts file.
|
||||
# https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#! /usr/bin/env python3
|
||||
|
||||
# run with python generate-domains-blacklist.py > list.txt.tmp && mv -f list.txt.tmp list
|
||||
# run with python generate-domains-blocklist.py > list.txt.tmp && mv -f list.txt.tmp list
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
|
|
@ -168,7 +168,7 @@ def has_suffix(names, name):
|
|||
return False
|
||||
|
||||
|
||||
def whitelist_from_url(url):
|
||||
def allowlist_from_url(url):
|
||||
if not url:
|
||||
return set()
|
||||
content, trusted = load_from_url(url)
|
||||
|
|
@ -177,16 +177,16 @@ def whitelist_from_url(url):
|
|||
return names
|
||||
|
||||
|
||||
def blacklists_from_config_file(
|
||||
file, whitelist, time_restricted_url, ignore_retrieval_failure, output_file
|
||||
def blocklists_from_config_file(
|
||||
file, allowlist, time_restricted_url, ignore_retrieval_failure, output_file
|
||||
):
|
||||
blacklists = {}
|
||||
whitelisted_names = set()
|
||||
blocklists = {}
|
||||
allowed_names = set()
|
||||
all_names = set()
|
||||
unique_names = set()
|
||||
all_globs = set()
|
||||
|
||||
# Load conf & blacklists
|
||||
# Load conf & blocklists
|
||||
with open(file) as fd:
|
||||
for line in fd:
|
||||
line = str.strip(line)
|
||||
|
|
@ -196,7 +196,7 @@ def blacklists_from_config_file(
|
|||
try:
|
||||
content, trusted = load_from_url(url)
|
||||
names, _time_restrictions, globs = parse_list(content, trusted)
|
||||
blacklists[url] = names
|
||||
blocklists[url] = names
|
||||
all_names |= names
|
||||
all_globs |= globs
|
||||
except Exception as e:
|
||||
|
|
@ -204,7 +204,7 @@ def blacklists_from_config_file(
|
|||
if not ignore_retrieval_failure:
|
||||
exit(1)
|
||||
|
||||
# Time-based blacklist
|
||||
# Time-based blocklist
|
||||
if time_restricted_url and not re.match(r"^[a-z0-9]+:", time_restricted_url):
|
||||
time_restricted_url = "file:" + time_restricted_url
|
||||
|
||||
|
|
@ -218,33 +218,33 @@ def blacklists_from_config_file(
|
|||
time_restricted_content)
|
||||
|
||||
if time_restricted_names:
|
||||
print("########## Time-based blacklist ##########\n",
|
||||
print("########## Time-based blocklist ##########\n",
|
||||
file=output_fd, end='\n')
|
||||
for name in time_restricted_names:
|
||||
print_restricted_name(output_fd, name, time_restrictions)
|
||||
|
||||
# Time restricted names should be whitelisted, or they could be always blocked
|
||||
whitelisted_names |= time_restricted_names
|
||||
# Time restricted names should be allowed, or they could be always blocked
|
||||
allowed_names |= time_restricted_names
|
||||
|
||||
# Whitelist
|
||||
if whitelist and not re.match(r"^[a-z0-9]+:", whitelist):
|
||||
whitelist = "file:" + whitelist
|
||||
# Allowed list
|
||||
if allowlist and not re.match(r"^[a-z0-9]+:", allowlist):
|
||||
allowlist = "file:" + allowlist
|
||||
|
||||
whitelisted_names |= whitelist_from_url(whitelist)
|
||||
allowed_names |= allowlist_from_url(allowlist)
|
||||
|
||||
# Process blacklists
|
||||
for url, names in blacklists.items():
|
||||
print("\n\n########## Blacklist from {} ##########\n".format(
|
||||
# Process blocklists
|
||||
for url, names in blocklists.items():
|
||||
print("\n\n########## Blocklist from {} ##########\n".format(
|
||||
url), file=output_fd, end='\n')
|
||||
ignored, glob_ignored, whitelisted = 0, 0, 0
|
||||
ignored, glob_ignored, allowed = 0, 0, 0
|
||||
list_names = list()
|
||||
for name in names:
|
||||
if covered_by_glob(all_globs, name):
|
||||
glob_ignored = glob_ignored + 1
|
||||
elif has_suffix(all_names, name) or name in unique_names:
|
||||
ignored = ignored + 1
|
||||
elif has_suffix(whitelisted_names, name) or name in whitelisted_names:
|
||||
whitelisted = whitelisted + 1
|
||||
elif has_suffix(allowed_names, name) or name in allowed_names:
|
||||
allowed = allowed + 1
|
||||
else:
|
||||
list_names.append(name)
|
||||
unique_names.add(name)
|
||||
|
|
@ -256,10 +256,10 @@ def blacklists_from_config_file(
|
|||
if glob_ignored:
|
||||
print("# Ignored due to overlapping local patterns: {}".format(
|
||||
glob_ignored), file=output_fd, end='\n')
|
||||
if whitelisted:
|
||||
if allowed:
|
||||
print(
|
||||
"# Ignored entries due to the whitelist: {}".format(whitelisted), file=output_fd, end='\n')
|
||||
if ignored or glob_ignored or whitelisted:
|
||||
"# Ignored entries due to the allowlist: {}".format(allowed), file=output_fd, end='\n')
|
||||
if ignored or glob_ignored or allowed:
|
||||
print(file=output_fd, end='\n')
|
||||
for name in list_names:
|
||||
print(name, file=output_fd, end='\n')
|
||||
|
|
@ -268,19 +268,19 @@ def blacklists_from_config_file(
|
|||
|
||||
|
||||
argp = argparse.ArgumentParser(
|
||||
description="Create a unified blacklist from a set of local and remote files"
|
||||
description="Create a unified blocklist from a set of local and remote files"
|
||||
)
|
||||
argp.add_argument(
|
||||
"-c",
|
||||
"--config",
|
||||
default="domains-blacklist.conf",
|
||||
help="file containing blacklist sources",
|
||||
default="domains-blocklist.conf",
|
||||
help="file containing blocklist sources",
|
||||
)
|
||||
argp.add_argument(
|
||||
"-w",
|
||||
"--whitelist",
|
||||
default="domains-whitelist.txt",
|
||||
help="file containing a set of names to exclude from the blacklist",
|
||||
"--allowlist",
|
||||
default="domains-allowlist.txt",
|
||||
help="file containing a set of names to exclude from the blocklist",
|
||||
)
|
||||
argp.add_argument(
|
||||
"-r",
|
||||
|
|
@ -298,17 +298,17 @@ argp.add_argument(
|
|||
"-o",
|
||||
"--output-file",
|
||||
default=None,
|
||||
help="save generated blacklist to a text file with the provided file name",
|
||||
help="save generated blocklist to a text file with the provided file name",
|
||||
)
|
||||
argp.add_argument("-t", "--timeout", default=30, help="URL open timeout")
|
||||
|
||||
args = argp.parse_args()
|
||||
|
||||
conf = args.config
|
||||
whitelist = args.whitelist
|
||||
allowlist = args.allowlist
|
||||
time_restricted = args.time_restricted
|
||||
ignore_retrieval_failure = args.ignore_retrieval_failure
|
||||
output_file = args.output_file
|
||||
|
||||
blacklists_from_config_file(
|
||||
conf, whitelist, time_restricted, ignore_retrieval_failure, output_file)
|
||||
blocklists_from_config_file(
|
||||
conf, allowlist, time_restricted, ignore_retrieval_failure, output_file)
|
||||
Loading…
Reference in New Issue