From 53e9c79194d7ce6e6ea3651e493902f5671f936b Mon Sep 17 00:00:00 2001 From: Benjamin Dos Santos Date: Sun, 11 Feb 2018 19:15:52 +0100 Subject: [PATCH] feat: add a flag to setup the open URL timeout Sometimes I randomly encounter a timeout when I generate blacklist. This commit add the ability to increase the timeout delay (default to 10s). --- .../generate-domains-blacklists/generate-domains-blacklist.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/generate-domains-blacklists/generate-domains-blacklist.py b/utils/generate-domains-blacklists/generate-domains-blacklist.py index 772b8849..30381179 100755 --- a/utils/generate-domains-blacklists/generate-domains-blacklist.py +++ b/utils/generate-domains-blacklists/generate-domains-blacklist.py @@ -44,7 +44,7 @@ def list_from_url(url): trusted = True response = None try: - response = urllib2.urlopen(req, timeout=10) + response = urllib2.urlopen(req, timeout=int(args.timeout)) except urllib2.URLError as err: raise Exception("[{}] could not be loaded: {}\n".format(url, err)) if trusted is False and response.getcode() != 200: @@ -131,6 +131,8 @@ argp.add_argument("-w", "--whitelist", default="domains-whitelist.txt", help="file containing a set of names to exclude from the blacklist") argp.add_argument("-i", "--ignore-retrieval-failure", action='store_true', help="generate list even if some urls couldn't be retrieved") +argp.add_argument("-t", "--timeout", default=10, + help="URL open timeout") args = argp.parse_args() conf = args.config