diff --git a/searx/__init__.py b/searx/__init__.py index 17da2f35..c4c363bf 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -17,6 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from os import environ from os.path import realpath, dirname, join, abspath +from searx.https_rewrite import load_https_rules try: from yaml import load except: @@ -34,7 +35,16 @@ if 'SEARX_SETTINGS_PATH' in environ: else: settings_path = join(searx_dir, 'settings.yml') +if 'SEARX_HTTPS_REWRITE_PATH' in environ: + https_rewrite_path = environ['SEARX_HTTPS_REWRITE_PATH'] +else: + https_rewrite_path = join(searx_dir, 'https_rules') # load settings with open(settings_path) as settings_yaml: settings = load(settings_yaml) + +# load https rules only if https rewrite is enabled +if settings.get('server', {}).get('https_rewrite'): + # loade https rules + load_https_rules(https_rewrite_path) diff --git a/searx/https_rewrite.py b/searx/https_rewrite.py index 44ada945..18405d87 100644 --- a/searx/https_rewrite.py +++ b/searx/https_rewrite.py @@ -1,14 +1,141 @@ +''' +searx is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +searx is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with searx. If not, see < http://www.gnu.org/licenses/ >. + +(C) 2013- by Adam Tauber, +''' + import re +from lxml import etree +from os import listdir +from os.path import isfile, join + # https://gitweb.torproject.org/\ # pde/https-everywhere.git/tree/4.0:/src/chrome/content/rules # HTTPS rewrite rules -https_rules = ( - # from - (re.compile(r'^http://(www\.|m\.|)?xkcd\.(?:com|org)/', re.I | re.U), - # to - r'https://\1xkcd.com/'), - (re.compile(r'^https?://(?:ssl)?imgs\.xkcd\.com/', re.I | re.U), - r'https://sslimgs.xkcd.com/'), -) +https_rules = [] + + +# load single ruleset from a xml file +def load_single_https_ruleset(filepath): + ruleset = () + + # init parser + parser = etree.XMLParser() + + # load and parse xml-file + try: + tree = etree.parse(filepath, parser) + except: + # TODO, error message + return () + + # get root node + root = tree.getroot() + + #print(etree.tostring(tree)) + + # check if root is a node with the name ruleset + # TODO improve parsing + if root.tag != 'ruleset': + return () + + # check if rule is deactivated by default + if root.attrib.get('default_off'): + return () + + # check if rule does only work for specific platforms + if root.attrib.get('platform'): + return () + + hosts = [] + rules = [] + exclusions = [] + + # parse childs from ruleset + for ruleset in root: + # this child define a target + if ruleset.tag == 'target': + # check if required tags available + if not ruleset.attrib.get('host'): + continue + + # convert host-rule to valid regex + host = ruleset.attrib.get('host').replace('.', '\.').replace('*', '.*') + + # append to host list + hosts.append(host) + + # this child define a rule + elif ruleset.tag == 'rule': + # check if required tags available + if not ruleset.attrib.get('from')\ + or not ruleset.attrib.get('to'): + continue + + # TODO hack, which convert a javascript regex group into a valid python regex group + rule_from = ruleset.attrib.get('from').replace('$', '\\') + rule_to = ruleset.attrib.get('to').replace('$', '\\') + + # TODO, not working yet because of the hack above, currently doing that in webapp.py + #rule_from_rgx = re.compile(rule_from, re.I) + + # append rule + rules.append((rule_from, rule_to)) + + # this child define an exclusion + elif ruleset.tag == 'exclusion': + # check if required tags available + if not ruleset.attrib.get('pattern'): + continue + + exclusion_rgx = re.compile(ruleset.attrib.get('pattern')) + + # append exclusion + exclusions.append(exclusion_rgx) + + # convert list of possible hosts to a simple regex + # TODO compress regex to improve performance + try: + target_hosts = re.compile('^(' + '|'.join(hosts) + ')', re.I | re.U) + except: + return () + + # return ruleset + return (target_hosts, rules, exclusions) + + +# load all https rewrite rules +def load_https_rules(rules_path): + # add / to path if not set yet + if rules_path[-1:] != '/': + rules_path += '/' + + # search all xml files which are stored in the https rule directory + xml_files = [ join(rules_path,f) for f in listdir(rules_path) if isfile(join(rules_path,f)) and f[-4:] == '.xml' ] + + # load xml-files + for ruleset_file in xml_files: + # calculate rewrite-rules + ruleset = load_single_https_ruleset(ruleset_file) + + # skip if no ruleset returned + if not ruleset: + continue + + # append ruleset + https_rules.append(ruleset) + + print(' * {n} https-rules loaded'.format(n=len(https_rules))) diff --git a/searx/https_rules/00README b/searx/https_rules/00README new file mode 100644 index 00000000..fcd8a772 --- /dev/null +++ b/searx/https_rules/00README @@ -0,0 +1,17 @@ + diff --git a/searx/https_rules/Bing.xml b/searx/https_rules/Bing.xml new file mode 100644 index 00000000..8b403f10 --- /dev/null +++ b/searx/https_rules/Bing.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Dailymotion.xml b/searx/https_rules/Dailymotion.xml new file mode 100644 index 00000000..743100cb --- /dev/null +++ b/searx/https_rules/Dailymotion.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Deviantart.xml b/searx/https_rules/Deviantart.xml new file mode 100644 index 00000000..7830fc20 --- /dev/null +++ b/searx/https_rules/Deviantart.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/DuckDuckGo.xml b/searx/https_rules/DuckDuckGo.xml new file mode 100644 index 00000000..173a9ad9 --- /dev/null +++ b/searx/https_rules/DuckDuckGo.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Flickr.xml b/searx/https_rules/Flickr.xml new file mode 100644 index 00000000..85c6e806 --- /dev/null +++ b/searx/https_rules/Flickr.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Github-Pages.xml b/searx/https_rules/Github-Pages.xml new file mode 100644 index 00000000..d3be58a4 --- /dev/null +++ b/searx/https_rules/Github-Pages.xml @@ -0,0 +1,11 @@ + + + + + + + + diff --git a/searx/https_rules/Github.xml b/searx/https_rules/Github.xml new file mode 100644 index 00000000..a9a3a1e5 --- /dev/null +++ b/searx/https_rules/Github.xml @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Google-mismatches.xml b/searx/https_rules/Google-mismatches.xml new file mode 100644 index 00000000..de9d3eb1 --- /dev/null +++ b/searx/https_rules/Google-mismatches.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Google.org.xml b/searx/https_rules/Google.org.xml new file mode 100644 index 00000000..d6cc4788 --- /dev/null +++ b/searx/https_rules/Google.org.xml @@ -0,0 +1,14 @@ + + + + + + + + + + \ No newline at end of file diff --git a/searx/https_rules/GoogleAPIs.xml b/searx/https_rules/GoogleAPIs.xml new file mode 100644 index 00000000..85a5a808 --- /dev/null +++ b/searx/https_rules/GoogleAPIs.xml @@ -0,0 +1,143 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleCanada.xml b/searx/https_rules/GoogleCanada.xml new file mode 100644 index 00000000..d5eefe81 --- /dev/null +++ b/searx/https_rules/GoogleCanada.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/searx/https_rules/GoogleImages.xml b/searx/https_rules/GoogleImages.xml new file mode 100644 index 00000000..0112001e --- /dev/null +++ b/searx/https_rules/GoogleImages.xml @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleMainSearch.xml b/searx/https_rules/GoogleMainSearch.xml new file mode 100644 index 00000000..df504d90 --- /dev/null +++ b/searx/https_rules/GoogleMainSearch.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleMaps.xml b/searx/https_rules/GoogleMaps.xml new file mode 100644 index 00000000..0f82c526 --- /dev/null +++ b/searx/https_rules/GoogleMaps.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleMelange.xml b/searx/https_rules/GoogleMelange.xml new file mode 100644 index 00000000..ec23cd45 --- /dev/null +++ b/searx/https_rules/GoogleMelange.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/searx/https_rules/GoogleSearch.xml b/searx/https_rules/GoogleSearch.xml new file mode 100644 index 00000000..66b7ffdb --- /dev/null +++ b/searx/https_rules/GoogleSearch.xml @@ -0,0 +1,135 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleServices.xml b/searx/https_rules/GoogleServices.xml new file mode 100644 index 00000000..704646b5 --- /dev/null +++ b/searx/https_rules/GoogleServices.xml @@ -0,0 +1,345 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleShopping.xml b/searx/https_rules/GoogleShopping.xml new file mode 100644 index 00000000..6ba69a91 --- /dev/null +++ b/searx/https_rules/GoogleShopping.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleSorry.xml b/searx/https_rules/GoogleSorry.xml new file mode 100644 index 00000000..72a19210 --- /dev/null +++ b/searx/https_rules/GoogleSorry.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/searx/https_rules/GoogleTranslate.xml b/searx/https_rules/GoogleTranslate.xml new file mode 100644 index 00000000..a004025a --- /dev/null +++ b/searx/https_rules/GoogleTranslate.xml @@ -0,0 +1,8 @@ + + + + + + + diff --git a/searx/https_rules/GoogleVideos.xml b/searx/https_rules/GoogleVideos.xml new file mode 100644 index 00000000..a5e88fcf --- /dev/null +++ b/searx/https_rules/GoogleVideos.xml @@ -0,0 +1,83 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/GoogleWatchBlog.xml b/searx/https_rules/GoogleWatchBlog.xml new file mode 100644 index 00000000..afec70c9 --- /dev/null +++ b/searx/https_rules/GoogleWatchBlog.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/searx/https_rules/Google_App_Engine.xml b/searx/https_rules/Google_App_Engine.xml new file mode 100644 index 00000000..851e051d --- /dev/null +++ b/searx/https_rules/Google_App_Engine.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/searx/https_rules/Googleplex.com.xml b/searx/https_rules/Googleplex.com.xml new file mode 100644 index 00000000..7ddbb5ba --- /dev/null +++ b/searx/https_rules/Googleplex.com.xml @@ -0,0 +1,16 @@ + + + + + + + + diff --git a/searx/https_rules/OpenStreetMap.xml b/searx/https_rules/OpenStreetMap.xml new file mode 100644 index 00000000..58a66182 --- /dev/null +++ b/searx/https_rules/OpenStreetMap.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + diff --git a/searx/https_rules/Rawgithub.com.xml b/searx/https_rules/Rawgithub.com.xml new file mode 100644 index 00000000..3868f332 --- /dev/null +++ b/searx/https_rules/Rawgithub.com.xml @@ -0,0 +1,14 @@ + + + + + + + + + + diff --git a/searx/https_rules/Soundcloud.xml b/searx/https_rules/Soundcloud.xml new file mode 100644 index 00000000..0baa5832 --- /dev/null +++ b/searx/https_rules/Soundcloud.xml @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/ThePirateBay.xml b/searx/https_rules/ThePirateBay.xml new file mode 100644 index 00000000..010387b6 --- /dev/null +++ b/searx/https_rules/ThePirateBay.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Torproject.xml b/searx/https_rules/Torproject.xml new file mode 100644 index 00000000..69269af7 --- /dev/null +++ b/searx/https_rules/Torproject.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Twitter.xml b/searx/https_rules/Twitter.xml new file mode 100644 index 00000000..3285f44e --- /dev/null +++ b/searx/https_rules/Twitter.xml @@ -0,0 +1,169 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Vimeo.xml b/searx/https_rules/Vimeo.xml new file mode 100644 index 00000000..f2a3e576 --- /dev/null +++ b/searx/https_rules/Vimeo.xml @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/WikiLeaks.xml b/searx/https_rules/WikiLeaks.xml new file mode 100644 index 00000000..977709d2 --- /dev/null +++ b/searx/https_rules/WikiLeaks.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/searx/https_rules/Wikimedia.xml b/searx/https_rules/Wikimedia.xml new file mode 100644 index 00000000..9f25831a --- /dev/null +++ b/searx/https_rules/Wikimedia.xml @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/Yahoo.xml b/searx/https_rules/Yahoo.xml new file mode 100644 index 00000000..33548c4a --- /dev/null +++ b/searx/https_rules/Yahoo.xml @@ -0,0 +1,2450 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/https_rules/YouTube.xml b/searx/https_rules/YouTube.xml new file mode 100644 index 00000000..bddc2a5f --- /dev/null +++ b/searx/https_rules/YouTube.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/searx/settings_robot.yml b/searx/settings_robot.yml index 98944a81..bb91dce8 100644 --- a/searx/settings_robot.yml +++ b/searx/settings_robot.yml @@ -4,6 +4,9 @@ server: debug : False request_timeout : 3.0 # seconds base_url: False + themes_path : "" + default_theme : default + https_rewrite : True engines: - name : general_dummy diff --git a/searx/webapp.py b/searx/webapp.py index 18dc89a3..830cf440 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -50,6 +50,9 @@ from searx.search import Search from searx.query import Query from searx.autocomplete import backends as autocomplete_backends +from urlparse import urlparse +import re + static_path, templates_path, themes =\ get_themes(settings['themes_path'] @@ -206,16 +209,60 @@ def index(): if not search.paging and engines[result['engine']].paging: search.paging = True + # check if HTTPS rewrite is required if settings['server']['https_rewrite']\ and result['parsed_url'].scheme == 'http': - for http_regex, https_url in https_rules: - if http_regex.match(result['url']): - result['url'] = http_regex.sub(https_url, result['url']) - # TODO result['parsed_url'].scheme + skip_https_rewrite = False + + # check if HTTPS rewrite is possible + for target, rules, exclusions in https_rules: + + # check if target regex match with url + if target.match(result['url']): + # process exclusions + for exclusion in exclusions: + # check if exclusion match with url + if exclusion.match(result['url']): + skip_https_rewrite = True + break + + # skip https rewrite if required + if skip_https_rewrite: + break + + # process rules + for rule in rules: + try: + # TODO, precompile rule + p = re.compile(rule[0]) + + # rewrite url if possible + new_result_url = p.sub(rule[1], result['url']) + except: + break + + # parse new url + new_parsed_url = urlparse(new_result_url) + + # continiue if nothing was rewritten + if result['url'] == new_result_url: + continue + + # get domainname from result + # TODO, does only work correct with TLD's like asdf.com, not for asdf.com.de + # TODO, using publicsuffix instead of this rewrite rule + old_result_domainname = '.'.join(result['parsed_url'].hostname.split('.')[-2:]) + new_result_domainname = '.'.join(new_parsed_url.hostname.split('.')[-2:]) + + # check if rewritten hostname is the same, to protect against wrong or malicious rewrite rules + if old_result_domainname == new_result_domainname: + # set new url + result['url'] = new_result_url + + # target has matched, do not search over the other rules break - # HTTPS rewrite if search.request_data.get('format', 'html') == 'html': if 'content' in result: result['content'] = highlight_content(result['content'],