devol
/
searx
mirror of https://github.com/searx/searx
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #2275 from dalf/mod-ahmia-filter 

[mod] ahmia_filter.py: minor changes
This commit is contained in:
Adam Tauber 2020-10-27 21:17:53 +01:00 committed by GitHub
parent db703a0283 5e7060053c
commit d3d50eff66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
searx/data/__init__.py
View File

@ -2,7 +2,7 @@ import json
from pathlib import Path
__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'bangs_loader']
__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'bangs_loader', 'ahmia_blacklist_loader']
data_dir = Path(__file__).parent
@ -16,6 +16,11 @@ def bangs_loader():
return load('bangs.json')
def ahmia_blacklist_loader():
with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as fd:
return fd.read().split()
ENGINES_LANGUAGES = load('engines_languages.json')
CURRENCIES = load('currencies.json')
USER_AGENTS = load('useragents.json')

11
searx/plugins/ahmia_filter.py
View File

@ -3,9 +3,7 @@
'''
from hashlib import md5
from os.path import join
from urllib.parse import urlparse
from searx import searx_dir
from searx.data import ahmia_blacklist_loader
name = "Ahmia blacklist"
description = "Filter out onion results that appear in Ahmia's blacklist. (See https://ahmia.fi/blacklist)"
@ -18,15 +16,14 @@ ahmia_blacklist = None
def get_ahmia_blacklist():
global ahmia_blacklist
if not ahmia_blacklist:
with open(join(join(searx_dir, "data"), "ahmia_blacklist.txt"), 'r') as f:
ahmia_blacklist = f.read().split()
ahmia_blacklist = ahmia_blacklist_loader()
return ahmia_blacklist
def not_blacklisted(result):
if not result.get('is_onion'):
if not result.get('is_onion') or not result.get('parsed_url'):
return True
result_hash = md5(urlparse(result.get('url')).hostname.encode()).hexdigest()
result_hash = md5(result['parsed_url'].hostname.encode()).hexdigest()
return result_hash not in get_ahmia_blacklist()