From e7de9674b14c2ff06fac009274742819d6468138 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sun, 8 Feb 2015 21:53:37 +0100 Subject: [PATCH 1/4] [enh] add safesearch functionality 2 = strict 1 = moderate 0 = none --- searx/engines/bing_images.py | 8 +++++++- searx/engines/blekko_images.py | 11 +++++++++++ searx/search.py | 5 +++++ searx/templates/courgette/preferences.html | 10 ++++++++++ searx/templates/default/preferences.html | 10 ++++++++++ searx/templates/oscar/preferences.html | 11 +++++++++++ searx/webapp.py | 12 ++++++++++++ 7 files changed, 66 insertions(+), 1 deletion(-) diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index 9d1c22f5..4798d9f3 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -21,12 +21,17 @@ import re # engine dependent config categories = ['images'] paging = True +safesearch = True # search-url base_url = 'https://www.bing.com/' search_string = 'images/search?{query}&count=10&first={offset}' thumb_url = "http://ts1.mm.bing.net/th?id={ihk}" +# safesearch definitions +safesearch_types = {2: 'STRICT', + 1: 'DEMOTE', + 0: 'OFF'} # do search-request def request(query, params): @@ -43,7 +48,8 @@ def request(query, params): offset=offset) params['cookies']['SRCHHPGUSR'] = \ - 'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0] + 'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0] +\ + '&ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE') params['url'] = base_url + search_path diff --git a/searx/engines/blekko_images.py b/searx/engines/blekko_images.py index 2bae9c35..4fbb9b30 100644 --- a/searx/engines/blekko_images.py +++ b/searx/engines/blekko_images.py @@ -14,11 +14,16 @@ from urllib import urlencode # engine dependent config categories = ['images'] paging = True +safesearch = True # search-url base_url = 'https://blekko.com' search_url = '/api/images?{query}&c={c}' +# safesearch definitions +safesearch_types = {2: '1', + 1: '', + 0: '0'} # do search-request def request(query, params): @@ -31,6 +36,12 @@ def request(query, params): if params['pageno'] != 1: params['url'] += '&page={pageno}'.format(pageno=(params['pageno']-1)) + # let Blekko know we wan't have profiling + params['cookies']['tag_lesslogging'] = '1' + + # parse safesearch argument + params['cookies']['safesearch'] = safesearch_types.get(params['safesearch'], '') + return params diff --git a/searx/search.py b/searx/search.py index 1b16ee02..79a8ae0a 100644 --- a/searx/search.py +++ b/searx/search.py @@ -457,6 +457,11 @@ class Search(object): request_params['started'] = time() request_params['pageno'] = self.pageno request_params['language'] = self.lang + try: + # 0 = None, 1 = Moderate, 2 = Strict + request_params['safesearch'] = int(request.cookies.get('safesearch', 1)) + except ValueError: + request_params['safesearch'] = 1 # update request parameters dependent on # search-engine (contained in engines folder) diff --git a/searx/templates/courgette/preferences.html b/searx/templates/courgette/preferences.html index 198bc7ea..18406672 100644 --- a/searx/templates/courgette/preferences.html +++ b/searx/templates/courgette/preferences.html @@ -59,6 +59,16 @@

+
+ {{ _('SafeSearch') }} +

+ +

+
{{ _('Themes') }}

diff --git a/searx/templates/default/preferences.html b/searx/templates/default/preferences.html index 112849c3..8ac6a895 100644 --- a/searx/templates/default/preferences.html +++ b/searx/templates/default/preferences.html @@ -60,6 +60,16 @@

+
+ {{ _('SafeSearch') }} +

+ +

+
{{ _('Themes') }}

diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html index 41656463..ecb483eb 100644 --- a/searx/templates/oscar/preferences.html +++ b/searx/templates/oscar/preferences.html @@ -87,6 +87,17 @@ {{ _('Change how forms are submited, learn more about request methods') }} +

+ +
+ +
+ {{ _('Filter explicite content') }} +
diff --git a/searx/webapp.py b/searx/webapp.py index 253e5048..7266de0f 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -267,6 +267,8 @@ def render(template_name, override_theme=None, **kwargs): kwargs['method'] = request.cookies.get('method', 'POST') + kwargs['safesearch'] = request.cookies.get('safesearch', '1') + # override url_for function in templates kwargs['url_for'] = url_for_theme @@ -455,6 +457,10 @@ def preferences(): Settings that are going to be saved as cookies.""" lang = None image_proxy = request.cookies.get('image_proxy', settings['server'].get('image_proxy')) + try: + savesearch = int(request.cookies.get('savesearch', 1)) + except ValueError: + savesearch = 1 if request.cookies.get('language')\ and request.cookies['language'] in (x[0] for x in language_codes): @@ -471,6 +477,8 @@ def preferences(): locale = None autocomplete = '' method = 'POST' + safesearch = '1' + for pd_name, pd in request.form.items(): if pd_name.startswith('category_'): category = pd_name[9:] @@ -489,6 +497,8 @@ def preferences(): lang = pd elif pd_name == 'method': method = pd + elif pd_name == 'safesearch': + safesearch = pd elif pd_name.startswith('engine_'): if pd_name.find('__') > -1: engine_name, category = pd_name.replace('engine_', '', 1).split('__', 1) @@ -529,6 +539,8 @@ def preferences(): ) resp.set_cookie('method', method, max_age=cookie_max_age) + + resp.set_cookie('safesearch', safesearch, max_age=cookie_max_age) resp.set_cookie('image_proxy', image_proxy, max_age=cookie_max_age) From 832ea60e3d19add2ffa857bc420ff33af83aac5c Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sun, 8 Feb 2015 22:01:24 +0100 Subject: [PATCH 2/4] [fix] pep8, tests --- searx/engines/bing_images.py | 1 + searx/engines/blekko_images.py | 1 + searx/tests/engines/test_bing_images.py | 1 + searx/tests/engines/test_blekko_images.py | 1 + searx/webapp.py | 6 +----- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index 4798d9f3..b8c61c15 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -33,6 +33,7 @@ safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'} + # do search-request def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 diff --git a/searx/engines/blekko_images.py b/searx/engines/blekko_images.py index 4fbb9b30..2e7ec904 100644 --- a/searx/engines/blekko_images.py +++ b/searx/engines/blekko_images.py @@ -25,6 +25,7 @@ safesearch_types = {2: '1', 1: '', 0: '0'} + # do search-request def request(query, params): c = (params['pageno'] - 1) * 48 diff --git a/searx/tests/engines/test_bing_images.py b/searx/tests/engines/test_bing_images.py index 59c13462..a1d96b06 100644 --- a/searx/tests/engines/test_bing_images.py +++ b/searx/tests/engines/test_bing_images.py @@ -12,6 +12,7 @@ class TestBingImagesEngine(SearxTestCase): dicto = defaultdict(dict) dicto['pageno'] = 1 dicto['language'] = 'fr_FR' + dicto['safesearch'] = 1 params = bing_images.request(query, dicto) self.assertTrue('url' in params) self.assertTrue(query in params['url']) diff --git a/searx/tests/engines/test_blekko_images.py b/searx/tests/engines/test_blekko_images.py index 6a5388aa..793fadba 100644 --- a/searx/tests/engines/test_blekko_images.py +++ b/searx/tests/engines/test_blekko_images.py @@ -10,6 +10,7 @@ class TestBlekkoImagesEngine(SearxTestCase): query = 'test_query' dicto = defaultdict(dict) dicto['pageno'] = 0 + dicto['safesearch'] = 1 params = blekko_images.request(query, dicto) self.assertTrue('url' in params) self.assertTrue(query in params['url']) diff --git a/searx/webapp.py b/searx/webapp.py index 7266de0f..861f3aa2 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -457,10 +457,6 @@ def preferences(): Settings that are going to be saved as cookies.""" lang = None image_proxy = request.cookies.get('image_proxy', settings['server'].get('image_proxy')) - try: - savesearch = int(request.cookies.get('savesearch', 1)) - except ValueError: - savesearch = 1 if request.cookies.get('language')\ and request.cookies['language'] in (x[0] for x in language_codes): @@ -539,7 +535,7 @@ def preferences(): ) resp.set_cookie('method', method, max_age=cookie_max_age) - + resp.set_cookie('safesearch', safesearch, max_age=cookie_max_age) resp.set_cookie('image_proxy', image_proxy, max_age=cookie_max_age) From 10666fd7c03ac3e9ba6507d36f04181fb90ae466 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sun, 8 Feb 2015 22:15:25 +0100 Subject: [PATCH 3/4] [enh] add safesearch to google_images --- searx/engines/google_images.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index 092ae663..31ad7e08 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -15,18 +15,25 @@ from json import loads # engine dependent config categories = ['images'] paging = True +safesearch = True # search-url url = 'https://ajax.googleapis.com/' -search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe=off&filter=off&{query}' +search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe={safesearch}&filter=off&{query}' # do search-request def request(query, params): offset = (params['pageno'] - 1) * 8 + if params['safesearch'] == 2: + safesearch = 'on' + else: + safesearch = 'off' + params['url'] = search_url.format(query=urlencode({'q': query}), - offset=offset) + offset=offset, + safesearch=safesearch) return params From 7ac6361b51199ce8b208dcd6e8ba62ec4f652186 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sun, 8 Feb 2015 22:29:26 +0100 Subject: [PATCH 4/4] [enh] set google safesearch filter more restictive --- searx/engines/google_images.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index 31ad7e08..1c0e62f5 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -26,10 +26,10 @@ search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&s def request(query, params): offset = (params['pageno'] - 1) * 8 - if params['safesearch'] == 2: - safesearch = 'on' - else: + if params['safesearch'] == 0: safesearch = 'off' + else: + safesearch = 'on' params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset,