From b786ac9a6360cc50472d572148343dc84b908830 Mon Sep 17 00:00:00 2001 From: Thomas Renard Date: Mon, 18 Jun 2018 07:53:06 +0200 Subject: [PATCH 1/6] Create an include for __common__/aboutextend.html for imprint purposes. --- searx/templates/__common__/about.html | 1 + 1 file changed, 1 insertion(+) diff --git a/searx/templates/__common__/about.html b/searx/templates/__common__/about.html index d8afab73..bf173335 100644 --- a/searx/templates/__common__/about.html +++ b/searx/templates/__common__/about.html @@ -60,3 +60,4 @@ Searx can be added to your browser's search bar; moreover, it can be set as the

Stats page contains some useful data about the engines used.

+{% include "__common__/aboutextend.html" ignore missing %} From 42d7094a87773447e70db350bc67e0b62b82ff02 Mon Sep 17 00:00:00 2001 From: Angristan Date: Sun, 19 Aug 2018 14:17:11 +0200 Subject: [PATCH 2/6] Update Docker image to Alpine 3.8 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 39d56edb..bf599a0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.7 +FROM alpine:3.8 MAINTAINER searx LABEL description "A privacy-respecting, hackable metasearch engine." From baacfac32b63ceb3ef784e3ff5f1b3ae51ee38ff Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 19 Aug 2018 15:06:05 +0200 Subject: [PATCH 3/6] [fix] remove publicly unavailable 500px engine - #1338 --- searx/engines/www500px.py | 73 ----------------------------- searx/settings.yml | 4 -- tests/unit/engines/test_www500px.py | 34 -------------- 3 files changed, 111 deletions(-) delete mode 100644 searx/engines/www500px.py delete mode 100644 tests/unit/engines/test_www500px.py diff --git a/searx/engines/www500px.py b/searx/engines/www500px.py deleted file mode 100644 index 7a2015ae..00000000 --- a/searx/engines/www500px.py +++ /dev/null @@ -1,73 +0,0 @@ -""" - 500px (Images) - - @website https://500px.com - @provide-api yes (https://developers.500px.com/) - - @using-api no - @results HTML - @stable no (HTML can change) - @parse url, title, thumbnail, img_src, content - - @todo rewrite to api -""" - -from json import loads -from searx.url_utils import urlencode, urljoin - -# engine dependent config -categories = ['images'] -paging = True - -# search-url -base_url = 'https://500px.com' -search_url = 'https://api.500px.com/v1/photos/search?type=photos'\ - '&{query}'\ - '&image_size%5B%5D=4'\ - '&image_size%5B%5D=20'\ - '&image_size%5B%5D=21'\ - '&image_size%5B%5D=1080'\ - '&image_size%5B%5D=1600'\ - '&image_size%5B%5D=2048'\ - '&include_states=true'\ - '&formats=jpeg%2Clytro'\ - '&include_tags=true'\ - '&exclude_nude=true'\ - '&page={pageno}'\ - '&rpp=50'\ - '&sdk_key=b68e60cff4c929bedea36ca978830c5caca790c3' - - -# do search-request -def request(query, params): - params['url'] = search_url.format(pageno=params['pageno'], - query=urlencode({'term': query})) - - return params - - -# get response from search-request -def response(resp): - results = [] - - response_json = loads(resp.text) - - # parse results - for result in response_json['photos']: - url = urljoin(base_url, result['url']) - title = result['name'] - # last index is the biggest resolution - img_src = result['image_url'][-1] - thumbnail_src = result['image_url'][0] - content = result['description'] or '' - - # append result - results.append({'url': url, - 'title': title, - 'img_src': img_src, - 'content': content, - 'thumbnail_src': thumbnail_src, - 'template': 'images.html'}) - - # return results - return results diff --git a/searx/settings.yml b/searx/settings.yml index 1168ad7a..e05d5f97 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -204,10 +204,6 @@ engines: shortcut : fa disabled : True - - name : 500px - engine : www500px - shortcut : px - - name : 1x engine : www1x shortcut : 1x diff --git a/tests/unit/engines/test_www500px.py b/tests/unit/engines/test_www500px.py deleted file mode 100644 index e50601da..00000000 --- a/tests/unit/engines/test_www500px.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -from collections import defaultdict -import mock -from searx.engines import www500px -from searx.testing import SearxTestCase - - -class TestWww500pxImagesEngine(SearxTestCase): - - def test_request(self): - query = 'test_query' - dicto = defaultdict(dict) - dicto['pageno'] = 1 - params = www500px.request(query, dicto) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('500px.com' in params['url']) - - def test_response(self): - self.assertRaises(AttributeError, www500px.response, None) - self.assertRaises(AttributeError, www500px.response, []) - self.assertRaises(AttributeError, www500px.response, '') - self.assertRaises(AttributeError, www500px.response, '[]') - - json = """ -{"current_page":1,"total_pages":1000,"total_items":862178,"photos":[{"id":64531569,"user_id":111147,"name":"Grand Canyon Afternoon","description":"Looking west on a very windy winter afternoon.","camera":"Canon EOS 5D Mark II","lens":"EF24-105mm f/4L IS USM","focal_length":"28","iso":"200","shutter_speed":"1/1250","aperture":"6.3","times_viewed":4809,"rating":48.5,"status":1,"created_at":"2014-03-22T03:44:46-04:00","category":8,"location":null,"latitude":36.0323916666667,"longitude":-111.85273,"taken_at":"2014-02-27T14:10:43-05:00","hi_res_uploaded":2,"for_sale":true,"width":5476,"height":3651,"votes_count":108,"favorites_count":35,"comments_count":5,"nsfw":false,"sales_count":0,"for_sale_date":null,"highest_rating":91.9,"highest_rating_date":"2014-03-22T22:34:54-04:00","license_type":0,"converted":31,"collections_count":10,"crop_version":0,"privacy":false,"profile":true,"image_url":["https://drscdn.500px.org/photo/64531569/w%3D70_h%3D70/449d50817f28d85395e23bbb415b3cdb?v=0","https://drscdn.500px.org/photo/64531569/q%3D50_w%3D140_h%3D140/3e3e123734a596644ede78105268bdb2?v=0","https://drscdn.500px.org/photo/64531569/q%3D80_h%3D300/2ce2f61714aebdca710967dfdc3efb04","https://drscdn.500px.org/photo/64531569/q%3D80_h%3D450/c8ec030441f2c68b9bd40a114903348a","https://drscdn.500px.org/photo/64531569/q%3D80_h%3D600/ab6562d0581b359679ecc8ef2e939396","https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1000/bd7dbc54a505e041a8c9a70dfa434272","https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1500/eb4d7f8f6a32d3e5c168c2cb55d29c12","https://drscdn.500px.org/photo/64531569/q%3D80_m%3D2000/d519f91b8a568e7357a8a7fa1aabbe74","https://drscdn.500px.org/photo/64531569/m%3D2048/4c52fb18cc2b2b6f91a0d04609786507","https://drscdn.500px.org/photo/64531569/m%3D900/fb620ae39569ab4a421e9170a94b1a0f","https://drscdn.500px.org/photo/64531569/m%3D900_s%3D1_k%3D1_a%3D1/02b95ce64db090c1f94f890960974612?v=0"],"images":[{"size":1,"url":"https://drscdn.500px.org/photo/64531569/w%3D70_h%3D70/449d50817f28d85395e23bbb415b3cdb?v=0","https_url":"https://drscdn.500px.org/photo/64531569/w%3D70_h%3D70/449d50817f28d85395e23bbb415b3cdb?v=0","format":"jpeg"},{"size":2,"url":"https://drscdn.500px.org/photo/64531569/q%3D50_w%3D140_h%3D140/3e3e123734a596644ede78105268bdb2?v=0","https_url":"https://drscdn.500px.org/photo/64531569/q%3D50_w%3D140_h%3D140/3e3e123734a596644ede78105268bdb2?v=0","format":"jpeg"},{"size":4,"url":"https://drscdn.500px.org/photo/64531569/m%3D900/fb620ae39569ab4a421e9170a94b1a0f","https_url":"https://drscdn.500px.org/photo/64531569/m%3D900/fb620ae39569ab4a421e9170a94b1a0f","format":"jpeg"},{"size":14,"url":"https://drscdn.500px.org/photo/64531569/m%3D900_s%3D1_k%3D1_a%3D1/02b95ce64db090c1f94f890960974612?v=0","https_url":"https://drscdn.500px.org/photo/64531569/m%3D900_s%3D1_k%3D1_a%3D1/02b95ce64db090c1f94f890960974612?v=0","format":"jpeg"},{"size":31,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D450/c8ec030441f2c68b9bd40a114903348a","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D450/c8ec030441f2c68b9bd40a114903348a","format":"jpeg"},{"size":32,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D300/2ce2f61714aebdca710967dfdc3efb04","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D300/2ce2f61714aebdca710967dfdc3efb04","format":"jpeg"},{"size":33,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D600/ab6562d0581b359679ecc8ef2e939396","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D600/ab6562d0581b359679ecc8ef2e939396","format":"jpeg"},{"size":34,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1000/bd7dbc54a505e041a8c9a70dfa434272","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1000/bd7dbc54a505e041a8c9a70dfa434272","format":"jpeg"},{"size":35,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1500/eb4d7f8f6a32d3e5c168c2cb55d29c12","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1500/eb4d7f8f6a32d3e5c168c2cb55d29c12","format":"jpeg"},{"size":36,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D2000/d519f91b8a568e7357a8a7fa1aabbe74","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D2000/d519f91b8a568e7357a8a7fa1aabbe74","format":"jpeg"},{"size":2048,"url":"https://drscdn.500px.org/photo/64531569/m%3D2048/4c52fb18cc2b2b6f91a0d04609786507","https_url":"https://drscdn.500px.org/photo/64531569/m%3D2048/4c52fb18cc2b2b6f91a0d04609786507","format":"jpeg"}],"url":"/photo/64531569/grand-canyon-afternoon-by-todd-hakala","positive_votes_count":108,"converted_bits":31,"tags":["landscape","river","arizona","canyon","grand","colorado","south","southwest","az","west","rim","CanonGetaway"],"watermark":false,"image_format":"jpeg","licensing_requested":false,"licensing_suggested":false,"is_free_photo":false,"user":{"id":111147,"username":"ToddHakala","firstname":"Todd","lastname":"Hakala","city":"Albuquerque","country":"US","usertype":0,"fullname":"Todd Hakala","userpic_url":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/1.jpg?2","userpic_https_url":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/1.jpg?2","cover_url":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/cover_2048.jpg?8","upgrade_status":1,"store_on":true,"affection":5217,"avatars":{"default":{"https":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/1.jpg?2"},"large":{"https":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/2.jpg?2"},"small":{"https":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/3.jpg?2"},"tiny":{"https":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/4.jpg?2"}},"followers_count":171}}]} - """ # noqa - response = mock.Mock(text=json) - results = www500px.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], u'Grand Canyon Afternoon') - self.assertEqual(results[0]['url'], 'https://500px.com/photo/64531569/grand-canyon-afternoon-by-todd-hakala') - self.assertEqual(results[0]['content'], u'Looking west on a very windy winter afternoon.') From 69d1dd3f862ee491518bc837325a498e66f925d0 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 19 Aug 2018 15:23:11 +0200 Subject: [PATCH 4/6] [enh] configurable default oscar style --- searx/preferences.py | 1 + searx/settings.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/searx/preferences.py b/searx/preferences.py index 5ff70191..fb46edd6 100644 --- a/searx/preferences.py +++ b/searx/preferences.py @@ -264,6 +264,7 @@ class Preferences(object): 'False': False, 'True': True}), 'doi_resolver': MultipleChoiceSetting(['oadoi.org'], choices=DOI_RESOLVERS), + 'oscar-style': EnumStringSetting(settings['ui'].get('theme_args', {}).get('oscar_style', 'logicodev'), choices=['', 'logicodev', 'logicodev-dark', 'pointhi']), } self.engines = EnginesSetting('engines', choices=engines) diff --git a/searx/settings.yml b/searx/settings.yml index e05d5f97..812da747 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -20,6 +20,8 @@ ui: templates_path : "" # Custom templates path - leave it blank if you didn't change default_theme : oscar # ui theme default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section + theme_args : + oscar_style : logicodev-dark # default style of oscar # searx supports result proxification using an external service: https://github.com/asciimoo/morty # uncomment below section if you have running morty proxy From f7bb45e8912a8529e59146afd3b9cf3413d70a67 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 19 Aug 2018 15:29:52 +0200 Subject: [PATCH 5/6] [enh] admin configurable suspend time of errored engines - closes #1266 --- searx/preferences.py | 4 +++- searx/search.py | 3 ++- searx/settings.yml | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/searx/preferences.py b/searx/preferences.py index fb46edd6..ed2cc402 100644 --- a/searx/preferences.py +++ b/searx/preferences.py @@ -264,7 +264,9 @@ class Preferences(object): 'False': False, 'True': True}), 'doi_resolver': MultipleChoiceSetting(['oadoi.org'], choices=DOI_RESOLVERS), - 'oscar-style': EnumStringSetting(settings['ui'].get('theme_args', {}).get('oscar_style', 'logicodev'), choices=['', 'logicodev', 'logicodev-dark', 'pointhi']), + 'oscar-style': EnumStringSetting( + settings['ui'].get('theme_args', {}).get('oscar_style', 'logicodev'), + choices=['', 'logicodev', 'logicodev-dark', 'pointhi']), } self.engines = EnginesSetting('engines', choices=engines) diff --git a/searx/search.py b/searx/search.py index 945f3219..f50391f1 100644 --- a/searx/search.py +++ b/searx/search.py @@ -147,7 +147,8 @@ def search_one_request_safe(engine_name, query, request_params, result_container if requests_exception: # update continuous_errors / suspend_end_time engine.continuous_errors += 1 - engine.suspend_end_time = time() + min(60, engine.continuous_errors) + engine.suspend_end_time = time() + min(settings['search']['max_ban_time_on_fail'], + engine.continuous_errors*settings['search']['ban_time_on_fail']) else: # no HTTP error (perhaps an engine error) # anyway, reset the suspend variables diff --git a/searx/settings.yml b/searx/settings.yml index 812da747..2575000c 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -6,6 +6,8 @@ search: safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default language : "en-US" + ban_time_on_fail : 5 # ban time in seconds after engine errors + max_ban_time_on_fail : 120 # max ban time in seconds after engine errors server: port : 8888 @@ -21,7 +23,7 @@ ui: default_theme : oscar # ui theme default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section theme_args : - oscar_style : logicodev-dark # default style of oscar + oscar_style : logicodev # default style of oscar # searx supports result proxification using an external service: https://github.com/asciimoo/morty # uncomment below section if you have running morty proxy From c5c812899f737082f65d0726fbc0ad21501e09e7 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 19 Aug 2018 15:32:32 +0200 Subject: [PATCH 6/6] [fix] pep8 --- searx/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/search.py b/searx/search.py index f50391f1..950a49f2 100644 --- a/searx/search.py +++ b/searx/search.py @@ -148,7 +148,7 @@ def search_one_request_safe(engine_name, query, request_params, result_container # update continuous_errors / suspend_end_time engine.continuous_errors += 1 engine.suspend_end_time = time() + min(settings['search']['max_ban_time_on_fail'], - engine.continuous_errors*settings['search']['ban_time_on_fail']) + engine.continuous_errors * settings['search']['ban_time_on_fail']) else: # no HTTP error (perhaps an engine error) # anyway, reset the suspend variables