Merge pull request #2641 from dalf/disable_http_by_default

[mod] by default allow only HTTPS, not HTTP
This commit is contained in:
Alexandre Flament 2021-03-12 19:21:46 +01:00 committed by GitHub
commit a1a492baed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 22 additions and 36 deletions

View File

@ -58,6 +58,8 @@ argument type information
name string name of search-engine name string name of search-engine
engine string name of searx-engine engine string name of searx-engine
(filename without ``.py``) (filename without ``.py``)
enable_http bool enable HTTP
(by default only HTTPS is enabled).
shortcut string shortcut of search-engine shortcut string shortcut of search-engine
timeout string specific timeout for search-engine timeout string specific timeout for search-engine
display_error_messages boolean display error messages on the web UI display_error_messages boolean display error messages on the web UI

View File

@ -50,6 +50,7 @@ engine_default_args = {'paging': False,
'timeout': settings['outgoing']['request_timeout'], 'timeout': settings['outgoing']['request_timeout'],
'shortcut': '-', 'shortcut': '-',
'disabled': False, 'disabled': False,
'enable_http': False,
'suspend_end_time': 0, 'suspend_end_time': 0,
'continuous_errors': 0, 'continuous_errors': 0,
'time_range_support': False, 'time_range_support': False,
@ -305,35 +306,3 @@ def initialize_engines(engine_list):
if init_fn: if init_fn:
logger.debug('%s engine: Starting background initialization', engine_name) logger.debug('%s engine: Starting background initialization', engine_name)
threading.Thread(target=engine_init, args=(engine_name, init_fn)).start() threading.Thread(target=engine_init, args=(engine_name, init_fn)).start()
_set_https_support_for_engine(engine)
def _set_https_support_for_engine(engine):
# check HTTPS support if it is not disabled
if engine.engine_type != 'offline' and not hasattr(engine, 'https_support'):
params = engine.request('http_test', {
'method': 'GET',
'headers': {},
'data': {},
'url': '',
'cookies': {},
'verify': True,
'auth': None,
'pageno': 1,
'time_range': None,
'language': '',
'safesearch': False,
'is_test': True,
'category': 'files',
'raise_for_status': True,
'engine_data': {},
})
if 'url' not in params:
return
parsed_url = urlparse(params['url'])
https_support = parsed_url.scheme == 'https'
setattr(engine, 'https_support', https_support)

View File

@ -91,9 +91,10 @@ class SessionSinglePool(requests.Session):
self.adapters.clear() self.adapters.clear()
https_adapter = threadLocal.__dict__.setdefault('https_adapter', next(https_adapters)) https_adapter = threadLocal.__dict__.setdefault('https_adapter', next(https_adapters))
http_adapter = threadLocal.__dict__.setdefault('http_adapter', next(http_adapters))
self.mount('https://', https_adapter) self.mount('https://', https_adapter)
self.mount('http://', http_adapter) if get_enable_http_protocol():
http_adapter = threadLocal.__dict__.setdefault('http_adapter', next(http_adapters))
self.mount('http://', http_adapter)
def close(self): def close(self):
"""Call super, but clear adapters since there are managed globaly""" """Call super, but clear adapters since there are managed globaly"""
@ -106,6 +107,17 @@ def set_timeout_for_thread(timeout, start_time=None):
threadLocal.start_time = start_time threadLocal.start_time = start_time
def set_enable_http_protocol(enable_http):
threadLocal.enable_http = enable_http
def get_enable_http_protocol():
try:
return threadLocal.enable_http
except AttributeError:
return False
def reset_time_for_thread(): def reset_time_for_thread():
threadLocal.total_time = 0 threadLocal.total_time = 0

View File

@ -131,6 +131,8 @@ class OnlineProcessor(EngineProcessor):
poolrequests.set_timeout_for_thread(timeout_limit, start_time=start_time) poolrequests.set_timeout_for_thread(timeout_limit, start_time=start_time)
# reset the HTTP total time # reset the HTTP total time
poolrequests.reset_time_for_thread() poolrequests.reset_time_for_thread()
# enable HTTP only if explicitly enabled
poolrequests.set_enable_http_protocol(self.engine.enable_http)
# suppose everything will be alright # suppose everything will be alright
requests_exception = False requests_exception = False

View File

@ -656,6 +656,7 @@ engines:
- name : library genesis - name : library genesis
engine : xpath engine : xpath
enable_http: True
search_url : http://libgen.rs/search.php?req={query} search_url : http://libgen.rs/search.php?req={query}
url_xpath : //a[contains(@href,"bookfi.net/md5")]/@href url_xpath : //a[contains(@href,"bookfi.net/md5")]/@href
title_xpath : //a[contains(@href,"book/")]/text()[1] title_xpath : //a[contains(@href,"book/")]/text()[1]

View File

@ -230,7 +230,7 @@
<td class="onoff-checkbox"> <td class="onoff-checkbox">
{{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in disabled_engines) }} {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in disabled_engines) }}
</td> </td>
<th scope="row">{% if not search_engine.https_support %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</td></th> <th scope="row">{% if search_engine.enable_http %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</td></th>
<td class="name">{{ shortcuts[search_engine.name] }} <td class="name">{{ shortcuts[search_engine.name] }}
<td>{{ support_toggle(stats[search_engine.name].supports_selected_language) }}</td> <td>{{ support_toggle(stats[search_engine.name].supports_selected_language) }}</td>
<td>{{ support_toggle(search_engine.safesearch==True) }}</td> <td>{{ support_toggle(search_engine.safesearch==True) }}</td>

View File

@ -121,7 +121,7 @@
{% set engine_id = 'engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_') %} {% set engine_id = 'engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_') %}
<tr> <tr>
<td class="engine_checkbox">{{ checkbox_onoff(engine_id, (search_engine.name, categ) in disabled_engines) }}</td> <td class="engine_checkbox">{{ checkbox_onoff(engine_id, (search_engine.name, categ) in disabled_engines) }}</td>
<th class="name">{% if not search_engine.https_support %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</th> <th class="name">{% if search_engine.enable_http %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</th>
<td class="shortcut">{{ shortcuts[search_engine.name] }}</td> <td class="shortcut">{{ shortcuts[search_engine.name] }}</td>
<td>{{ checkbox(engine_id + '_supported_languages', current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages, true, true) }}</td> <td>{{ checkbox(engine_id + '_supported_languages', current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages, true, true) }}</td>
<td>{{ checkbox(engine_id + '_safesearch', search_engine.safesearch==True, true, true) }}</td> <td>{{ checkbox(engine_id + '_safesearch', search_engine.safesearch==True, true, true) }}</td>