From ca76f3119a4d66cb8aa74829ca5f0a0a72f0f96b Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 17 Jan 2021 16:11:26 +0100 Subject: [PATCH 1/3] [fix] error_recorder: record code and lineno about the engine since the PR #2225 , code and lineno were sometimes meaningless see /stats/errors --- searx/metrology/error_recorder.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/searx/metrology/error_recorder.py b/searx/metrology/error_recorder.py index 65dbf33c..bd8404ad 100644 --- a/searx/metrology/error_recorder.py +++ b/searx/metrology/error_recorder.py @@ -51,15 +51,12 @@ def add_error_context(engine_name: str, error_context: ErrorContext) -> None: def get_trace(traces): - previous_trace = traces[-1] for trace in reversed(traces): - if trace.filename.endswith('searx/search.py'): - if previous_trace.filename.endswith('searx/poolrequests.py'): - return trace - if previous_trace.filename.endswith('requests/models.py'): - return trace - return previous_trace - previous_trace = trace + split_filename = trace.filename.split('/') + if len(split_filename) > 3 and '/'.join(split_filename[-3:-1]) == 'searx/engines': + return trace + if len(split_filename) > 3 and '/'.join(split_filename[-4:-1]) == 'searx/search/processors': + return trace return traces[-1] From d473407ec97107022ccbdd366559001b1ec162d8 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 17 Jan 2021 16:14:16 +0100 Subject: [PATCH 2/3] [fix] checker: fix engine statistics Without this commit, the URL /stats/errors shows percentage above 100% after the checker has run. --- searx/metrology/error_recorder.py | 4 ++-- searx/search/checker/impl.py | 3 +++ searx/search/processors/abstract.py | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/searx/metrology/error_recorder.py b/searx/metrology/error_recorder.py index bd8404ad..f533e4e8 100644 --- a/searx/metrology/error_recorder.py +++ b/searx/metrology/error_recorder.py @@ -53,9 +53,9 @@ def add_error_context(engine_name: str, error_context: ErrorContext) -> None: def get_trace(traces): for trace in reversed(traces): split_filename = trace.filename.split('/') - if len(split_filename) > 3 and '/'.join(split_filename[-3:-1]) == 'searx/engines': + if '/'.join(split_filename[-3:-1]) == 'searx/engines': return trace - if len(split_filename) > 3 and '/'.join(split_filename[-4:-1]) == 'searx/search/processors': + if '/'.join(split_filename[-4:-1]) == 'searx/search/processors': return trace return traces[-1] diff --git a/searx/search/checker/impl.py b/searx/search/checker/impl.py index 71a941f7..244536f1 100644 --- a/searx/search/checker/impl.py +++ b/searx/search/checker/impl.py @@ -4,6 +4,7 @@ import typing import types import functools import itertools +import threading from time import time from urllib.parse import urlparse @@ -377,6 +378,8 @@ class Checker: engineref_category = search_query.engineref_list[0].category params = self.processor.get_params(search_query, engineref_category) if params is not None: + with threading.RLock(): + self.processor.engine.stats['sent_search_count'] += 1 self.processor.search(search_query.query, params, result_container, time(), 5) return result_container diff --git a/searx/search/processors/abstract.py b/searx/search/processors/abstract.py index eb8d296e..3a853d49 100644 --- a/searx/search/processors/abstract.py +++ b/searx/search/processors/abstract.py @@ -1,13 +1,13 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -from abc import abstractmethod +from abc import abstractmethod, ABC from searx import logger logger = logger.getChild('searx.search.processor') -class EngineProcessor: +class EngineProcessor(ABC): def __init__(self, engine, engine_name): self.engine = engine From 67a1aab0d5b8ab93b60be5da2390039f6c861505 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 17 Jan 2021 16:56:36 +0100 Subject: [PATCH 3/3] [fix] /stats/checker : remove the timestamp field when the checker is disabled --- searx/search/checker/background.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/search/checker/background.py b/searx/search/checker/background.py index e41bff5f..22a51b8b 100644 --- a/searx/search/checker/background.py +++ b/searx/search/checker/background.py @@ -98,7 +98,7 @@ def initialize(): signal.signal(signal.SIGUSR1, _signal_handler) # disabled by default - _set_result({'status': 'disabled'}) + _set_result({'status': 'disabled'}, include_timestamp=False) # special case when debug is activate if searx_debug and settings.get('checker', {}).get('off_when_debug', True):