From b8cd3264644208d7afa1a239f829222d45226334 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 25 Feb 2021 17:42:52 +0100 Subject: [PATCH] Add searx_extra package Split the utils directory into: * searx_extra contains update scripts, standalone_searx.py * utils contains the files to build and setup searx. --- .github/workflows/data-update.yml | 16 +++++++-------- Makefile | 4 ++-- docs/index.rst | 1 + docs/searx_extra/index.rst | 14 +++++++++++++ docs/searx_extra/standalone_searx.py.rst | 9 +++++++++ docs/utils/index.rst | 10 ++++------ docs/utils/standalone_searx.py.rst | 11 ---------- searx_extra/__init__.py | 0 {utils => searx_extra}/google_search.py | 0 {utils => searx_extra}/standalone_searx.py | 2 +- searx_extra/update/__init__.py | 0 .../update/update_ahmia_blacklist.py | 5 +---- .../update/update_currencies.py | 1 - .../update/update_engine_descriptions.py | 6 +----- .../update/update_external_bangs.py | 6 +----- .../update/update_firefox_version.py | 7 +------ .../update/update_languages.py | 4 +--- .../update/update_translations.sh | 0 .../update/update_wikidata_units.py | 4 +--- setup.py | 2 +- tests/unit/test_standalone_searx.py | 20 +------------------ 21 files changed, 47 insertions(+), 75 deletions(-) create mode 100644 docs/searx_extra/index.rst create mode 100644 docs/searx_extra/standalone_searx.py.rst delete mode 100644 docs/utils/standalone_searx.py.rst create mode 100644 searx_extra/__init__.py rename {utils => searx_extra}/google_search.py (100%) rename {utils => searx_extra}/standalone_searx.py (99%) create mode 100644 searx_extra/update/__init__.py rename utils/fetch_ahmia_blacklist.py => searx_extra/update/update_ahmia_blacklist.py (84%) rename utils/fetch_currencies.py => searx_extra/update/update_currencies.py (98%) mode change 100644 => 100755 rename utils/fetch_engine_descriptions.py => searx_extra/update/update_engine_descriptions.py (98%) mode change 100644 => 100755 rename utils/fetch_external_bangs.py => searx_extra/update/update_external_bangs.py (97%) rename utils/fetch_firefox_version.py => searx_extra/update/update_firefox_version.py (93%) rename utils/fetch_languages.py => searx_extra/update/update_languages.py (99%) mode change 100644 => 100755 rename utils/update-translations.sh => searx_extra/update/update_translations.sh (100%) rename utils/fetch_wikidata_units.py => searx_extra/update/update_wikidata_units.py (92%) mode change 100644 => 100755 diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index c9c6b29a..eb9bed8c 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -11,12 +11,12 @@ jobs: strategy: matrix: fetch: - - ahmia_blacklist - - currencies - - external_bangs - - firefox_version - - languages - - wikidata_units + - update_ahmia_blacklist.py + - update_currencies.py + - update_external_bangs.py + - update_firefox_version.py + - update_languages.py + - update_wikidata_units.py steps: - name: Checkout uses: actions/checkout@v2 @@ -45,10 +45,10 @@ jobs: - name: Fetch data env: - FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py + FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }} run: | source local/py3/bin/activate - python $FETCH_SCRIPT + $FETCH_SCRIPT - name: Create Pull Request id: cpr diff --git a/Makefile b/Makefile index 4e451b7c..9917da78 100644 --- a/Makefile +++ b/Makefile @@ -195,8 +195,8 @@ PYLINT_FILES=\ searx/engines/google_videos.py \ searx/engines/google_images.py \ searx/engines/mediathekviewweb.py \ - utils/fetch_external_bangs.py \ - searx/engines/google_scholar.py + searx/engines/google_scholar.py \ + searx_extra/update/update_external_bangs.py test.pylint: pyenvinstall $(call cmd,pylint,$(PYLINT_FILES)) diff --git a/docs/index.rst b/docs/index.rst index 9e590867..a406da19 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`. user/index admin/index dev/index + searx_extra/index utils/index blog/index diff --git a/docs/searx_extra/index.rst b/docs/searx_extra/index.rst new file mode 100644 index 00000000..64d0b904 --- /dev/null +++ b/docs/searx_extra/index.rst @@ -0,0 +1,14 @@ +.. _searx_extra: + +====================================================== +Tooling box ``searx_extra`` for developers and users +====================================================== + +In the folder :origin:`searx_extra/` we maintain some tools useful for +developers and users. + +.. toctree:: + :maxdepth: 2 + :caption: Contents + + standalone_searx.py diff --git a/docs/searx_extra/standalone_searx.py.rst b/docs/searx_extra/standalone_searx.py.rst new file mode 100644 index 00000000..ff4b5338 --- /dev/null +++ b/docs/searx_extra/standalone_searx.py.rst @@ -0,0 +1,9 @@ + +.. _standalone_searx.py: + +=================================== +``searx_extra/standalone_searx.py`` +=================================== + +.. automodule:: searx_extra.standalone_searx + :members: diff --git a/docs/utils/index.rst b/docs/utils/index.rst index 28515318..32baa570 100644 --- a/docs/utils/index.rst +++ b/docs/utils/index.rst @@ -1,12 +1,11 @@ .. _searx_utils: .. _toolboxing: -======================= -Tooling box ``utils/*`` -======================= +======================================== +Tooling box ``utils`` for administrators +======================================== -In the folder :origin:`utils/` we maintain some tools useful for admins and -developers. +In the folder :origin:`utils/` we maintain some tools useful for administrators. .. toctree:: :maxdepth: 2 @@ -16,7 +15,6 @@ developers. filtron.sh morty.sh lxc.sh - standalone_searx.py .. _toolboxing common: diff --git a/docs/utils/standalone_searx.py.rst b/docs/utils/standalone_searx.py.rst deleted file mode 100644 index 557c4b75..00000000 --- a/docs/utils/standalone_searx.py.rst +++ /dev/null @@ -1,11 +0,0 @@ - -.. _standalone_searx.py: - -============================= -``utils/standalone_searx.py`` -============================= - -.. automodule:: standalone_searx - :members: - - diff --git a/searx_extra/__init__.py b/searx_extra/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/google_search.py b/searx_extra/google_search.py similarity index 100% rename from utils/google_search.py rename to searx_extra/google_search.py diff --git a/utils/standalone_searx.py b/searx_extra/standalone_searx.py similarity index 99% rename from utils/standalone_searx.py rename to searx_extra/standalone_searx.py index 89023f41..f52b7e80 100755 --- a/utils/standalone_searx.py +++ b/searx_extra/standalone_searx.py @@ -15,7 +15,7 @@ Example to use this script: .. code:: bash - $ python3 utils/standalone_searx.py rain + $ python3 searx_extra/standalone_searx.py rain Example to run it from python: diff --git a/searx_extra/update/__init__.py b/searx_extra/update/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/fetch_ahmia_blacklist.py b/searx_extra/update/update_ahmia_blacklist.py similarity index 84% rename from utils/fetch_ahmia_blacklist.py rename to searx_extra/update/update_ahmia_blacklist.py index 3e393edb..f645880e 100755 --- a/utils/fetch_ahmia_blacklist.py +++ b/searx_extra/update/update_ahmia_blacklist.py @@ -4,11 +4,8 @@ # More info in https://ahmia.fi/blacklist/ # set path -from sys import path -from os.path import realpath, dirname, join -path.append(realpath(dirname(realpath(__file__)) + '/../')) +from os.path import join -# import requests from searx import searx_dir diff --git a/utils/fetch_currencies.py b/searx_extra/update/update_currencies.py old mode 100644 new mode 100755 similarity index 98% rename from utils/fetch_currencies.py rename to searx_extra/update/update_currencies.py index 8811049a..0cfb7a95 --- a/utils/fetch_currencies.py +++ b/searx_extra/update/update_currencies.py @@ -7,7 +7,6 @@ import json # set path from sys import path from os.path import realpath, dirname, join -path.append(realpath(dirname(realpath(__file__)) + '/../')) from searx import searx_dir, settings from searx.engines.wikidata import send_wikidata_query diff --git a/utils/fetch_engine_descriptions.py b/searx_extra/update/update_engine_descriptions.py old mode 100644 new mode 100755 similarity index 98% rename from utils/fetch_engine_descriptions.py rename to searx_extra/update/update_engine_descriptions.py index 9ca001d4..109fdbfa --- a/utils/fetch_engine_descriptions.py +++ b/searx_extra/update/update_engine_descriptions.py @@ -3,13 +3,9 @@ import sys import json from urllib.parse import quote, urlparse -from os.path import realpath, dirname -import cld3 +import detect_language from lxml.html import fromstring -# set path -sys.path.append(realpath(dirname(realpath(__file__)) + '/../')) - from searx.engines.wikidata import send_wikidata_query from searx.utils import extract_text import searx diff --git a/utils/fetch_external_bangs.py b/searx_extra/update/update_external_bangs.py similarity index 97% rename from utils/fetch_external_bangs.py rename to searx_extra/update/update_external_bangs.py index ba6f51e7..e9dc0ff1 100755 --- a/utils/fetch_external_bangs.py +++ b/searx_extra/update/update_external_bangs.py @@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION """ # pylint: disable=C0116 -import sys import json import re -from os.path import realpath, dirname, join +from os.path import join import requests -# set path -sys.path.append(realpath(dirname(realpath(__file__)) + '/../')) - from searx import searx_dir # pylint: disable=E0401 C0413 diff --git a/utils/fetch_firefox_version.py b/searx_extra/update/update_firefox_version.py similarity index 93% rename from utils/fetch_firefox_version.py rename to searx_extra/update/update_firefox_version.py index 997a752b..6acfe76c 100755 --- a/utils/fetch_firefox_version.py +++ b/searx_extra/update/update_firefox_version.py @@ -1,14 +1,9 @@ #!/usr/bin/env python -# set path -from sys import path -from os.path import realpath, dirname, join -path.append(realpath(dirname(realpath(__file__)) + '/../')) - -# import json import requests import re +from os.path import dirname, join from urllib.parse import urlparse, urljoin from distutils.version import LooseVersion, StrictVersion from lxml import html diff --git a/utils/fetch_languages.py b/searx_extra/update/update_languages.py old mode 100644 new mode 100755 similarity index 99% rename from utils/fetch_languages.py rename to searx_extra/update/update_languages.py index 582e0ae0..e6328258 --- a/utils/fetch_languages.py +++ b/searx_extra/update/update_languages.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +#!/usr/bin/env python # This script generates languages.py from intersecting each engine's supported languages. # @@ -7,11 +7,9 @@ import json from pathlib import Path from pprint import pformat -from sys import path from babel import Locale, UnknownLocaleError from babel.languages import get_global -path.append('../searx') # noqa from searx import settings, searx_dir from searx.engines import initialize_engines, engines diff --git a/utils/update-translations.sh b/searx_extra/update/update_translations.sh similarity index 100% rename from utils/update-translations.sh rename to searx_extra/update/update_translations.sh diff --git a/utils/fetch_wikidata_units.py b/searx_extra/update/update_wikidata_units.py old mode 100644 new mode 100755 similarity index 92% rename from utils/fetch_wikidata_units.py rename to searx_extra/update/update_wikidata_units.py index 69ae8ab2..1e6b8b9c --- a/utils/fetch_wikidata_units.py +++ b/searx_extra/update/update_wikidata_units.py @@ -4,9 +4,7 @@ import json import collections # set path -from sys import path -from os.path import realpath, dirname, join -path.append(realpath(dirname(realpath(__file__)) + '/../')) +from os.path import join from searx import searx_dir from searx.engines.wikidata import send_wikidata_query diff --git a/setup.py b/setup.py index 09a3021e..61227d19 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setup( author='Adam Tauber', author_email='asciimoo@gmail.com', license='GNU Affero General Public License', - packages=find_packages(exclude=["tests*"]), + packages=find_packages(exclude=["tests*", "searx_extra"]), zip_safe=False, install_requires=requirements, extras_require={ diff --git a/tests/unit/test_standalone_searx.py b/tests/unit/test_standalone_searx.py index 6cc230e6..a69353c0 100644 --- a/tests/unit/test_standalone_searx.py +++ b/tests/unit/test_standalone_searx.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Test utils/standalone_searx.py""" import datetime -import importlib.util import io import sys @@ -10,16 +9,7 @@ from nose2.tools import params from searx.search import SearchQuery, EngineRef, initialize from searx.testing import SearxTestCase - - -def get_standalone_searx_module(): - """Get standalone_searx module.""" - module_name = 'utils.standalone_searx' - filename = 'utils/standalone_searx.py' - spec = importlib.util.spec_from_file_location(module_name, filename) - sas = importlib.util.module_from_spec(spec) - spec.loader.exec_module(sas) - return sas +from searx_extra import standalone_searx as sas class StandaloneSearx(SearxTestCase): @@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase): def test_parse_argument_no_args(self): """Test parse argument without args.""" - sas = get_standalone_searx_module() with patch.object(sys, 'argv', ['standalone_searx']), \ self.assertRaises(SystemExit): sys.stderr = io.StringIO() @@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase): def test_parse_argument_basic_args(self): """Test parse argument with basic args.""" - sas = get_standalone_searx_module() query = 'red box' exp_dict = { 'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1, @@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase): def test_to_dict(self): """test to_dict.""" - sas = get_standalone_searx_module() self.assertEqual( sas.to_dict( sas.get_search_query(sas.parse_argument(['red box']))), @@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase): def test_to_dict_with_mock(self): """test to dict.""" - sas = get_standalone_searx_module() with patch.object(sas.searx.search, 'Search') as mock_s: m_search = mock_s().search() m_sq = Mock() @@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase): def test_get_search_query(self): """test get_search_query.""" - sas = get_standalone_searx_module() args = sas.parse_argument(['rain', ]) search_q = sas.get_search_query(args) self.assertTrue(search_q) @@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase): def test_no_parsed_url(self): """test no_parsed_url func""" - sas = get_standalone_searx_module() self.assertEqual( sas.no_parsed_url([{'parsed_url': 'http://example.com'}]), [{}] @@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase): ) def test_json_serial(self, arg, exp_res): """test json_serial func""" - sas = get_standalone_searx_module() self.assertEqual(sas.json_serial(arg), exp_res) def test_json_serial_error(self): """test error on json_serial.""" - sas = get_standalone_searx_module() with self.assertRaises(TypeError): sas.json_serial('a')