Browse Source

Add searx_extra package

Split the utils directory into:
* searx_extra contains update scripts, standalone_searx.py
* utils contains the files to build and setup searx.
tags/1.0.0
Alexandre Flament 3 months ago
parent
commit
b8cd326464
21 changed files with 47 additions and 75 deletions
  1. +8
    -8
      .github/workflows/data-update.yml
  2. +2
    -2
      Makefile
  3. +1
    -0
      docs/index.rst
  4. +14
    -0
      docs/searx_extra/index.rst
  5. +9
    -0
      docs/searx_extra/standalone_searx.py.rst
  6. +4
    -6
      docs/utils/index.rst
  7. +0
    -11
      docs/utils/standalone_searx.py.rst
  8. +0
    -0
      searx_extra/__init__.py
  9. +0
    -0
      searx_extra/google_search.py
  10. +1
    -1
      searx_extra/standalone_searx.py
  11. +0
    -0
      searx_extra/update/__init__.py
  12. +1
    -4
      searx_extra/update/update_ahmia_blacklist.py
  13. +0
    -1
      searx_extra/update/update_currencies.py
  14. +1
    -5
      searx_extra/update/update_engine_descriptions.py
  15. +1
    -5
      searx_extra/update/update_external_bangs.py
  16. +1
    -6
      searx_extra/update/update_firefox_version.py
  17. +1
    -3
      searx_extra/update/update_languages.py
  18. +0
    -0
      searx_extra/update/update_translations.sh
  19. +1
    -3
      searx_extra/update/update_wikidata_units.py
  20. +1
    -1
      setup.py
  21. +1
    -19
      tests/unit/test_standalone_searx.py

+ 8
- 8
.github/workflows/data-update.yml View File

@@ -11,12 +11,12 @@ jobs:
strategy:
matrix:
fetch:
- ahmia_blacklist
- currencies
- external_bangs
- firefox_version
- languages
- wikidata_units
- update_ahmia_blacklist.py
- update_currencies.py
- update_external_bangs.py
- update_firefox_version.py
- update_languages.py
- update_wikidata_units.py
steps:
- name: Checkout
uses: actions/checkout@v2
@@ -45,10 +45,10 @@ jobs:

- name: Fetch data
env:
FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py
FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
run: |
source local/py3/bin/activate
python $FETCH_SCRIPT
$FETCH_SCRIPT

- name: Create Pull Request
id: cpr


+ 2
- 2
Makefile View File

@@ -195,8 +195,8 @@ PYLINT_FILES=\
searx/engines/google_videos.py \
searx/engines/google_images.py \
searx/engines/mediathekviewweb.py \
utils/fetch_external_bangs.py \
searx/engines/google_scholar.py
searx/engines/google_scholar.py \
searx_extra/update/update_external_bangs.py

test.pylint: pyenvinstall
$(call cmd,pylint,$(PYLINT_FILES))


+ 1
- 0
docs/index.rst View File

@@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`.
user/index
admin/index
dev/index
searx_extra/index
utils/index
blog/index



+ 14
- 0
docs/searx_extra/index.rst View File

@@ -0,0 +1,14 @@
.. _searx_extra:

======================================================
Tooling box ``searx_extra`` for developers and users
======================================================

In the folder :origin:`searx_extra/` we maintain some tools useful for
developers and users.

.. toctree::
:maxdepth: 2
:caption: Contents

standalone_searx.py

+ 9
- 0
docs/searx_extra/standalone_searx.py.rst View File

@@ -0,0 +1,9 @@

.. _standalone_searx.py:

===================================
``searx_extra/standalone_searx.py``
===================================

.. automodule:: searx_extra.standalone_searx
:members:

+ 4
- 6
docs/utils/index.rst View File

@@ -1,12 +1,11 @@
.. _searx_utils:
.. _toolboxing:

=======================
Tooling box ``utils/*``
=======================
========================================
Tooling box ``utils`` for administrators
========================================

In the folder :origin:`utils/` we maintain some tools useful for admins and
developers.
In the folder :origin:`utils/` we maintain some tools useful for administrators.

.. toctree::
:maxdepth: 2
@@ -16,7 +15,6 @@ developers.
filtron.sh
morty.sh
lxc.sh
standalone_searx.py

.. _toolboxing common:



+ 0
- 11
docs/utils/standalone_searx.py.rst View File

@@ -1,11 +0,0 @@

.. _standalone_searx.py:

=============================
``utils/standalone_searx.py``
=============================

.. automodule:: standalone_searx
:members:



+ 0
- 0
searx_extra/__init__.py View File


utils/google_search.py → searx_extra/google_search.py View File


utils/standalone_searx.py → searx_extra/standalone_searx.py View File

@@ -15,7 +15,7 @@ Example to use this script:

.. code:: bash

$ python3 utils/standalone_searx.py rain
$ python3 searx_extra/standalone_searx.py rain

Example to run it from python:


+ 0
- 0
searx_extra/update/__init__.py View File


utils/fetch_ahmia_blacklist.py → searx_extra/update/update_ahmia_blacklist.py View File

@@ -4,11 +4,8 @@
# More info in https://ahmia.fi/blacklist/

# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from os.path import join

#
import requests
from searx import searx_dir


utils/fetch_currencies.py → searx_extra/update/update_currencies.py View File

@@ -7,7 +7,6 @@ import json
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))

from searx import searx_dir, settings
from searx.engines.wikidata import send_wikidata_query

utils/fetch_engine_descriptions.py → searx_extra/update/update_engine_descriptions.py View File

@@ -3,13 +3,9 @@
import sys
import json
from urllib.parse import quote, urlparse
from os.path import realpath, dirname
import cld3
import detect_language
from lxml.html import fromstring

# set path
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))

from searx.engines.wikidata import send_wikidata_query
from searx.utils import extract_text
import searx

utils/fetch_external_bangs.py → searx_extra/update/update_external_bangs.py View File

@@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION
"""
# pylint: disable=C0116

import sys
import json
import re
from os.path import realpath, dirname, join
from os.path import join

import requests

# set path
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))

from searx import searx_dir # pylint: disable=E0401 C0413



utils/fetch_firefox_version.py → searx_extra/update/update_firefox_version.py View File

@@ -1,14 +1,9 @@
#!/usr/bin/env python

# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))

#
import json
import requests
import re
from os.path import dirname, join
from urllib.parse import urlparse, urljoin
from distutils.version import LooseVersion, StrictVersion
from lxml import html

utils/fetch_languages.py → searx_extra/update/update_languages.py View File

@@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
#!/usr/bin/env python

# This script generates languages.py from intersecting each engine's supported languages.
#
@@ -7,11 +7,9 @@
import json
from pathlib import Path
from pprint import pformat
from sys import path
from babel import Locale, UnknownLocaleError
from babel.languages import get_global

path.append('../searx') # noqa
from searx import settings, searx_dir
from searx.engines import initialize_engines, engines


utils/update-translations.sh → searx_extra/update/update_translations.sh View File


utils/fetch_wikidata_units.py → searx_extra/update/update_wikidata_units.py View File

@@ -4,9 +4,7 @@ import json
import collections

# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
from os.path import join

from searx import searx_dir
from searx.engines.wikidata import send_wikidata_query

+ 1
- 1
setup.py View File

@@ -41,7 +41,7 @@ setup(
author='Adam Tauber',
author_email='asciimoo@gmail.com',
license='GNU Affero General Public License',
packages=find_packages(exclude=["tests*"]),
packages=find_packages(exclude=["tests*", "searx_extra"]),
zip_safe=False,
install_requires=requirements,
extras_require={


+ 1
- 19
tests/unit/test_standalone_searx.py View File

@@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
"""Test utils/standalone_searx.py"""
import datetime
import importlib.util
import io
import sys

@@ -10,16 +9,7 @@ from nose2.tools import params

from searx.search import SearchQuery, EngineRef, initialize
from searx.testing import SearxTestCase


def get_standalone_searx_module():
"""Get standalone_searx module."""
module_name = 'utils.standalone_searx'
filename = 'utils/standalone_searx.py'
spec = importlib.util.spec_from_file_location(module_name, filename)
sas = importlib.util.module_from_spec(spec)
spec.loader.exec_module(sas)
return sas
from searx_extra import standalone_searx as sas


class StandaloneSearx(SearxTestCase):
@@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase):

def test_parse_argument_no_args(self):
"""Test parse argument without args."""
sas = get_standalone_searx_module()
with patch.object(sys, 'argv', ['standalone_searx']), \
self.assertRaises(SystemExit):
sys.stderr = io.StringIO()
@@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase):

def test_parse_argument_basic_args(self):
"""Test parse argument with basic args."""
sas = get_standalone_searx_module()
query = 'red box'
exp_dict = {
'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,
@@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase):

def test_to_dict(self):
"""test to_dict."""
sas = get_standalone_searx_module()
self.assertEqual(
sas.to_dict(
sas.get_search_query(sas.parse_argument(['red box']))),
@@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase):

def test_to_dict_with_mock(self):
"""test to dict."""
sas = get_standalone_searx_module()
with patch.object(sas.searx.search, 'Search') as mock_s:
m_search = mock_s().search()
m_sq = Mock()
@@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase):

def test_get_search_query(self):
"""test get_search_query."""
sas = get_standalone_searx_module()
args = sas.parse_argument(['rain', ])
search_q = sas.get_search_query(args)
self.assertTrue(search_q)
@@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase):

def test_no_parsed_url(self):
"""test no_parsed_url func"""
sas = get_standalone_searx_module()
self.assertEqual(
sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),
[{}]
@@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase):
)
def test_json_serial(self, arg, exp_res):
"""test json_serial func"""
sas = get_standalone_searx_module()
self.assertEqual(sas.json_serial(arg), exp_res)

def test_json_serial_error(self):
"""test error on json_serial."""
sas = get_standalone_searx_module()
with self.assertRaises(TypeError):
sas.json_serial('a')

Loading…
Cancel
Save