Searx is no longer maintained

Bumps [pallets-sphinx-themes](https://github.com/pallets/pallets-sphinx-themes) from 2.0.2 to 2.0.3.
- [Release notes](https://github.com/pallets/pallets-sphinx-themes/releases)
- [Changelog](https://github.com/pallets/pallets-sphinx-themes/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/pallets-sphinx-themes/compare/2.0.2...2.0.3)

---
updated-dependencies:
- dependency-name: pallets-sphinx-themes
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.config.sh

@@ -28,7 +28,10 @@ fi
 # SEARX_INTERNAL_URL="127.0.0.1:8888"
 # SEARX_SETTINGS_TEMPLATE="${REPO_ROOT}/utils/templates/etc/searx/use_default_settings.yml"
 
-# Only change, if you maintain a searx brand in your searx fork.
+# Only change, if you maintain a searx brand in your searx fork (GIT_URL) which
+# is not hold by branch 'master'.  The branch has to be a local branch, in the
+# repository from which you install (which is most often the case).  If you want
+# to install branch 'foo', don't forget to run 'git branch foo origin/foo' once.
 # GIT_BRANCH="${GIT_BRANCH:-master}"
 
 # filtron.sh

.dir-locals.el

@@ -25,7 +25,7 @@
 ;; Alternatively create the virtualenv, source it and install jedi + epc
 ;; (required by `emacs-jedi <https://tkf.github.io/emacs-jedi>`_)::
 ;;
-;;     $ virtualenv --python=python3  "--no-site-packages" ./local/py3
+;;     $ python -m venv ./local/py3
 ;;     ...
 ;;     $ source ./local/py3/bin/activate
 ;;     (py3)$ # now install into the activated 'py3' environment ..

.github/ISSUE_TEMPLATE/bug-report.md

@@ -6,6 +6,8 @@ labels: bug
 assignees: ''
 
 ---
+<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SEARX -->
+
 **Version of Searx, commit number if you are using on master branch and stipulate if you forked Searx**
 <!-- If you are running on master branch using git execute this command
 in order to fetch the latest commit ID:
@@ -34,4 +36,4 @@ or manually by executing the searx/webapp.py file? -->
 <!-- If applicable, add screenshots, logs to help explain your problem. -->
 
 **Additional context**
-<!-- Add any other context about the problem here. -->
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/engine-request.md

@@ -6,6 +6,8 @@ labels: enhancement, engine request
 assignees: ''
 
 ---
+<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SEARX -->
+
 **Working URL to the engine**
 <!-- Please check if the engine is responding correctly before submitting it. -->
 
@@ -26,4 +28,4 @@ general, files, images, it, map, music, news, science, social media and videos.
 You can add multiple categories at the same time. -->
 
 **Additional context**
-<!-- Add any other context about this engine here. -->
+<!-- Add any other context about this engine here. -->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -6,6 +6,8 @@ labels: enhancement
 assignees: ''
 
 ---
+<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SEARX -->
+
 **Is your feature request related to a problem? Please describe.**
 <!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
 
@@ -16,4 +18,4 @@ assignees: ''
 <!-- A clear and concise description of any alternative solutions or features you've considered. -->
 
 **Additional context**
-<!-- Add any other context or screenshots about the feature request here. -->
+<!-- Add any other context or screenshots about the feature request here. -->

.github/dependabot.yml

@@ -0,0 +1,10 @@
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "friday"
+    open-pull-requests-limit: 5
+    target-branch: "master"

.github/workflows/data-update.yml

@@ -0,0 +1,69 @@
+name: "Update searx.data"
+on:
+  schedule:
+    - cron: "05 06 1 * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  updateData:
+    name: Update data - ${{ matrix.fetch }}
+    runs-on: ubuntu-20.04
+    if: ${{ github.repository_owner == 'searx'}}
+    strategy:
+      matrix:
+        fetch:
+          - update_ahmia_blacklist.py
+          - update_currencies.py
+          - update_external_bangs.py
+          - update_firefox_version.py
+          - update_languages.py
+          - update_wikidata_units.py
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Ubuntu packages
+        run: |
+          sudo ./utils/searx.sh install packages
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+          architecture: 'x64'
+
+      - name: Install Python dependencies
+        run: |
+          make V=1 install
+
+      - name: Fetch data
+        env:
+          FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
+        run: |
+          V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT"
+
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.DATA_PR_TOKEN }}
+          commit-message: Update searx.data - ${{ matrix.fetch }}
+          committer: searx-bot <noreply@github.com>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          signoff: false
+          branch: update_data_${{ matrix.fetch }}
+          delete-branch: true
+          draft: false
+          title: 'Update searx.data - ${{ matrix.fetch }}'
+          body: |
+            Update searx.data - ${{ matrix.fetch }}
+          labels: |
+            data
+
+      - name: Check outputs
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

.github/workflows/integration.yml

@@ -1,36 +1,50 @@
 name: Integration
 
-on: [push, pull_request]
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+    branches: ["master"]
+
+permissions:
+  contents: read
 
 jobs:
   python:
     name: Python ${{ matrix.python-version }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        os: [ubuntu-latest]
-        python-version: [3.5, 3.6, 3.7, 3.8]
+        os: [ubuntu-20.04]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
     steps:
-    - name: Checkout 🛎️
+    - name: Checkout
       uses: actions/checkout@v2
-    - name: Install Ubuntu packages 🧰
+    - name: Install Ubuntu packages
       run: |
         sudo ./utils/searx.sh install packages
         sudo apt install firefox
-    - name: Set up Python 🧰
+    - name: Set up Python
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
         architecture: 'x64'
-    - name: Install Python dependencies 🧰
+    - name: Cache Python dependencies
+      id: cache-python
+      uses: actions/cache@v2
+      with:
+        path: ./local
+        key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }}
+    - name: Install Python dependencies
+      if: steps.cache-python.outputs.cache-hit != 'true'
       run: |
         make V=1 install
         make V=1 gecko.driver
-    - name: Run tests 🏗️
-      run: make V=1 test
-    - name: Test coverage 🗺️
+    - name: Run tests
+      run: make V=1 ci.test
+    - name: Test coverage
       run: make V=1 test.coverage
-    - name: Store coverage result 🗺️
+    - name: Store coverage result
       uses: actions/upload-artifact@v2
       with:
         name: coverage-${{ matrix.python-version }}
@@ -39,41 +53,62 @@ jobs:
 
   themes:
     name: Themes
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
-    - name: Checkout 🛎️
+    - name: Checkout
       uses: actions/checkout@v2
-    - name: Install Ubuntu packages 🧰
+    - name: Install Ubuntu packages
       run: sudo ./utils/searx.sh install packages
-    - name: Install node dependencies 🧰
-      run: make V=1 node.env
-    - name: Build themes 🏗️
-      run: make V=1 themes
-
-  documentation:
-    name: Documentation
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout 🛎️
-      uses: actions/checkout@v2
-      with:
-        persist-credentials: false
-    - name: Install Ubuntu packages 🧰
-      run: sudo ./utils/searx.sh install buildhost
-    - name: Set up Python 🧰
+    - name: Set up Python
       uses: actions/setup-python@v2
       with:
         python-version: '3.9'
         architecture: 'x64'
-    - name: Build documentation 🏗️
-      run: SEARX_DEBUG=1 make V=1 travis-gh-pages
-    - name: Deploy 🚀
+    - name: Cache Python dependencies
+      id: cache-python
+      uses: actions/cache@v2
+      with:
+        path: ./local
+        key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
+    - name: Install node dependencies
+      run: make V=1 node.env
+    - name: Build themes
+      run: make V=1 themes.all
+
+  documentation:
+    permissions:
+      contents: write  # for JamesIves/github-pages-deploy-action to push changes in repo
+    name: Documentation
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: '0'
+        persist-credentials: false
+    - name: Install Ubuntu packages
+      run: sudo ./utils/searx.sh install buildhost
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.10'
+        architecture: 'x64'
+    - name: Cache Python dependencies
+      id: cache-python
+      uses: actions/cache@v2
+      with:
+        path: ./local
+        key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
+    - name: Build documentation
+      run: |
+        make V=1 docs.clean docs.html
+    - name: Deploy
       if:  github.ref == 'refs/heads/master'
       uses: JamesIves/github-pages-deploy-action@3.7.1
       with:
         GITHUB_TOKEN: ${{ github.token }}
         BRANCH: gh-pages
-        FOLDER: gh-pages
+        FOLDER: dist/docs
         CLEAN: true # Automatically remove deleted files from the deploy branch
 
   dockers:
@@ -85,26 +120,37 @@ jobs:
       - documentation
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
-      - name: Checkout 🛎️
+      - name: Checkout
         if: env.DOCKERHUB_USERNAME != null
         uses: actions/checkout@v2
         with:
           # make sure "make docker.push" can get the git history
           fetch-depth: '0'
-      - name: Set up QEMU 🧰
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - name: Cache Python dependencies
+        id: cache-python
+        uses: actions/cache@v2
+        with:
+          path: ./local
+          key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
+      - name: Set up QEMU
         if: env.DOCKERHUB_USERNAME != null
         uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx 🧰
+      - name: Set up Docker Buildx
         if: env.DOCKERHUB_USERNAME != null
         uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub 🔒
+      - name: Login to DockerHub
         if: env.DOCKERHUB_USERNAME != null
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push 🐳
+      - name: Build and push
         if: env.DOCKERHUB_USERNAME != null
         run: make -e GIT_URL=$(git remote get-url origin) docker.push

.gitignore

@@ -26,3 +26,5 @@ dist/
 local/
 gh-pages/
 searx.egg-info/
+.env
+geckodriver.log

.pylintrc

@@ -59,7 +59,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=bad-whitespace, duplicate-code
+disable=duplicate-code, consider-using-f-string
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -105,39 +105,18 @@ max-nested-blocks=5
 
 [BASIC]
 
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input
-
-# Naming hint for argument names
-argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
-# Regular expression matching correct argument names
-argument-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$
-
-# Naming hint for attribute names
-attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
 # Regular expression matching correct attribute names
 attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*)|([A-Z0-9_]*))$
 
 # Bad variable names which should always be refused, separated by a comma
 bad-names=foo,bar,baz,toto,tutu,tata
 
-# Naming hint for class attribute names
-class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
-
 # Regular expression matching correct class attribute names
 class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
 
-# Naming hint for class names
-class-name-hint=[A-Z_][a-zA-Z0-9]+$
-
 # Regular expression matching correct class names
 class-rgx=[A-Z_][a-zA-Z0-9]+$
 
-# Naming hint for constant names
-const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
-
 # Regular expression matching correct constant names
 const-rgx=(([a-zA-Z_][a-zA-Z0-9_]*)|(__.*__))$
 #const-rgx=[f]?[A-Z_][a-zA-Z0-9_]{2,30}$
@@ -146,9 +125,6 @@ const-rgx=(([a-zA-Z_][a-zA-Z0-9_]*)|(__.*__))$
 # ones are exempt.
 docstring-min-length=-1
 
-# Naming hint for function names
-function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
 # Regular expression matching correct function names
 function-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$
 
@@ -158,21 +134,12 @@ good-names=i,j,k,ex,Run,_,log,cfg,id
 # Include a hint for the correct naming format with invalid-name
 include-naming-hint=no
 
-# Naming hint for inline iteration names
-inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
-
 # Regular expression matching correct inline iteration names
 inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
 
-# Naming hint for method names
-method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
 # Regular expression matching correct method names
 method-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$
 
-# Naming hint for module names
-module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
-
 # Regular expression matching correct module names
 #module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 module-rgx=([a-z_][a-z0-9_]*)$
@@ -189,9 +156,6 @@ no-docstring-rgx=^_
 # to this list to register other decorators that produce valid properties.
 property-classes=abc.abstractproperty
 
-# Naming hint for variable names
-variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
 # Regular expression matching correct variable names
 variable-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*)|([a-z]))$
 
@@ -217,12 +181,6 @@ max-line-length=120
 # Maximum number of lines in a module
 max-module-lines=2000
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,dict-separator
-
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.No config file found, using default configuration
 

AUTHORS.rst

@@ -1,4 +1,4 @@
-Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament, Noémi Ványi, @pofilo, Gaspard d'Hautefeuille and Markus Heiser.
+Searx was created by Adam Tauber and is maintained by Adam Tauber, Noémi Ványi, @pofilo, Gaspard d'Hautefeuille and Émilien Devos.
 
 Major contributing authors:
 
@@ -12,8 +12,9 @@ Major contributing authors:
 - @pofilo
 - Markus Heiser @return42
 - Émilien Devos @unixfox
+- Alexandre Flament
 
-People who have submitted patches/translates, reported bugs, consulted features or
+People who have submitted patches/translations, reported bugs, consulted features or
 generally made searx better:
 
 - Laszlo Hammerl
@@ -66,7 +67,6 @@ generally made searx better:
 - @firebovine
 - Lorenzo J. Lucchini @luccoj
 - @eig8phei
-- Joachim Cherqui
 - @maxigas
 - Jannik Winkel @kiney
 - @juanitobananas
@@ -154,6 +154,42 @@ generally made searx better:
 - @mrwormo
 - Xiaoyu WEI @xywei
 - @joshu9h
-
-
-
+- Daniel Hones
+- @cyclaero
+- @thezeroalpha
+- @Tobi823
+- @archiecodes
+- @BBaoVanC
+- @datagram1
+- @lucky13820
+- @jhigginbotham
+- @xenrox
+- @OliveiraHermogenes
+- Paul Alcock @Guilvareux
+- Ben Collerson
+- @3nprob
+- @plague-doctor
+- @CicadaCinema
+- @mikamp116
+- @Zackptg5
+- @darkmagic13
+- @CrocodileCroco
+- @allendema
+- Jordan Webb @jordemort
+- Samuel Dudik @dudik
+- @c1492
+- @nav1s
+- Igor Rzegocki @ajgon
+- Dmitrii Faiazov @scientia-ac-labore
+- @noctux
+- @jecarr
+- @israelyago
+- Georg @tacerus
+- Dario Nuevo @narcoticfresh
+- Andy Jones @andyljones
+- Maciej Urbański @rooterkyberian
+- @ilyakooo0
+- Eric Zhang @EricZhang456
+- @nathannaveen
+- @liimee
+- @james-still

CHANGELOG.rst

@@ -1,3 +1,257 @@
+1.1.0 2022.08.07
+================
+
+It has been a while since we released a new version of searx. Thus, we have lots of new things to offer, like new engines, autocompleter, plugins, etc. We got numerous contributions from ~30 new developers, but also we got many PRs from our recurring contributors.
+
+Thank you so much for you support! We couldn't have release so many awesome things without you!
+
+Core
+~~~~
+
+- Drop Python 3.6 support #3133
+- Run tests under python 3.10 #3035
+- Reduce redundant docker build steps #2725
+- Allow overriding Docker repository when building docker image #2726
+- Add healthcheck endpoint for Docker #2992
+
+New engines
+~~~~~~~~~~~
+
+- Wordnik.com #2735
+- Bandcamp #2763
+- SJP - Słownik języka polskiego #2736
+- Wikimini #2819
+- Dogpile #2822
+- PyPI XPATH engine #2830
+- ManKier #2829
+- Kaufland.de #2915
+- ask.com #2898
+- gpodder.net (JSON) #2885
+- woxikon.de synonyme (xpath) #2883
+- Petalsearch.com engine (xpath) #2897
+- whaleslide.com #2861
+- azlyrics.com #2955
+- IMDB #2980
+- Prowlarr #3118
+- Tineye reverse image search #3040
+- Onesearch #3065
+- TVmaze #3246
+- Emojipedia #3278
+- Psychonautwiki by @dimethyltriptamine @kvch
+
+Fixed engines
+~~~~~~~~~~~~~
+
+- Remove hubsbpot tracking URL params #2723
+- Fix URL to solidtorrent result page #2786
+- Update onion engines to v3 #2904
+- Fix Seznam engine #2905
+- Add pagination support for Library Genesis #2887
+- Fix uppercase ip query #2991
+- Fix Libgen + Uncomment Ebay and Urbandictionary #2986
+- Fixed Hoogle engine #3146
+- Fix Digg engine #3150
+- Don't lump all search suggestions together in Yahoo #3208
+- Fix DDG safe search #3247
+- Fix Qwant: Remove extra q from URL #3091
+
+New plugins
+~~~~~~~~~~~
+
+- hostname_replace plugin to rewrite result hostnames #2724
+- search_operators plugin to filter search results using -, site: and -site: #3311
+
+Fixed plugins
+~~~~~~~~~~~~~
+
+- Fix default_doi_resolver in preferences #2707
+- Add DOI resolver from sci-hub and replace default DOI #2706
+
+Themes
+~~~~~~
+
+- Fix dark "expand" button from infobox #2702
+- fix url_for(..., _external=True) in templates #2656
+- [enh] oscar: image thumbnail layout #2675
+- Improve text overflow of custom select #2985
+- Products results: add possibility to show if a product is in stock or not #3120
+- Configurable autofocus of search input (#1984) #3285
+- archive.today option for results page #3308
+- Fix keyboard hints for category tabs (#1187) #3276
+
+Enhancements
+~~~~~~~~~~~~
+
+- Allow overriding env vars SEARX_SETTINGS_PATH, UWSGI_SETTINGS_PATH #2717
+- correct typo/grammatical mistake #2744
+- Fix bug for 'FileNotFoundError' in 'standalone_searx.py' #2764
+- Fix grammar mistake in debug log output #2759
+- Fix typo #2768
+- Fix redirect when saving preferences #2760
+- Replace Makefile boilerplate by shell scripts #2594
+- Fix Qwant's fetch_languages function #2799
+- Apply HTTPS where possible + fix small typo #2922
+- Сhange in user-agent Firefox versions to latest #3008
+- Use engine-type when looking up supported_languages from JSON files #3002
+- Update about section of Invidious and Rumble + Change filtron error wording #2959
+- Verify that Tor proxy works every time searx starts #3015
+- Update settings_loader.get_user_settings_path() #3056
+- Fix wrong func call #3058
+- Improve ranking based on language #3053
+
+1.0.0 2021.03.27
+================
+
+Core
+~~~~
+
+- drop support for Python 3.5 ( #2459 )
+- add support for Python 3.9 ( #2397 #2459 )
+- update Python dependencies ( #2428 #2459 #2206 ) ⚠️ pyopenssl is not longer required
+- automatic update of searx.data ( #2555 #2585 #2595 #2592 #2600 )
+- update searx.data ( #2604 #2605 #2606 #2607 #2415 )
+- add ability to send engine data to subsequent requests ( #2615 )
+- add checker ( #2419 #2476 #2481 #2682 #2682 #2657 )
+- by default allow only HTTPS, not HTTP ( #2641 #2659 )
+- replace /translations.js with an embedded JSON ( #2660 )
+- activate raise_for_error by default ( #2557 )
+- don't dump traceback of SearxEngineResponseException on init ( #2635 )
+
+Documentation
+~~~~~~~~~~~~~
+
+- update nginx configuration ( #2618 )
+- document workaround for using 2 languages simultaneously ( #2479 )
+- improve admin-docs about result proxy (morty) configuration ( #2509 )
+- fixed typo ( #2457 )
+
+New settings.yml
+~~~~~~~~~~~~~~~~
+
+- `general.contact_url` : add link to contact instance maintainer to footer of each page ( #2391 14c7cc0e118f1d0873b32b34793cdec2c5c9c13e #2412 )
+- `brand` : move brand options from Makefile to settings.yml ( #2408 #2473 )
+
+Themes
+~~~~~~
+
+- oscar: Hide links panel in mobile screens ( #2458 )
+- oscar: upgrade dependencies ( #2346 #2673 #2662 )
+- remove legacy, courgette and pix-art themes ( #2344 )
+- add hyperlink to searx instances list in error message ( #2387 )
+- preferences: a tooltip is shown when the mouse is over the engine names ( #2661 )
+- Ignore double-quotes when highlighting query parts ( #2553 )
+- update autocomplete ( #2593 )
+
+New engines
+~~~~~~~~~~~
+
+- ccengine ( #2533 )
+- mediathekviewweb ( #2541 )
+- solidtorrents ( #2626 )
+- solr ( #2652 )
+- rumble ( #2588 )
+- wiby.me ( #2452 )
+
+Fixed engines
+~~~~~~~~~~~~~
+
+- apk_mirror ( #2556 #2642 )
+- bing ( #2602 )
+- duckduckgo ( #2560 #2559 )
+- library genesis ( #2448 )
+- ina ( 0ba71c3644c4d20f70528c10eed1385399ec1c82 )
+- invidious ( #2451 )
+- json_engine ( #2562 )
+- google ( #2482 )
+- google images ( #2482 )
+- google play apps ( 88657fe9c2a41b9be38ee5146e5870672416db12 )
+- google play movies ( 50ba2b9e87ef61e96da124f906d3aff4c7870e3f )
+- google news ( #2483 #2498 )
+- google scholar ( #2611 )
+- google video ( #2482 )
+- hoogle ( 6255b33c9dcf0d28f0a3307af988565f69259ce2 )
+- naver ( #2542 )
+- semantic schollar ( f596f5767bed915a5c3bed59ae26283e53f975ca f596f5767bed915a5c3bed59ae26283e53f975ca )
+- startpage ( #2396 )
+- seznam ( #2564 28286cf3f2308113bf440fb6e7cf326c6ed07889 )
+- wikipedia ( #2554 #2565 #2681 #2681 )
+- yacy ( #2669 )
+- yahoo news ( #2640 #2655 )
+
+Updated engines
+~~~~~~~~~~~~~~~
+
+- duckduckgo ( 5f450fda74e80bf350eb1493f66cfa61deaf5cea )
+- geektimes ( 45f0e1a859fa12ce2ae0c24dc356922fcad50c8d )
+- lobste.rs ( 06b754ad67aa6066aed6df77b5ffb74aabebb040 )
+- soundcloud ( #2671 )
+- peertube ( #2570 )
+- recoll ( #2539 )
+- yggtorrent ( #2573 )
+
+Removed engines
+~~~~~~~~~~~~~~~
+
+- acgsou ( #2654 )
+- google_play_music ( #2558 )
+- metager ( #2538 )
+- voat ( #2445 )
+- yandex ( #2566 )
+
+Bug fixes
+~~~~~~~~~
+
+- Fix empty colon in query from selecting Chinese ( #2454 )
+- Get correct locale with country from browser ( #2531 )
+
+Code refactoring / reduce the technical debt
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- refactor searx.search.SearchQuery and searx.search.EngineRef ( #2398 )
+- dynamically set language_support variable ( #2499 )
+- engines: add about variable ( #2460 )
+- processors ( #2225 5c6a5407a0b124c3323e73c33b81ec1fbd7d2fce )
+- remove Fabric file ( #2494 )
+- use unittest from py3, remove unittest2 from py2 ( #2608 )
+
+Github
+~~~~~~
+
+- add notice for the issue templates ( #2447 )
+- every Sunday, call utils/fetch_*.py scripts and create a PR automatically ( #2500 728e09676400221a064627509a31470d8f6e33bf )
+- minor change: replace "travis" by "CI" ( #2528 )
+
+Build scripts
+~~~~~~~~~~~~~
+
+- update secret key check ( #2411 )
+- fix makefile targets `books/{name}.*`  and `books/user.pdf` ( #2420 #2530 )
+- upload-pypi-test & linuxdoc has been released on PyPi ( #2456 )
+- fix makefile target `gh-pages` : flatten history of branch gh.pages ( #2514 )
+- optimize creation of the virtualenv & pyenvinstall targets ( #2421 )
+- update pyenv pyenvinstall Make targets ( #2517 )
+- makefile.python: remove duplicate pyenv-(un)install targets ( #2418 )
+- [fix] make targets engines.languages and useragents.update ( #2643 )
+- [fix] utils/serax.sh create_pyenv() - drop duplicate 'pip install .' ( #2621 )
+
+Install scripts
+~~~~~~~~~~~~~~~
+
+- drop Ubuntu 16.04 (Xenial Xerus) support ( #2619 )
+- replace ubu1910 image by ubu2010 image ( #2435 )
+- LXC switch to Fedora 33 / Fedora 31 reached its EOL #2634 (  #2634 )
+- add package which to CentOS-7 boilerplate ( #2623 )
+- use SEARX_SETTINGS_TEMPLATE from .config environment ( #2417 )
+- determine path to makefile.lxc in a LXC ( #2399 )
+- remove unused code ( #2401 #2497 )
+- support git versions <v2.22 ( #2620 )
+
+Announcement
+~~~~~~~~~~~~
+
+We, the searx maintainer team, would like to say a huge thank you for everybody who had been involved in the development of searx or supported us in the past 7 years - making our first stable release available. Special thanks to [NLNet](https://nlnet.nl) for sponsoring multiple features of this release.
+
+
 0.18.0 2020.12.14
 =================
 
@@ -28,7 +282,7 @@ New settings.yml
 - ``server.method: "POST"`` - Make default query submission method configurable ( #2130 )
 - ``server.default_http_headers`` - add default http headers ( #2295 )
 - ``engines.*.proxies`` - Using proxy only for specific engines ( #1827 #2319 ), see https://searx.github.io/searx/dev/engine_overview.html#settings-yml
-- ``enabled_plugins`` - Enabled plugins ( a05c660e3036ad8d02072fc6731af54c2ed6151c ) 
+- ``enabled_plugins`` - Enabled plugins ( a05c660e3036ad8d02072fc6731af54c2ed6151c )
 - ``preferences.lock`` - Let admins lock user preferences ( #2270 )
 
 Oscar theme
@@ -38,10 +292,10 @@ Oscar theme
 
   - Make infoboxes shorter by default.
   - Hide the main image by default as well and set a maximum height even when expanded.
-  - Add a toggle at the bottom of the infobox to expand it or to shrink it again. 
+  - Add a toggle at the bottom of the infobox to expand it or to shrink it again.
   - Fix pointhi style
 - query suggestion does not keep the language tag of the original query  ( #1314 )
-- fix the clear button ( #2306 ) 
+- fix the clear button ( #2306 )
 
 Simple theme
 ~~~~~~~~~~~~
@@ -170,7 +424,7 @@ Special thanks to `NLNet <https://nlnet.nl>`__ for sponsoring multiple features
  - Removed engines: faroo
 
 Special thanks to `NLNet <https://nlnet.nl>`__ for sponsoring multiple features of this release.
-Special thanks to https://www.accessibility.nl/english for making accessibilty audit.
+Special thanks to https://www.accessibility.nl/english for making accessibility audit.
 
 News
 ~~~~

Dockerfile

@@ -1,29 +1,22 @@
-FROM alpine:3.12
+FROM alpine:3.15
 ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]
 EXPOSE 8080
 VOLUME /etc/searx
 VOLUME /var/log/uwsgi
 
-ARG GIT_URL=unknown
-ARG VERSION_GITCOMMIT=unknown
-ARG SEARX_GIT_VERSION=unknown
-
 ARG SEARX_GID=977
 ARG SEARX_UID=977
 
 RUN addgroup -g ${SEARX_GID} searx && \
     adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx
 
-ARG TIMESTAMP_SETTINGS=0
-ARG TIMESTAMP_UWSGI=0
-ARG LABEL_VCS_REF=
-ARG LABEL_VCS_URL=
-
 ENV INSTANCE_NAME=searx \
     AUTOCOMPLETE= \
     BASE_URL= \
     MORTY_KEY= \
-    MORTY_URL=
+    MORTY_URL= \
+    SEARX_SETTINGS_PATH=/etc/searx/settings.yml \
+    UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini
 
 WORKDIR /usr/local/searx
 
@@ -53,14 +46,19 @@ RUN apk upgrade --no-cache \
     uwsgi \
     uwsgi-python3 \
     brotli \
- && pip3 install --upgrade pip \
+ && pip3 install --upgrade pip wheel setuptools \
  && pip3 install --no-cache -r requirements.txt \
  && apk del build-dependencies \
  && rm -rf /root/.cache
 
-COPY --chown=searx:searx . .
+COPY searx ./searx
+COPY dockerfiles ./dockerfiles
 
-RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
+ARG TIMESTAMP_SETTINGS=0
+ARG TIMESTAMP_UWSGI=0
+ARG VERSION_GITCOMMIT=unknown
+
+RUN /usr/bin/python3 -m compileall -q searx; \
     touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \
     touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \
     if [ ! -z $VERSION_GITCOMMIT ]; then\
@@ -70,8 +68,12 @@ RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
     -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \
     -type f -exec gzip -9 -k {} \+ -exec brotli --best {} \+
 
-# Keep this argument at the end since it change each time
+# Keep these arguments at the end to prevent redundant layer rebuilds
 ARG LABEL_DATE=
+ARG GIT_URL=unknown
+ARG SEARX_GIT_VERSION=unknown
+ARG LABEL_VCS_REF=
+ARG LABEL_VCS_URL=
 LABEL maintainer="searx <${GIT_URL}>" \
       description="A privacy-respecting, hackable metasearch engine." \
       version="${SEARX_GIT_VERSION}" \

Makefile

@@ -1,281 +1,107 @@
 # -*- coding: utf-8; mode: makefile-gmake -*-
-.DEFAULT_GOAL=help
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
-# START Makefile setup
-export GIT_URL=https://github.com/searx/searx
-export GIT_BRANCH=master
-export SEARX_URL=https://searx.me
-export DOCS_URL=https://searx.github.io/searx
-# END Makefile setup
+.DEFAULT_GOAL=help
+export MTOOLS=./manage
 
 include utils/makefile.include
 
-PYOBJECTS = searx
-DOC       = docs
-PY_SETUP_EXTRAS ?= \[test\]
-PYLINT_SEARX_DISABLE_OPTION := I,C,R,W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401
-PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES := supported_languages,language_aliases
-
-include utils/makefile.python
-include utils/makefile.sphinx
-
 all: clean install
 
-PHONY += help-min help-all help
+PHONY += help
 
-help: help-min
-	@echo  ''
-	@echo  'to get more help:  make help-all'
+help:
+	@./manage --help
+	@echo '----'
+	@echo 'run            - run developer instance'
+	@echo 'install        - developer install of searx into virtualenv'
+	@echo 'uninstall      - uninstall developer installation'
+	@echo 'clean          - clean up working tree'
+	@echo 'search.checker - check search engines'
+	@echo 'test           - run shell & CI tests'
+	@echo 'test.sh        - test shell scripts'
+	@echo 'ci.test        - run CI tests'
 
-help-min:
-	@echo  '  test      - run developer tests'
-	@echo  '  docs      - build documentation'
-	@echo  '  docs-live - autobuild HTML documentation while editing'
-	@echo  '  run       - run developer instance'
-	@echo  '  install   - developer install (./local)'
-	@echo  '  uninstall - uninstall (./local)'
-	@echo  '  gh-pages  - build docs & deploy on gh-pages branch'
-	@echo  '  clean     - drop builds and environments'
-	@echo  '  project   - re-build generic files of the searx project'
-	@echo  '  buildenv  - re-build environment files (aka brand)'
-	@echo  '  themes    - re-build build the source of the themes'
-	@echo  '  docker    - build Docker image'
-	@echo  '  node.env  - download & install npm dependencies locally'
-	@echo  ''
-	@echo  'environment'
-	@echo  '  SEARX_URL = $(SEARX_URL)'
-	@echo  '  GIT_URL   = $(GIT_URL)'
-	@echo  '  DOCS_URL  = $(DOCS_URL)'
-	@echo  ''
-	@$(MAKE) -e -s make-help
-
-help-all: help-min
-	@echo  ''
-	@$(MAKE) -e -s python-help
-	@echo  ''
-	@$(MAKE) -e -s docs-help
-
-PHONY += install
-install: buildenv pyenvinstall
-
-PHONY += uninstall
-uninstall: pyenvuninstall
-
-PHONY += clean
-clean: pyclean docs-clean node.clean test.clean
-	$(call cmd,common_clean)
 
 PHONY += run
-run:  buildenv pyenvinstall
+run:  install
 	$(Q) ( \
 	sleep 2 ; \
 	xdg-open http://127.0.0.1:8888/ ; \
 	) &
-	SEARX_DEBUG=1 $(PY_ENV)/bin/python ./searx/webapp.py
+	SEARX_DEBUG=1 ./manage pyenv.cmd python ./searx/webapp.py
 
-# docs
-# ----
+PHONY += install uninstall
+install uninstall:
+	$(Q)./manage pyenv.$@
 
-sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes
+PHONY += clean
+clean: py.clean docs.clean node.clean test.clean
+	$(Q)./manage build_msg CLEAN  "common files"
+	$(Q)find . -name '*.orig' -exec rm -f {} +
+	$(Q)find . -name '*.rej' -exec rm -f {} +
+	$(Q)find . -name '*~' -exec rm -f {} +
+	$(Q)find . -name '*.bak' -exec rm -f {} +
 
-PHONY += docs
-docs:  sphinx-doc-prebuilds
-	$(call cmd,sphinx,html,docs,docs)
+PHONY += search.checker search.checker.%
+search.checker: install
+	$(Q)./manage pyenv.cmd searx-checker -v
 
-PHONY += docs-live
-docs-live:  sphinx-doc-prebuilds
-	$(call cmd,sphinx_autobuild,html,docs,docs)
+search.checker.%: install
+	$(Q)./manage pyenv.cmd searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
 
-PHONY += prebuild-includes
-prebuild-includes:
-	$(Q)mkdir -p $(DOCS_BUILD)/includes
-	$(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst
-	$(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst
-	$(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst
-
-
-$(GH_PAGES)::
-	@echo "doc available at --> $(DOCS_URL)"
-
-# update project files
-# --------------------
-
-PHONY += project engines.languages useragents.update buildenv
-
-project: buildenv useragents.update engines.languages
-
-engines.languages:  pyenvinstall
-	$(Q)echo "fetch languages .."
-	$(Q)$(PY_ENV_ACT); python utils/fetch_languages.py
-	$(Q)echo "update searx/data/engines_languages.json"
-	$(Q)mv engines_languages.json searx/data/engines_languages.json
-	$(Q)echo "update searx/languages.py"
-	$(Q)mv languages.py searx/languages.py
-
-useragents.update:  pyenvinstall
-	$(Q)echo "Update searx/data/useragents.json with the most recent versions of Firefox."
-	$(Q)$(PY_ENV_ACT); python utils/fetch_firefox_version.py
-
-buildenv:
-	$(Q)echo "build searx/brand.py"
-	$(Q)echo "GIT_URL = '$(GIT_URL)'"  > searx/brand.py
-	$(Q)echo "GIT_BRANCH = '$(GIT_BRANCH)'"  >> searx/brand.py
-	$(Q)echo "ISSUE_URL = 'https://github.com/searx/searx/issues'" >> searx/brand.py
-	$(Q)echo "SEARX_URL = '$(SEARX_URL)'" >> searx/brand.py
-	$(Q)echo "DOCS_URL = '$(DOCS_URL)'" >> searx/brand.py
-	$(Q)echo "PUBLIC_INSTANCES = 'https://searx.space'" >> searx/brand.py
-	$(Q)echo "build utils/brand.env"
-	$(Q)echo "export GIT_URL='$(GIT_URL)'"  > utils/brand.env
-	$(Q)echo "export GIT_BRANCH='$(GIT_BRANCH)'"  >> utils/brand.env
-	$(Q)echo "export ISSUE_URL='https://github.com/searx/searx/issues'" >> utils/brand.env
-	$(Q)echo "export SEARX_URL='$(SEARX_URL)'" >> utils/brand.env
-	$(Q)echo "export DOCS_URL='$(DOCS_URL)'" >> utils/brand.env
-	$(Q)echo "export PUBLIC_INSTANCES='https://searx.space'" >> utils/brand.env
-
-
-# node / npm
-# ----------
-
-node.env: buildenv
-	$(Q)./manage.sh npm_packages
-
-node.clean:
-	$(Q)echo "CLEAN     locally installed npm dependencies"
-	$(Q)rm -rf \
-	  ./node_modules  \
-	  ./package-lock.json \
-	  ./searx/static/themes/oscar/package-lock.json \
-	  ./searx/static/themes/oscar/node_modules \
-	  ./searx/static/themes/simple/package-lock.json \
-	  ./searx/static/themes/simple/node_modules
-
-# build themes
-# ------------
-
-PHONY += themes.bootstrap themes themes.oscar themes.simple themes.legacy themes.courgette themes.pixart
-themes: buildenv themes.bootstrap themes.oscar themes.simple themes.legacy themes.courgette themes.pixart
-
-quiet_cmd_lessc = LESSC     $3
-      cmd_lessc = PATH="$$(npm bin):$$PATH" \
-	lessc --clean-css="--s1 --advanced --compatibility=ie9" "searx/static/$2" "searx/static/$3"
-
-quiet_cmd_grunt = GRUNT     $2
-      cmd_grunt = PATH="$$(npm bin):$$PATH" \
-	grunt --gruntfile  "$2"
-
-themes.oscar: node.env
-	$(Q)echo '[!] build oscar theme'
-	$(call cmd,grunt,searx/static/themes/oscar/gruntfile.js)
-
-themes.simple: node.env
-	$(Q)echo '[!] build simple theme'
-	$(call cmd,grunt,searx/static/themes/simple/gruntfile.js)
-
-themes.legacy: node.env
-	$(Q)echo '[!] build legacy theme'
-	$(call cmd,lessc,themes/legacy/less/style-rtl.less,themes/legacy/css/style-rtl.css)
-	$(call cmd,lessc,themes/legacy/less/style.less,themes/legacy/css/style.css)
-
-themes.courgette: node.env
-	$(Q)echo '[!] build courgette theme'
-	$(call cmd,lessc,themes/courgette/less/style.less,themes/courgette/css/style.css)
-	$(call cmd,lessc,themes/courgette/less/style-rtl.less,themes/courgette/css/style-rtl.css)
-
-themes.pixart: node.env
-	$(Q)echo '[!] build pixart theme'
-	$(call cmd,lessc,themes/pix-art/less/style.less,themes/pix-art/css/style.css)
-
-themes.bootstrap: node.env
-	$(call cmd,lessc,less/bootstrap/bootstrap.less,css/bootstrap.min.css)
-
-
-# docker
-# ------
-
-PHONY += docker
-docker: buildenv
-	$(Q)./manage.sh docker_build
-
-docker.push: buildenv
-	$(Q)./manage.sh docker_build push
-
-# gecko
-# -----
-
-PHONY += gecko.driver
-gecko.driver:
-	$(PY_ENV_ACT); ./manage.sh install_geckodriver
-
-# test
-# ----
-
-PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot
-test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot
-
-PYLINT_FILES=\
-	searx/preferences.py \
-	searx/testing.py \
-	searx/engines/gigablast.py \
-	searx/engines/deviantart.py \
-	searx/engines/digg.py
-
-test.pylint: pyenvinstall
-	$(call cmd,pylint,$(PYLINT_FILES))
-	$(call cmd,pylint,\
-		--disable=$(PYLINT_SEARX_DISABLE_OPTION) \
-		--additional-builtins=$(PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES) \
-		searx/engines \
-	)
-	$(call cmd,pylint,\
-		--disable=$(PYLINT_SEARX_DISABLE_OPTION) \
-		--ignore=searx/engines \
-		searx tests \
-	)
-
-# ignored rules:
-#  E402 module level import not at top of file
-#  W503 line break before binary operator
-
-# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported!
+PHONY += ci.test test test.sh
+ci.test: test.pep8 test.pylint test.unit test.robot
+test: ci.test
 test.sh:
-	shellcheck -x -s bash utils/brand.env
-	shellcheck -x utils/lib.sh
-	shellcheck -x utils/filtron.sh
-	shellcheck -x utils/searx.sh
-	shellcheck -x utils/morty.sh
-	shellcheck -x utils/lxc.sh
-	shellcheck -x utils/lxc-searx.env
-	shellcheck -x .config.sh
-
-test.pep8: pyenvinstall
-	@echo "TEST      pycodestyle (formerly pep8)"
-	$(Q)$(PY_ENV_ACT); pycodestyle --exclude='searx/static, searx/languages.py, $(foreach f,$(PYLINT_FILES),$(f),)' \
-        --max-line-length=120 --ignore "E117,E252,E402,E722,E741,W503,W504,W605" searx tests
-
-test.unit: pyenvinstall
-	@echo "TEST      tests/unit"
-	$(Q)$(PY_ENV_ACT); python -m nose2 -s tests/unit
-
-test.coverage:  pyenvinstall
-	@echo "TEST      unit test coverage"
-	$(Q)$(PY_ENV_ACT); \
-	python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit \
-	&& coverage report \
-	&& coverage html \
-
-test.robot: pyenvinstall gecko.driver
-	@echo "TEST      robot"
-	$(Q)$(PY_ENV_ACT); PYTHONPATH=. python searx/testing.py robot
-
-test.clean:
-	@echo "CLEAN     intermediate test stuff"
-	$(Q)rm -rf geckodriver.log .coverage coverage/
+	$(Q)shellcheck -x -s bash \
+		utils/brand.env \
+		./manage \
+		utils/lib.sh \
+	        utils/filtron.sh \
+	        utils/searx.sh \
+	        utils/morty.sh \
+	        utils/lxc.sh \
+	        utils/lxc-searx.env \
+	        .config.sh
+	$(Q)./manage build_msg TEST "$@ OK"
 
 
-# travis
-# ------
+# wrap ./manage script
 
-travis.codecov:
-	$(Q)$(PY_ENV_BIN)/python -m pip install codecov
+MANAGE += buildenv
+MANAGE += babel.compile
+MANAGE += data.all data.languages data.useragents
+MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
+MANAGE += docker.build docker.push
+MANAGE += gecko.driver
+MANAGE += node.env node.clean
+MANAGE += py.build py.clean
+MANAGE += pyenv pyenv.install pyenv.uninstall
+MANAGE += pypi.upload pypi.upload.test
+MANAGE += test.pylint test.pep8 test.unit test.coverage test.robot test.clean
+MANAGE += themes.all themes.oscar themes.simple themes.bootstrap
 
-.PHONY: $(PHONY)
+PHONY += $(MANAGE)
+
+$(MANAGE):
+	$(Q)$(MTOOLS) $@
+
+# deprecated
+
+PHONY += docs docs-clean docs-live docker themes
+
+docs: docs.html
+	$(Q)./manage build_msg WARN $@ is deprecated use docs.html
+
+docs-clean: docs.clean
+	$(Q)./manage build_msg WARN $@ is deprecated use docs.clean
+
+docs-live: docs.live
+	$(Q)./manage build_msg WARN $@ is deprecated use docs.live
+
+docker:  docker.build
+	$(Q)./manage build_msg WARN $@ is deprecated use docker.build
+
+themes: themes.all
+	$(Q)./manage build_msg WARN $@ is deprecated use themes.all

PULL_REQUEST_TEMPLATE.md

@@ -16,7 +16,7 @@
 
 ## Author's checklist
 
-<!-- additional notes for reviewiers -->
+<!-- additional notes for reviewers -->
 
 ## Related issues
 

README.rst

@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: AGPL-3.0-or-later
 
+Searx is no longer maintained. Thank you for your support and all your contributions.
+
 .. figure:: https://raw.githubusercontent.com/searx/searx/master/searx/static/themes/oscar/img/logo_searx_a.png
    :target: https://searx.github.io/searx/
    :alt: searX
@@ -17,7 +19,7 @@
 |OpenCollective searx backers|
 |OpenCollective searx sponsors|
 
-Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **səːks**.
+Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **sɜːks**.
 
 .. _metasearch engine: https://en.wikipedia.org/wiki/Metasearch_engine
 
@@ -61,13 +63,64 @@ our homepage_.
 .. _homepage: https://searx.github.io/searx
 
 contact:
-  openhub_ // twitter_ // IRC: #searx @ freenode
+  openhub_ // twitter_ // IRC: #searx @ Libera (irc.libera.chat)
 
 .. _openhub: https://www.openhub.net/p/searx
 .. _twitter: https://twitter.com/Searx_engine
 
--------
+**************************
+Frequently asked questions
+**************************
 
-|gluten free|
+Is searx in maintenance mode?
+#############################
 
-.. |gluten free| image:: https://forthebadge.com/images/featured/featured-gluten-free.svg
+No, searx is no longer maintained.
+
+What is the difference between searx and SearxNG?
+#################################################
+
+TL;DR: SearXNG is for users that want more features and bugs getting fixed quicker.
+If you prefer a minimalist software and stable experience, use searx.
+
+SearxNG is a fork of searx, created by a former maintainer of searx. The fork
+was created because the majority of the maintainers at the time did not find
+the new proposed features privacy respecting enough. The most significant issue is with
+engine metrics.
+
+Searx is built for privacy conscious users. It comes with a unique set of
+challenges. One of the problems we face is that users rather not report bugs,
+because they do not want to publicly share what engines they use or what search
+query triggered a problem. It is a challenge we accepted.
+
+The new metrics feature collects more information to make engine maintenance easier.
+We could have had better and more error reports to benefit searx maintainers.
+However, we believe that the users of searx must come first, not the
+software. We are willing to compromise on the lack of issue reports to avoid
+violating the privacy of users.
+
+Furthermore, SearxNG is under heavy refactoring and dependencies are constantly updated, even
+if it is unnecessary. It increases the risk of introducing regressions. In searx
+we strive for stability, rather than moving fast and breaking things.
+
+Is searx for me?
+################
+
+Are you privacy conscious user? Then yes.
+
+In searx we decided to double down on being privacy respecting. We are picking
+engine changes from SearxNG, but we are not implementing engine detailed
+monitoring and not adding a new UI that relies on Javascript.
+
+If you are willing to give up some privacy respecting features, we encourage you to
+adopt SearxNG. Searx is targeted for privacy conscious users who run their
+instances locally, instead of using public instances.
+
+Why should I use SearxNG?
+#########################
+
+SearxNG has rolling releases, dependencies updated more frequently, and engines are fixed
+faster. It is easy to set up your own public instance, and monitor its
+performance and metrics. It is simple to maintain as an instance administrator.
+
+As a user, it provides a prettier user interface and nicer experience.

dockerfiles/docker-entrypoint.sh

@@ -24,9 +24,6 @@ if [ -z "${BIND_ADDRESS}" ]; then
     export BIND_ADDRESS="${DEFAULT_BIND_ADDRESS}"
 fi
 
-export UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini
-export SEARX_SETTINGS_PATH=/etc/searx/settings.yml
-
 # Parse special command line
 # see docs/admin/installation-docker.rst
 # display the help message without the version
@@ -103,7 +100,7 @@ update_conf() {
 	    # There is a new version
 	    if [ $FORCE_CONF_UPDATE -ne 0 ]; then
 		# Replace the current configuration
-		printf '⚠️  Automaticaly update %s to the new version\n' "${CONF}"
+		printf '⚠️  Automatically update %s to the new version\n' "${CONF}"
 		if [ ! -f "${OLD_CONF}" ]; then
 		    printf 'The previous configuration is saved to %s\n' "${OLD_CONF}"
 		    mv "${CONF}" "${OLD_CONF}"

dockerfiles/uwsgi.ini

@@ -9,7 +9,7 @@ workers = 4
 # The right granted on the created socket
 chmod-socket = 666
 
-# Plugin to use and interpretor config
+# Plugin to use and interpreter config
 single-interpreter = true
 master = true
 plugin = python3
@@ -42,3 +42,6 @@ static-map = /static=/usr/local/searx/searx/static
 static-expires = /* 864000
 static-gzip-all = True
 offload-threads = %k
+
+# Cache
+cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1

docs/_themes/searx/static/searx.css

@@ -138,32 +138,3 @@ caption {
   caption-side: top;
   text-align: left;
 }
-
-/* bugs since sphinx 3.1
-
-See sphinx-doc project, PR 7838 & 7484 with elementary patch to the basic CSS:
-
-- https://github.com/sphinx-doc/sphinx/issues/7838#issuecomment-646009605
-- https://github.com/sphinx-doc/sphinx/pull/7484#issuecomment-646058972
-
-*/
-
-li > p:first-child {
-    margin-top: 0;
-}
-
-li > p:last-child {
-    margin-bottom: 0;
-}
-
-div.admonition dl {
-    margin-bottom: 0;
-}
-
-div.sidebar {
-    clear: none;
-}
-
-div.admonition, div.topic, pre {
-    clear: none;
-}

docs/admin/buildhosts.rst

@@ -49,9 +49,9 @@ Build docs
    - dvisvgm_
 
 Most of the sphinx requirements are installed from :origin:`setup.py` and the
-docs can be build from scratch with ``make docs``.  For better math and image
-processing additional packages are needed.  The XeTeX_ needed not only for PDF
-creation, its also needed for :ref:`math` when HTML output is build.
+docs can be build from scratch with ``make docs.html``.  For better math and
+image processing additional packages are needed.  The XeTeX_ needed not only for
+PDF creation, its also needed for :ref:`math` when HTML output is build.
 
 To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
 as images (``sphinx.ext.imgmath`` extension).
@@ -64,7 +64,7 @@ to ``imgmath``:
    :start-after: # sphinx.ext.imgmath setup
    :end-before: # sphinx.ext.imgmath setup END
 
-If your docs build (``make docs``) shows warnings like this::
+If your docs build (``make docs.html``) shows warnings like this::
 
    WARNING: dot(1) not found, for better output quality install \
             graphviz from https://www.graphviz.org

docs/admin/command-engine.rst

@@ -0,0 +1,129 @@
+=====================================
+Run shell commands from your instance
+=====================================
+
+Command line engines are custom engines that run commands in the shell of the
+host. In this article you can learn how to create a command engine and how to
+customize the result display.
+
+The command
+===========
+
+When specifyng commands, you must make sure the commands are available on the
+searx host. Searx will not install anything for you. Also, make sure that the
+``searx`` user on your host is allowed to run the selected command and has
+access to the required files.
+
+Access control
+==============
+
+Be careful when creating command engines if you are running a public
+instance. Do not expose any sensitive information. You can restrict access by
+configuring a list of access tokens under tokens in your ``settings.yml``.
+
+Available settings
+==================
+
+* ``command``: A comma separated list of the elements of the command. A special
+  token ``{{QUERY}}`` tells searx where to put the search terms of the
+  user. Example: ``['ls', '-l', '-h', '{{QUERY}}']``
+* ``query_type``: The expected type of user search terms. Possible values:
+  ``path`` and ``enum``. ``path`` checks if the uesr provided path is inside the
+  working directory. If not the query is not executed. ``enum`` is a list of
+  allowed search terms. If the user submits something which is not included in
+  the list, the query returns an error.
+* ``delimiter``: A dict containing a delimiter char and the "titles" of each
+  element in keys.
+* ``parse_regex``: A dict containing the regular expressions for each result
+  key.
+* ``query_enum``: A list containing allowed search terms if ``query_type`` is
+  set to ``enum``.
+* ``working_dir``: The directory where the command has to be executed. Default:
+  ``.``
+* ``result_separator``: The character that separates results. Default: ``\n``
+
+Customize the result template
+=============================
+
+There is a default result template for displaying key-value pairs coming from
+command engines. If you want something more tailored to your result types, you
+can design your own template.
+
+Searx relies on `Jinja2 <https://jinja.palletsprojects.com/>`_ for
+templating. If you are familiar with Jinja, you will not have any issues
+creating templates. You can access the result attributes with ``{{
+result.attribute_name }}``.
+
+In the example below the result has two attributes: ``header`` and ``content``.
+To customize their diplay, you need the following template (you must define
+these classes yourself):
+
+.. code:: html
+
+    <div class="result">
+        <div class="result-header">
+            {{ result.header }}
+        </div>
+        <div class="result-content">
+            {{ result.content }}
+        </div>
+    </div>
+
+Then put your template under ``searx/templates/{theme-name}/result_templates``
+named ``your-template-name.html``. You can select your custom template with the
+option ``result_template``.
+
+.. code:: yaml
+
+  - name: your engine name
+    engine: command
+    result_template: your-template-name.html
+
+Examples
+========
+
+Find files by name
+------------------
+
+The first example is to find files on your searx host. It uses the command
+`find` available on most Linux distributions. It expects a path type query. The
+path in the search request must be inside the ``working_dir``.
+
+The results are displayed with the default `key-value.html` template.  A result
+is displayed in a single row table with the key "line".
+
+.. code:: yaml
+
+  - name : find
+    engine : command
+    command : ['find', '.', '-name', '{{QUERY}}']
+    query_type : path
+    shortcut : fnd
+    tokens : []
+    disabled : True
+    delimiter :
+        chars : ' '
+        keys : ['line']
+
+
+Find files by contents
+-----------------------
+
+In the second example, we define an engine that searches in the contents of the
+files under the ``working_dir``. The search type is not defined, so the user can
+input any string they want. To restrict the input, you can set the ``query_type``
+to ``enum`` and only allow a set of search terms to protect
+yourself. Alternatively, make the engine private, so no one malevolent accesses
+the engine.
+
+.. code:: yaml
+
+  - name : regex search in files
+    engine : command
+    command : ['grep', '{{QUERY}}']
+    shortcut : gr
+    tokens : []
+    disabled : True
+    delimiter :
+        chars : ' '
+        keys : ['line']

docs/admin/engines.rst

@@ -33,11 +33,11 @@ Engine        ..          Paging support       **P**
 ------------------------- -------------------- ------------
 Shortcut      **S**       Language support     **L**
 Timeout       **TO**      Time range support   **TR**
-Disabled      **D**       Offline              **O**
+Disabled      **D**       Engine type          **ET**
 ------------- ----------- -------------------- ------------
 Safe search   **SS**
 ------------- ----------- ---------------------------------
-Weigth        **W**
+Weight        **W**
 ------------- ----------- ---------------------------------
 Disabled      **D**
 ------------- ----------- ---------------------------------
@@ -46,7 +46,7 @@ Show errors   **DE**
 
 .. _configured engines:
 
-.. jinja:: webapp
+.. jinja:: searx
 
    .. flat-table:: Engines configured at built time (defaults)
       :header-rows: 1
@@ -62,10 +62,10 @@ Show errors   **DE**
         - SS
         - D
         - TR
-        - O
-	- W
-	- D
-	- DE
+        - ET
+        - W
+        - D
+        - DE
 
       {% for name, mod in engines.items() %}
 
@@ -79,10 +79,67 @@ Show errors   **DE**
         - {{(mod.safesearch and "y") or ""}}
         - {{(mod.disabled and "y") or ""}}
         - {{(mod.time_range_support and "y") or ""}}
-        - {{(mod.offline and "y") or ""}}
+        - {{mod.engine_type or ""}}
         - {{mod.weight or 1 }}
         - {{(mod.disabled and "y") or ""}}
         - {{(mod.display_error_messages and "y") or ""}}
 
      {% endfor %}
 
+   .. flat-table:: Additional engines (commented out in settings.yml)
+      :header-rows: 1
+      :stub-columns: 2
+
+      * - Name
+        - Base URL
+        - Host
+        - Port
+        - Paging
+
+      * - elasticsearch
+        - localhost:9200
+        - 
+        - 
+        - False
+
+      * - meilicsearch
+        - localhost:7700
+        - 
+        - 
+        - True
+
+      * - mongodb
+        - 
+        - 127.0.0.1
+        - 21017
+        - True
+
+      * - mysql_server
+        - 
+        - 127.0.0.1
+        - 3306
+        - True
+
+      * - postgresql
+        - 
+        - 127.0.0.1
+        - 5432
+        - True
+
+      * - redis_server
+        - 
+        - 127.0.0.1
+        - 6379
+        - False
+
+      * - solr
+        - localhost:8983
+        - 
+        - 
+        - True
+
+      * - sqlite
+        - 
+        - 
+        - 
+        - True

docs/admin/engines/recoll.rst

@@ -39,7 +39,7 @@ Example
 Scenario:
 
 #. Recoll indexes a local filesystem mounted in ``/export/documents/reference``,
-#. the Recoll search inteface can be reached at https://recoll.example.org/ and
+#. the Recoll search interface can be reached at https://recoll.example.org/ and
 #. the contents of this filesystem can be reached though https://download.example.org/reference
 
 .. code:: yaml

docs/admin/filtron.rst

@@ -173,7 +173,7 @@ Use it along with ``nginx`` with the following example configuration.
    location /searx {
        proxy_pass         http://127.0.0.1:4004/;
 
-       proxy_set_header   Host             $http_host;
+       proxy_set_header   Host             $host;
        proxy_set_header   Connection       $http_connection;
        proxy_set_header   X-Real-IP        $remote_addr;
        proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;

docs/admin/index.rst

@@ -19,5 +19,9 @@ Administrator documentation
    filtron
    morty
    engines
+   private-engines
+   command-engine
+   indexer-engines
+   no-sql-engines
    plugins
    buildhosts

docs/admin/indexer-engines.rst

@@ -0,0 +1,89 @@
+==================
+Search in indexers
+==================
+
+Searx supports three popular indexer search engines:
+
+* Elasticsearch
+* Meilisearch
+* Solr
+
+Elasticsearch
+=============
+
+Make sure that the Elasticsearch user has access to the index you are querying.
+If you are not using TLS during your connection, set ``enable_http`` to ``True``.
+
+.. code:: yaml
+
+  - name : elasticsearch
+    shortcut : es
+    engine : elasticsearch
+    base_url : http://localhost:9200
+    username : elastic
+    password : changeme
+    index : my-index
+    query_type : match
+    enable_http : True
+
+Available settings
+------------------
+
+* ``base_url``: URL of Elasticsearch instance. By default it is set to ``http://localhost:9200``.
+* ``index``: Name of the index to query. Required.
+* ``query_type``: Elasticsearch query method to use. Available: ``match``,
+  ``simple_query_string``, ``term``, ``terms``, ``custom``.
+* ``custom_query_json``: If you selected ``custom`` for ``query_type``, you must
+  provide the JSON payload in this option.
+* ``username``: Username in Elasticsearch
+* ``password``: Password for the Elasticsearch user
+
+Meilisearch
+===========
+
+If you are not using TLS during connection, set ``enable_http`` to ``True``.
+
+.. code:: yaml
+
+  - name : meilisearch
+    engine : meilisearch
+    shortcut: mes
+    base_url : http://localhost:7700
+    index : my-index
+    enable_http: True
+
+Available settings
+------------------
+
+* ``base_url``: URL of the Meilisearch instance. By default it is set to http://localhost:7700
+* ``index``: Name of the index to query. Required.
+* ``auth_key``: Key required for authentication.
+* ``facet_filters``: List of facets to search in.
+
+Solr
+====
+
+If you are not using TLS during connection, set ``enable_http`` to ``True``.
+
+.. code:: yaml
+
+  - name : solr
+    engine : solr
+    shortcut : slr
+    base_url : http://localhost:8983
+    collection : my-collection
+    sort : asc
+    enable_http : True
+
+Available settings
+------------------
+
+* ``base_url``: URL of the Meilisearch instance. By default it is set to http://localhost:8983
+* ``collection``: Name of the collection to query. Required.
+* ``sort``: Sorting of the results. Available: ``asc``, ``desc``.
+* ``rows``: Maximum number of results from a query. Default value: 10.
+* ``field_list``: List of fields returned from the query.
+* ``default_fields``: Default fields to query.
+* ``query_fields``: List of fields with a boost factor. The bigger the boost
+  factor of a field, the more important the field is in the query. Example:
+  ``qf="field1^2.3 field2"``

docs/admin/installation-apache.rst

@@ -180,10 +180,6 @@ modules and create a `Location`_ configuration for the searx site.  In most
 distributions you have to un-comment the lines in the main configuration file,
 except in :ref:`The Debian Layout`.
 
-To pass the HTTP HOST header 
-With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the
-proxied host.
-
 .. tabs::
 
    .. group-tab:: Ubuntu / debian
@@ -231,6 +227,11 @@ proxied host.
          LoadModule proxy_module modules/mod_proxy.so
          LoadModule proxy_http_module modules/mod_proxy_http.so
 
+With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the
+proxied host.
+
+.. _apache searx via filtron plus morty:
+
 .. tabs::
 
    .. group-tab:: searx via filtron plus morty
@@ -285,15 +286,15 @@ proxied host.
 
          </Location>
 
-      Note that reverse proxy advised to be used in case of single-user or
-      low-traffic instances.  For a fully result proxification add :ref:`morty's
-      <searx morty>` **public URL** to your :origin:`searx/settings.yml`:
+      For a fully result proxification add :ref:`morty's <searx morty>` **public
+      URL** to your :origin:`searx/settings.yml`:
 
       .. code:: yaml
 
          result_proxy:
              # replace example.org with your server's public name
              url : https://example.org/morty
+             key : !!binary "insert_your_morty_proxy_key_here"
 
          server:
              image_proxy : True

docs/admin/installation-docker.rst

@@ -51,7 +51,7 @@ It's also possible to build searx from the embedded Dockerfile.
 
    git clone https://github.com/searx/searx.git
    cd searx
-   make docker
+   make docker.build
 
 
 Public instance

docs/admin/installation-nginx.rst

@@ -163,6 +163,8 @@ Started wiki`_ is always a good resource *to keep in the pocket*.
       Create configuration at ``/etc/nginx/conf.d/searx`` and place a
       symlink to sites-enabled:
 
+.. _nginx searx via filtron plus morty:
+
 .. tabs::
 
    .. group-tab:: searx via filtron plus morty
@@ -180,7 +182,7 @@ Started wiki`_ is always a good resource *to keep in the pocket*.
 	 location /searx {
 	     proxy_pass         http://127.0.0.1:4004/;
 
-	     proxy_set_header   Host             $http_host;
+	     proxy_set_header   Host             $host;
 	     proxy_set_header   Connection       $http_connection;
 	     proxy_set_header   X-Real-IP        $remote_addr;
 	     proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
@@ -188,8 +190,8 @@ Started wiki`_ is always a good resource *to keep in the pocket*.
 	     proxy_set_header   X-Script-Name    /searx;
 	 }
 
-	 location /searx/static {
-	     alias /usr/local/searx/searx-src/searx/static;
+	 location /searx/static/ {
+	     alias /usr/local/searx/searx-src/searx/static/;
 	 }
 
 
@@ -203,28 +205,28 @@ Started wiki`_ is always a good resource *to keep in the pocket*.
 	 location /morty {
              proxy_pass         http://127.0.0.1:3000/;
 
-             proxy_set_header   Host             $http_host;
+             proxy_set_header   Host             $host;
              proxy_set_header   Connection       $http_connection;
              proxy_set_header   X-Real-IP        $remote_addr;
              proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
              proxy_set_header   X-Scheme         $scheme;
          }
 
-      Note that reverse proxy advised to be used in case of single-user or
-      low-traffic instances.  For a fully result proxification add :ref:`morty's
-      <searx morty>` **public URL** to your :origin:`searx/settings.yml`:
+      For a fully result proxification add :ref:`morty's <searx morty>` **public
+      URL** to your :origin:`searx/settings.yml`:
 
       .. code:: yaml
 
          result_proxy:
              # replace example.org with your server's public name
              url : https://example.org/morty
+             key : !!binary "insert_your_morty_proxy_key_here"
 
          server:
              image_proxy : True
 
 
-   .. group-tab:: proxy or uWSGI 
+   .. group-tab:: proxy or uWSGI
 
       Be warned, with this setup, your instance isn't :ref:`protected <searx
       filtron>`.  Nevertheless it is good enough for intranet usage and it is a
@@ -307,8 +309,8 @@ Started wiki`_ is always a good resource *to keep in the pocket*.
              proxy_buffering off;
          }
 
-         location /searx/static {
-             alias /usr/local/searx/searx-src/searx/static;
+         location /searx/static/ {
+             alias /usr/local/searx/searx-src/searx/static/;
          }
 
       The ``X-Script-Name /searx`` is needed by the searx implementation to
@@ -326,8 +328,8 @@ Started wiki`_ is always a good resource *to keep in the pocket*.
              uwsgi_pass unix:/run/uwsgi/app/searx/socket;
          }
 
-         location /searx/static {
-             alias /usr/local/searx/searx-src/searx;
+         location /searx/static/ {
+             alias /usr/local/searx/searx-src/searx/;
          }
 
       For searx to work correctly the ``base_url`` must be set in the

docs/admin/installation-searx.rst

@@ -61,7 +61,7 @@ from the login (*~/.profile*):
 
 .. tip::
 
-   Open a second terminal for the configuration tasks and left the ``(searx)$``
+   Open a second terminal for the configuration tasks and leave the ``(searx)$``
    terminal open for the tasks below.
 
 
@@ -70,13 +70,20 @@ from the login (*~/.profile*):
 Configuration
 =============
 
+.. sidebar:: ``use_default_settings: True``
+
+   - :ref:`settings global`
+   - :ref:`settings location`
+   - :ref:`settings use_default_settings`
+   - :origin:`/etc/searx/settings.yml <utils/templates/etc/searx/use_default_settings.yml>`
+
 To create a initial ``/etc/searx/settings.yml`` you can start with a copy of the
 file :origin:`utils/templates/etc/searx/use_default_settings.yml`.  This setup
-:option:ref:`use default settings <settings use_default_settings>` from
+:ref:`use default settings <settings use_default_settings>` from
 :origin:`searx/settings.yml` and is recommended since :pull:`2291` is merged.
 
-For minimal Setup, configure like shown below – replace ``searx@\$(uname -n)``
-with a name of your choice, set ``ultrasecretkey`` -- *and/or* edit
+For a *minimal setup*, configure like shown below – replace ``searx@$(uname
+-n)`` with a name of your choice, set ``ultrasecretkey`` -- *and/or* edit
 ``/etc/searx/settings.yml`` to your needs.
 
 .. kernel-include:: $DOCS_BUILD/includes/searx.rst

docs/admin/installation-uwsgi.rst

@@ -94,8 +94,8 @@ My experience is, that this command is a bit buggy.
 
 .. _uwsgi configuration:
 
-Alltogether
-===========
+All together
+============
 
 Create the configuration ini-file according to your distribution (see below) and
 restart the uwsgi application.

docs/admin/installation.rst

@@ -39,13 +39,18 @@ install from ``root``, take into account that the scripts are creating a
 these new created users do need read access to the clone of searx, which is not
 the case if you clone into a folder below ``/root``.
 
-
 .. code:: bash
 
    $ cd ~/Downloads
    $ git clone https://github.com/searx/searx searx
    $ cd searx
 
+.. sidebar:: further read
+
+   - :ref:`toolboxing`
+   - :ref:`update searx`
+   - :ref:`inspect searx`
+
 **Install** :ref:`searx service <searx.sh>`
 
 This installs searx as described in :ref:`installation basic`.
@@ -76,6 +81,6 @@ If all services are running fine, you can add it to your HTTP server:
 .. tip::
 
    About script's installation options have a look at chapter :ref:`toolboxing
-   setup`.  How to brand your instance see chapter :ref:`makefile setup`.  To
+   setup`.  How to brand your instance see chapter :ref:`settings global`.  To
    *stash* your instance's setup, `git stash`_ your clone's :origin:`Makefile`
    and :origin:`.config.sh` file .

docs/admin/morty.rst

@@ -16,15 +16,22 @@ By default searx can only act as an image proxy for result images, but it is
 possible to proxify all the result URLs with an external service, morty_.
 
 To use this feature, morty has to be installed and activated in searx's
-``settings.yml``.
-
-Add the following snippet to your ``settings.yml`` and restart searx:
+``settings.yml``.  Add the following snippet to your ``settings.yml`` and
+restart searx:
 
 .. code:: yaml
 
     result_proxy:
         url : http://127.0.0.1:3000/
-        key : your_morty_proxy_key
+        key : !!binary "insert_your_morty_proxy_key_here"
+
+Note that the example above (``http://127.0.0.1:3000``) is only for single-user
+instances without a HTTP proxy.  If your morty service is public, the url is the
+address of the reverse proxy (e.g ``https://example.org/morty``).
+
+For more information about *result proxy* have a look at *"searx via filtron
+plus morty"* in the :ref:`nginx <nginx searx via filtron plus morty>` and
+:ref:`apache <apache searx via filtron plus morty>` sections.
 
 ``url``
   Is the address of the running morty service.

docs/admin/no-sql-engines.rst

@@ -0,0 +1,170 @@
+===========================
+Query SQL and NoSQL servers
+===========================
+
+SQL
+===
+
+SQL servers are traditional databases with predefined data schema. Furthermore,
+modern versions also support BLOB data.
+
+You can search in the following servers:
+
+* `PostgreSQL`_
+* `MySQL`_
+* `SQLite`_
+
+The configuration of the new database engines are similar. You must put a valid
+SELECT SQL query in ``query_str``. At the moment you can only bind at most
+one parameter in your query.
+
+Do not include LIMIT or OFFSET in your SQL query as the engines
+rely on these keywords during paging.
+
+PostgreSQL
+----------
+
+Required PyPi package: ``psychopg2``
+
+You can find an example configuration below:
+
+.. code:: yaml
+
+  - name : postgresql
+    engine : postgresql
+    database : my_database
+    username : searx
+    password : password
+    query_str : 'SELECT * from my_table WHERE my_column = %(query)s'
+    shortcut : psql
+
+
+Available options
+~~~~~~~~~~~~~~~~~
+* ``host``: IP address of the host running PostgreSQL. By default it is ``127.0.0.1``.
+* ``port``: Port number PostgreSQL is listening on. By default it is ``5432``.
+* ``database``: Name of the database you are connecting to.
+* ``username``: Name of the user connecting to the database.
+* ``password``: Password of the database user.
+* ``query_str``: Query string to run. Keywords like ``LIMIT`` and ``OFFSET`` are not allowed. Required.
+* ``limit``: Number of returned results per page. By default it is 10.
+
+MySQL
+-----
+
+Required PyPi package: ``mysql-connector-python``
+
+This is an example configuration for quering a MySQL server:
+
+.. code:: yaml
+
+  - name : mysql
+    engine : mysql_server
+    database : my_database
+    username : searx
+    password : password
+    limit : 5
+    query_str : 'SELECT * from my_table WHERE my_column=%(query)s'
+    shortcut : mysql
+
+
+Available options
+~~~~~~~~~~~~~~~~~
+* ``host``: IP address of the host running MySQL. By default it is ``127.0.0.1``.
+* ``port``: Port number MySQL is listening on. By default it is ``3306``.
+* ``database``: Name of the database you are connecting to.
+* ``auth_plugin``: Authentication plugin to use. By default it is ``caching_sha2_password``.
+* ``username``: Name of the user connecting to the database.
+* ``password``: Password of the database user.
+* ``query_str``: Query string to run. Keywords like ``LIMIT`` and ``OFFSET`` are not allowed. Required.
+* ``limit``: Number of returned results per page. By default it is 10.
+
+SQLite
+------
+
+You can read from your database ``my_database`` using this example configuration:
+
+.. code:: yaml
+
+  - name : sqlite
+    engine : sqlite
+    shortcut: sq
+    database : my_database
+    query_str : 'SELECT * FROM my_table WHERE my_column=:query'
+
+
+Available options
+~~~~~~~~~~~~~~~~~
+* ``database``: Name of the database you are connecting to.
+* ``query_str``: Query string to run. Keywords like ``LIMIT`` and ``OFFSET`` are not allowed. Required.
+* ``limit``: Number of returned results per page. By default it is 10.
+
+NoSQL
+=====
+
+NoSQL data stores are used for storing arbitrary data without first defining their
+structure. To query the supported servers, you must install their drivers using PyPi.
+
+You can search in the following servers:
+
+* `Redis`_
+* `MongoDB`_
+
+Redis
+-----
+
+Reqired PyPi package: ``redis``
+
+Example configuration:
+
+.. code:: yaml
+
+  - name : mystore
+    engine : redis_server
+    exact_match_only : True
+    host : 127.0.0.1
+    port : 6379
+    password : secret-password
+    db : 0
+    shortcut : rds
+    enable_http : True
+
+Available options
+~~~~~~~~~~~~~~~~~
+
+* ``host``: IP address of the host running Redis. By default it is ``127.0.0.1``.
+* ``port``: Port number Redis is listening on. By default it is ``6379``.
+* ``password``: Password if required by Redis.
+* ``db``: Number of the database you are connecting to.
+* ``exact_match_only``: Enable if you need exact matching. By default it is ``True``.
+
+
+MongoDB
+-------
+
+Required PyPi package: ``pymongo``
+
+Below is an example configuration for using a MongoDB collection:
+
+.. code:: yaml
+
+  - name : mymongo
+    engine : mongodb
+    shortcut : icm
+    host : '127.0.0.1'
+    port : 27017
+    database : personal
+    collection : income
+    key : month
+    enable_http: True
+
+
+Available options
+~~~~~~~~~~~~~~~~~
+
+* ``host``: IP address of the host running MongoDB. By default it is ``127.0.0.1``.
+* ``port``: Port number MongoDB is listening on. By default it is ``27017``.
+* ``password``: Password if required by Redis.
+* ``database``: Name of the database you are connecting to.
+* ``collection``: Name of the collection you want to search in.
+* ``exact_match_only``: Enable if you need exact matching. By default it is ``True``.

docs/admin/plugins.rst

@@ -14,7 +14,7 @@ Configuration defaults (at built time):
 
 .. _configured plugins:
 
-.. jinja:: webapp
+.. jinja:: searx
 
    .. flat-table:: Plugins configured at built time (defaults)
       :header-rows: 1

docs/admin/private-engines.rst

@@ -0,0 +1,44 @@
+=============================
+How to create private engines
+=============================
+
+If you are running your public searx instance, you might want to restrict access
+to some engines. Maybe you are afraid of bots might abusing the engine. Or the
+engine might return private results you do not want to share with strangers.
+
+Server side configuration
+=========================
+
+You can make any engine private by setting a list of tokens in your settings.yml
+file. In the following example, we set two different tokens that provide access
+to the engine.
+
+.. code:: yaml
+
+    - name: my-private-google
+      engine: google
+      shortcut: pgo
+      tokens: ['my-secret-token-1', 'my-secret-token-2']
+
+
+To access the private engine, you must distribute the tokens to your searx
+users. It is up to you how you let them know what the access token is you
+created.
+
+Client side configuration
+=========================
+
+As a searx instance user, you can add any number of access tokens on the
+Preferences page. You have to set a comma separated lists of strings in "Engine
+tokens" input, then save your new preferences.
+
+.. image:: prefernces-private.png
+    :width: 600px
+    :align: center
+    :alt: location of token textarea
+
+Once the Preferences page is loaded again, you can see the information of the
+private engines you got access to. If you cannot see the expected engines in the
+engines list, double check your token. If there is no issue with the token,
+contact your instance administrator.
+

docs/admin/settings.rst

@@ -27,7 +27,8 @@ First, searx will try to load settings.yml from these locations:
 1. the full path specified in the ``SEARX_SETTINGS_PATH`` environment variable.
 2. ``/etc/searx/settings.yml``
 
-If these files don't exist (or are empty or can't be read), searx uses the :origin:`searx/settings.yml` file.
+If these files don't exist (or are empty or can't be read), searx uses the
+:origin:`searx/settings.yml` file.
 
 
 .. _settings global:
@@ -35,16 +36,46 @@ If these files don't exist (or are empty or can't be read), searx uses the :orig
 Global Settings
 ===============
 
+``general:``
+------------
+
 .. code:: yaml
 
    general:
        debug : False # Debug mode, only for development
        instance_name : "searx" # displayed name
+       git_url: https://github.com/searx/searx
+       git_branch: master
+       issue_url: https://github.com/searx/searx/issues
+       docs_url: https://searx.github.io/searx
+       public_instances: https://searx.space
+       contact_url: False # mailto:contact@example.com
+       wiki_url: https://github.com/searx/searx/wiki
+       twitter_url: https://twitter.com/Searx_engine
 
 ``debug`` :
   Allow a more detailed log if you run searx directly. Display *detailed* error
   messages in the browser too, so this must be deactivated in production.
 
+``contact_url``:
+  Contact ``mailto:`` address or WEB form.
+
+``git_url`` and ``git_branch``:
+  Changes this, to point to your searx fork (branch).
+
+``docs_url``
+  If you host your own documentation, change this URL.
+
+``wiki_url``:
+  Link to your wiki (or ``False``)
+
+``twitter_url``:
+  Link to your tweets (or ``False``)
+
+
+``server:``
+-----------
+
 .. code:: yaml
 
    server:
@@ -90,13 +121,15 @@ Global Settings
 ``default_http_headers``:
   Set additional HTTP headers, see `#755 <https://github.com/searx/searx/issues/715>`__
 
+``outgoing:``
+-------------
 
 .. code:: yaml
 
    outgoing: # communication with search engines
        request_timeout : 2.0        # default timeout in seconds, can be override by engine
        # max_request_timeout: 10.0  # the maximum timeout in seconds
-       useragent_suffix : ""        # informations like an email address to the administrator
+       useragent_suffix : ""        # information like an email address to the administrator
        pool_connections : 100       # Number of different hosts
        pool_maxsize : 10            # Number of simultaneous requests by host
    # uncomment below section if you want to use a proxy
@@ -139,6 +172,10 @@ Global Settings
   If you use multiple network interfaces, define from which IP the requests must
   be made. This parameter is ignored when ``proxies`` is set.
 
+
+``locales:``
+------------
+
 .. code:: yaml
 
    locales:
@@ -244,61 +281,76 @@ Engine settings
 use_default_settings
 ====================
 
-.. note::
+.. sidebar:: ``use_default_settings: True``
 
-   If searx is cloned from a git repository, most probably there is no need to have an user settings.
+   - :ref:`settings location`
+   - :ref:`use_default_settings.yml`
+   - :origin:`/etc/searx/settings.yml <utils/templates/etc/searx/use_default_settings.yml>`
 
-The user defined settings.yml can relied on the default configuration :origin:`searx/settings.yml` using ``use_default_settings: True``.
+The user defined ``settings.yml`` is loaded from the :ref:`settings location`
+and can relied on the default configuration :origin:`searx/settings.yml` using:
 
-In the following example, the actual settings are the default settings defined in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and the ``bind_address``:
+ ``use_default_settings: True``
 
-.. code-block:: yaml
+``server:``
+  In the following example, the actual settings are the default settings defined
+  in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and
+  the ``bind_address``:
 
-  use_default_settings: True
-  server:
-      secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
-      bind_address: "0.0.0.0"
+  .. code-block:: yaml
 
-With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``.
+    use_default_settings: True
+    server:
+        secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
+        bind_address: "0.0.0.0"
 
-In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token<private engines>`:
+``engines:``
+  With ``use_default_settings: True``, each settings can be override in a
+  similar way, the ``engines`` section is merged according to the engine
+  ``name``.  In this example, searx will load all the engine and the arch linux
+  wiki engine has a :ref:`token<private engines>`:
 
-.. code-block:: yaml
+  .. code-block:: yaml
 
-  use_default_settings: True
-  server:
-      secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
-  engines:
-    - name: arch linux wiki
-      tokens: ['$ecretValue']
+    use_default_settings: True
+    server:
+        secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
+    engines:
+      - name: arch linux wiki
+        tokens: ['$ecretValue']
 
-It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine:
+``engines:`` / ``remove:``
+  It is possible to remove some engines from the default settings. The following
+  example is similar to the above one, but searx doesn't load the the google
+  engine:
 
-.. code-block:: yaml
+  .. code-block:: yaml
 
-  use_default_settings:
-      engines:
-         remove:
-           - google
-  server:
-      secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
-  engines:
-    - name: arch linux wiki
-      tokens: ['$ecretValue']
+    use_default_settings:
+        engines:
+           remove:
+             - google
+    server:
+        secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
+    engines:
+      - name: arch linux wiki
+        tokens: ['$ecretValue']
 
-As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines:
+``engines:`` / ``keep_only:``
+  As an alternative, it is possible to specify the engines to keep. In the
+  following example, searx has only two engines:
 
-.. code-block:: yaml
+  .. code-block:: yaml
 
-  use_default_settings:
-      engines:
-         keep_only:
-           - google
-           - duckduckgo
-  server:
-      secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
-  engines:
-    - name: google
-      tokens: ['$ecretValue']
-    - name: duckduckgo
-      tokens: ['$ecretValue']
+    use_default_settings:
+        engines:
+           keep_only:
+             - google
+             - duckduckgo
+    server:
+        secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
+    engines:
+      - name: google
+        tokens: ['$ecretValue']
+      - name: duckduckgo
+        tokens: ['$ecretValue']

docs/admin/update-searx.rst

@@ -4,20 +4,56 @@
 How to update
 =============
 
+How to update depends on the :ref:`installation` method.  If you have used the
+:ref:`installation scripts`, use ``update`` command from the scripts.
+
+**Update** :ref:`searx service <searx.sh>`
+
 .. code:: sh
 
-    sudo -H -u searx -i
-    (searx)$ git stash
-    (searx)$ git pull origin master
-    (searx)$ git stash apply
-    (searx)$ ./manage.sh update_packages
+    sudo -H ./utils/searx.sh update searx
 
-Restart uwsgi:
+**Update** :ref:`filtron reverse proxy <filtron.sh>`
 
-.. tabs::
+.. code:: sh
 
-   .. group-tab:: Ubuntu / debian
+    sudo -H ./utils/filtron.sh update filtron
 
-      .. code:: sh
+**Update** :ref:`result proxy <morty.sh>`
+
+.. code:: bash
+
+    sudo -H ./utils/morty.sh update morty
+
+.. _inspect searx:
+
+======================
+How to inspect & debug
+======================
+
+.. sidebar:: further read
+
+   - :ref:`toolboxing`
+   - :ref:`Makefile`
+
+How to debug depends on the :ref:`installation` method.  If you have used the
+:ref:`installation scripts`, use ``inspect`` command from the scripts.
+
+**Inspect** :ref:`searx service <searx.sh>`
+
+.. code:: sh
+
+    sudo -H ./utils/searx.sh inspect service
+
+**Inspect** :ref:`filtron reverse proxy <filtron.sh>`
+
+.. code:: sh
+
+    sudo -H ./utils/filtron.sh inspect service
+
+**Inspect** :ref:`result proxy <morty.sh>`
+
+.. code:: bash
+
+    sudo -H ./utils/morty.sh inspect service
 
-         sudo -H systemctl restart uwsgi

docs/blog/documentation-offline-engines.rst

@@ -0,0 +1,48 @@
+=================================
+Private searx project is finished
+=================================
+
+We are officially finished with the Private searx project. The goal was to
+extend searx capabilities beyond just searching on the Internet. We added
+support for offline engines. These engines do not connect to the Internet,
+they find results locally.
+
+As some of the offline engines run commands on the searx host, we added an
+option to protect any engine by making them private. Private engines can only be
+accessed using a token.
+
+After searx was prepared to run offline queries we added numerous new engines:
+
+1. Command line engine
+2. MySQL
+3. PostgreSQL
+4. SQLite
+5. Redis
+6. MongoDB
+
+We also added new engines that communicate over HTTP, but you might want to keep
+them private:
+
+1. Elasticsearch
+2. Meilisearch
+3. Solr
+
+The last step was to document this work. We added new tutorials on creating
+command engines, making engines private and also adding a custom result template
+to your own engines.
+
+Acknowledgement
+===============
+
+The project was sponsored by `Search and Discovery Fund`_ of `NLnet
+Foundation`_. We would like to thank the NLnet for not only the funds, but the
+conversations and their ideas. They were truly invested and passionate about
+supporting searx.
+
+.. _Search and Discovery Fund: https://nlnet.nl/discovery
+.. _NLnet Foundation: https://nlnet.nl/
+
+
+| Happy hacking.
+| kvch // 2022.09.30 23:15
+

docs/blog/index.rst

@@ -3,12 +3,16 @@ Blog
 ====
 
 .. toctree::
-   :maxdepth: 2
-   :caption: Contents
+   :titlesonly:
+   :reversed:
 
-   lxcdev-202006
    python3
    admin
    intro-offline
    private-engines
+   lxcdev-202006
    command-line-engines
+   search-indexer-engines
+   sql-engines
+   search-database-engines
+   documentation-offline-engines

docs/blog/lxcdev-202006.rst

@@ -31,7 +31,7 @@ might fail in some aspects we should not overlook.
 
     The environment in which we run all our development processes matters!
 
-The :ref:`makefile` and the :ref:`make pyenv` encapsulate a lot for us, but they
+The :ref:`makefile` and the :ref:`make install` encapsulate a lot for us, but they
 do not have access to all prerequisites.  For example, there may have
 dependencies on packages that are installed on the developer's desktop, but
 usually are not preinstalled on a server or client system.  Another examples
@@ -207,7 +207,7 @@ debug services from filtron and morty analogous use:
 Another point we have to notice is that each service (:ref:`searx <searx.sh>`,
 :ref:`filtron <filtron.sh>` and :ref:`morty <morty.sh>`) runs under dedicated
 system user account with the same name (compare :ref:`create searx user`).  To
-get a shell from theses accounts, simply call one of the scripts:
+get a shell from these accounts, simply call one of the scripts:
 
 .. tabs::
 
@@ -259,8 +259,8 @@ suite.  For this, we have to keep an eye on the :ref:`installation basic`:
 - virtualenv in: ``/usr/local/searx/searx-pyenv``
 - searx software in: ``/usr/local/searx/searx-src``
 
-The searx software is a clone of the ``GIT_URL`` (see :ref:`makefile setup`) and
-the working tree is checked out from the ``GIT_BRANCH``.  With the use of the
+The searx software is a clone of the ``git_url`` (see :ref:`settings global`) and
+the working tree is checked out from the ``git_branch``.  With the use of the
 :ref:`searx.sh` the searx service was installed as :ref:`uWSGI application
 <searx uwsgi>`.  To maintain this service, we can use ``systemctl`` (compare
 :ref:`service architectures on distributions <uwsgi configuration>`).
@@ -311,7 +311,7 @@ of the container:
 
 Now we can develop as usual in the working tree of our desktop system.  Every
 time the software was changed, you have to restart the searx service (in the
-conatiner):
+container):
 
 .. tabs::
 
@@ -356,7 +356,7 @@ daily usage:
      .. code:: sh
 
         $ sudo -H ./utils/lxc.sh cmd searx-archlinux \
-          make docs
+          make docs.html
 
 .. _blog-lxcdev-202006 abstract:
 
@@ -370,7 +370,7 @@ We build up a fully functional searx suite in a archlinux container:
    $ sudo -H ./utils/lxc.sh install suite searx-archlinux
 
 To access HTTP from the desktop we installed nginx for the services inside the
-conatiner:
+container:
 
 .. tabs::
 
@@ -407,7 +407,7 @@ To get remarks from the suite of the archlinux container we can use:
         ...
         [searx-archlinux]  INFO:  (eth0) filtron:    http://10.174.184.156:4004/ http://10.174.184.156/searx
         [searx-archlinux]  INFO:  (eth0) morty:      http://10.174.184.156:3000/
-        [searx-archlinux]  INFO:  (eth0) docs-live:  http://10.174.184.156:8080/
+        [searx-archlinux]  INFO:  (eth0) docs.live:  http://10.174.184.156:8080/
         [searx-archlinux]  INFO:  (eth0) IPv6:       http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65]
         ...
 

docs/blog/search-database-engines.rst

@@ -0,0 +1,95 @@
+===============================
+Query more of your NoSQL stores
+===============================
+
+From now on, searx lets you to query your NoSQL data stores:
+
+* `Redis`_
+* `MongoDB`_
+
+The reference configuration of the engines are included ``settings.yml`` just commented out,
+as you have to set various options and install dependencies before using them.
+
+By default, the engines use ``key-value`` template for displaying results.
+If you are not satisfied with the original result layout,
+you can use your owm template by placing the template under
+``searx/templates/{theme_name}/result_templates/{template_name}`` and setting
+``result_template`` attribute to ``{template_name}``.
+
+Furthermore, if you do not want to expose these engines on a public instance, you can
+still add them and limit the access by setting ``tokens`` as described in the `blog post about
+private engines`_.
+
+Configuring searx to use the stores
+===================================
+
+NoSQL data stores are used for storing arbitrary data without first defining their
+structure.
+
+Redis
+-----
+
+Required package: ``redis``
+
+Redis is a key value based data store usually stored in memory. 
+
+Select a database to search in and set its index in the option ``db``. You can
+either look for exact matches or use partial keywords to find what you are looking for
+by configuring ``exact_match_only``.
+
+In this example you can search for exact matches in your first database:
+
+.. code:: yaml
+
+  - name : mystore
+    engine : redis_server
+    exact_match_only : True
+    host : 127.0.0.1
+    port : 6379
+    password : secret-password
+    db : 0
+    shortcut : rds
+    enable_http : True
+
+
+MongoDB
+-------
+
+Required package: ``pymongo``
+
+MongoDB is a document based database program that handles JSON like data. 
+
+In order to query MongoDB, you have to select a ``database`` and a ``collection``. Furthermore,
+you have to select a ``key`` that is going to be searched. MongoDB also supports the option ``exact_match_only``, so configure it
+as you wish.
+
+Above is an example configuration for using a MongoDB collection:
+
+.. code:: yaml
+
+  - name : mymongo
+    engine : mongodb
+    shortcut : md
+    host : '127.0.0.1'
+    port : 27017
+    database : personal
+    collection : income
+    key : month
+    enable_http: True
+
+
+Acknowledgement
+===============
+
+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .
+
+.. _Redis: https://redis.io/
+.. _MongoDB: https://mongodb.com/
+.. _blog post about private engines: private-engines.html#private-engines
+.. _Search and Discovery Fund: https://nlnet.nl/discovery
+.. _NLnet Foundation: https://nlnet.nl/
+
+
+| Happy hacking.
+| kvch // 2021.07.13 23:16
+

docs/blog/search-indexer-engines.rst

@@ -0,0 +1,114 @@
+===============================
+Query your local search engines
+===============================
+
+From now on, searx lets you to query your locally running search engines. The following
+ones are supported now:
+
+* `Elasticsearch`_
+* `Meilisearch`_
+* `Solr`_
+
+All of the engines above are added to ``settings.yml`` just commented out, as you have to
+``base_url`` for all them.
+
+Please note that if you are not using HTTPS to access these engines, you have to enable
+HTTP requests by setting ``enable_http`` to ``True``.
+
+Furthermore, if you do not want to expose these engines on a public instance, you can
+still add them and limit the access by setting ``tokens`` as described in the `blog post about
+private engines`_.
+
+Configuring searx for search engines
+====================================
+
+Each search engine is powerful, capable of full-text search.
+
+Elasticsearch
+-------------
+
+Elasticsearch supports numerous ways to query the data it is storing. At the moment
+the engine supports the most popular search methods: ``match``, ``simple_query_string``, ``term`` and ``terms``.
+
+If none of the methods fit your use case, you can select ``custom`` query type and provide the JSON payload
+searx has to submit to Elasticsearch in ``custom_query_json``.
+
+The following is an example configuration for an Elasticsearch instance with authentication
+configured to read from ``my-index`` index.
+
+.. code:: yaml
+
+  - name : elasticsearch
+    shortcut : es
+    engine : elasticsearch
+    base_url : http://localhost:9200
+    username : elastic
+    password : changeme
+    index : my-index
+    query_type : match
+    enable_http : True
+
+
+Meilisearch
+-----------
+
+This search engine is aimed at individuals and small companies. It is designed for
+small-scale (less than 10 million documents) data collections. E.g. it is great for storing
+web pages you have visited and searching in the contents later.
+
+The engine supports faceted search, so you can search in a subset of documents of the collection.
+Furthermore, you can search in Meilisearch instances that require authentication by setting ``auth_token``.
+
+Here is a simple example to query a Meilisearch instance:
+
+.. code:: yaml
+
+  - name : meilisearch
+    engine : meilisearch
+    shortcut: mes
+    base_url : http://localhost:7700
+    index : my-index
+    enable_http: True
+
+
+Solr
+----
+
+Solr is a popular search engine based on Lucene, just like Elasticsearch.
+But instead of searching in indices, you can search in collections.
+
+This is an example configuration for searching in the collection ``my-collection`` and get
+the results in ascending order.
+
+.. code:: yaml
+
+  - name : solr
+    engine : solr
+    shortcut : slr
+    base_url : http://localhost:8983
+    collection : my-collection
+    sort : asc
+    enable_http : True
+
+
+Next steps
+==========
+
+The next step is to add support for various SQL databases.
+
+Acknowledgement
+===============
+
+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .
+
+.. _blog post about private engines: private-engines.html#private-engines
+.. _Elasticsearch: https://www.elastic.co/elasticsearch/
+.. _Meilisearch: https://www.meilisearch.com/
+.. _Solr: https://solr.apache.org/
+.. _Search and Discovery Fund: https://nlnet.nl/discovery
+.. _NLnet Foundation: https://nlnet.nl/
+
+
+| Happy hacking.
+| kvch // 2021.04.07 23:16
+

docs/blog/sql-engines.rst

@@ -0,0 +1,117 @@
+=================
+Query SQL servers 
+=================
+
+Now you can query SQL servers using searx. The following ones are supported:
+
+* `PostgreSQL`_
+* `MySQL`_
+* `SQLite`_
+
+All of the engines above are added to ``settings.yml`` just commented out, as you have to
+set the required attributes for the engines, e.g. ``database``. By default, the engines use
+``key-value`` template for displaying results. If you are not satisfied with the original result layout,
+you can use your owm template by placing the template under
+``searx/templates/{theme_name}/result_templates/{template_name}`` and setting
+``result_template`` attribute to ``{template_name}``.
+
+As mentioned in previous blog posts, if you do not wish to expose these engines on a
+public instance, you can still add them and limit the access by setting ``tokens``
+as described in the `blog post about private engines`_.
+
+Configure the engines
+=====================
+
+The configuration of the new database engines are similar. You must put a valid
+SELECT SQL query in ``query_str``. At the moment you can only bind at most
+one parameter in your query. By setting the attribute ``limit`` you can
+define how many results you want from the SQL server. Basically, it
+is the same as the LIMIT keyword in SQL.
+
+Please, do not include LIMIT or OFFSET in your SQL query as the engines
+rely on these keywords during paging. If you want to configure the number of returned results
+use the option ``limit``.
+
+PostgreSQL
+----------
+
+PostgreSQL is a powerful and robust open source database.
+
+Before configuring the PostgreSQL engine, you must install the dependency ``psychopg2``.
+
+You can find an example configuration below:
+
+.. code:: yaml
+
+  - name : postgresql
+    engine : postgresql
+    database : my_database
+    username : searx
+    password : password
+    query_str : 'SELECT * from my_table WHERE my_column = %(query)s'
+    shortcut : psql
+
+
+MySQL
+-----
+
+MySQL is said to be the most popular open source database. 
+
+Before enabling MySQL engine, you must install the package ``mysql-connector-python``.
+
+The authentication plugin is configurable by setting ``auth_plugin`` in the attributes.
+By default it is set to ``caching_sha2_password``.
+
+This is an example configuration for querying a MySQL server:
+
+.. code:: yaml
+
+  - name : mysql
+    engine : mysql_server
+    database : my_database
+    username : searx
+    password : password
+    limit : 5
+    query_str : 'SELECT * from my_table WHERE my_column=%(query)s'
+    shortcut : mysql
+
+
+SQLite
+------
+
+SQLite is a small, fast and reliable SQL database engine. It does not require
+any extra dependency.
+
+You can read from your database ``my_database`` using this example configuration:
+
+.. code:: yaml
+
+  - name : sqlite
+    engine : sqlite
+    shortcut: sq
+    database : my_database
+    query_str : 'SELECT * FROM my_table WHERE my_column=:query'
+
+
+Next steps
+==========
+
+The next step is to add support for more data stores, e.g. Redis and MongoDB.
+
+Acknowledgement
+===============
+
+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .
+
+.. _PostgreSQL: https://www.postgresql.org/
+.. _MySQL: https://www.mysql.com/
+.. _SQLite: https://www.sqlite.org/index.html
+.. _blog post about private engines: private-engines.html#private-engines
+.. _Search and Discovery Fund: https://nlnet.nl/discovery
+.. _NLnet Foundation: https://nlnet.nl/
+
+
+| Happy hacking.
+| kvch // 2021.05.23 23:16
+
+

docs/build-templates/morty.rst

@@ -32,7 +32,7 @@
        (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
        (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_TAR}\" \\
                    \"${GO_PKG_URL}\"
-       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf \"${GO_TAR}\"
+       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_TAR}\"
        (${SERVICE_USER}) $ which go
        ${SERVICE_HOME}/local/go/bin/go
 

docs/build-templates/searx.rst

@@ -116,6 +116,7 @@ ${fedora_build}
        pip install -U pip
        pip install -U setuptools
        pip install -U wheel
+       pip install -U pyyaml
 
        # jump to searx's working tree and install searx into virtualenv
        (${SERVICE_USER})$ cd \"$SEARX_SRC\"

docs/conf.py

@@ -1,20 +1,16 @@
 # -*- coding: utf-8 -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
 import  sys, os
-from sphinx_build_tools import load_sphinx_config
-from searx.version import VERSION_STRING
 from pallets_sphinx_themes import ProjectLink
 
-from searx.brand import GIT_URL
-GIT_BRANCH = os.environ.get("GIT_BRANCH", "master")
-from searx.brand import SEARX_URL
-from searx.brand import DOCS_URL
-
+from searx import brand
+from searx.version import VERSION_STRING
 
 # Project --------------------------------------------------------------
 
 project = u'searx'
-copyright = u'2015-2020, Adam Tauber, Noémi Ványi'
+copyright = u'2015-2022, Adam Tauber, Noémi Ványi'
 author = u'Adam Tauber'
 release, version = VERSION_STRING, VERSION_STRING
 highlight_language = 'none'
@@ -27,35 +23,41 @@ numfig = True
 
 exclude_patterns = ['build-templates/*.rst']
 
-from searx import webapp
+import searx.search
+import searx.engines
+import searx.plugins
+searx.search.initialize()
 jinja_contexts = {
-    'webapp': dict(**webapp.__dict__),
+    'searx': {
+        'engines': searx.engines.engines,
+        'plugins': searx.plugins.plugins
+    },
 }
 
 # usage::   lorem :patch:`f373169` ipsum
 extlinks = {}
 
 # upstream links
-extlinks['wiki'] = ('https://github.com/searx/searx/wiki/%s', ' ')
-extlinks['pull'] = ('https://github.com/searx/searx/pull/%s', 'PR ')
+extlinks['wiki'] = ('https://github.com/searx/searx/wiki/%s', '%s')
+extlinks['pull'] = ('https://github.com/searx/searx/pull/%s', 'PR %s')
 
 # links to custom brand
-extlinks['origin'] = (GIT_URL + '/blob/' + GIT_BRANCH + '/%s', 'git://')
-extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
-extlinks['search'] = (SEARX_URL + '/%s', '#')
-extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
-extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: ')
-extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '')
+extlinks['origin'] = (brand.GIT_URL + '/blob/' + brand.GIT_BRANCH + '/%s', 'Origin: %s')
+extlinks['patch'] = (brand.GIT_URL + '/commit/%s', 'path %s')
+extlinks['search'] = (brand.SEARX_URL + '/%s', 'URL: %s')
+extlinks['docs'] = (brand.DOCS_URL + '/%s', 'docs: %s')
+extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: %s')
+extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', 'man: %s')
 #extlinks['role'] = (
 #    'https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-%s', '')
 extlinks['duref'] = (
-    'https://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '')
+    'https://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '%s')
 extlinks['durole'] = (
-    'https://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '')
+    'https://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '%s')
 extlinks['dudir'] =  (
-    'https://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '')
+    'https://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '%s')
 extlinks['ctan'] =  (
-    'https://ctan.org/pkg/%s', 'CTAN: ')
+    'https://ctan.org/pkg/%s', 'CTAN: %s')
 
 extensions = [
     'sphinx.ext.imgmath',
@@ -65,7 +67,7 @@ extensions = [
     "sphinx.ext.intersphinx",
     "pallets_sphinx_themes",
     "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
-    "sphinxcontrib.jinja",  # https://github.com/tardyp/sphinx-jinja
+    "sphinx_jinja",  # https://github.com/tardyp/sphinx-jinja
     "sphinxcontrib.programoutput",  # https://github.com/NextThought/sphinxcontrib-programoutput
     'linuxdoc.kernel_include',  # Implementation of the 'kernel-include' reST-directive.
     'linuxdoc.rstFlatTable',    # Implementation of the 'flat-table' reST-directive.
@@ -98,14 +100,19 @@ imgmath_font_size = 14
 # sphinx.ext.imgmath setup END
 
 html_theme_options = {"index_sidebar_logo": True}
-html_context = {
-    "project_links": [
-        ProjectLink("Source", GIT_URL),
-        ProjectLink("Wiki", "https://github.com/searx/searx/wiki"),
-        ProjectLink("Public instances", "https://searx.space/"),
-        ProjectLink("Twitter", "https://twitter.com/Searx_engine"),
-    ]
-}
+html_context = {"project_links": [] }
+html_context["project_links"].append(ProjectLink("Blog", brand.DOCS_URL + "/blog/index.html"))
+if brand.GIT_URL:
+    html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))
+if brand.WIKI_URL:
+    html_context["project_links"].append(ProjectLink("Wiki", brand.WIKI_URL))
+if brand.TWITTER_URL:
+    html_context["project_links"].append(ProjectLink("Twitter", brand.TWITTER_URL))
+if brand.ISSUE_URL:
+    html_context["project_links"].append(ProjectLink("Issue Tracker", brand.ISSUE_URL))
+if brand.CONTACT_URL:
+    html_context["project_links"].append(ProjectLink("Contact", brand.CONTACT_URL))
+
 html_sidebars = {
     "**": ["project.html", "relations.html", "searchbox.html"],
 }
@@ -120,9 +127,3 @@ html_show_sourcelink = False
 latex_documents = [
     (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")
 ]
-
-# ------------------------------------------------------------------------------
-# Since loadConfig overwrites settings from the global namespace, it has to be
-# the last statement in the conf.py file
-# ------------------------------------------------------------------------------
-load_sphinx_config(globals())

docs/dev/contribution_guide.rst

@@ -132,11 +132,11 @@ Here is an example which makes a complete rebuild:
 
 .. code:: sh
 
-   $ make docs-clean docs
+   $ make docs.clean docs.html
    ...
    The HTML pages are in dist/docs.
 
-.. _make docs-live:
+.. _make docs.live:
 
 live build
 ----------
@@ -144,19 +144,19 @@ live build
 .. _sphinx-autobuild:
    https://github.com/executablebooks/sphinx-autobuild/blob/master/README.md
 
-.. sidebar:: docs-clean
+.. sidebar:: docs.clean
 
    It is recommended to assert a complete rebuild before deploying (use
-   ``docs-clean``).
+   ``docs.clean``).
 
 Live build is like WYSIWYG.  If you want to edit the documentation, its
-recommended to use.  The Makefile target ``docs-live`` builds the docs, opens
+recommended to use.  The Makefile target ``docs.live`` builds the docs, opens
 URL in your favorite browser and rebuilds every time a reST file has been
 changed.
 
 .. code:: sh
 
-   $ make docs-live
+   $ make docs.live
    ...
    The HTML pages are in dist/docs.
    ... Serving on http://0.0.0.0:8000
@@ -169,7 +169,7 @@ argument.  E.g to find and use a free port, use:
 
 .. code:: sh
 
-   $ SPHINXOPTS="--port 0" make docs-live
+   $ SPHINXOPTS="--port 0" make docs.live
    ...
    ... Serving on http://0.0.0.0:50593
    ...
@@ -180,21 +180,10 @@ argument.  E.g to find and use a free port, use:
 deploy on github.io
 -------------------
 
-To deploy documentation at :docs:`github.io <.>` use Makefile target
-:ref:`make gh-pages`, which will builds the documentation, clones searx into a sub
-folder ``gh-pages``, cleans it, copies the doc build into and runs all the
-needed git add, commit and push:
+To deploy documentation at :docs:`github.io <.>` use Makefile target :ref:`make
+docs.gh-pages`, which builds the documentation and runs all the needed git add,
+commit and push:
 
 .. code:: sh
 
-   $ make docs-clean gh-pages
-   ...
-   SPHINX    docs --> file://<...>/dist/docs
-   The HTML pages are in dist/docs.
-   ...
-   Cloning into 'gh-pages' ...
-   ...
-   cd gh-pages; git checkout gh-pages >/dev/null
-   Switched to a new branch 'gh-pages'
-   ...
-   doc available at --> https://searx.github.io/searx
+   $ make docs.clean docs.gh-pages

docs/dev/engine_overview.rst

@@ -37,15 +37,15 @@ settings.  However, the standard way is the following:
 engine file
 -----------
 
-======================= =========== ===========================================
+======================= =========== ========================================================
 argument                type        information
-======================= =========== ===========================================
+======================= =========== ========================================================
 categories              list        pages, in which the engine is working
-paging                  boolean     support multible pages
-language_support        boolean     support language choosing
+paging                  boolean     support multiple pages
 time_range_support      boolean     support search time range
-offline                 boolean     engine runs offline
-======================= =========== ===========================================
+engine_type             str         ``online`` by default, other possibles values are 
+                                    ``offline``, ``online_dictionary``, ``online_currency``
+======================= =========== ========================================================
 
 .. _engine settings:
 
@@ -58,6 +58,8 @@ argument                type        information
 name                    string      name of search-engine
 engine                  string      name of searx-engine
                                     (filename without ``.py``)
+enable_http             bool        enable HTTP
+                                    (by default only HTTPS is enabled).
 shortcut                string      shortcut of search-engine
 timeout                 string      specific timeout for search-engine
 display_error_messages  boolean     display error messages on the web UI
@@ -96,7 +98,6 @@ example code
    # engine dependent config
    categories = ['general']
    paging = True
-   language_support = True
 
 
 making a request
@@ -111,38 +112,66 @@ passed arguments
 These arguments can be used to construct the search query.  Furthermore,
 parameters with default value can be redefined for special purposes.
 
+If the ``engine_type`` is ``online```:
+
+====================== ============== ========================================================================
+argument               type           default-value, information
+====================== ============== ========================================================================
+url                    str            ``''``
+method                 str            ``'GET'``
+headers                set            ``{}``
+data                   set            ``{}``
+cookies                set            ``{}``
+verify                 bool           ``True``
+headers.User-Agent     str            a random User-Agent
+category               str            current category, like ``'general'``
+safesearch             int            ``0``, between ``0`` and ``2`` (normal, moderate, strict)
+time_range             Optional[str]  ``None``, can be ``day``, ``week``, ``month``, ``year``
+pageno                 int            current pagenumber
+language               str            specific language code like ``'en_US'``, or ``'all'`` if unspecified
+====================== ============== ========================================================================
+
+
+If the ``engine_type`` is ``online_dictionary```, in addition to the ``online`` arguments:
+
 ====================== ============ ========================================================================
 argument               type         default-value, information
 ====================== ============ ========================================================================
-url                    string       ``''``
-method                 string       ``'GET'``
-headers                set          ``{}``
-data                   set          ``{}``
-cookies                set          ``{}``
-verify                 boolean      ``True``
-headers.User-Agent     string       a random User-Agent
-category               string       current category, like ``'general'``
-started                datetime     current date-time
-pageno                 int          current pagenumber
-language               string       specific language code like ``'en_US'``, or ``'all'`` if unspecified
+from_lang              str          specific language code like ``'en_US'``
+to_lang                str          specific language code like ``'en_US'``
+query                  str          the text query without the languages
 ====================== ============ ========================================================================
 
+If the ``engine_type`` is ``online_currency```, in addition to the ``online`` arguments:
+
+====================== ============ ========================================================================
+argument               type         default-value, information
+====================== ============ ========================================================================
+amount                 float        the amount to convert
+from                   str          ISO 4217 code
+to                     str          ISO 4217 code
+from_name              str          currency name
+to_name                str          currency name
+====================== ============ ========================================================================
+
+
 parsed arguments
 ----------------
 
 The function ``def request(query, params):`` always returns the ``params``
-variable.  Inside searx, the following paramters can be used to specify a search
+variable.  Inside searx, the following parameters can be used to specify a search
 request:
 
 =================== =========== ==========================================================================
 argument            type        information
 =================== =========== ==========================================================================
-url                 string      requested url
-method              string      HTTP request method
+url                 str         requested url
+method              str         HTTP request method
 headers             set         HTTP header information
-data                set         HTTP data information (parsed if ``method != 'GET'``)
+data                set         HTTP data information
 cookies             set         HTTP cookies
-verify              boolean     Performing SSL-Validity check
+verify              bool        Performing SSL-Validity check
+follow_redirects     bool        Follow redirects
 max_redirects       int         maximum redirects, hard limit
 soft_max_redirects  int         maximum redirects, soft limit. Record an error but don't stop the engine
 raise_for_httperror bool        True by default: raise an exception if the HTTP code of response is >= 300

docs/dev/makefile.rst

@@ -1,65 +1,33 @@
 .. _makefile:
 
-================
-Makefile Targets
-================
+========
+Makefile
+========
 
 .. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction
 
 .. sidebar:: build environment
 
-   Before looking deeper at the targets, first read about :ref:`makefile setup`
-   and :ref:`make pyenv`.
+   Before looking deeper at the targets, first read about :ref:`make
+   install`.
 
    To install system requirements follow :ref:`buildhosts`.
 
-With the aim to simplify development cycles, started with :pull:`1756` a
-``Makefile`` based boilerplate was added.  If you are not familiar with
-Makefiles, we recommend to read gnu-make_ introduction.
+All relevant build tasks are implemented in :origin:`manage.sh` and for CI or
+IDE integration a small ``Makefile`` wrapper is available.  If you are not
+familiar with Makefiles, we recommend to read gnu-make_ introduction.
 
 The usage is simple, just type ``make {target-name}`` to *build* a target.
 Calling the ``help`` target gives a first overview (``make help``):
 
 .. program-output:: bash -c "cd ..; make --no-print-directory help"
 
-
 .. contents:: Contents
    :depth: 2
    :local:
    :backlinks: entry
 
-
-.. _makefile setup:
-
-Makefile setup
-==============
-
-.. _git stash: https://git-scm.com/docs/git-stash
-
-.. sidebar:: fork & upstream
-
-   Commit changes in your (local) branch, fork or whatever, but do not push them
-   upstream / `git stash`_ is your friend.
-
-The main setup is done in the :origin:`Makefile`.
-
-.. literalinclude:: ../../Makefile
-   :start-after: START Makefile setup
-   :end-before: END Makefile setup
-
-:GIT_URL:    Changes this, to point to your searx fork.
-:GIT_BRANCH: Changes this, to point to your searx branch.
-:SEARX_URL:  Changes this, to point to your searx instance.
-:DOCS_URL:   If you host your own (*brand*) documentation, change this URL.
-
-If you change any of this build environment variables, you have to run ``make
-buildenv``::
-
-  $ make buildenv
-  build searx/brand.py
-  build utils/brand.env
-
-.. _make pyenv:
+.. _make install:
 
 Python environment
 ==================
@@ -68,31 +36,42 @@ Python environment
 
    ``source ./local/py3/bin/activate``
 
-With Makefile we do no longer need to build up the virtualenv manually (as
-described in the :ref:`devquickstart` guide).  Jump into your git working tree
-and release a ``make pyenv``:
-
-.. code:: sh
+We do no longer need to build up the virtualenv manually.  Jump into your git
+working tree and release a ``make install`` to get a virtualenv with a
+*developer install* of searx (:origin:`setup.py`). ::
 
    $ cd ~/searx-clone
-   $ make pyenv
-   PYENV     usage: source ./local/py3/bin/activate
+   $ make install
+   PYENV     [virtualenv] installing ./requirements*.txt into local/py3
    ...
+   PYENV     OK
+   PYENV     [install] pip install -e 'searx[test]'
+   ...
+   Successfully installed argparse-1.4.0 searx
+   BUILDENV  INFO:searx:load the default settings from ./searx/settings.yml
+   BUILDENV  INFO:searx:Initialisation done
+   BUILDENV  build utils/brand.env
 
-With target ``pyenv`` a development environment (aka virtualenv) was build up in
-``./local/py3/``.  To make a *developer install* of searx (:origin:`setup.py`)
-into this environment, use make target ``install``:
-
-.. code:: sh
+If you release ``make install`` multiple times the installation will only
+rebuild if the sha256 sum of the *requirement files* fails.  With other words:
+the check fails if you edit the requirements listed in
+:origin:`requirements-dev.txt` and :origin:`requirements.txt`). ::
 
    $ make install
-   PYENV     usage: source ./local/py3/bin/activate
-   PYENV     using virtualenv from ./local/py3
-   PYENV     install .
-
-You have never to think about intermediate targets like ``pyenv`` or
-``install``, the ``Makefile`` chains them as requisites.  Just run your main
-target.
+   PYENV     OK
+   PYENV     [virtualenv] requirements.sha256 failed
+             [virtualenv] - 6cea6eb6def9e14a18bf32f8a3e...  ./requirements-dev.txt
+             [virtualenv] - 471efef6c73558e391c3adb35f4...  ./requirements.txt
+   ...
+   PYENV     [virtualenv] installing ./requirements*.txt into local/py3
+   ...
+   PYENV     OK
+   PYENV     [install] pip install -e 'searx[test]'
+   ...
+   Successfully installed argparse-1.4.0 searx
+   BUILDENV  INFO:searx:load the default settings from ./searx/settings.yml
+   BUILDENV  INFO:searx:Initialisation done
+   BUILDENV  build utils/brand.env
 
 .. sidebar:: drop environment
 
@@ -100,10 +79,7 @@ target.
    <make clean>` first.
 
 If you think, something goes wrong with your ./local environment or you change
-the :origin:`setup.py` file (or the requirements listed in
-:origin:`requirements-dev.txt` and :origin:`requirements.txt`), you have to call
-:ref:`make clean`.
-
+the :origin:`setup.py` file, you have to call :ref:`make clean`.
 
 .. _make run:
 
@@ -113,88 +89,113 @@ the :origin:`setup.py` file (or the requirements listed in
 To get up a running a developer instance simply call ``make run``.  This enables
 *debug* option in :origin:`searx/settings.yml`, starts a ``./searx/webapp.py``
 instance, disables *debug* option again and opens the URL in your favorite WEB
-browser (:man:`xdg-open`):
+browser (:man:`xdg-open`)::
 
-.. code:: sh
-
-  $ make run
-  PYENV     usage: source ./local/py3/bin/activate
-  PYENV     install .
-  ./local/py3/bin/python ./searx/webapp.py
-  ...
-  INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit)
-  ...
+   $ make run
+   PYENV     OK
+   SEARX_DEBUG=1 ./manage.sh pyenv.cmd python ./searx/webapp.py
+   ...
+   INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit)
 
 .. _make clean:
 
 ``make clean``
 ==============
 
-Drop all intermediate files, all builds, but keep sources untouched.  Includes
-target ``pyclean`` which drops ./local environment.  Before calling ``make
-clean`` stop all processes using :ref:`make pyenv`.
-
-.. code:: sh
+Drop all intermediate files, all builds, but keep sources untouched.  Before
+calling ``make clean`` stop all processes using :ref:`make install`. ::
 
    $ make clean
-   CLEAN     pyclean
-   CLEAN     clean
+   CLEAN     pyenv
+   PYENV     [virtualenv] drop ./local/py3
+   CLEAN     docs -- ./build/docs ./dist/docs
+   CLEAN     locally installed npm dependencies
+   CLEAN     test stuff
+   CLEAN     common files
 
 .. _make docs:
 
-``make docs docs-live docs-clean``
-==================================
+``make docs docs.autobuild docs.clean``
+=======================================
 
-We describe the usage of the ``doc*`` targets in the :ref:`How to contribute /
+We describe the usage of the ``doc.*`` targets in the :ref:`How to contribute /
 Documentation <contrib docs>` section.  If you want to edit the documentation
-read our :ref:`make docs-live` section.  If you are working in your own brand,
-adjust your :ref:`Makefile setup <makefile setup>`.
+read our :ref:`make docs.live` section.  If you are working in your own brand,
+adjust your :ref:`settings global`.
 
+.. _make docs.gh-pages:
 
-.. _make gh-pages:
+``make docs.gh-pages``
+======================
 
-``make gh-pages``
-=================
-
-To deploy on github.io first adjust your :ref:`Makefile setup <makefile
-setup>`.  For any further read :ref:`deploy on github.io`.
+To deploy on github.io first adjust your :ref:`settings global`.  For any
+further read :ref:`deploy on github.io`.
 
 .. _make test:
 
 ``make test``
 =============
 
-Runs a series of tests: ``test.pep8``, ``test.unit``, ``test.robot`` and does
-additional :ref:`pylint checks <make pylint>`.  You can run tests selective,
-e.g.:
-
-.. code:: sh
+Runs a series of tests: :ref:`make test.pylint`, ``test.pep8``, ``test.unit``
+and ``test.robot``.  You can run tests selective, e.g.::
 
   $ make test.pep8 test.unit test.sh
-  . ./local/py3/bin/activate; ./manage.sh pep8_check
-  [!] Running pep8 check
-  . ./local/py3/bin/activate; ./manage.sh unit_tests
-  [!] Running unit tests
+  TEST      test.pep8 OK
+  ...
+  TEST      test.unit OK
+  ...
+  TEST      test.sh OK
 
-.. _make pylint:
+.. _make test.sh:
 
-``make pylint``
-===============
+``make test.sh``
+================
+
+:ref:`sh lint` / if you have changed some bash scripting run this test before
+commit.
+
+.. _make test.pylint:
+
+``make test.pylint``
+====================
 
 .. _Pylint: https://www.pylint.org/
 
-Before commiting its recommend to do some (more) linting.  Pylint_ is known as
-one of the best source-code, bug and quality checker for the Python programming
-language.  Pylint_ is not yet a quality gate within our searx project (like
-:ref:`test.pep8 <make test>` it is), but Pylint_ can help to improve code
-quality anyway.  The pylint profile we use at searx project is found in
-project's root folder :origin:`.pylintrc`.
+Pylint_ is known as one of the best source-code, bug and quality checker for the
+Python programming language.  The pylint profile we use at searx project is
+found in project's root folder :origin:`.pylintrc`.
 
-Code quality is a ongoing process.  Don't try to fix all messages from Pylint,
-run Pylint and check if your changed lines are bringing up new messages.  If so,
-fix it.  By this, code quality gets incremental better and if there comes the
-day, the linting is balanced out, we might decide to add Pylint as a quality
-gate.
+.. _make search.checker:
+
+``search.checker.{engine name}``
+================================
+
+To check all engines::
+
+    make search.checker
+
+To check a engine with whitespace in the name like *google news* replace space
+by underline::
+
+    make search.checker.google_news
+
+To see HTTP requests and more use SEARX_DEBUG::
+
+    make SEARX_DEBUG=1 search.checker.google_news
+
+.. _3xx: https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_redirection
+
+To filter out HTTP redirects (3xx_)::
+
+    make SEARX_DEBUG=1 search.checker.google_news | grep -A1 "HTTP/1.1\" 3[0-9][0-9]"
+    ...
+    Engine google news                   Checking
+    https://news.google.com:443 "GET /search?q=life&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
+    https://news.google.com:443 "GET /search?q=life&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
+    --
+    https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
+    https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
+    --
 
 
 ``make pybuild``
@@ -203,9 +204,7 @@ gate.
 .. _PyPi: https://pypi.org/
 .. _twine: https://twine.readthedocs.io/en/latest/
 
-Build Python packages in ``./dist/py``.
-
-.. code:: sh
+Build Python packages in ``./dist/py``::
 
   $ make pybuild
   ...
@@ -213,9 +212,11 @@ Build Python packages in ``./dist/py``.
   running sdist
   running egg_info
   ...
-  $ ls  ./dist/py/
-  searx-0.15.0-py3-none-any.whl  searx-0.15.0.tar.gz
+  running bdist_wheel
 
-To upload packages to PyPi_, there is also a ``upload-pypi`` target.  It needs
-twine_ to be installed.  Since you are not the owner of :pypi:`searx` you will
-never need the latter.
+  $ ls  ./dist
+  searx-0.18.0-py3-none-any.whl  searx-0.18.0.tar.gz
+
+To upload packages to PyPi_, there is also a ``pypi.upload`` target (to test use
+``pypi.upload.test``).  Since you are not the owner of :pypi:`searx` you will
+never need to upload.

docs/dev/reST.rst

@@ -15,8 +15,8 @@ generated and deployed at :docs:`github.io <.>`.  For build prerequisites read
 :ref:`docs build`.
 
 The source files of Searx's documentation are located at :origin:`docs`.  Sphinx
-assumes source files to be encoded in UTF-8 by defaul.  Run :ref:`make docs-live
-<make docs-live>` to build HTML while editing.
+assumes source files to be encoded in UTF-8 by default.  Run :ref:`make docs.live
+<make docs.live>` to build HTML while editing.
 
 .. sidebar:: Further reading
 
@@ -227,13 +227,13 @@ To refer anchors use the `ref role`_ markup:
 
 .. code:: reST
 
-   Visit chapter :ref:`reST anchor`.  Or set hyperlink text manualy :ref:`foo
+   Visit chapter :ref:`reST anchor`.  Or set hyperlink text manually :ref:`foo
    bar <reST anchor>`.
 
 .. admonition:: ``:ref:`` role
    :class: rst-example
 
-   Visist chapter :ref:`reST anchor`.  Or set hyperlink text manualy :ref:`foo
+   Visist chapter :ref:`reST anchor`.  Or set hyperlink text manually :ref:`foo
    bar <reST anchor>`.
 
 .. _reST ordinary ref:
@@ -319,6 +319,9 @@ To list all anchors of the inventory (e.g. ``python``) use:
 .. code:: sh
 
    $ python -m sphinx.ext.intersphinx https://docs.python.org/3/objects.inv
+   ...
+   $ python -m sphinx.ext.intersphinx https://searx.github.io/searx/objects.inv
+   ...
 
 Literal blocks
 ==============
@@ -491,8 +494,8 @@ Figures & Images
    is flexible.  To get best results in the generated output format, install
    ImageMagick_ and Graphviz_.
 
-Searx's sphinx setup includes: :ref:`linuxdoc:kfigure`.  Scaleable here means;
-scaleable in sense of the build process.  Normally in absence of a converter
+Searx's sphinx setup includes: :ref:`linuxdoc:kfigure`.  Scalable here means;
+scalable in sense of the build process.  Normally in absence of a converter
 tool, the build process will break.  From the authors POV it’s annoying to care
 about the build process when handling with images, especially since he has no
 access to the build process.  With :ref:`linuxdoc:kfigure` the build process
@@ -500,7 +503,7 @@ continues and scales output quality in dependence of installed image processors.
 
 If you want to add an image, you should use the ``kernel-figure`` (inheritance
 of :dudir:`figure`) and ``kernel-image`` (inheritance of :dudir:`image`)
-directives.  E.g. to insert a figure with a scaleable image format use SVG
+directives.  E.g. to insert a figure with a scalable image format use SVG
 (:ref:`svg image example`):
 
 .. code:: reST
@@ -1182,7 +1185,7 @@ and *targets* (e.g. a ref to :ref:`row 2 of table's body <row body 2>`).
         - cell 4.4
 
       * - row 5
-        - cell 5.1 with automatic span to rigth end
+        - cell 5.1 with automatic span to right end
 
       * - row 6
         - cell 6.1
@@ -1234,7 +1237,7 @@ and *targets* (e.g. a ref to :ref:`row 2 of table's body <row body 2>`).
         - cell 4.4
 
       * - row 5
-        - cell 5.1 with automatic span to rigth end
+        - cell 5.1 with automatic span to right end
 
       * - row 6
         - cell 6.1
@@ -1273,28 +1276,33 @@ Templating
 
 .. sidebar:: Build environment
 
-   All *generic-doc* tasks are running in the :ref:`build environment <make
-   pyenv>`.
+   All *generic-doc* tasks are running in the :ref:`make install`.
 
 Templating is suitable for documentation which is created generic at the build
-time.  The sphinx-jinja_ extension evaluates jinja_ templates in the :ref:`build
-environment <make pyenv>` (with searx modules installed).  We use this e.g. to
-build chapter: :ref:`engines generic`.  Below the jinja directive from the
+time.  The sphinx-jinja_ extension evaluates jinja_ templates in the :ref:`make
+install` (with searx modules installed).  We use this e.g. to build chapter:
+:ref:`engines generic`.  Below the jinja directive from the
 :origin:`docs/admin/engines.rst` is shown:
 
 .. literalinclude:: ../admin/engines.rst
    :language: reST
    :start-after: .. _configured engines:
 
-The context for the template is selected in the line ``.. jinja:: webapp``.  In
-sphinx's build configuration (:origin:`docs/conf.py`) the ``webapp`` context
-points to the name space of the python module: ``webapp``.
+The context for the template is selected in the line ``.. jinja:: searx``.  In
+sphinx's build configuration (:origin:`docs/conf.py`) the ``searx`` context
+contains the ``engines`` and ``plugins``.
 
 .. code:: py
 
-   from searx import webapp
+   import searx.search
+   import searx.engines
+   import searx.plugins
+   searx.search.initialize()
    jinja_contexts = {
-       'webapp': dict(**webapp.__dict__)
+      'searx': {
+         'engines': searx.engines.engines,
+         'plugins': searx.plugins.plugins
+      },
    }
 
 

docs/dev/search_api.rst

@@ -6,7 +6,7 @@ Search API
 
 The search supports both ``GET`` and ``POST``.
 
-Furthermore, two enpoints ``/`` and ``/search`` are available for querying.
+Furthermore, two endpoints ``/`` and ``/search`` are available for querying.
 
 
 ``GET /``
@@ -76,7 +76,7 @@ Parameters
   supports safe search in the preferences page of an instance.
 
 ``theme`` : default ``oscar``
-  [ ``oscar``, ``simple``, ``legacy``, ``pix-art``, ``courgette`` ]
+  [ ``oscar``, ``simple`` ]
 
   Theme of instance.
 

docs/index.rst

@@ -8,9 +8,6 @@ Searx is a free internet metasearch engine which aggregates results from more
 than 70 search services.  Users are neither tracked nor profiled.  Additionally,
 searx can be used over Tor for online anonymity.
 
-Get started with searx by using one of the Searx-instances_.  If you don't trust
-anyone, you can set up your own, see :ref:`installation`.
-
 .. sidebar::  Features
 
    - Self hosted
@@ -30,7 +27,6 @@ anyone, you can set up your own, see :ref:`installation`.
    user/index
    admin/index
    dev/index
+   searx_extra/index
    utils/index
    blog/index
-
-.. _Searx-instances: https://searx.space

docs/searx_extra/index.rst

@@ -0,0 +1,14 @@
+.. _searx_extra:
+
+======================================================
+Tooling box ``searx_extra`` for developers and users
+======================================================
+
+In the folder :origin:`searx_extra/` we maintain some tools useful for
+developers and users.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents
+
+   standalone_searx.py

docs/searx_extra/standalone_searx.py.rst

@@ -0,0 +1,9 @@
+
+.. _standalone_searx.py:
+
+===================================
+``searx_extra/standalone_searx.py``
+===================================
+
+.. automodule:: searx_extra.standalone_searx
+  :members:

docs/user/conf.py

@@ -1,19 +0,0 @@
-# -*- coding: utf-8; mode: python -*-
-"""Configuration for the Searx user handbook
-"""
-project   = 'Searx User-HB'
-version   = release = VERSION_STRING
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    ('index'                       # startdocname
-     , 'searx-user-hb.tex'         # targetname
-     , ''                          # take title from .rst
-     , author                      # author
-     , 'howto'                     # documentclass
-     , False                       # toctree_only
-    ),
-]
-

docs/user/own-instance.rst

@@ -56,9 +56,9 @@ results.
 I see. What about private instances?
 ------------------------------------
 
-If users run their own instances, everything is in their control: the source
-code, logging settings and private data.  Unknown instance administrators do not
-have to be trusted.
+If users run their :ref:`own instances <installation>`, everything is in their
+control: the source code, logging settings and private data.  Unknown instance
+administrators do not have to be trusted.
 
 Furthermore, as the default settings of their instance is editable, there is no
 need to use cookies to tailor searx to their needs.  So preferences will not be

docs/user/search_syntax.rst

@@ -17,7 +17,7 @@ Prefix: ``:``
 Prefix: ``?``
   to add engines and categories to the currently selected categories
 
-Abbrevations of the engines and languages are also accepted.  Engine/category
+Abbreviations of the engines and languages are also accepted.  Engine/category
 modifiers are chainable and inclusive (e.g. with :search:`!it !ddg !wp qwer
 <?q=%21it%20%21ddg%20%21wp%20qwer>` search in IT category **and** duckduckgo
 **and** wikipedia for ``qwer``).
@@ -40,3 +40,27 @@ Image search:
 Custom language in wikipedia:
 
 - :search:`:hu !wp hackerspace <?q=%3Ahu%20%21wp%20hackerspace>`
+
+Multilingual Search
+===================
+
+Searx does not support true multilingual search.
+You have to use the language prefix in your search query when searching in a different language.
+
+But there is a workaround:
+By adding a new search engine with a different language, Searx will search in your default and other language.
+
+Example configuration in settings.yml for a German and English speaker:
+ .. code-block:: yaml
+
+    search:
+        language : "de"
+        ...
+
+    engines:
+      - name : google english
+        engine : google
+        language : english
+        ...
+
+When searching, the default google engine will return German results and "google english" will return English results.

docs/utils/index.rst

@@ -1,12 +1,11 @@
 .. _searx_utils:
 .. _toolboxing:
 
-=======================
-Tooling box ``utils/*``
-=======================
+===================
+Admin's tooling box
+===================
 
-In the folder :origin:`utils/` we maintain some tools useful for admins and
-developers.
+In the folder :origin:`utils/` we maintain some tools useful for administrators.
 
 .. toctree::
    :maxdepth: 2
@@ -16,7 +15,6 @@ developers.
    filtron.sh
    morty.sh
    lxc.sh
-   standalone_searx.py
 
 .. _toolboxing common:
 
@@ -47,8 +45,8 @@ Scripts to maintain services often dispose of common commands and environments.
 Tooling box setup
 =================
 
-The main setup is done in the :origin:`.config.sh` (read also :ref:`makefile
-setup`).
+The main setup is done in the :origin:`.config.sh` (read also :ref:`settings
+global`).
 
 .. literalinclude:: ../../.config.sh
    :language: bash

docs/utils/lxc.sh.rst

@@ -119,15 +119,15 @@ of coffee).::
 
 To build (live) documentation inside a archlinux_ container::
 
-  sudo -H ./utils/lxc.sh cmd searx-archlinux make docs-clean docs-live
+  sudo -H ./utils/lxc.sh cmd searx-archlinux make docs.clean docs.live
   ...
   [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080
 
 To get IP of the container and the port number *live docs* is listening::
 
-  $ sudo ./utils/lxc.sh show suite | grep docs-live
+  $ sudo ./utils/lxc.sh show suite | grep docs.live
   ...
-  [searx-archlinux]  INFO:  (eth0) docs-live:  http://n.n.n.12:8080/
+  [searx-archlinux]  INFO:  (eth0) docs.live:  http://n.n.n.12:8080/
 
 
 .. _lxc.sh help:

docs/utils/standalone_searx.py.rst

@@ -1,11 +0,0 @@
-
-.. _standalone_searx.py:
-
-=============================
-``utils/standalone_searx.py``
-=============================
-
-.. automodule:: standalone_searx
-  :members:
-
-

manage

@@ -0,0 +1,498 @@
+#!/usr/bin/env bash
+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# shellcheck disable=SC2031
+
+# shellcheck source=utils/lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
+# shellcheck source=utils/brand.env
+source "${REPO_ROOT}/utils/brand.env"
+source_dot_config
+
+# config
+
+PY_SETUP_EXTRAS='[test]'
+NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"
+GECKODRIVER_VERSION="v0.30.0"
+# SPHINXOPTS=
+
+# These py files are linted by test.pylint(), all other files are linted by
+# test.pep8()
+PYLINT_FILES=(
+    searx/preferences.py
+    searx/testing.py
+    searx/engines/gigablast.py
+    searx/engines/deviantart.py
+    searx/engines/digg.py
+    searx/engines/google.py
+    searx/engines/google_news.py
+    searx/engines/google_videos.py
+    searx/engines/google_images.py
+    searx/engines/mediathekviewweb.py
+    searx/engines/meilisearch.py
+    searx/engines/solidtorrents.py
+    searx/engines/solr.py
+    searx/engines/sqlite.py
+    searx/engines/springer.py
+    searx/engines/google_scholar.py
+    searx/engines/yahoo_news.py
+    searx/engines/apkmirror.py
+    searx/engines/core.py
+    searx_extra/update/update_external_bangs.py
+)
+
+PYLINT_SEARX_DISABLE_OPTION="\
+I,C,R,\
+W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
+E1136"
+PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
+PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
+
+help() {
+    cat <<EOF
+buildenv
+  rebuild ./utils/brand.env
+babel.compile
+  pybabel compile ./searx/translations
+data.*
+  all       : update searx/languages.py and ./data/*
+  languages : update searx/data/engines_languages.json & searx/languages.py
+  useragents: update searx/data/useragents.json with the most recent versions of Firefox.
+docs.*
+  html      : build HTML documentation
+  live      : autobuild HTML documentation while editing
+  gh-pages  : deploy on gh-pages branch
+  prebuild  : build reST include files (./${DOCS_BUILD}/includes)
+  clean     : clean documentation build
+docker
+  build     : build docker image
+  push      : build and push docker image
+gecko.driver
+  download & install geckodriver if not already installed (required for
+  robot_tests)
+node.*
+  env       : download & install npm dependencies locally
+  clean     : drop npm installations
+py.*
+  build     : Build python packages at ./${PYDIST}
+  clean     : delete virtualenv and intermediate py files
+pyenv.* :
+  install   : developer install of searx into virtualenv
+  uninstall : uninstall developer installation
+  cmd ...   : run command ... in virtualenv
+  OK        : test if virtualenv is OK
+pypi.upload:
+  Upload python packages to PyPi (to test use pypi.upload.test)
+test.* :
+  pylint    : lint PYLINT_FILES, searx/engines, searx & tests
+  pep8      : pycodestyle (pep8) for all files except PYLINT_FILES
+  unit      : run unit tests
+  coverage  : run unit tests with coverage
+  robot     : run robot test
+  clean     : clean intermediate test stuff
+themes.* :
+  all       : build all themes
+  oscar     : build oscar theme
+  simple    : build simple theme
+EOF
+}
+
+
+if [ "$VERBOSE" = "1" ]; then
+    SPHINX_VERBOSE="-v"
+    PYLINT_VERBOSE="-v"
+fi
+
+# needed by sphinx-docs
+export DOCS_BUILD
+
+buildenv() {
+    SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1
+    return "${PIPESTATUS[0]}"
+}
+
+babel.compile() {
+    build_msg BABEL compile
+    pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
+    dump_return $?
+}
+
+
+data.all() {
+    data.languages
+    data.useragents
+    build_msg DATA "update searx/data/ahmia_blacklist.txt"
+    pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py
+    build_msg DATA "update searx/data/wikidata_units.json"
+    pyenv.cmd python searx_extra/update/update_wikidata_units.py
+    build_msg DATA "update searx/data/currencies.json"
+    pyenv.cmd python searx_extra/update/update_currencies.py
+}
+
+
+data.languages() {
+    (   set -e
+        build_msg ENGINES "fetch languages .."
+        pyenv.cmd python searx_extra/update/update_languages.py
+        build_msg ENGINES "update update searx/languages.py"
+        build_msg DATA "update searx/data/engines_languages.json"
+    )
+    dump_return $?
+}
+
+data.useragents() {
+    build_msg DATA "update searx/data/useragents.json"
+    pyenv.cmd python searx_extra/update/update_firefox_version.py
+    dump_return $?
+}
+
+docs.prebuild() {
+    build_msg DOCS "build ${DOCS_BUILD}/includes"
+    (
+        set -e
+        [ "$VERBOSE" = "1" ] && set -x
+        mkdir -p "${DOCS_BUILD}/includes"
+        ./utils/searx.sh doc   | cat > "${DOCS_BUILD}/includes/searx.rst"
+        ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
+        ./utils/morty.sh doc   | cat > "${DOCS_BUILD}/includes/morty.rst"
+    )
+    dump_return $?
+}
+
+docker.push() {
+    docker.build push
+}
+
+# shellcheck disable=SC2119
+docker.build() {
+    pyenv.install
+
+    build_msg DOCKER build
+    # run installation in a subprocess and activate pyenv
+
+    # See https://www.shellcheck.net/wiki/SC1001 and others ..
+    # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
+    (   set -e
+        # shellcheck source=/dev/null
+        source "${PY_ENV_BIN}/activate"
+
+        # Check if it is a git repository
+        if [ ! -d .git ]; then
+	    die 1 "This is not Git repository"
+        fi
+        if [ ! -x "$(which git)" ]; then
+	    die 1 "git is not installed"
+        fi
+
+        if ! git remote get-url origin 2> /dev/null; then
+	    die 1 "there is no remote origin"
+        fi
+
+        # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
+        # awk to remove the "v" and the "g"
+        SEARX_GIT_VERSION=$(git describe --tags | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); }  print}')
+
+        # add the suffix "-dirty" if the repository has uncommitted change
+        # /!\ HACK for searx/searx: ignore utils/brand.env
+        git update-index -q --refresh
+        if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
+	    SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
+        fi
+
+        # Get the last git commit id, will be added to the Searx version (see Dockerfile)
+        VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
+        build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
+
+        # Check consistency between the git tag and the searx/version.py file
+        # /! HACK : parse Python file with bash /!
+        # otherwise it is not possible build the docker image without all Python
+        # dependencies ( version.py loads __init__.py )
+        # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
+        SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
+        if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
+	    err_msg "git tag:          $SEARX_GIT_VERSION"
+	    err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
+	    die 1 "Inconsistency between the last git tag and the searx/version.py file"
+        fi
+
+        # define the docker image name
+        GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
+        SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"
+
+        # build Docker image
+        build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
+        docker build \
+         --build-arg GIT_URL="${GIT_URL}" \
+         --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
+         --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
+         --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+         --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
+         --build-arg LABEL_VCS_URL="${GIT_URL}" \
+	 --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
+	 --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
+         -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
+
+        if [ "$1" = "push" ]; then
+	    docker push "${SEARX_IMAGE_NAME}:latest"
+	    docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
+        fi
+    )
+    dump_return $?
+}
+
+# shellcheck disable=SC2119
+gecko.driver() {
+    pyenv.install
+
+    build_msg INSTALL "gecko.driver"
+    # run installation in a subprocess and activate pyenv
+    (   set -e
+        # shellcheck source=/dev/null
+        source "${PY_ENV_BIN}/activate"
+
+        # TODO : check the current geckodriver version
+        geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
+        set +e
+        if [ -z "$NOTFOUND" ]; then
+            build_msg INSTALL "geckodriver already installed"
+            return
+        fi
+        PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
+        case "$PLATFORM" in
+            "linux 32bit" | "linux2 32bit") ARCH="linux32";;
+            "linux 64bit" | "linux2 64bit") ARCH="linux64";;
+            "windows 32 bit") ARCH="win32";;
+            "windows 64 bit") ARCH="win64";;
+            "mac 64bit") ARCH="macos";;
+        esac
+        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
+
+        build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
+
+        FILE="$(mktemp)"
+        wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
+        rm -- "$FILE"
+        chmod 755 -- "${PY_ENV_BIN}/geckodriver"
+    )
+    dump_return $?
+}
+
+node.env() {
+    local err=0
+    pushd "${REPO_ROOT}" &> /dev/null
+    # shellcheck disable=SC2230
+    which npm &> /dev/null || die 1 'node.env - npm is not found!'
+
+    (   set -e
+
+        build_msg INSTALL "npm install $NPM_PACKAGES"
+        # shellcheck disable=SC2086
+        npm install $NPM_PACKAGES
+
+        cd "${REPO_ROOT}/searx/static/themes/oscar"
+        build_msg INSTALL "($(pwd)) npm install"
+        npm install
+
+        build_msg INSTALL "($(pwd)) npm install"
+        cd "${REPO_ROOT}/searx/static/themes/simple"
+        npm install
+    )
+    err=$?
+    popd &> /dev/null
+    dump_return "$err"
+}
+
+node.clean() {
+
+    build_msg CLEAN "locally installed npm dependencies"
+    rm -rf \
+       ./node_modules  \
+       ./package-lock.json \
+       ./searx/static/themes/oscar/package-lock.json \
+       ./searx/static/themes/oscar/node_modules \
+       ./searx/static/themes/simple/package-lock.json \
+       ./searx/static/themes/simple/node_modules
+    dump_return $?
+}
+
+py.build() {
+    build_msg BUILD "[pylint] python package ${PYDIST}"
+    pyenv.cmd python setup.py \
+              sdist -d "${PYDIST}" \
+              bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
+}
+
+py.clean() {
+    build_msg CLEAN pyenv
+    (   set -e
+        pyenv.drop
+        [ "$VERBOSE" = "1" ] && set -x
+        rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
+        find . -name '*.pyc' -exec rm -f {} +
+        find . -name '*.pyo' -exec rm -f {} +
+        find . -name __pycache__ -exec rm -rf {} +
+    )
+}
+
+pyenv.check() {
+    cat  <<EOF
+import yaml
+print('import yaml --> OK')
+EOF
+}
+
+pyenv.install() {
+
+    if ! pyenv.OK; then
+        py.clean > /dev/null
+    fi
+    if pyenv.install.OK > /dev/null; then
+        return 0
+    fi
+    (   set -e
+        pyenv
+    	build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
+    	"${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
+        buildenv
+    ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"
+}
+
+pyenv.uninstall() {
+    build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
+    pyenv.cmd python setup.py develop --uninstall 2>&1 \
+        | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+
+}
+
+pypi.upload() {
+    py.clean
+    py.build
+    # https://github.com/pypa/twine
+    pyenv.cmd twine upload "${PYDIST}"/*
+}
+
+pypi.upload.test() {
+    py.clean
+    py.build
+    pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
+}
+
+test.pylint() {
+    # shellcheck disable=SC2086
+    (   set -e
+        build_msg TEST "[pylint] \$PYLINT_FILES"
+        pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            "${PYLINT_FILES[@]}"
+
+        build_msg TEST "[pylint] searx/engines"
+        pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
+            --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
+            searx/engines
+
+        build_msg TEST "[pylint] searx tests"
+        pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
+	    --ignore=searx/engines \
+	    tests
+    )
+    dump_return $?
+}
+
+test.pep8() {
+    build_msg TEST 'pycodestyle (formerly pep8)'
+    local _exclude=""
+    printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
+    pyenv.cmd pycodestyle \
+              --exclude="searx/static, searx/languages.py, $_exclude " \
+              --max-line-length=120 \
+              --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
+              searx tests
+    dump_return $?
+}
+
+test.unit() {
+    build_msg TEST 'tests/unit'
+    pyenv.cmd python -m nose2 -s tests/unit
+    dump_return $?
+}
+
+test.coverage() {
+    build_msg TEST 'unit test coverage'
+    (   set -e
+        pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
+        pyenv.cmd coverage report
+        pyenv.cmd coverage html
+    )
+    dump_return $?
+}
+
+test.robot() {
+    build_msg TEST 'robot'
+    gecko.driver
+    PYTHONPATH=. pyenv.cmd python searx/testing.py robot
+    dump_return $?
+}
+
+test.clean() {
+    build_msg CLEAN  "test stuff"
+    rm -rf geckodriver.log .coverage coverage/
+    dump_return $?
+}
+
+themes.all() {
+    (   set -e
+        node.env
+        themes.oscar
+        themes.simple
+    )
+    dump_return $?
+}
+
+themes.oscar() {
+    local gruntfile=searx/static/themes/oscar/gruntfile.js
+    build_msg GRUNT "${gruntfile}"
+    PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
+    dump_return $?
+}
+
+themes.simple() {
+    local gruntfile=searx/static/themes/simple/gruntfile.js
+    build_msg GRUNT "${gruntfile}"
+    PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
+    dump_return $?
+}
+
+# shellcheck disable=SC2119
+main() {
+
+    local _type
+    local cmd="$1"; shift
+
+    if [ "$cmd" == "" ]; then
+        help
+        err_msg "missing command"
+        return 42
+    fi
+
+    case "$cmd" in
+        --getenv) var="$1"; echo "${!var}";;
+        --help) help;;
+        --*)
+            help
+            err_msg "unknown option $cmd"
+            return 42
+            ;;
+        *)
+            _type="$(type -t "$cmd")"
+            if [ "$_type" != 'function' ]; then
+                err_msg "unknown command $1 / use --help"
+                return 42
+            else
+                "$cmd" "$@"
+            fi
+            ;;
+    esac
+}
+
+main "$@"

manage.sh

@@ -1,205 +0,0 @@
-#!/bin/sh
-
-export LANG=C
-
-BASE_DIR="$(dirname -- "`readlink -f -- "$0"`")"
-
-cd -- "$BASE_DIR"
-set -e
-
-# subshell
-PYTHONPATH="$BASE_DIR"
-SEARX_DIR="$BASE_DIR/searx"
-ACTION="$1"
-
-. "${BASE_DIR}/utils/brand.env"
-
-#
-# Python
-#
-
-update_packages() {
-    pip install --upgrade pip
-    pip install --upgrade setuptools
-    pip install -Ur "$BASE_DIR/requirements.txt"
-}
-
-update_dev_packages() {
-    update_packages
-    pip install -Ur "$BASE_DIR/requirements-dev.txt"
-}
-
-install_geckodriver() {
-    echo '[!] Checking geckodriver'
-    # TODO : check the current geckodriver version
-    set -e
-    geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
-    set +e
-    if [ -z "$NOTFOUND" ]; then
-        return
-    fi
-    GECKODRIVER_VERSION="v0.24.0"
-    PLATFORM="`python3 -c "import platform; print(platform.system().lower(), platform.architecture()[0])"`"
-    case "$PLATFORM" in
-        "linux 32bit" | "linux2 32bit") ARCH="linux32";;
-        "linux 64bit" | "linux2 64bit") ARCH="linux64";;
-        "windows 32 bit") ARCH="win32";;
-        "windows 64 bit") ARCH="win64";;
-        "mac 64bit") ARCH="macos";;
-    esac
-    GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
-
-    if [ -z "$1" ]; then
-        if [ -z "$VIRTUAL_ENV" ]; then
-            printf "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n  %s" "$GECKODRIVER_URL"
-            exit
-        else
-            GECKODRIVER_DIR="$VIRTUAL_ENV/bin"
-        fi
-    else
-        GECKODRIVER_DIR="$1"
-        mkdir -p -- "$GECKODRIVER_DIR"
-    fi
-
-    printf "Installing %s/geckodriver from\n  %s" "$GECKODRIVER_DIR" "$GECKODRIVER_URL"
-
-    FILE="`mktemp`"
-    wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "$GECKODRIVER_DIR" -f "$FILE" geckodriver
-    rm -- "$FILE"
-    chmod 777 -- "$GECKODRIVER_DIR/geckodriver"
-}
-
-locales() {
-    pybabel compile -d "$SEARX_DIR/translations"
-}
-
-
-#
-# Web
-#
-
-npm_path_setup() {
-    which npm || (printf 'Error: npm is not found\n'; exit 1)
-    export PATH="$(npm bin)":$PATH
-}
-
-npm_packages() {
-    npm_path_setup
-
-    echo '[!] install NPM packages'
-    cd -- "$BASE_DIR"
-    npm install less@2.7 less-plugin-clean-css grunt-cli
-
-    echo '[!] install NPM packages for oscar theme'
-    cd -- "$BASE_DIR/searx/static/themes/oscar"
-    npm install
-
-    echo '[!] install NPM packages for simple theme'
-    cd -- "$BASE_DIR/searx/static/themes/simple"
-    npm install
-}
-
-docker_build() {
-    # Check if it is a git repository
-    if [ ! -d .git ]; then
-	echo "This is not Git repository"
-	exit 1
-    fi
-
-    if [ ! -x "$(which git)" ]; then
-	echo "git is not installed"
-	exit 1
-    fi
-
-    if [ ! git remote get-url origin 2> /dev/null ]; then
-	echo "there is no remote origin"
-	exit 1
-    fi
-
-    # This is a git repository
-
-    # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
-    # awk to remove the "v" and the "g"
-    SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); }  print}')
-
-    # add the suffix "-dirty" if the repository has uncommited change
-    # /!\ HACK for searx/searx: ignore searx/brand.py and utils/brand.env
-    git update-index -q --refresh
-    if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'searx/brand.py' | grep -v 'utils/brand.env')" ]; then
-	SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
-    fi
-
-    # Get the last git commit id, will be added to the Searx version (see Dockerfile)
-    VERSION_GITCOMMIT=$(echo $SEARX_GIT_VERSION | cut -d- -f2-4)
-    echo "Last commit : $VERSION_GITCOMMIT"
-
-    # Check consistency between the git tag and the searx/version.py file
-    # /!\ HACK : parse Python file with bash /!\
-    # otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py )
-    # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
-    SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
-    if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then
-	echo "Inconsistency between the last git tag and the searx/version.py file"
-	echo "git tag:          $SEARX_GIT_VERSION"
-	echo "searx/version.py: $SEARX_PYTHON_VERSION"
-	exit 1
-    fi
-
-    # define the docker image name
-    GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
-    SEARX_IMAGE_NAME="${GITHUB_USER:-searx}/searx"
-
-    # build Docker image
-    echo "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
-    sudo docker build \
-         --build-arg GIT_URL="${GIT_URL}" \
-         --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
-         --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
-         --build-arg LABEL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
-         --build-arg LABEL_VCS_REF=$(git rev-parse HEAD) \
-         --build-arg LABEL_VCS_URL="${GIT_URL}" \
-	 --build-arg TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- searx/settings.yml) \
-	 --build-arg TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini) \
-         -t ${SEARX_IMAGE_NAME}:latest -t ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} .
-
-    if [ "$1" = "push" ]; then
-	sudo docker push ${SEARX_IMAGE_NAME}:latest
-	sudo docker push ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}
-    fi
-}
-
-#
-# Help
-#
-
-help() {
-    [ -z "$1" ] || printf 'Error: %s\n' "$1"
-    echo "Searx manage.sh help
-
-Commands
-========
-    help                 - This text
-
-    Build requirements
-    ------------------
-    update_packages      - Check & update production dependency changes
-    update_dev_packages  - Check & update development and production dependency changes
-    install_geckodriver  - Download & install geckodriver if not already installed (required for robot_tests)
-    npm_packages         - Download & install npm dependencies
-
-    Build
-    -----
-    locales              - Compile locales
-
-Environment:
-    GIT_URL:          ${GIT_URL}
-    ISSUE_URL:        ${ISSUE_URL}
-    SEARX_URL:        ${SEARX_URL}
-    DOCS_URL:         ${DOCS_URL}
-    PUBLIC_INSTANCES: ${PUBLIC_INSTANCES}
-"
-}
-
-[ "$(command -V "$ACTION" | grep ' function$')" = "" ] \
-    && help "action not found" \
-    || "$ACTION" "$2"

requirements-dev.txt

@@ -1,24 +1,19 @@
-mock==2.0.0
-nose2[coverage_plugin]==0.9.2
+mock==5.0.1
+nose2[coverage_plugin]==0.12.0
 cov-core==1.15.0
-pycodestyle==2.6.0
-pylint==2.4.4
-plone.testing==5.0.0
-splinter==0.11.0
-transifex-client==0.12.2
-unittest2==1.1.0
-zope.testrunner==4.5.1
-selenium==3.141.0
-twine==3.2.0; python_version >= "3.6"
-twine==1.15.0; python_version < "3.6"
-Pallets-Sphinx-Themes==1.2.3
-Sphinx==3.2.1; python_version >= '3.6'
-Sphinx==3.0.1; python_version < '3.6'
-sphinx-issues==1.2.0
-sphinx-jinja==1.1.1
-sphinx-tabs==1.3.0; python_version >= '3.6'
-sphinx-tabs==1.1.13; python_version < '3.6'
-sphinxcontrib-programoutput==0.16
-sphinx-autobuild==2020.9.1; python_version >= '3.6'
-sphinx-autobuild==0.7.1; python_version < '3.6'
-linuxdoc @ git+http://github.com/return42/linuxdoc.git@70673dcf69e705e08d81f53794895dc15c4920b3#egg=linuxdoc
+pycodestyle==2.10.0
+pylint==2.15.9
+splinter==0.19.0
+transifex-client==0.14.3; python_version < '3.10'
+transifex-client==0.12.5; python_version == '3.10'
+selenium==4.8.3
+twine==4.0.2
+Pallets-Sphinx-Themes==2.0.3
+docutils==0.18
+Sphinx==5.3.0
+sphinx-issues==3.0.1
+sphinx-jinja==2.0.2
+sphinx-tabs==3.4.1
+sphinxcontrib-programoutput==0.17
+sphinx-autobuild==2021.3.14
+linuxdoc==20221127

requirements.txt

@@ -1,12 +1,13 @@
-certifi==2020.6.20
-babel==2.7.0
-flask-babel==1.0.0
-flask==1.1.2
-idna==2.10
-jinja2==2.11.1
-lxml==4.6.1
-pygments==2.1.3
-pyopenssl==19.1.0
-python-dateutil==2.8.0
-pyyaml==5.3.1
-requests[socks]==2.24.0
+Brotli==1.0.9
+babel==2.11.0
+certifi==2022.12.7
+flask-babel==2.0.0
+flask==2.2.2
+jinja2==3.1.2
+langdetect==1.0.9
+lxml==4.9.2
+pygments==2.12.0
+python-dateutil==2.8.2
+pyyaml==6.0
+requests[socks]==2.28.2
+setproctitle==1.3.2

searx/__init__.py

@@ -19,23 +19,33 @@ import logging
 import searx.settings_loader
 from os import environ
 from os.path import realpath, dirname, join, abspath, isfile
+from sys import exit
 
+from searx.exceptions import SearxSettingsException
 
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
 static_path = abspath(join(dirname(__file__), 'static'))
-settings, settings_load_message = searx.settings_loader.load_settings()
+
+settings, settings_outgoing = {}, ''
+try:
+    settings, settings_load_message = searx.settings_loader.load_settings()
+except SearxSettingsException as e:
+    logger = logging.getLogger('searx')
+    logger.error('Failed to load settings file: {}'.format(str(e)))
+    exit(1)
+
 
 if settings['ui']['static_path']:
     static_path = settings['ui']['static_path']
 
 '''
 enable debug if
-the environnement variable SEARX_DEBUG is 1 or true
+the environment variable SEARX_DEBUG is 1 or true
 (whatever the value in settings.yml)
 or general.debug=True in settings.yml
 disable debug if
-the environnement variable SEARX_DEBUG is 0 or false
+the environment variable SEARX_DEBUG is 0 or false
 (whatever the value in settings.yml)
 or general.debug=False in settings.yml
 '''
@@ -61,6 +71,48 @@ if 'SEARX_SECRET' in environ:
 if 'SEARX_BIND_ADDRESS' in environ:
     settings['server']['bind_address'] = environ['SEARX_BIND_ADDRESS']
 
-if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
-    logger.error('server.secret_key is not changed. Please use something else instead of ultrasecretkey.')
-    exit(1)
+
+class _brand_namespace:
+
+    @classmethod
+    def get_val(cls, group, name, default=''):
+        return settings.get(group, {}).get(name) or default
+
+    @property
+    def SEARX_URL(self):
+        return self.get_val('server', 'base_url')
+
+    @property
+    def CONTACT_URL(self):
+        return self.get_val('general', 'contact_url')
+
+    @property
+    def GIT_URL(self):
+        return self.get_val('brand', 'git_url')
+
+    @property
+    def GIT_BRANCH(self):
+        return self.get_val('brand', 'git_branch')
+
+    @property
+    def ISSUE_URL(self):
+        return self.get_val('brand', 'issue_url')
+
+    @property
+    def DOCS_URL(self):
+        return self.get_val('brand', 'docs_url')
+
+    @property
+    def PUBLIC_INSTANCES(self):
+        return self.get_val('brand', 'public_instances')
+
+    @property
+    def WIKI_URL(self):
+        return self.get_val('brand', 'wiki_url')
+
+    @property
+    def TWITTER_URL(self):
+        return self.get_val('brand', 'twitter_url')
+
+
+brand = _brand_namespace()

searx/answerers/__init__.py

@@ -32,7 +32,7 @@ def ask(query):
     results = []
     query_parts = list(filter(None, query.query.split()))
 
-    if query_parts[0] not in answerers_by_keywords:
+    if not query_parts or query_parts[0] not in answerers_by_keywords:
         return results
 
     for answerer in answerers_by_keywords[query_parts[0]]:

searx/autocomplete.py

@@ -20,95 +20,34 @@ from lxml import etree
 from json import loads
 from urllib.parse import urlencode
 
+from requests import RequestException
+
 from searx import settings
-from searx.languages import language_codes
-from searx.engines import (
-    categories, engines, engine_shortcuts
-)
 from searx.poolrequests import get as http_get
+from searx.exceptions import SearxEngineResponseException
 
 
 def get(*args, **kwargs):
     if 'timeout' not in kwargs:
         kwargs['timeout'] = settings['outgoing']['request_timeout']
-
+    kwargs['raise_for_httperror'] = True
     return http_get(*args, **kwargs)
 
 
-def searx_bang(full_query):
-    '''check if the searchQuery contain a bang, and create fitting autocompleter results'''
-    # check if there is a query which can be parsed
-    if len(full_query.getQuery()) == 0:
-        return []
+def brave(query, lang):
+    # brave search autocompleter
+    url = 'https://search.brave.com/api/suggest?{query}'
+
+    resp = get(url.format(query=urlencode({'q': query})))
 
     results = []
 
-    # check if current query stats with !bang
-    first_char = full_query.getQuery()[0]
-    if first_char == '!' or first_char == '?':
-        if len(full_query.getQuery()) == 1:
-            # show some example queries
-            # TODO, check if engine is not avaliable
-            results.append(first_char + "images")
-            results.append(first_char + "wikipedia")
-            results.append(first_char + "osm")
-        else:
-            engine_query = full_query.getQuery()[1:]
+    if resp.ok:
+        data = loads(resp.text)
+        for suggestion in data[1]:
+            results.append(suggestion)
 
-            # check if query starts with categorie name
-            for categorie in categories:
-                if categorie.startswith(engine_query):
-                    results.append(first_char + '{categorie}'.format(categorie=categorie))
-
-            # check if query starts with engine name
-            for engine in engines:
-                if engine.startswith(engine_query.replace('_', ' ')):
-                    results.append(first_char + '{engine}'.format(engine=engine.replace(' ', '_')))
-
-            # check if query starts with engine shortcut
-            for engine_shortcut in engine_shortcuts:
-                if engine_shortcut.startswith(engine_query):
-                    results.append(first_char + '{engine_shortcut}'.format(engine_shortcut=engine_shortcut))
-
-    # check if current query stats with :bang
-    elif first_char == ':':
-        if len(full_query.getQuery()) == 1:
-            # show some example queries
-            results.append(":en")
-            results.append(":en_us")
-            results.append(":english")
-            results.append(":united_kingdom")
-        else:
-            engine_query = full_query.getQuery()[1:]
-
-            for lc in language_codes:
-                lang_id, lang_name, country, english_name = map(str.lower, lc)
-
-                # check if query starts with language-id
-                if lang_id.startswith(engine_query):
-                    if len(engine_query) <= 2:
-                        results.append(':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
-                    else:
-                        results.append(':{lang_id}'.format(lang_id=lang_id))
-
-                # check if query starts with language name
-                if lang_name.startswith(engine_query) or english_name.startswith(engine_query):
-                    results.append(':{lang_name}'.format(lang_name=lang_name))
-
-                # check if query starts with country
-                if country.startswith(engine_query.replace('_', ' ')):
-                    results.append(':{country}'.format(country=country.replace(' ', '_')))
-
-    # remove duplicates
-    result_set = set(results)
-
-    # remove results which are already contained in the query
-    for query_part in full_query.query_parts:
-        if query_part in result_set:
-            result_set.remove(query_part)
-
-    # convert result_set back to list
-    return list(result_set)
+    return results
 
 
 def dbpedia(query, lang):
@@ -153,12 +92,11 @@ def google(query, lang):
 
 def startpage(query, lang):
     # startpage autocompleter
-    url = 'https://startpage.com/do/suggest?{query}'
-
-    resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')
-    if len(resp) > 1:
-        return resp
-    return []
+    lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')
+    url = 'https://startpage.com/suggestions?{query}'
+    resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))
+    data = resp.json()
+    return [e['text'] for e in data.get('suggestions', []) if 'text' in e]
 
 
 def swisscows(query, lang):
@@ -196,7 +134,8 @@ def wikipedia(query, lang):
     return []
 
 
-backends = {'dbpedia': dbpedia,
+backends = {'brave': brave,
+            'dbpedia': dbpedia,
             'duckduckgo': duckduckgo,
             'google': google,
             'startpage': startpage,
@@ -204,3 +143,14 @@ backends = {'dbpedia': dbpedia,
             'qwant': qwant,
             'wikipedia': wikipedia
             }
+
+
+def search_autocomplete(backend_name, query, lang):
+    backend = backends.get(backend_name)
+    if backend is None:
+        return []
+
+    try:
+        return backend(query, lang)
+    except (RequestException, SearxEngineResponseException):
+        return []

searx/brand.py

@@ -1,6 +0,0 @@
-GIT_URL = 'https://github.com/searx/searx'
-GIT_BRANCH = 'master'
-ISSUE_URL = 'https://github.com/searx/searx/issues'
-SEARX_URL = 'https://searx.me'
-DOCS_URL = 'https://searx.github.io/searx'
-PUBLIC_INSTANCES = 'https://searx.space'

searx/data/__init__.py

@@ -1,29 +1,50 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""This module holds the *data* created by::
+
+  make data.all
+
+"""
+
+__all__ = [
+    'ENGINES_LANGUAGES',
+    'CURRENCIES',
+    'USER_AGENTS',
+    'EXTERNAL_URLS',
+    'WIKIDATA_UNITS',
+    'EXTERNAL_BANGS',
+    'OSM_KEYS_TAGS',
+    'ahmia_blacklist_loader',
+]
+
 import json
 from pathlib import Path
 
-
-__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'EXTERNAL_URLS', 'WIKIDATA_UNITS',
-            'bangs_loader', 'ahmia_blacklist_loader']
 data_dir = Path(__file__).parent
 
 
-def load(filename):
-    # add str(...) for Python 3.5
-    with open(str(data_dir / filename), encoding='utf-8') as fd:
-        return json.load(fd)
-
-
-def bangs_loader():
-    return load('bangs.json')
+def _load(filename):
+    with open(data_dir / filename, encoding='utf-8') as f:
+        return json.load(f)
 
 
 def ahmia_blacklist_loader():
-    with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as fd:
-        return fd.read().split()
+    """Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion
+    names.  The MD5 values are fetched by::
+
+      searx_extra/update/update_ahmia_blacklist.py
+
+    This function is used by :py:mod:`searx.plugins.ahmia_filter`.
+
+    """
+    with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as f:
+        return f.read().split()
 
 
-ENGINES_LANGUAGES = load('engines_languages.json')
-CURRENCIES = load('currencies.json')
-USER_AGENTS = load('useragents.json')
-EXTERNAL_URLS = load('external_urls.json')
-WIKIDATA_UNITS = load('wikidata_units.json')
+ENGINES_LANGUAGES = _load('engines_languages.json')
+CURRENCIES = _load('currencies.json')
+USER_AGENTS = _load('useragents.json')
+EXTERNAL_URLS = _load('external_urls.json')
+WIKIDATA_UNITS = _load('wikidata_units.json')
+EXTERNAL_BANGS = _load('external_bangs.json')
+OSM_KEYS_TAGS = _load('osm_keys_tags.json')

searx/data/useragents.json

@@ -1,12 +1,13 @@
 {
     "versions": [
-        "75.0",
-        "74.0.1",
-        "74.0"
+        "111.0.1",
+        "111.0",
+        "110.0.1",
+        "110.0"
     ],
     "os": [
         "Windows NT 10.0; WOW64",
         "X11; Linux x86_64"
     ],
     "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
-}
+}

searx/data/wikidata_units.json

@@ -1,7 +1,64 @@
 {
     "Q199": "1",
+    "Q100036106": "int nmi",
     "Q100149279": "°We",
     "Q100995": "lb",
+    "Q101194838": "GHz/V",
+    "Q101427873": "pk (US)",
+    "Q101427917": "pk (UK)",
+    "Q101463141": "ym²",
+    "Q101463237": "zm²",
+    "Q101463321": "am²",
+    "Q101463409": "fm²",
+    "Q101463496": "pm²",
+    "Q101463679": "hm²",
+    "Q101464050": "Mm²",
+    "Q101464215": "Gm²",
+    "Q101464369": "Tm²",
+    "Q101464499": "Pm²",
+    "Q101464624": "Em²",
+    "Q101464753": "Zm²",
+    "Q101464875": "Ym²",
+    "Q101515060": "g/J",
+    "Q101875087": "cd/cm²",
+    "Q101879174": "dm/s",
+    "Q102068844": "cm⁻³",
+    "Q102130673": "ym/s",
+    "Q102130674": "zm/s",
+    "Q102130677": "am/s",
+    "Q102130679": "fm/s",
+    "Q102130681": "pm/s",
+    "Q102130684": "nm/s",
+    "Q102130686": "μm/s",
+    "Q102130688": "mm/s",
+    "Q102130690": "dam/s",
+    "Q102130692": "hm/s",
+    "Q102130694": "Mm/s",
+    "Q102130696": "Gm/s",
+    "Q102130698": "Tm/s",
+    "Q102130700": "Pm/s",
+    "Q102130702": "Em/s",
+    "Q102130704": "Zm/s",
+    "Q102130706": "Ym/s",
+    "Q102130743": "ym/s²",
+    "Q102130745": "zm/s²",
+    "Q102130747": "am/s²",
+    "Q102130748": "fm/s²",
+    "Q102130751": "pm/s²",
+    "Q102130753": "nm/s²",
+    "Q102130755": "μm/s²",
+    "Q102130756": "mm/s²",
+    "Q102130758": "dm/s²",
+    "Q102130759": "dam/s²",
+    "Q102130761": "hm/s²",
+    "Q102130762": "km/s²",
+    "Q102130765": "Mm/s²",
+    "Q102130767": "Gm/s²",
+    "Q102130769": "Tm/s²",
+    "Q102130771": "Pm/s²",
+    "Q102130773": "Em/s²",
+    "Q102130775": "Zm/s²",
+    "Q102130777": "Ym/s²",
     "Q1022113": "cm³",
     "Q102573": "Bq",
     "Q103246": "Sv",
@@ -9,57 +66,207 @@
     "Q10380431": "TJ",
     "Q1040401": "das",
     "Q1040427": "hs",
-    "Q1042866": "Zibit",
+    "Q104117265": "Bi",
+    "Q1042866": "Zib",
+    "Q104628312": "vars",
+    "Q104629193": "m/Es²",
+    "Q104816263": "kg/dm³",
+    "Q104907398": "μN m",
+    "Q104907399": "mN m",
     "Q1050958": "inHg",
     "Q1051665": "m/s²",
     "Q1052397": "rad",
     "Q1054140": "Mm",
     "Q10543042": "Ym",
+    "Q105519288": "B SPL",
     "Q1057069": "hg",
+    "Q105761866": "mV/K",
+    "Q105816142": "tsp",
+    "Q105816269": "cup (US)",
+    "Q105840138": "BU",
     "Q1063756": "rad/s",
     "Q1063786": "in²",
     "Q1065153": "mrad",
+    "Q106589841": "MHz/T",
+    "Q106611785": "g/dm³",
+    "Q106611903": "J/g",
     "Q1066138": "Ps",
+    "Q106617289": "GBq/kg",
+    "Q106617513": "mGy/s",
+    "Q106617558": "mSv/s",
+    "Q106617559": "μSv/s",
+    "Q106617560": "nSv/s",
+    "Q106617579": "kBq/kg",
+    "Q106617585": "MBq/kg",
+    "Q106623539": "Mg/m³",
+    "Q106623548": "μg/m³",
+    "Q106623562": "kg/cm³",
+    "Q106623580": "kg/kmol",
+    "Q106623615": "kg cm²",
+    "Q106623620": "kg mm²",
+    "Q106639792": "dm³/kg",
+    "Q106645176": "dz",
+    "Q106645210": "mg/g",
+    "Q106645232": "g/mm",
+    "Q106645237": "kg/mm",
+    "Q106645241": "mg/m",
+    "Q106645245": "kg/km",
+    "Q106645257": "MN m",
+    "Q106645261": "kN m",
+    "Q106645290": "dN m",
     "Q1067722": "Fg",
+    "Q106777906": "μS/m",
+    "Q106777917": "S/cm",
+    "Q106777923": "mS/cm",
+    "Q106777925": "MS/m",
+    "Q106777933": "kS/m",
+    "Q106777943": "nS/m",
+    "Q106777952": "nS/cm",
+    "Q106777957": "pS/m",
+    "Q106808096": "kJ/g",
+    "Q106808114": "GC/m³",
+    "Q106808120": "C/mm³",
+    "Q106808129": "MC/m³",
+    "Q106808138": "C/cm³",
+    "Q106808144": "kC/m³",
+    "Q106808151": "mC/m³",
+    "Q106808156": "μC/m³",
+    "Q106808167": "MC/m²",
+    "Q106808174": "C/mm²",
+    "Q106808180": "C/cm²",
+    "Q106808187": "kC/m²",
+    "Q106808194": "mC/m²",
+    "Q106808200": "μC/m²",
+    "Q106808221": "mC/kg",
+    "Q106885926": "N/mm",
+    "Q106886632": "hbar",
+    "Q106886776": "N/mm²",
     "Q1069725": "p.",
+    "Q106998070": "cm³/m³",
+    "Q106998079": "dm³/m³",
+    "Q107028522": "μg/hg",
+    "Q107028836": "MK⁻¹",
+    "Q107133620": "mg/m²",
+    "Q107133637": "g/cm²",
+    "Q107133667": "mg/cm²",
+    "Q107133676": "g/m²",
+    "Q107164998": "cd mm²/m²",
+    "Q107210119": "g/s",
+    "Q107210344": "mg/s",
+    "Q107226391": "cm⁻¹",
+    "Q1072404": "K",
+    "Q107244316": "mm⁻¹",
+    "Q107244475": "dm⁻¹",
+    "Q107244557": "km⁻¹",
+    "Q107256474": "l.",
+    "Q107299747": "kvar",
+    "Q107299751": "Mvar",
+    "Q107313731": "μg/kg",
+    "Q107313738": "ng/kg",
+    "Q107313800": "cm³/s",
+    "Q107325155": "kWb/m",
+    "Q107325173": "Wb/mm",
+    "Q107361007": "kJ/s",
+    "Q107361082": "MJ/m³",
+    "Q107361092": "J/cm²",
+    "Q107361171": "mW/m²",
+    "Q107361180": "μW/m²",
+    "Q107361187": "pW/m²",
+    "Q107378499": "kN/m²",
+    "Q107410680": "cN m",
+    "Q107410689": "N cm",
+    "Q107410785": "g/mm²",
+    "Q107410801": "g/(cm s)",
+    "Q107410895": "kJ/hg",
+    "Q107440604": "W/cm²",
+    "Q107440662": "mmol/kg",
+    "Q107440685": "mmol/g",
+    "Q107440698": "kmol/kg",
+    "Q107440839": "g/g",
+    "Q107440910": "g/hg",
+    "Q107441004": "mg/hg",
+    "Q107460729": "mm³/mm³",
+    "Q107460790": "kg/GJ",
+    "Q107460866": "g/m",
+    "Q107460882": "mg/km",
+    "Q107461064": "MJ/m²",
+    "Q107461092": "g/(m² s)",
+    "Q107461119": "μg/(m² s)",
+    "Q107461139": "μg/J",
+    "Q107461146": "g/MJ",
+    "Q107538710": "μH/m",
+    "Q107538724": "nH/m",
+    "Q107970291": "mol/dm³",
     "Q1084321": "Tb/s",
     "Q1086691": "fg",
+    "Q108730765": "kW a",
+    "Q108888186": "eV/c²",
+    "Q108888198": "keV/c²",
+    "Q108888206": "MeV/c²",
+    "Q108888224": "GeV/c²",
     "Q1091257": "tex",
     "Q1092296": "a",
-    "Q1104069": "CAD$",
+    "Q110143852": "Ω cm",
+    "Q110143896": "cm³/g",
+    "Q1104069": "$",
     "Q11061003": "μm²",
     "Q11061005": "nm²",
+    "Q110742003": "dppx",
     "Q1131660": "st",
     "Q1137675": "cr",
-    "Q1140444": "Zbit",
-    "Q1140577": "Ybit",
-    "Q1152074": "Pbit",
-    "Q1152323": "Tbit",
+    "Q114002440": "𒄀",
+    "Q114002534": "𒃻",
+    "Q114002639": "𒈨𒊑",
+    "Q114002796": "𒂆",
+    "Q114002930": "𒀺",
+    "Q114002955": "𒀹𒃷",
+    "Q114002974": "𒃷",
+    "Q1140444": "Zb",
+    "Q1140577": "Yb",
+    "Q114589269": "A",
+    "Q1152074": "Pb",
+    "Q1152323": "Tb",
+    "Q115277430": "QB",
+    "Q115280832": "RB",
+    "Q115359862": "qg",
+    "Q115359863": "rg",
+    "Q115359865": "Rg",
+    "Q115359866": "Qg",
+    "Q115359910": "Rm",
+    "Q115533751": "rm",
+    "Q115533764": "qm",
+    "Q115533776": "Qm",
+    "Q116432446": "ᵐ",
+    "Q116432563": "ˢ",
+    "Q116443090": "ʰ",
     "Q1165799": "mil",
     "Q11776930": "Mg",
     "Q11830636": "psf",
     "Q11929860": "kpc",
     "Q1194225": "lbf",
-    "Q1194580": "Mibit",
-    "Q1195111": "Ebit",
+    "Q1194580": "Mib",
+    "Q1195111": "Eb",
     "Q1196837": "ω_P",
     "Q1197459": "Ms",
     "Q11982285": "Em³",
     "Q11982288": "Zm³",
     "Q11982289": "Tm³",
     "Q12011178": "Zs",
-    "Q1204894": "Gibit",
+    "Q12034595": "oz (ap.)",
+    "Q1204894": "Gib",
     "Q12257695": "Eb/s",
     "Q12257696": "EB/s",
     "Q12261466": "kB/s",
+    "Q12263659": "mgal",
     "Q12265780": "Pb/s",
     "Q12265783": "PB/s",
     "Q12269121": "Yb/s",
     "Q12269122": "YB/s",
     "Q12269308": "Zb/s",
     "Q12269309": "ZB/s",
+    "Q1238720": "vols.",
     "Q1247300": "cm H₂O",
-    "Q12714022": "sh cwt",
+    "Q12714022": "cwt",
     "Q12789864": "GeV",
     "Q12874593": "W h",
     "Q128822": "kn",
@@ -67,13 +274,14 @@
     "Q130964": "cal",
     "Q131255": "F",
     "Q13147228": "g/cm³",
+    "Q131723": "₿",
     "Q1322380": "Ts",
     "Q1323615": "oz t",
     "Q132643": "kr",
     "Q13400897": "g",
-    "Q13479685": "mm wg",
-    "Q1351253": "Eibit",
-    "Q1351334": "Pibit",
+    "Q13479685": "mm H2O",
+    "Q1351253": "Eib",
+    "Q1351334": "Pib",
     "Q13542672": "Ry",
     "Q13548586": "THz",
     "Q13582667": "kgf/cm²",
@@ -86,16 +294,17 @@
     "Q1396128": "F",
     "Q1413142": "Gb",
     "Q14158377": "A_P",
+    "Q1427899": "U",
     "Q14623803": "MDa",
     "Q14623804": "kDa",
-    "Q1472674": "Sv",
+    "Q1472674": "S",
     "Q14754979": "Zg",
     "Q14786969": "MJ",
+    "Q14850704": "℧",
     "Q14913554": "Ys",
     "Q14914907": "th",
     "Q14916719": "Gpc",
     "Q14923662": "Pm³",
-    "Q1511773": "LSd",
     "Q15120301": "l atm",
     "Q1542309": "xu",
     "Q1545979": "ft³",
@@ -103,21 +312,20 @@
     "Q15551713": "Sh",
     "Q1569733": "St",
     "Q15784325": "apc",
-    "Q160680": "Br",
     "Q160857": "hp",
+    "Q162525": "°E",
     "Q1628990": "hph",
     "Q163343": "T",
     "Q163354": "H",
     "Q1640501": "hyl",
     "Q1645498": "μg",
-    "Q16859309": "lb·ft",
+    "Q16859309": "lb ft",
     "Q169893": "S",
     "Q170804": "Wb",
     "Q17093295": "m/h",
     "Q17255465": "v_P",
     "Q173117": "R$",
     "Q1741429": "kpm",
-    "Q174467": "Lm",
     "Q174728": "cm",
     "Q174789": "mm",
     "Q175821": "μm",
@@ -130,6 +338,7 @@
     "Q177974": "atm",
     "Q178506": "bbl",
     "Q178674": "nm",
+    "Q1790908": "mi3",
     "Q1793863": "sn",
     "Q179836": "lx",
     "Q180154": "km/h",
@@ -140,14 +349,13 @@
     "Q182429": "m/s",
     "Q1826195": "dl",
     "Q18413919": "cm/s",
-    "Q184172": "FF",
     "Q185078": "a",
     "Q185153": "erg",
     "Q185648": "Torr",
     "Q185759": "span",
     "Q1872619": "zs",
-    "Q189097": "₧",
     "Q190095": "Gy",
+    "Q19017495": "mm²",
     "Q190951": "S$",
     "Q191118": "t",
     "Q1913097": "fg",
@@ -161,6 +369,7 @@
     "Q194339": "B$",
     "Q1970718": "mam",
     "Q1972579": "pdl",
+    "Q19877834": "cd-ft",
     "Q199462": "LE",
     "Q199471": "Afs",
     "Q200323": "dm",
@@ -171,11 +380,11 @@
     "Q2029519": "hl",
     "Q203567": "₦",
     "Q2042279": "m H₂O",
-    "Q204737": "៛",
+    "Q204992": "L.",
     "Q2051195": "GWh",
     "Q2055118": "ppb",
     "Q2064166": "fc",
-    "Q206600": "ރ",
+    "Q206600": "MRF",
     "Q20706220": "cmm",
     "Q20706221": "dmm",
     "Q2080811": "vol%",
@@ -183,6 +392,7 @@
     "Q208528": "gon",
     "Q208634": "kat",
     "Q208788": "fm",
+    "Q2090348": "Kib/s",
     "Q209351": "b",
     "Q209426": "′",
     "Q21006887": "ppm",
@@ -192,13 +402,14 @@
     "Q21061369": "g/kg",
     "Q21062777": "MPa",
     "Q21064807": "kPa",
-    "Q21064845": "mol/L",
     "Q21075844": "ml/l",
     "Q21077820": "mg/m³",
     "Q21091747": "mg/kg",
-    "Q211256": "mph",
+    "Q211256": "mi/h",
+    "Q21154419": "PD",
     "Q211580": "BTU (th)",
-    "Q212120": "A h",
+    "Q212120": "A⋅h",
+    "Q213005": "G$",
     "Q2140397": "in³",
     "Q214377": "ell",
     "Q2143992": "kHz",
@@ -211,7 +422,7 @@
     "Q215571": "N m",
     "Q21604951": "g/m³",
     "Q2165290": "yd³",
-    "Q216880": "kp",
+    "Q216880": "kgf",
     "Q217208": "a",
     "Q2175964": "dm³",
     "Q218593": "in",
@@ -226,15 +437,17 @@
     "Q2269250": "kb/s",
     "Q2282891": "μl",
     "Q2282906": "ng",
+    "Q22934083": "nC",
     "Q229354": "Ci",
     "Q232291": "mi²",
     "Q2332346": "ml",
+    "Q235729": "y (365 days)",
     "Q23823681": "TW",
     "Q23925410": "gal (UK)",
     "Q23925413": "gal (US)",
     "Q23931040": "dam²",
     "Q23931103": "nmi²",
-    "Q2414435": "$b.",
+    "Q240468": "syr£",
     "Q242988": "Lib$",
     "Q2438073": "ag",
     "Q2448803": "mV",
@@ -246,16 +459,18 @@
     "Q249439": "q_P",
     "Q2518569": "nSv",
     "Q253276": "mi",
+    "Q254532": "deg²",
     "Q25472681": "GB/s",
     "Q25472693": "TB/s",
     "Q25499149": "oct",
     "Q25511288": "mb",
     "Q2553708": "MV",
     "Q2554092": "kV",
-    "Q259502": "AU$",
+    "Q259502": "A$",
     "Q260126": "rem",
     "Q2612219": "Pg",
     "Q261247": "ct",
+    "Q26162546": "mm²/s",
     "Q2619500": "foe",
     "Q2636421": "nH",
     "Q2637946": "dal",
@@ -276,6 +491,7 @@
     "Q2756030": "pF",
     "Q2757753": "PW h",
     "Q2762458": "ys",
+    "Q2784622": "T",
     "Q27864215": "μW h",
     "Q2793566": "GV",
     "Q27949241": "R",
@@ -306,8 +522,12 @@
     "Q30001831": "aV",
     "Q30001832": "aW",
     "Q30001833": "aWb",
-    "Q3013059": "kyr",
-    "Q3194304": "kbit",
+    "Q3013059": "ka",
+    "Q304479": "tr",
+    "Q305896": "DPI",
+    "Q3095010": "γ",
+    "Q31889818": "ppq",
+    "Q3194304": "kb",
     "Q3207456": "mW",
     "Q321017": "R",
     "Q3221356": "ym",
@@ -330,10 +550,10 @@
     "Q3312063": "fL",
     "Q3320608": "kW",
     "Q3331719": "dm²",
-    "Q3332689": "ToR",
-    "Q3332814": "Mbit",
+    "Q3332689": "RT",
+    "Q3332814": "Mb",
     "Q3396758": "daa",
-    "Q3414243": "rps",
+    "Q3414243": "qps",
     "Q3421309": "R_J",
     "Q3495543": "mbar",
     "Q355198": "px",
@@ -343,11 +563,11 @@
     "Q376660": "nat",
     "Q37732658": "°R",
     "Q3773454": "Mpc",
-    "Q3815076": "Kibit",
+    "Q3815076": "Kib",
     "Q3833309": "£",
-    "Q3858002": "mA h",
+    "Q3858002": "mA⋅h",
     "Q3867152": "ft/s²",
-    "Q389062": "Tibit",
+    "Q389062": "Tib",
     "Q3902688": "pl",
     "Q3902709": "ps",
     "Q39360235": "US lea",
@@ -359,7 +579,7 @@
     "Q39462789": "µin²",
     "Q39467934": "kgf/m²",
     "Q39469927": "N/m²",
-    "Q39617688": "cwt long",
+    "Q39617688": "cwt",
     "Q39617818": "t lb",
     "Q39628023": "y",
     "Q39699418": "cm/s²",
@@ -367,14 +587,13 @@
     "Q39709980": "bd",
     "Q39710113": "bhp EDR",
     "Q3972226": "kL",
-    "Q4041686": "iwg",
+    "Q4041686": "in H20",
     "Q4068266": "Ʒ",
     "Q4176683": "aC",
-    "Q420266": "oz. fl.",
     "Q42319606": "people/m²",
     "Q4243638": "km³",
     "Q4456994": "mF",
-    "Q469356": "tn. sh.",
+    "Q469356": "T",
     "Q476572": "Ha",
     "Q482798": "yd",
     "Q483261": "Da",
@@ -383,6 +602,7 @@
     "Q4861171": "H",
     "Q494083": "fur",
     "Q4989854": "kJ",
+    "Q4992853": "kt",
     "Q500515": "Gal",
     "Q5042194": "£",
     "Q50808017": "kg m²",
@@ -390,19 +610,24 @@
     "Q514845": "pz",
     "Q5195628": "hm³",
     "Q5198770": "dam³",
-    "Q524410": "byr",
+    "Q524410": "Ga",
+    "Q5299480": "DPCm",
     "Q53393488": "PHz",
     "Q53393490": "EHz",
     "Q53393494": "ZHz",
     "Q53393498": "YHz",
     "Q53393659": "ML",
     "Q53393664": "GL",
+    "Q53393669": "El",
     "Q53393674": "ZL",
     "Q53393678": "YL",
+    "Q53393768": "zl",
     "Q53393771": "yL",
     "Q53393868": "GJ",
     "Q53393886": "PJ",
     "Q53393890": "EJ",
+    "Q53393893": "ZJ",
+    "Q53393898": "YJ",
     "Q53448786": "yHz",
     "Q53448790": "zHz",
     "Q53448794": "fHz",
@@ -416,6 +641,7 @@
     "Q53448826": "hHz",
     "Q53448828": "yJ",
     "Q53448832": "zJ",
+    "Q53448835": "fJ",
     "Q53448842": "pJ",
     "Q53448844": "nJ",
     "Q53448847": "μJ",
@@ -478,6 +704,7 @@
     "Q53951982": "Mt",
     "Q53952048": "kt",
     "Q54006645": "ZWb",
+    "Q54081354": "ZT",
     "Q54081925": "ZSv",
     "Q54082468": "ZS",
     "Q54083144": "ZΩ",
@@ -492,7 +719,7 @@
     "Q54083813": "Zkat",
     "Q5409016": "MVA",
     "Q5465723": "ft-pdl",
-    "Q549389": "bit/s",
+    "Q549389": "b/s",
     "Q550341": "V A",
     "Q552299": "ch",
     "Q55442349": "U/L",
@@ -502,8 +729,6 @@
     "Q56157046": "nmol",
     "Q56157048": "pmol",
     "Q56160603": "fmol",
-    "Q56302633": "UM",
-    "Q56317116": "mgal",
     "Q56317622": "Q_P",
     "Q56318907": "kbar",
     "Q56349362": "Bs.S",
@@ -523,6 +748,8 @@
     "Q6170164": "yg",
     "Q6171168": "zg",
     "Q61756607": "yd",
+    "Q61771602": "ft",
+    "Q61771670": "in",
     "Q61793198": "rd",
     "Q61794766": "ch (US survey)",
     "Q61994988": "Wth",
@@ -534,13 +761,12 @@
     "Q6414556": "kip",
     "Q648908": "bya",
     "Q64996135": "gal (US)/min",
-    "Q65028392": "mm/yr",
+    "Q65028392": "mm/a",
     "Q651336": "M_J",
     "Q6517513": "dag",
     "Q667419": "UK t",
-    "Q681996": "M⊕",
+    "Q681996": "M🜨",
     "Q685662": "p_P",
-    "Q6859652": "mm Hg",
     "Q686163": "$",
     "Q68725821": "°Rø",
     "Q68726230": "°De",
@@ -588,14 +814,17 @@
     "Q732707": "MHz",
     "Q73408": "K",
     "Q7350781": "Mb/s",
+    "Q7398951": "PPI",
     "Q743895": "bpm",
-    "Q748716": "ft/s",
-    "Q750178": "‱",
+    "Q748716": "fps",
+    "Q752079": "RT",
     "Q752197": "kJ/mol",
+    "Q7574000": "sp",
     "Q7672057": "TU",
     "Q777017": "dBm",
+    "Q780456": "Td",
     "Q78754556": "rot",
-    "Q78756901": "rev",
+    "Q78756901": "r",
     "Q78757683": "windings",
     "Q79726": "kB",
     "Q79735": "MB",
@@ -637,14 +866,18 @@
     "Q848856": "dam",
     "Q851872": "o",
     "Q854546": "Gm",
-    "Q855161": "Yibit",
+    "Q855161": "Yib",
     "Q856240": "ft³/min",
     "Q857027": "ft²",
     "Q85854198": "MN",
-    "Q864818": "abA",
     "Q87262709": "kΩ",
     "Q87416053": "MΩ",
     "Q88296091": "tsp",
+    "Q89187604": "bbl (US)",
+    "Q89473028": "bu (UK)",
+    "Q89662131": "pt (UK)",
+    "Q901492": "ph",
+    "Q902274": "cp",
     "Q9026416": "MWth",
     "Q9048643": "nl",
     "Q905912": "L",
@@ -653,8 +886,10 @@
     "Q911730": "nx",
     "Q914151": "P_P",
     "Q915169": "F_P",
-    "Q93318": "nmi",
-    "Q940052": "q",
+    "Q93318": "M",
+    "Q933427": "B",
+    "Q93678895": "gill (US)",
+    "Q93679498": "gill (UK)",
     "Q94076025": "dalm",
     "Q94076717": "dakat",
     "Q942092": "BWI$",
@@ -664,6 +899,7 @@
     "Q94415255": "GC",
     "Q94415438": "Yrad",
     "Q94415526": "YC",
+    "Q94415561": "krad",
     "Q94415782": "Mrad",
     "Q94416260": "GN",
     "Q94416535": "cN",
@@ -775,7 +1011,6 @@
     "Q95379580": "hC",
     "Q95379588": "dC",
     "Q95379596": "EC",
-    "Q95445986": "nC",
     "Q95446327": "pC",
     "Q95446670": "fC",
     "Q95447079": "zC",
@@ -943,6 +1178,7 @@
     "Q96106385": "h°C",
     "Q96106393": "M°C",
     "Q96236286": "G°C",
+    "Q96312779": "μas",
     "Q97059641": "p°C",
     "Q97059652": "T°C",
     "Q97143826": "P°C",
@@ -953,9 +1189,18 @@
     "Q97143843": "z°C",
     "Q97143849": "Y°C",
     "Q97143851": "a°C",
-    "Q98538634": "eV/m²",
-    "Q98635536": "eV/m",
-    "Q98642859": "eV m²/kg",
+    "Q98492214": "den",
+    "Q98793302": "qt (UK)",
+    "Q98793408": "liq qt (US)",
+    "Q98793687": "dry qt (US)",
+    "Q99476928": "gf",
+    "Q99487704": "ppt",
+    "Q99490009": "BTU (IT)",
+    "Q99490479": "BTU (39 °F)",
+    "Q99490986": "BTU (59 °F)",
+    "Q99491193": "BTU (60 °F)",
+    "Q99491447": "BTU (mean)",
+    "Q99492167": "m Hg",
     "Q11229": "%",
     "Q11570": "kg",
     "Q11573": "m",
@@ -964,9 +1209,8 @@
     "Q11582": "L",
     "Q12129": "pc",
     "Q12438": "N",
-    "Q16068": "DM",
-    "Q1811": "ua",
-    "Q20764": "Myr",
+    "Q1811": "AU",
+    "Q20764": "Ma",
     "Q2101": "e",
     "Q25235": "h",
     "Q25236": "W",
@@ -979,28 +1223,29 @@
     "Q25517": "m³",
     "Q33680": "rad",
     "Q35852": "ha",
-    "Q36384": "equiv",
+    "Q36384": "Eq",
     "Q3710": "ft",
     "Q39274": "Sv",
     "Q39369": "Hz",
     "Q41509": "mol",
     "Q41803": "g",
     "Q42289": "°F",
-    "Q4406": "TV$",
+    "Q4406": "$T",
     "Q44395": "Pa",
-    "Q4587": "Le",
     "Q4588": "WS$",
     "Q4592": "F$",
     "Q4596": "Rs",
     "Q4597": "$",
     "Q47083": "Ω",
     "Q48013": "oz",
+    "Q4917": "US$",
     "Q50094": "Np",
     "Q50098": "B",
-    "Q531": "ly",
+    "Q531": "l.y.",
     "Q5329": "dB",
     "Q573": "d",
     "Q577": "a",
     "Q7727": "min",
-    "Q8799": "B"
+    "Q8799": "B",
+    "Q8805": "bit"
 }

searx/engines/1337x.py

@@ -1,7 +1,21 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ 1337x
+"""
+
 from urllib.parse import quote, urljoin
 from lxml import html
 from searx.utils import extract_text, get_torrent_size, eval_xpath, eval_xpath_list, eval_xpath_getindex
 
+# about
+about = {
+    "website": 'https://1337x.to/',
+    "wikidata_id": 'Q28134166',
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
 
 url = 'https://1337x.to/'
 search_url = url + 'search/{search_term}/{pageno}/'

searx/engines/__init__.py

@@ -26,8 +26,9 @@ from operator import itemgetter
 from searx import settings
 from searx import logger
 from searx.data import ENGINES_LANGUAGES
+from searx.exceptions import SearxEngineResponseException
 from searx.poolrequests import get, get_proxy_cycles
-from searx.utils import load_module, match_language, get_engine_from_settings
+from searx.utils import load_module, match_language, get_engine_from_settings, gen_useragent
 
 
 logger = logger.getChild('engines')
@@ -44,16 +45,16 @@ babel_langs = [lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else
 engine_shortcuts = {}
 engine_default_args = {'paging': False,
                        'categories': ['general'],
-                       'language_support': True,
                        'supported_languages': [],
                        'safesearch': False,
                        'timeout': settings['outgoing']['request_timeout'],
                        'shortcut': '-',
                        'disabled': False,
+                       'enable_http': False,
                        'suspend_end_time': 0,
                        'continuous_errors': 0,
                        'time_range_support': False,
-                       'offline': False,
+                       'engine_type': 'online',
                        'display_error_messages': True,
                        'tokens': []}
 
@@ -109,8 +110,8 @@ def load_engine(engine_data):
             sys.exit(1)
 
     # assign supported languages from json file
-    if engine_data['name'] in ENGINES_LANGUAGES:
-        setattr(engine, 'supported_languages', ENGINES_LANGUAGES[engine_data['name']])
+    if engine_data['engine'] in ENGINES_LANGUAGES:
+        setattr(engine, 'supported_languages', ENGINES_LANGUAGES[engine_data['engine']])
 
     # find custom aliases for non standard language codes
     if hasattr(engine, 'supported_languages'):
@@ -127,14 +128,21 @@ def load_engine(engine_data):
 
         setattr(engine, 'language_aliases', language_aliases)
 
+    # language_support
+    setattr(engine, 'language_support', len(getattr(engine, 'supported_languages', [])) > 0)
+
     # assign language fetching method if auxiliary method exists
     if hasattr(engine, '_fetch_supported_languages'):
+        headers = {
+            'User-Agent': gen_useragent(),
+            'Accept-Language': 'ja-JP,ja;q=0.8,en-US;q=0.5,en;q=0.3',  # bing needs a non-English language
+        }
         setattr(engine, 'fetch_supported_languages',
-                lambda: engine._fetch_supported_languages(get(engine.supported_languages_url)))
+                lambda: engine._fetch_supported_languages(get(engine.supported_languages_url, headers=headers)))
 
     engine.stats = {
         'sent_search_count': 0,  # sent search
-        'search_count': 0,  # succesful search
+        'search_count': 0,  # successful search
         'result_count': 0,
         'engine_time': 0,
         'engine_time_count': 0,
@@ -142,7 +150,9 @@ def load_engine(engine_data):
         'errors': 0
     }
 
-    if not engine.offline:
+    engine_type = getattr(engine, 'engine_type', 'online')
+
+    if engine_type != 'offline':
         engine.stats['page_load_time'] = 0
         engine.stats['page_load_count'] = 0
 
@@ -161,7 +171,7 @@ def load_engine(engine_data):
         categories.setdefault(category_name, []).append(engine)
 
     if engine.shortcut in engine_shortcuts:
-        logger.error('Engine config error: ambigious shortcut: {0}'.format(engine.shortcut))
+        logger.error('Engine config error: ambiguous shortcut: {0}'.format(engine.shortcut))
         sys.exit(1)
 
     engine_shortcuts[engine.shortcut] = engine.name
@@ -209,7 +219,7 @@ def get_engines_stats(preferences):
         else:
             score = score_per_result = 0.0
 
-        if not engine.offline:
+        if engine.engine_type != 'offline':
             load_times = 0
             if engine.stats['page_load_count'] != 0:
                 load_times = engine.stats['page_load_time'] / float(engine.stats['page_load_count'])  # noqa
@@ -267,7 +277,7 @@ def get_engines_stats(preferences):
 
 
 def load_engines(engine_list):
-    global engines, engine_shortcuts
+    global engines, engine_shortcuts  # pylint: disable=global-variable-not-assigned
     engines.clear()
     engine_shortcuts.clear()
     for engine_data in engine_list:
@@ -283,6 +293,8 @@ def initialize_engines(engine_list):
     def engine_init(engine_name, init_fn):
         try:
             init_fn(get_engine_from_settings(engine_name))
+        except SearxEngineResponseException as exc:
+            logger.warn('%s engine: Fail to initialize // %s', engine_name, exc)
         except Exception:
             logger.exception('%s engine: Fail to initialize', engine_name)
         else:
@@ -294,34 +306,3 @@ def initialize_engines(engine_list):
             if init_fn:
                 logger.debug('%s engine: Starting background initialization', engine_name)
                 threading.Thread(target=engine_init, args=(engine_name, init_fn)).start()
-
-        _set_https_support_for_engine(engine)
-
-
-def _set_https_support_for_engine(engine):
-    # check HTTPS support if it is not disabled
-    if not engine.offline and not hasattr(engine, 'https_support'):
-        params = engine.request('http_test', {
-            'method': 'GET',
-            'headers': {},
-            'data': {},
-            'url': '',
-            'cookies': {},
-            'verify': True,
-            'auth': None,
-            'pageno': 1,
-            'time_range': None,
-            'language': '',
-            'safesearch': False,
-            'is_test': True,
-            'category': 'files',
-            'raise_for_status': True,
-        })
-
-        if 'url' not in params:
-            return
-
-        parsed_url = urlparse(params['url'])
-        https_support = parsed_url.scheme == 'https'
-
-        setattr(engine, 'https_support', https_support)

searx/engines/acgsou.py

@@ -1,70 +0,0 @@
-"""
- Acgsou (Japanese Animation/Music/Comics Bittorrent tracker)
-
- @website      https://www.acgsou.com/
- @provide-api  no
- @using-api    no
- @results      HTML
- @stable       no (HTML can change)
- @parse        url, title, content, seed, leech, torrentfile
-"""
-
-from urllib.parse import urlencode
-from lxml import html
-from searx.utils import extract_text, get_torrent_size, eval_xpath_list, eval_xpath_getindex
-
-# engine dependent config
-categories = ['files', 'images', 'videos', 'music']
-paging = True
-
-# search-url
-base_url = 'https://www.acgsou.com/'
-search_url = base_url + 'search.php?{query}&page={offset}'
-# xpath queries
-xpath_results = '//table[contains(@class, "list_style table_fixed")]//tr[not(th)]'
-xpath_category = './/td[2]/a[1]'
-xpath_title = './/td[3]/a[last()]'
-xpath_torrent_links = './/td[3]/a'
-xpath_filesize = './/td[4]/text()'
-
-
-def request(query, params):
-    query = urlencode({'keyword': query})
-    params['url'] = search_url.format(query=query, offset=params['pageno'])
-    return params
-
-
-def response(resp):
-    results = []
-    dom = html.fromstring(resp.text)
-    for result in eval_xpath_list(dom, xpath_results):
-        # defaults
-        filesize = 0
-        magnet_link = "magnet:?xt=urn:btih:{}&tr=https://tracker.acgsou.com:2710/announce"
-
-        category = extract_text(eval_xpath_getindex(result, xpath_category, 0, default=[]))
-        page_a = eval_xpath_getindex(result, xpath_title, 0)
-        title = extract_text(page_a)
-        href = base_url + page_a.attrib.get('href')
-
-        magnet_link = magnet_link.format(page_a.attrib.get('href')[5:-5])
-
-        filesize_info = eval_xpath_getindex(result, xpath_filesize, 0, default=None)
-        if filesize_info:
-            try:
-                filesize = filesize_info[:-2]
-                filesize_multiplier = filesize_info[-2:]
-                filesize = get_torrent_size(filesize, filesize_multiplier)
-            except:
-                pass
-        # I didn't add download/seed/leech count since as I figured out they are generated randomly everytime
-        content = 'Category: "{category}".'
-        content = content.format(category=category)
-
-        results.append({'url': href,
-                        'title': title,
-                        'content': content,
-                        'filesize': filesize,
-                        'magnetlink': magnet_link,
-                        'template': 'torrent.html'})
-    return results

searx/engines/ahmia.py

@@ -1,26 +1,29 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Ahmia (Onions)
-
- @website      http://msydqstlz2kzerdg.onion
- @provides-api no
-
- @using-api    no
- @results      HTML
- @stable       no
- @parse        url, title, content
 """
 
 from urllib.parse import urlencode, urlparse, parse_qs
 from lxml.html import fromstring
 from searx.engines.xpath import extract_url, extract_text, eval_xpath_list, eval_xpath
 
+# about
+about = {
+    "website": 'http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion',
+    "wikidata_id": 'Q18693938',
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
 # engine config
 categories = ['onions']
 paging = True
 page_size = 10
 
 # search url
-search_url = 'http://msydqstlz2kzerdg.onion/search/?{query}'
+search_url = 'http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion/search/?{query}'
 time_range_support = True
 time_range_dict = {'day': 1,
                    'week': 7,

searx/engines/apkmirror.py

@@ -1,25 +1,33 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""APKMirror
 """
- APK Mirror
 
- @website     https://www.apkmirror.com
-
- @using-api   no
- @results     HTML
- @stable      no (HTML can change)
- @parse       url, title, thumbnail_src
-"""
+# pylint: disable=invalid-name, missing-function-docstring
 
 from urllib.parse import urlencode
 from lxml import html
-from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
 
+from searx import logger
+from searx.utils import (
+    eval_xpath_list,
+    eval_xpath_getindex,
+    extract_text,
+)
+
+logger = logger.getChild('APKMirror engine')
+
+about = {
+    "website": 'https://www.apkmirror.com',
+    "wikidata_id": None,
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
 
 # engine dependent config
-categories = ['it']
+categories = ['files']
 paging = True
-
-# I am not 100% certain about this, as apkmirror appears to be a wordpress site,
-# which might support time_range searching. If you want to implement it, go ahead.
 time_range_support = False
 
 # search-url
@@ -27,37 +35,34 @@ base_url = 'https://www.apkmirror.com'
 search_url = base_url + '/?post_type=app_release&searchtype=apk&page={pageno}&{query}'
 
 
-# do search-request
 def request(query, params):
-
-    params['url'] = search_url.format(pageno=params['pageno'],
-                                      query=urlencode({'s': query}))
+    params['url'] = search_url.format(
+        pageno = params['pageno'],
+        query = urlencode({'s': query}),
+    )
+    logger.debug("query_url --> %s", params['url'])
     return params
 
 
-# get response from search-request
 def response(resp):
     results = []
 
     dom = html.fromstring(resp.text)
 
     # parse results
-    for result in eval_xpath_list(dom, './/div[@id="content"]/div[@class="listWidget"]/div[@class="appRow"]'):
+    for result in eval_xpath_list(dom, "//div[@id='content']//div[@class='listWidget']/div/div[@class='appRow']"):
 
         link = eval_xpath_getindex(result, './/h5/a', 0)
+
         url = base_url + link.attrib.get('href') + '#downloads'
         title = extract_text(link)
-        thumbnail_src = base_url\
-            + eval_xpath_getindex(result, './/img', 0).attrib.get('src').replace('&w=32&h=32', '&w=64&h=64')
-
+        img_src = base_url + eval_xpath_getindex(result, './/img/@src', 0)
         res = {
             'url': url,
             'title': title,
-            'thumbnail_src': thumbnail_src
+            'img_src': img_src
         }
 
-        # append result
         results.append(res)
 
-    # return results
     return results

searx/engines/archlinux.py

@@ -1,23 +1,26 @@
-# -*- coding: utf-8 -*-
-
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Arch Linux Wiki
 
- @website      https://wiki.archlinux.org
- @provide-api  no (Mediawiki provides API, but Arch Wiki blocks access to it
- @using-api    no
- @results      HTML
- @stable       no (HTML can change)
- @parse        url, title
+ API: Mediawiki provides API, but Arch Wiki blocks access to it
 """
 
 from urllib.parse import urlencode, urljoin
 from lxml import html
 from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
 
+# about
+about = {
+    "website": 'https://wiki.archlinux.org/',
+    "wikidata_id": 'Q101445877',
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
 # engine dependent config
 categories = ['it']
-language_support = True
 paging = True
 base_url = 'https://wiki.archlinux.org'
 

searx/engines/arxiv.py

@@ -1,20 +1,21 @@
-#!/usr/bin/env python
-
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  ArXiV (Scientific preprints)
- @website     https://arxiv.org
- @provide-api yes (export.arxiv.org/api/query)
- @using-api   yes
- @results     XML-RSS
- @stable      yes
- @parse       url, title, publishedDate, content
- More info on api: https://arxiv.org/help/api/user-manual
 """
 
 from lxml import html
 from datetime import datetime
 from searx.utils import eval_xpath_list, eval_xpath_getindex
 
+# about
+about = {
+    "website": 'https://arxiv.org',
+    "wikidata_id": 'Q118398',
+    "official_api_documentation": 'https://arxiv.org/help/api',
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": 'XML-RSS',
+}
 
 categories = ['science']
 paging = True

searx/engines/bandcamp.py

@@ -0,0 +1,73 @@
+"""
+Bandcamp (Music)
+
+@website     https://bandcamp.com/
+@provide-api no
+@results     HTML
+@parse       url, title, content, publishedDate, embedded, thumbnail
+"""
+
+from urllib.parse import urlencode, urlparse, parse_qs
+from dateutil.parser import parse as dateparse
+from lxml import html
+from searx.utils import extract_text
+
+categories = ['music']
+paging = True
+
+base_url = "https://bandcamp.com/"
+search_string = search_string = 'search?{query}&page={page}'
+embedded_url = '''<iframe width="100%" height="166"
+    scrolling="no" frameborder="no"
+    data-src="https://bandcamp.com/EmbeddedPlayer/{type}={result_id}/size=large/bgcol=ffffff/linkcol=0687f5/tracklist=false/artwork=small/transparent=true/"
+></iframe>'''
+
+
+def request(query, params):
+    '''pre-request callback
+    params<dict>:
+      method  : POST/GET
+      headers : {}
+      data    : {} # if method == POST
+      url     : ''
+      category: 'search category'
+      pageno  : 1 # number of the requested page
+    '''
+
+    search_path = search_string.format(
+        query=urlencode({'q': query}),
+        page=params['pageno'])
+
+    params['url'] = base_url + search_path
+
+    return params
+
+
+def response(resp):
+    '''post-response callback
+    resp: requests response object
+    '''
+    results = []
+    tree = html.fromstring(resp.text)
+    search_results = tree.xpath('//li[contains(@class, "searchresult")]')
+    for result in search_results:
+        link = result.xpath('.//div[@class="itemurl"]/a')[0]
+        result_id = parse_qs(urlparse(link.get('href')).query)["search_item_id"][0]
+        title = result.xpath('.//div[@class="heading"]/a/text()')
+        date = dateparse(result.xpath('//div[@class="released"]/text()')[0].replace("released ", ""))
+        content = result.xpath('.//div[@class="subhead"]/text()')
+        new_result = {
+            "url": extract_text(link),
+            "title": extract_text(title),
+            "content": extract_text(content),
+            "publishedDate": date,
+        }
+        thumbnail = result.xpath('.//div[@class="art"]/img/@src')
+        if thumbnail:
+            new_result['thumbnail'] = thumbnail[0]
+        if "album" in result.classes:
+            new_result["embedded"] = embedded_url.format(type='album', result_id=result_id)
+        elif "track" in result.classes:
+            new_result["embedded"] = embedded_url.format(type='track', result_id=result_id)
+        results.append(new_result)
+    return results

searx/engines/base.py

@@ -1,16 +1,6 @@
-#!/usr/bin/env python
-
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  BASE (Scholar publications)
-
- @website     https://base-search.net
- @provide-api yes with authorization (https://api.base-search.net/)
-
- @using-api   yes
- @results     XML
- @stable      ?
- @parse       url, title, publishedDate, content
- More info on api: http://base-search.net/about/download/base_interface.pdf
 """
 
 from urllib.parse import urlencode
@@ -19,6 +9,15 @@ from datetime import datetime
 import re
 from searx.utils import searx_useragent
 
+# about
+about = {
+    "website": 'https://base-search.net',
+    "wikidata_id": 'Q448335',
+    "official_api_documentation": 'https://api.base-search.net/',
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": 'XML',
+}
 
 categories = ['science']
 

searx/engines/bing.py

@@ -1,16 +1,6 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Bing (Web)
-
- @website     https://www.bing.com
- @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
-              max. 5000 query/month
-
- @using-api   no (because of query limit)
- @results     HTML (using search portal)
- @stable      no (HTML can change)
- @parse       url, title, content
-
- @todo        publishedDate
 """
 
 import re
@@ -21,10 +11,19 @@ from searx.utils import eval_xpath, extract_text, match_language
 
 logger = logger.getChild('bing engine')
 
+# about
+about = {
+    "website": 'https://www.bing.com',
+    "wikidata_id": 'Q182496',
+    "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-web-search-api',
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
 # engine dependent config
 categories = ['general']
 paging = True
-language_support = True
 supported_languages_url = 'https://www.bing.com/account/general'
 language_aliases = {'zh-CN': 'zh-CHS', 'zh-TW': 'zh-CHT', 'zh-HK': 'zh-CHT'}
 
@@ -53,6 +52,7 @@ def request(query, params):
         offset=offset)
 
     params['url'] = base_url + search_path
+    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
 
     return params
 
@@ -67,11 +67,13 @@ def response(resp):
     for result in eval_xpath(dom, '//div[@class="sa_cc"]'):
         link = eval_xpath(result, './/h3/a')[0]
         url = link.attrib.get('href')
+        pretty_url = extract_text(eval_xpath(result, './/cite'))
         title = extract_text(link)
         content = extract_text(eval_xpath(result, './/p'))
 
         # append result
         results.append({'url': url,
+                        'pretty_url': pretty_url,
                         'title': title,
                         'content': content})
 
@@ -79,11 +81,13 @@ def response(resp):
     for result in eval_xpath(dom, '//li[@class="b_algo"]'):
         link = eval_xpath(result, './/h2/a')[0]
         url = link.attrib.get('href')
+        pretty_url = extract_text(eval_xpath(result, './/cite'))
         title = extract_text(link)
         content = extract_text(eval_xpath(result, './/p'))
 
         # append result
         results.append({'url': url,
+                        'pretty_url': pretty_url,
                         'title': title,
                         'content': content})
 

searx/engines/bing_images.py

@@ -1,15 +1,6 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Bing (Images)
-
- @website     https://www.bing.com/images
- @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
-              max. 5000 query/month
-
- @using-api   no (because of query limit)
- @results     HTML (using search portal)
- @stable      no (HTML can change)
- @parse       url, title, img_src
-
 """
 
 from urllib.parse import urlencode
@@ -20,12 +11,21 @@ from searx.utils import match_language
 from searx.engines.bing import language_aliases
 from searx.engines.bing import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import
 
+# about
+about = {
+    "website": 'https://www.bing.com/images',
+    "wikidata_id": 'Q182496',
+    "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-image-search-api',
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
 # engine dependent config
 categories = ['images']
 paging = True
 safesearch = True
 time_range_support = True
-language_support = True
 supported_languages_url = 'https://www.bing.com/account/general'
 number_of_results = 28
 
@@ -35,7 +35,7 @@ search_string = 'images/search'\
     '?{query}'\
     '&count={count}'\
     '&first={first}'\
-    '&FORM=IBASEP'
+    '&tsc=ImageHoverTitle'
 time_range_string = '&qft=+filterui:age-lt{interval}'
 time_range_dict = {'day': '1440',
                    'week': '10080',

searx/engines/bing_news.py

@@ -1,14 +1,6 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Bing (News)
-
- @website     https://www.bing.com/news
- @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
-              max. 5000 query/month
-
- @using-api   no (because of query limit)
- @results     RSS (using search portal)
- @stable      yes (except perhaps for the images)
- @parse       url, title, content, publishedDate, thumbnail
 """
 
 from datetime import datetime
@@ -20,10 +12,19 @@ from searx.utils import match_language, eval_xpath_getindex
 from searx.engines.bing import language_aliases
 from searx.engines.bing import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import
 
+# about
+about = {
+    "website": 'https://www.bing.com/news',
+    "wikidata_id": 'Q2878637',
+    "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-news-search-api',
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'RSS',
+}
+
 # engine dependent config
 categories = ['news']
 paging = True
-language_support = True
 time_range_support = True
 
 # search-url

searx/engines/bing_videos.py

@@ -1,13 +1,6 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Bing (Videos)
-
- @website     https://www.bing.com/videos
- @provide-api yes (http://datamarket.azure.com/dataset/bing/search)
-
- @using-api   no
- @results     HTML
- @stable      no
- @parse       url, title, content, thumbnail
 """
 
 from json import loads
@@ -18,12 +11,21 @@ from searx.utils import match_language
 from searx.engines.bing import language_aliases
 from searx.engines.bing import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import
 
+# about
+about = {
+    "website": 'https://www.bing.com/videos',
+    "wikidata_id": 'Q4914152',
+    "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-video-search-api',
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
 categories = ['videos']
 paging = True
 safesearch = True
 time_range_support = True
 number_of_results = 28
-language_support = True
 
 base_url = 'https://www.bing.com/'
 search_string = 'videos/search'\
@@ -68,7 +70,7 @@ def request(query, params):
     if params['time_range'] in time_range_dict:
         params['url'] += time_range_string.format(interval=time_range_dict[params['time_range']])
 
-    # bing videos did not like "older" versions < 70.0.1 when selectin other
+    # bing videos did not like "older" versions < 70.0.1 when selecting other
     # languages then 'en' .. very strange ?!?!
     params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:73.0.1) Gecko/20100101 Firefox/73.0.1'
 

searx/engines/btdigg.py

@@ -1,19 +1,25 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  BTDigg (Videos, Music, Files)
-
- @website     https://btdig.com
- @provide-api yes (on demand)
-
- @using-api   no
- @results     HTML (using search portal)
- @stable      no (HTML can change)
- @parse       url, title, content, seed, leech, magnetlink
 """
 
 from lxml import html
 from urllib.parse import quote, urljoin
 from searx.utils import extract_text, get_torrent_size
 
+# about
+about = {
+    "website": 'https://btdig.com',
+    "wikidata_id": 'Q4836698',
+    "official_api_documentation": {
+        'url': 'https://btdig.com/contacts',
+        'comment': 'on demand'
+    },
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
 # engine dependent config
 categories = ['videos', 'music', 'files']
 paging = True

searx/engines/command.py

@@ -1,18 +1,7 @@
-'''
-searx is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-searx is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with searx. If not, see < http://www.gnu.org/licenses/ >.
-'''
-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ Command (offline)
+"""
 
 import re
 from os.path import expanduser, isabs, realpath, commonprefix
@@ -23,7 +12,7 @@ from threading import Thread
 from searx import logger
 
 
-offline = True
+engine_type = 'offline'
 paging = True
 command = []
 delimiter = {}
@@ -46,7 +35,7 @@ def init(engine_settings):
     if 'command' not in engine_settings:
         raise ValueError('engine command : missing configuration key: command')
 
-    global command, working_dir, result_template, delimiter, parse_regex, timeout, environment_variables
+    global command, working_dir, delimiter, parse_regex, environment_variables
 
     command = engine_settings['command']
 

searx/engines/core.py

@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""CORE (science)
+
+"""
+# pylint: disable=missing-function-docstring
+
+from json import loads
+from datetime import datetime
+from urllib.parse import urlencode
+
+from searx import logger
+from searx.exceptions import SearxEngineAPIException
+
+logger = logger.getChild('CORE engine')
+
+about = {
+    "website": 'https://core.ac.uk',
+    "wikidata_id": None,
+    "official_api_documentation": 'https://core.ac.uk/documentation/api/',
+    "use_official_api": True,
+    "require_api_key": True,
+    "results": 'JSON',
+}
+
+categories = ['science']
+paging = True
+nb_per_page = 10
+
+api_key = 'unset'
+
+logger = logger.getChild('CORE engine')
+
+base_url = 'https://core.ac.uk:443/api-v2/search/'
+search_string = '{query}?page={page}&pageSize={nb_per_page}&apiKey={apikey}'
+
+def request(query, params):
+
+    if api_key == 'unset':
+        raise SearxEngineAPIException('missing CORE API key')
+
+    search_path = search_string.format(
+        query = urlencode({'q': query}),
+        nb_per_page = nb_per_page,
+        page = params['pageno'],
+        apikey = api_key,
+    )
+    params['url'] = base_url + search_path
+
+    logger.debug("query_url --> %s", params['url'])
+    return params
+
+def response(resp):
+    results = []
+    json_data = loads(resp.text)
+
+    for result in json_data['data']:
+
+        source = result['_source']
+        time = source['publishedDate'] or source['depositedDate']
+        if time :
+            date = datetime.fromtimestamp(time / 1000)
+        else:
+            date = None
+
+        metadata = []
+        if source['publisher'] and len(source['publisher']) > 3:
+            metadata.append(source['publisher'])
+        if source['topics']:
+            metadata.append(source['topics'][0])
+        if source['doi']:
+            metadata.append(source['doi'])
+        metadata = ' / '.join(metadata)
+
+        results.append({
+            'url': source['urls'][0].replace('http://', 'https://', 1),
+            'title': source['title'],
+            'content': source['description'],
+            'publishedDate': date,
+            'metadata' : metadata,
+        })
+
+    return results

searx/engines/currency_convert.py

@@ -1,54 +1,30 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ currency convert (DuckDuckGo)
+"""
+
 import json
-import re
-import unicodedata
-from searx.data import CURRENCIES  # NOQA
 
+# about
+about = {
+    "website": 'https://duckduckgo.com/',
+    "wikidata_id": 'Q12805',
+    "official_api_documentation": 'https://duckduckgo.com/api',
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'JSONP',
+}
 
+engine_type = 'online_currency'
 categories = []
 url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'
 weight = 100
 
-parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)
 https_support = True
 
 
-def normalize_name(name):
-    name = name.lower().replace('-', ' ').rstrip('s')
-    name = re.sub(' +', ' ', name)
-    return unicodedata.normalize('NFKD', name).lower()
-
-
-def name_to_iso4217(name):
-    global CURRENCIES
-
-    name = normalize_name(name)
-    currency = CURRENCIES['names'].get(name, [name])
-    return currency[0]
-
-
-def iso4217_to_name(iso4217, language):
-    global CURRENCIES
-
-    return CURRENCIES['iso4217'].get(iso4217, {}).get(language, iso4217)
-
-
 def request(query, params):
-    m = parser_re.match(query)
-    if not m:
-        # wrong query
-        return params
-    amount, from_currency, to_currency = m.groups()
-    amount = float(amount)
-    from_currency = name_to_iso4217(from_currency.strip())
-    to_currency = name_to_iso4217(to_currency.strip())
-
-    params['url'] = url.format(from_currency, to_currency)
-    params['amount'] = amount
-    params['from'] = from_currency
-    params['to'] = to_currency
-    params['from_name'] = iso4217_to_name(from_currency, 'en')
-    params['to_name'] = iso4217_to_name(to_currency, 'en')
-
+    params['url'] = url.format(params['from'], params['to'])
     return params
 
 

searx/engines/dailymotion.py

@@ -1,15 +1,6 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Dailymotion (Videos)
-
- @website     https://www.dailymotion.com
- @provide-api yes (http://www.dailymotion.com/developer)
-
- @using-api   yes
- @results     JSON
- @stable      yes
- @parse       url, title, thumbnail, publishedDate, embedded
-
- @todo        set content-parameter with correct data
 """
 
 from json import loads
@@ -17,10 +8,19 @@ from datetime import datetime
 from urllib.parse import urlencode
 from searx.utils import match_language, html_to_text
 
+# about
+about = {
+    "website": 'https://www.dailymotion.com',
+    "wikidata_id": 'Q769222',
+    "official_api_documentation": 'https://www.dailymotion.com/developer',
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": 'JSON',
+}
+
 # engine dependent config
 categories = ['videos']
 paging = True
-language_support = True
 
 # search-url
 # see http://www.dailymotion.com/doc/api/obj-video.html

searx/engines/deezer.py

@@ -1,18 +1,21 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Deezer (Music)
-
- @website     https://deezer.com
- @provide-api yes (http://developers.deezer.com/api/)
-
- @using-api   yes
- @results     JSON
- @stable      yes
- @parse       url, title, content, embedded
 """
 
 from json import loads
 from urllib.parse import urlencode
 
+# about
+about = {
+    "website": 'https://deezer.com',
+    "wikidata_id": 'Q602243',
+    "official_api_documentation": 'https://developers.deezer.com/',
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": 'JSON',
+}
+
 # engine dependent config
 categories = ['music']
 paging = True

searx/engines/deviantart.py

@@ -1,21 +1,22 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Deviantart (Images)
-
- @website     https://www.deviantart.com/
- @provide-api yes (https://www.deviantart.com/developers/) (RSS)
-
- @using-api   no (TODO, rewrite to api)
- @results     HTML
- @stable      no (HTML can change)
- @parse       url, title, img_src
-
- @todo        rewrite to api
 """
 # pylint: disable=missing-function-docstring
 
 from urllib.parse import urlencode
 from lxml import html
 
+# about
+about = {
+    "website": 'https://www.deviantart.com/',
+    "wikidata_id": 'Q46523',
+    "official_api_documentation": 'https://www.deviantart.com/developers/',
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
 # engine dependent config
 categories = ['images']
 paging = True

searx/engines/dictzone.py

@@ -1,44 +1,33 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 """
  Dictzone
-
- @website     https://dictzone.com/
- @provide-api no
- @using-api   no
- @results     HTML (using search portal)
- @stable      no (HTML can change)
- @parse       url, title, content
 """
 
-import re
 from urllib.parse import urljoin
 from lxml import html
-from searx.utils import is_valid_lang, eval_xpath
+from searx.utils import eval_xpath
 
+# about
+about = {
+    "website": 'https://dictzone.com/',
+    "wikidata_id": None,
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
+engine_type = 'online_dictionary'
 categories = ['general']
 url = 'https://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'
 weight = 100
 
-parser_re = re.compile('.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)
 results_xpath = './/table[@id="r"]/tr'
 https_support = True
 
 
 def request(query, params):
-    m = parser_re.match(query)
-    if not m:
-        return params
-
-    from_lang, to_lang, query = m.groups()
-
-    from_lang = is_valid_lang(from_lang)
-    to_lang = is_valid_lang(to_lang)
-
-    if not from_lang or not to_lang:
-        return params
-
-    params['url'] = url.format(from_lang=from_lang[2],
-                               to_lang=to_lang[2],
-                               query=query)
+    params['url'] = url.format(from_lang=params['from_lang'][2], to_lang=params['to_lang'][2], query=params['query'])
 
     return params
 
@@ -60,10 +49,12 @@ def response(resp):
             if t.strip():
                 to_results.append(to_result.text_content())
 
-        results.append({
-            'url': urljoin(resp.url, '?%d' % k),
-            'title': from_result.text_content(),
-            'content': '; '.join(to_results)
-        })
+        results.append(
+            {
+                'url': urljoin(str(resp.url), '?%d' % k),
+                'title': from_result.text_content(),
+                'content': '; '.join(to_results),
+            }
+        )
 
     return results