Searx is no longer maintained

Bumps [pallets-sphinx-themes](https://github.com/pallets/pallets-sphinx-themes) from 2.0.2 to 2.0.3.
- [Release notes](https://github.com/pallets/pallets-sphinx-themes/releases)
- [Changelog](https://github.com/pallets/pallets-sphinx-themes/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/pallets-sphinx-themes/compare/2.0.2...2.0.3)

---
updated-dependencies:
- dependency-name: pallets-sphinx-themes
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/data-update.yml

@@ -4,6 +4,9 @@ on:
     - cron: "05 06 1 * *"
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   updateData:
     name: Update data - ${{ matrix.fetch }}
@@ -29,7 +32,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.10'
           architecture: 'x64'
 
       - name: Install Python dependencies
@@ -40,8 +43,7 @@ jobs:
         env:
           FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
         run: |
-          source local/py3/bin/activate
-          $FETCH_SCRIPT
+          V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT"
 
       - name: Create Pull Request
         id: cpr

.github/workflows/integration.yml

@@ -6,6 +6,9 @@ on:
   pull_request:
     branches: ["master"]
 
+permissions:
+  contents: read
+
 jobs:
   python:
     name: Python ${{ matrix.python-version }}
@@ -13,7 +16,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -56,35 +59,56 @@ jobs:
       uses: actions/checkout@v2
     - name: Install Ubuntu packages
       run: sudo ./utils/searx.sh install packages
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+        architecture: 'x64'
+    - name: Cache Python dependencies
+      id: cache-python
+      uses: actions/cache@v2
+      with:
+        path: ./local
+        key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
     - name: Install node dependencies
       run: make V=1 node.env
     - name: Build themes
-      run: make V=1 themes
+      run: make V=1 themes.all
 
   documentation:
+    permissions:
+      contents: write  # for JamesIves/github-pages-deploy-action to push changes in repo
     name: Documentation
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout
       uses: actions/checkout@v2
       with:
+        fetch-depth: '0'
         persist-credentials: false
     - name: Install Ubuntu packages
       run: sudo ./utils/searx.sh install buildhost
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.9'
+        python-version: '3.10'
         architecture: 'x64'
+    - name: Cache Python dependencies
+      id: cache-python
+      uses: actions/cache@v2
+      with:
+        path: ./local
+        key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
     - name: Build documentation
-      run: SEARX_DEBUG=1 make V=1 ci-gh-pages
+      run: |
+        make V=1 docs.clean docs.html
     - name: Deploy
       if:  github.ref == 'refs/heads/master'
       uses: JamesIves/github-pages-deploy-action@3.7.1
       with:
         GITHUB_TOKEN: ${{ github.token }}
         BRANCH: gh-pages
-        FOLDER: build/gh-pages
+        FOLDER: dist/docs
         CLEAN: true # Automatically remove deleted files from the deploy branch
 
   dockers:
@@ -96,7 +120,7 @@ jobs:
       - documentation
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         if: env.DOCKERHUB_USERNAME != null
@@ -104,6 +128,17 @@ jobs:
         with:
           # make sure "make docker.push" can get the git history
           fetch-depth: '0'
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - name: Cache Python dependencies
+        id: cache-python
+        uses: actions/cache@v2
+        with:
+          path: ./local
+          key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
       - name: Set up QEMU
         if: env.DOCKERHUB_USERNAME != null
         uses: docker/setup-qemu-action@v1

.gitignore

@@ -26,3 +26,5 @@ dist/
 local/
 gh-pages/
 searx.egg-info/
+.env
+geckodriver.log

.pylintrc

@@ -59,7 +59,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=bad-whitespace, duplicate-code
+disable=duplicate-code, consider-using-f-string
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -105,39 +105,18 @@ max-nested-blocks=5
 
 [BASIC]
 
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input
-
-# Naming hint for argument names
-argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
-# Regular expression matching correct argument names
-argument-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$
-
-# Naming hint for attribute names
-attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
 # Regular expression matching correct attribute names
 attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*)|([A-Z0-9_]*))$
 
 # Bad variable names which should always be refused, separated by a comma
 bad-names=foo,bar,baz,toto,tutu,tata
 
-# Naming hint for class attribute names
-class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
-
 # Regular expression matching correct class attribute names
 class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
 
-# Naming hint for class names
-class-name-hint=[A-Z_][a-zA-Z0-9]+$
-
 # Regular expression matching correct class names
 class-rgx=[A-Z_][a-zA-Z0-9]+$
 
-# Naming hint for constant names
-const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
-
 # Regular expression matching correct constant names
 const-rgx=(([a-zA-Z_][a-zA-Z0-9_]*)|(__.*__))$
 #const-rgx=[f]?[A-Z_][a-zA-Z0-9_]{2,30}$
@@ -146,9 +125,6 @@ const-rgx=(([a-zA-Z_][a-zA-Z0-9_]*)|(__.*__))$
 # ones are exempt.
 docstring-min-length=-1
 
-# Naming hint for function names
-function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
 # Regular expression matching correct function names
 function-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$
 
@@ -158,21 +134,12 @@ good-names=i,j,k,ex,Run,_,log,cfg,id
 # Include a hint for the correct naming format with invalid-name
 include-naming-hint=no
 
-# Naming hint for inline iteration names
-inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
-
 # Regular expression matching correct inline iteration names
 inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
 
-# Naming hint for method names
-method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
 # Regular expression matching correct method names
 method-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*))$
 
-# Naming hint for module names
-module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
-
 # Regular expression matching correct module names
 #module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 module-rgx=([a-z_][a-z0-9_]*)$
@@ -189,9 +156,6 @@ no-docstring-rgx=^_
 # to this list to register other decorators that produce valid properties.
 property-classes=abc.abstractproperty
 
-# Naming hint for variable names
-variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
-
 # Regular expression matching correct variable names
 variable-rgx=(([a-z][a-zA-Z0-9_]{2,30})|(_[a-z0-9_]*)|([a-z]))$
 
@@ -217,12 +181,6 @@ max-line-length=120
 # Maximum number of lines in a module
 max-module-lines=2000
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,dict-separator
-
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.No config file found, using default configuration
 

AUTHORS.rst

@@ -1,4 +1,4 @@
-Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament, Noémi Ványi, @pofilo, Gaspard d'Hautefeuille and Markus Heiser.
+Searx was created by Adam Tauber and is maintained by Adam Tauber, Noémi Ványi, @pofilo, Gaspard d'Hautefeuille and Émilien Devos.
 
 Major contributing authors:
 
@@ -12,8 +12,9 @@ Major contributing authors:
 - @pofilo
 - Markus Heiser @return42
 - Émilien Devos @unixfox
+- Alexandre Flament
 
-People who have submitted patches/translates, reported bugs, consulted features or
+People who have submitted patches/translations, reported bugs, consulted features or
 generally made searx better:
 
 - Laszlo Hammerl
@@ -164,3 +165,31 @@ generally made searx better:
 - @jhigginbotham
 - @xenrox
 - @OliveiraHermogenes
+- Paul Alcock @Guilvareux
+- Ben Collerson
+- @3nprob
+- @plague-doctor
+- @CicadaCinema
+- @mikamp116
+- @Zackptg5
+- @darkmagic13
+- @CrocodileCroco
+- @allendema
+- Jordan Webb @jordemort
+- Samuel Dudik @dudik
+- @c1492
+- @nav1s
+- Igor Rzegocki @ajgon
+- Dmitrii Faiazov @scientia-ac-labore
+- @noctux
+- @jecarr
+- @israelyago
+- Georg @tacerus
+- Dario Nuevo @narcoticfresh
+- Andy Jones @andyljones
+- Maciej Urbański @rooterkyberian
+- @ilyakooo0
+- Eric Zhang @EricZhang456
+- @nathannaveen
+- @liimee
+- @james-still

CHANGELOG.rst

@@ -1,3 +1,104 @@
+1.1.0 2022.08.07
+================
+
+It has been a while since we released a new version of searx. Thus, we have lots of new things to offer, like new engines, autocompleter, plugins, etc. We got numerous contributions from ~30 new developers, but also we got many PRs from our recurring contributors.
+
+Thank you so much for you support! We couldn't have release so many awesome things without you!
+
+Core
+~~~~
+
+- Drop Python 3.6 support #3133
+- Run tests under python 3.10 #3035
+- Reduce redundant docker build steps #2725
+- Allow overriding Docker repository when building docker image #2726
+- Add healthcheck endpoint for Docker #2992
+
+New engines
+~~~~~~~~~~~
+
+- Wordnik.com #2735
+- Bandcamp #2763
+- SJP - Słownik języka polskiego #2736
+- Wikimini #2819
+- Dogpile #2822
+- PyPI XPATH engine #2830
+- ManKier #2829
+- Kaufland.de #2915
+- ask.com #2898
+- gpodder.net (JSON) #2885
+- woxikon.de synonyme (xpath) #2883
+- Petalsearch.com engine (xpath) #2897
+- whaleslide.com #2861
+- azlyrics.com #2955
+- IMDB #2980
+- Prowlarr #3118
+- Tineye reverse image search #3040
+- Onesearch #3065
+- TVmaze #3246
+- Emojipedia #3278
+- Psychonautwiki by @dimethyltriptamine @kvch
+
+Fixed engines
+~~~~~~~~~~~~~
+
+- Remove hubsbpot tracking URL params #2723
+- Fix URL to solidtorrent result page #2786
+- Update onion engines to v3 #2904
+- Fix Seznam engine #2905
+- Add pagination support for Library Genesis #2887
+- Fix uppercase ip query #2991
+- Fix Libgen + Uncomment Ebay and Urbandictionary #2986
+- Fixed Hoogle engine #3146
+- Fix Digg engine #3150
+- Don't lump all search suggestions together in Yahoo #3208
+- Fix DDG safe search #3247
+- Fix Qwant: Remove extra q from URL #3091
+
+New plugins
+~~~~~~~~~~~
+
+- hostname_replace plugin to rewrite result hostnames #2724
+- search_operators plugin to filter search results using -, site: and -site: #3311
+
+Fixed plugins
+~~~~~~~~~~~~~
+
+- Fix default_doi_resolver in preferences #2707
+- Add DOI resolver from sci-hub and replace default DOI #2706
+
+Themes
+~~~~~~
+
+- Fix dark "expand" button from infobox #2702
+- fix url_for(..., _external=True) in templates #2656
+- [enh] oscar: image thumbnail layout #2675
+- Improve text overflow of custom select #2985
+- Products results: add possibility to show if a product is in stock or not #3120
+- Configurable autofocus of search input (#1984) #3285
+- archive.today option for results page #3308
+- Fix keyboard hints for category tabs (#1187) #3276
+
+Enhancements
+~~~~~~~~~~~~
+
+- Allow overriding env vars SEARX_SETTINGS_PATH, UWSGI_SETTINGS_PATH #2717
+- correct typo/grammatical mistake #2744
+- Fix bug for 'FileNotFoundError' in 'standalone_searx.py' #2764
+- Fix grammar mistake in debug log output #2759
+- Fix typo #2768
+- Fix redirect when saving preferences #2760
+- Replace Makefile boilerplate by shell scripts #2594
+- Fix Qwant's fetch_languages function #2799
+- Apply HTTPS where possible + fix small typo #2922
+- Сhange in user-agent Firefox versions to latest #3008
+- Use engine-type when looking up supported_languages from JSON files #3002
+- Update about section of Invidious and Rumble + Change filtron error wording #2959
+- Verify that Tor proxy works every time searx starts #3015
+- Update settings_loader.get_user_settings_path() #3056
+- Fix wrong func call #3058
+- Improve ranking based on language #3053
+
 1.0.0 2021.03.27
 ================
 
@@ -323,7 +424,7 @@ Special thanks to `NLNet <https://nlnet.nl>`__ for sponsoring multiple features
  - Removed engines: faroo
 
 Special thanks to `NLNet <https://nlnet.nl>`__ for sponsoring multiple features of this release.
-Special thanks to https://www.accessibility.nl/english for making accessibilty audit.
+Special thanks to https://www.accessibility.nl/english for making accessibility audit.
 
 News
 ~~~~

Dockerfile

@@ -1,29 +1,22 @@
-FROM alpine:3.12
+FROM alpine:3.15
 ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]
 EXPOSE 8080
 VOLUME /etc/searx
 VOLUME /var/log/uwsgi
 
-ARG GIT_URL=unknown
-ARG VERSION_GITCOMMIT=unknown
-ARG SEARX_GIT_VERSION=unknown
-
 ARG SEARX_GID=977
 ARG SEARX_UID=977
 
 RUN addgroup -g ${SEARX_GID} searx && \
     adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx
 
-ARG TIMESTAMP_SETTINGS=0
-ARG TIMESTAMP_UWSGI=0
-ARG LABEL_VCS_REF=
-ARG LABEL_VCS_URL=
-
 ENV INSTANCE_NAME=searx \
     AUTOCOMPLETE= \
     BASE_URL= \
     MORTY_KEY= \
-    MORTY_URL=
+    MORTY_URL= \
+    SEARX_SETTINGS_PATH=/etc/searx/settings.yml \
+    UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini
 
 WORKDIR /usr/local/searx
 
@@ -53,14 +46,19 @@ RUN apk upgrade --no-cache \
     uwsgi \
     uwsgi-python3 \
     brotli \
- && pip3 install --upgrade pip \
+ && pip3 install --upgrade pip wheel setuptools \
  && pip3 install --no-cache -r requirements.txt \
  && apk del build-dependencies \
  && rm -rf /root/.cache
 
-COPY --chown=searx:searx . .
+COPY searx ./searx
+COPY dockerfiles ./dockerfiles
 
-RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
+ARG TIMESTAMP_SETTINGS=0
+ARG TIMESTAMP_UWSGI=0
+ARG VERSION_GITCOMMIT=unknown
+
+RUN /usr/bin/python3 -m compileall -q searx; \
     touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \
     touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \
     if [ ! -z $VERSION_GITCOMMIT ]; then\
@@ -70,8 +68,12 @@ RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
     -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \
     -type f -exec gzip -9 -k {} \+ -exec brotli --best {} \+
 
-# Keep this argument at the end since it change each time
+# Keep these arguments at the end to prevent redundant layer rebuilds
 ARG LABEL_DATE=
+ARG GIT_URL=unknown
+ARG SEARX_GIT_VERSION=unknown
+ARG LABEL_VCS_REF=
+ARG LABEL_VCS_URL=
 LABEL maintainer="searx <${GIT_URL}>" \
       description="A privacy-respecting, hackable metasearch engine." \
       version="${SEARX_GIT_VERSION}" \

Makefile

@@ -1,265 +1,107 @@
 # -*- coding: utf-8; mode: makefile-gmake -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
 .DEFAULT_GOAL=help
+export MTOOLS=./manage
 
 include utils/makefile.include
 
-PYOBJECTS = searx
-DOC       = docs
-PY_SETUP_EXTRAS ?= \[test\]
-PYLINT_SEARX_DISABLE_OPTION := I,C,R,W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,E1136
-PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES := supported_languages,language_aliases
-
-include utils/makefile.python
-include utils/makefile.sphinx
-
 all: clean install
 
-PHONY += help-min help-all help
+PHONY += help
 
-help: help-min
-	@echo  ''
-	@echo  'to get more help:  make help-all'
+help:
+	@./manage --help
+	@echo '----'
+	@echo 'run            - run developer instance'
+	@echo 'install        - developer install of searx into virtualenv'
+	@echo 'uninstall      - uninstall developer installation'
+	@echo 'clean          - clean up working tree'
+	@echo 'search.checker - check search engines'
+	@echo 'test           - run shell & CI tests'
+	@echo 'test.sh        - test shell scripts'
+	@echo 'ci.test        - run CI tests'
 
-help-min:
-	@echo  '  test      - run developer tests'
-	@echo  '  docs      - build documentation'
-	@echo  '  docs-live - autobuild HTML documentation while editing'
-	@echo  '  run       - run developer instance'
-	@echo  '  install   - developer install (./local)'
-	@echo  '  uninstall - uninstall (./local)'
-	@echo  '  gh-pages  - build docs & deploy on gh-pages branch'
-	@echo  '  clean     - drop builds and environments'
-	@echo  '  project   - re-build generic files of the searx project'
-	@echo  '  buildenv  - re-build environment files (aka brand)'
-	@echo  '  themes    - re-build build the source of the themes'
-	@echo  '  docker    - build Docker image'
-	@echo  '  node.env  - download & install npm dependencies locally'
-	@echo  ''
-	@$(MAKE) -e -s make-help
-
-help-all: help-min
-	@echo  ''
-	@$(MAKE) -e -s python-help
-	@echo  ''
-	@$(MAKE) -e -s docs-help
-
-PHONY += install
-install: buildenv pyenvinstall
-
-PHONY += uninstall
-uninstall: pyenvuninstall
-
-PHONY += clean
-clean: pyclean docs-clean node.clean test.clean
-	$(call cmd,common_clean)
 
 PHONY += run
-run:  buildenv pyenvinstall
+run:  install
 	$(Q) ( \
 	sleep 2 ; \
 	xdg-open http://127.0.0.1:8888/ ; \
 	) &
-	SEARX_DEBUG=1 $(PY_ENV)/bin/python ./searx/webapp.py
+	SEARX_DEBUG=1 ./manage pyenv.cmd python ./searx/webapp.py
 
-# docs
-# ----
+PHONY += install uninstall
+install uninstall:
+	$(Q)./manage pyenv.$@
 
-sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes
+PHONY += clean
+clean: py.clean docs.clean node.clean test.clean
+	$(Q)./manage build_msg CLEAN  "common files"
+	$(Q)find . -name '*.orig' -exec rm -f {} +
+	$(Q)find . -name '*.rej' -exec rm -f {} +
+	$(Q)find . -name '*~' -exec rm -f {} +
+	$(Q)find . -name '*.bak' -exec rm -f {} +
 
-PHONY += docs
-docs:  sphinx-doc-prebuilds
-	$(call cmd,sphinx,html,docs,docs)
+PHONY += search.checker search.checker.%
+search.checker: install
+	$(Q)./manage pyenv.cmd searx-checker -v
 
-PHONY += docs-live
-docs-live:  sphinx-doc-prebuilds
-	$(call cmd,sphinx_autobuild,html,docs,docs)
+search.checker.%: install
+	$(Q)./manage pyenv.cmd searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
 
-PHONY += prebuild-includes
-prebuild-includes:
-	$(Q)mkdir -p $(DOCS_BUILD)/includes
-	$(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst
-	$(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst
-	$(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst
-
-
-$(GH_PAGES)::
-	@echo "doc available at --> $(DOCS_URL)"
-
-# update project files
-# --------------------
-
-PHONY += project engines.languages useragents.update buildenv
-
-project: buildenv useragents.update engines.languages
-
-engines.languages:  pyenvinstall
-	$(Q)echo "fetch languages .."
-	$(Q)$(PY_ENV_ACT); python ./searx_extra/update/update_languages.py
-	$(Q)echo "updated searx/data/engines_languages.json"
-	$(Q)echo "updated searx/languages.py"
-
-useragents.update:  pyenvinstall
-	$(Q)echo "fetch useragents .."
-	$(Q)$(PY_ENV_ACT); python ./searx_extra/update/update_firefox_version.py
-	$(Q)echo "updated searx/data/useragents.json with the most recent versions of Firefox."
-
-buildenv: pyenv
-	$(Q)$(PY_ENV_ACT); SEARX_DEBUG=1 python utils/build_env.py
-
-# node / npm
-# ----------
-
-node.env: buildenv
-	$(Q)./manage.sh npm_packages
-
-node.clean:
-	$(Q)echo "CLEAN     locally installed npm dependencies"
-	$(Q)rm -rf \
-	  ./node_modules  \
-	  ./package-lock.json \
-	  ./searx/static/themes/oscar/package-lock.json \
-	  ./searx/static/themes/oscar/node_modules \
-	  ./searx/static/themes/simple/package-lock.json \
-	  ./searx/static/themes/simple/node_modules
-
-# build themes
-# ------------
-
-PHONY += themes themes.oscar themes.simple
-themes: buildenv themes.oscar themes.simple
-
-quiet_cmd_lessc = LESSC     $3
-      cmd_lessc = PATH="$$(npm bin):$$PATH" \
-	lessc --clean-css="--s1 --advanced --compatibility=ie9" "searx/static/$2" "searx/static/$3"
-
-quiet_cmd_grunt = GRUNT     $2
-      cmd_grunt = PATH="$$(npm bin):$$PATH" \
-	grunt --gruntfile  "$2"
-
-themes.oscar: node.env
-	$(Q)echo '[!] build oscar theme'
-	$(call cmd,grunt,searx/static/themes/oscar/gruntfile.js)
-
-themes.simple: node.env
-	$(Q)echo '[!] build simple theme'
-	$(call cmd,grunt,searx/static/themes/simple/gruntfile.js)
-
-
-# docker
-# ------
-
-PHONY += docker
-docker: buildenv
-	$(Q)./manage.sh docker_build
-
-docker.push: buildenv
-	$(Q)./manage.sh docker_build push
-
-# gecko
-# -----
-
-PHONY += gecko.driver
-gecko.driver:
-	$(PY_ENV_ACT); ./manage.sh install_geckodriver
-
-# search.checker
-# --------------
-
-search.checker: pyenvinstall
-	$(Q)$(PY_ENV_ACT); searx-checker -v
-
-ENGINE_TARGETS=$(patsubst searx/engines/%.py,search.checker.%,$(wildcard searx/engines/[!_]*.py))
-
-$(ENGINE_TARGETS): pyenvinstall
-	$(Q)$(PY_ENV_ACT); searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
-
-
-# test
-# ----
-
-PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot
-test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot
-
-PYLINT_FILES=\
-	searx/preferences.py \
-	searx/testing.py \
-	searx/engines/gigablast.py \
-	searx/engines/deviantart.py \
-	searx/engines/digg.py \
-	searx/engines/google.py \
-	searx/engines/google_news.py \
-	searx/engines/google_videos.py \
-	searx/engines/google_images.py \
-	searx/engines/mediathekviewweb.py \
-	searx/engines/solidtorrents.py \
-	searx/engines/solr.py \
-	searx/engines/google_scholar.py \
-	searx/engines/yahoo_news.py \
-	searx/engines/apkmirror.py \
-	searx_extra/update/update_external_bangs.py
-
-test.pylint: pyenvinstall
-	$(call cmd,pylint,$(PYLINT_FILES))
-	$(call cmd,pylint,\
-		--disable=$(PYLINT_SEARX_DISABLE_OPTION) \
-		--additional-builtins=$(PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES) \
-		searx/engines \
-	)
-	$(call cmd,pylint,\
-		--disable=$(PYLINT_SEARX_DISABLE_OPTION) \
-		--ignore=searx/engines \
-		searx tests \
-	)
-
-# ignored rules:
-#  E402 module level import not at top of file
-#  W503 line break before binary operator
-
-# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported!
+PHONY += ci.test test test.sh
+ci.test: test.pep8 test.pylint test.unit test.robot
+test: ci.test
 test.sh:
-	shellcheck -x -s bash utils/brand.env
-	shellcheck -x utils/lib.sh
-	shellcheck -x utils/filtron.sh
-	shellcheck -x utils/searx.sh
-	shellcheck -x utils/morty.sh
-	shellcheck -x utils/lxc.sh
-	shellcheck -x utils/lxc-searx.env
-	shellcheck -x .config.sh
-
-test.pep8: pyenvinstall
-	@echo "TEST      pycodestyle (formerly pep8)"
-	$(Q)$(PY_ENV_ACT); pycodestyle --exclude='searx/static, searx/languages.py, $(foreach f,$(PYLINT_FILES),$(f),)' \
-        --max-line-length=120 --ignore "E117,E252,E402,E722,E741,W503,W504,W605" searx tests
-
-test.unit: pyenvinstall
-	@echo "TEST      tests/unit"
-	$(Q)$(PY_ENV_ACT); python -m nose2 -s tests/unit
-
-test.coverage:  pyenvinstall
-	@echo "TEST      unit test coverage"
-	$(Q)$(PY_ENV_ACT); \
-	python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit \
-	&& coverage report \
-	&& coverage html \
-
-test.robot: pyenvinstall gecko.driver
-	@echo "TEST      robot"
-	$(Q)$(PY_ENV_ACT); PYTHONPATH=. python searx/testing.py robot
-
-test.clean:
-	@echo "CLEAN     intermediate test stuff"
-	$(Q)rm -rf geckodriver.log .coverage coverage/
+	$(Q)shellcheck -x -s bash \
+		utils/brand.env \
+		./manage \
+		utils/lib.sh \
+	        utils/filtron.sh \
+	        utils/searx.sh \
+	        utils/morty.sh \
+	        utils/lxc.sh \
+	        utils/lxc-searx.env \
+	        .config.sh
+	$(Q)./manage build_msg TEST "$@ OK"
 
 
-# travis
-# ------
+# wrap ./manage script
 
-PHONY += ci.test
-ci.test:
-	$(PY_ENV_BIN)/python -c "import yaml"  || make clean
-	$(MAKE) test
+MANAGE += buildenv
+MANAGE += babel.compile
+MANAGE += data.all data.languages data.useragents
+MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
+MANAGE += docker.build docker.push
+MANAGE += gecko.driver
+MANAGE += node.env node.clean
+MANAGE += py.build py.clean
+MANAGE += pyenv pyenv.install pyenv.uninstall
+MANAGE += pypi.upload pypi.upload.test
+MANAGE += test.pylint test.pep8 test.unit test.coverage test.robot test.clean
+MANAGE += themes.all themes.oscar themes.simple themes.bootstrap
 
-travis.codecov:
-	$(Q)$(PY_ENV_BIN)/python -m pip install codecov
+PHONY += $(MANAGE)
 
-.PHONY: $(PHONY)
+$(MANAGE):
+	$(Q)$(MTOOLS) $@
+
+# deprecated
+
+PHONY += docs docs-clean docs-live docker themes
+
+docs: docs.html
+	$(Q)./manage build_msg WARN $@ is deprecated use docs.html
+
+docs-clean: docs.clean
+	$(Q)./manage build_msg WARN $@ is deprecated use docs.clean
+
+docs-live: docs.live
+	$(Q)./manage build_msg WARN $@ is deprecated use docs.live
+
+docker:  docker.build
+	$(Q)./manage build_msg WARN $@ is deprecated use docker.build
+
+themes: themes.all
+	$(Q)./manage build_msg WARN $@ is deprecated use themes.all

PULL_REQUEST_TEMPLATE.md

@@ -16,7 +16,7 @@
 
 ## Author's checklist
 
-<!-- additional notes for reviewiers -->
+<!-- additional notes for reviewers -->
 
 ## Related issues
 

README.rst

@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: AGPL-3.0-or-later
 
+Searx is no longer maintained. Thank you for your support and all your contributions.
+
 .. figure:: https://raw.githubusercontent.com/searx/searx/master/searx/static/themes/oscar/img/logo_searx_a.png
    :target: https://searx.github.io/searx/
    :alt: searX
@@ -17,7 +19,7 @@
 |OpenCollective searx backers|
 |OpenCollective searx sponsors|
 
-Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **səːks**.
+Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **sɜːks**.
 
 .. _metasearch engine: https://en.wikipedia.org/wiki/Metasearch_engine
 
@@ -61,13 +63,64 @@ our homepage_.
 .. _homepage: https://searx.github.io/searx
 
 contact:
-  openhub_ // twitter_ // IRC: #searx @ freenode
+  openhub_ // twitter_ // IRC: #searx @ Libera (irc.libera.chat)
 
 .. _openhub: https://www.openhub.net/p/searx
 .. _twitter: https://twitter.com/Searx_engine
 
--------
+**************************
+Frequently asked questions
+**************************
 
-|gluten free|
+Is searx in maintenance mode?
+#############################
 
-.. |gluten free| image:: https://forthebadge.com/images/featured/featured-gluten-free.svg
+No, searx is no longer maintained.
+
+What is the difference between searx and SearxNG?
+#################################################
+
+TL;DR: SearXNG is for users that want more features and bugs getting fixed quicker.
+If you prefer a minimalist software and stable experience, use searx.
+
+SearxNG is a fork of searx, created by a former maintainer of searx. The fork
+was created because the majority of the maintainers at the time did not find
+the new proposed features privacy respecting enough. The most significant issue is with
+engine metrics.
+
+Searx is built for privacy conscious users. It comes with a unique set of
+challenges. One of the problems we face is that users rather not report bugs,
+because they do not want to publicly share what engines they use or what search
+query triggered a problem. It is a challenge we accepted.
+
+The new metrics feature collects more information to make engine maintenance easier.
+We could have had better and more error reports to benefit searx maintainers.
+However, we believe that the users of searx must come first, not the
+software. We are willing to compromise on the lack of issue reports to avoid
+violating the privacy of users.
+
+Furthermore, SearxNG is under heavy refactoring and dependencies are constantly updated, even
+if it is unnecessary. It increases the risk of introducing regressions. In searx
+we strive for stability, rather than moving fast and breaking things.
+
+Is searx for me?
+################
+
+Are you privacy conscious user? Then yes.
+
+In searx we decided to double down on being privacy respecting. We are picking
+engine changes from SearxNG, but we are not implementing engine detailed
+monitoring and not adding a new UI that relies on Javascript.
+
+If you are willing to give up some privacy respecting features, we encourage you to
+adopt SearxNG. Searx is targeted for privacy conscious users who run their
+instances locally, instead of using public instances.
+
+Why should I use SearxNG?
+#########################
+
+SearxNG has rolling releases, dependencies updated more frequently, and engines are fixed
+faster. It is easy to set up your own public instance, and monitor its
+performance and metrics. It is simple to maintain as an instance administrator.
+
+As a user, it provides a prettier user interface and nicer experience.

dockerfiles/docker-entrypoint.sh

@@ -24,9 +24,6 @@ if [ -z "${BIND_ADDRESS}" ]; then
     export BIND_ADDRESS="${DEFAULT_BIND_ADDRESS}"
 fi
 
-export UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini
-export SEARX_SETTINGS_PATH=/etc/searx/settings.yml
-
 # Parse special command line
 # see docs/admin/installation-docker.rst
 # display the help message without the version
@@ -103,7 +100,7 @@ update_conf() {
 	    # There is a new version
 	    if [ $FORCE_CONF_UPDATE -ne 0 ]; then
 		# Replace the current configuration
-		printf '⚠️  Automaticaly update %s to the new version\n' "${CONF}"
+		printf '⚠️  Automatically update %s to the new version\n' "${CONF}"
 		if [ ! -f "${OLD_CONF}" ]; then
 		    printf 'The previous configuration is saved to %s\n' "${OLD_CONF}"
 		    mv "${CONF}" "${OLD_CONF}"

dockerfiles/uwsgi.ini

@@ -9,7 +9,7 @@ workers = 4
 # The right granted on the created socket
 chmod-socket = 666
 
-# Plugin to use and interpretor config
+# Plugin to use and interpreter config
 single-interpreter = true
 master = true
 plugin = python3

docs/admin/buildhosts.rst

@@ -49,9 +49,9 @@ Build docs
    - dvisvgm_
 
 Most of the sphinx requirements are installed from :origin:`setup.py` and the
-docs can be build from scratch with ``make docs``.  For better math and image
-processing additional packages are needed.  The XeTeX_ needed not only for PDF
-creation, its also needed for :ref:`math` when HTML output is build.
+docs can be build from scratch with ``make docs.html``.  For better math and
+image processing additional packages are needed.  The XeTeX_ needed not only for
+PDF creation, its also needed for :ref:`math` when HTML output is build.
 
 To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
 as images (``sphinx.ext.imgmath`` extension).
@@ -64,7 +64,7 @@ to ``imgmath``:
    :start-after: # sphinx.ext.imgmath setup
    :end-before: # sphinx.ext.imgmath setup END
 
-If your docs build (``make docs``) shows warnings like this::
+If your docs build (``make docs.html``) shows warnings like this::
 
    WARNING: dot(1) not found, for better output quality install \
             graphviz from https://www.graphviz.org

docs/admin/command-engine.rst

@@ -0,0 +1,129 @@
+=====================================
+Run shell commands from your instance
+=====================================
+
+Command line engines are custom engines that run commands in the shell of the
+host. In this article you can learn how to create a command engine and how to
+customize the result display.
+
+The command
+===========
+
+When specifyng commands, you must make sure the commands are available on the
+searx host. Searx will not install anything for you. Also, make sure that the
+``searx`` user on your host is allowed to run the selected command and has
+access to the required files.
+
+Access control
+==============
+
+Be careful when creating command engines if you are running a public
+instance. Do not expose any sensitive information. You can restrict access by
+configuring a list of access tokens under tokens in your ``settings.yml``.
+
+Available settings
+==================
+
+* ``command``: A comma separated list of the elements of the command. A special
+  token ``{{QUERY}}`` tells searx where to put the search terms of the
+  user. Example: ``['ls', '-l', '-h', '{{QUERY}}']``
+* ``query_type``: The expected type of user search terms. Possible values:
+  ``path`` and ``enum``. ``path`` checks if the uesr provided path is inside the
+  working directory. If not the query is not executed. ``enum`` is a list of
+  allowed search terms. If the user submits something which is not included in
+  the list, the query returns an error.
+* ``delimiter``: A dict containing a delimiter char and the "titles" of each
+  element in keys.
+* ``parse_regex``: A dict containing the regular expressions for each result
+  key.
+* ``query_enum``: A list containing allowed search terms if ``query_type`` is
+  set to ``enum``.
+* ``working_dir``: The directory where the command has to be executed. Default:
+  ``.``
+* ``result_separator``: The character that separates results. Default: ``\n``
+
+Customize the result template
+=============================
+
+There is a default result template for displaying key-value pairs coming from
+command engines. If you want something more tailored to your result types, you
+can design your own template.
+
+Searx relies on `Jinja2 <https://jinja.palletsprojects.com/>`_ for
+templating. If you are familiar with Jinja, you will not have any issues
+creating templates. You can access the result attributes with ``{{
+result.attribute_name }}``.
+
+In the example below the result has two attributes: ``header`` and ``content``.
+To customize their diplay, you need the following template (you must define
+these classes yourself):
+
+.. code:: html
+
+    <div class="result">
+        <div class="result-header">
+            {{ result.header }}
+        </div>
+        <div class="result-content">
+            {{ result.content }}
+        </div>
+    </div>
+
+Then put your template under ``searx/templates/{theme-name}/result_templates``
+named ``your-template-name.html``. You can select your custom template with the
+option ``result_template``.
+
+.. code:: yaml
+
+  - name: your engine name
+    engine: command
+    result_template: your-template-name.html
+
+Examples
+========
+
+Find files by name
+------------------
+
+The first example is to find files on your searx host. It uses the command
+`find` available on most Linux distributions. It expects a path type query. The
+path in the search request must be inside the ``working_dir``.
+
+The results are displayed with the default `key-value.html` template.  A result
+is displayed in a single row table with the key "line".
+
+.. code:: yaml
+
+  - name : find
+    engine : command
+    command : ['find', '.', '-name', '{{QUERY}}']
+    query_type : path
+    shortcut : fnd
+    tokens : []
+    disabled : True
+    delimiter :
+        chars : ' '
+        keys : ['line']
+
+
+Find files by contents
+-----------------------
+
+In the second example, we define an engine that searches in the contents of the
+files under the ``working_dir``. The search type is not defined, so the user can
+input any string they want. To restrict the input, you can set the ``query_type``
+to ``enum`` and only allow a set of search terms to protect
+yourself. Alternatively, make the engine private, so no one malevolent accesses
+the engine.
+
+.. code:: yaml
+
+  - name : regex search in files
+    engine : command
+    command : ['grep', '{{QUERY}}']
+    shortcut : gr
+    tokens : []
+    disabled : True
+    delimiter :
+        chars : ' '
+        keys : ['line']

docs/admin/engines.rst

@@ -37,7 +37,7 @@ Disabled      **D**       Engine type          **ET**
 ------------- ----------- -------------------- ------------
 Safe search   **SS**
 ------------- ----------- ---------------------------------
-Weigth        **W**
+Weight        **W**
 ------------- ----------- ---------------------------------
 Disabled      **D**
 ------------- ----------- ---------------------------------
@@ -86,3 +86,60 @@ Show errors   **DE**
 
      {% endfor %}
 
+   .. flat-table:: Additional engines (commented out in settings.yml)
+      :header-rows: 1
+      :stub-columns: 2
+
+      * - Name
+        - Base URL
+        - Host
+        - Port
+        - Paging
+
+      * - elasticsearch
+        - localhost:9200
+        - 
+        - 
+        - False
+
+      * - meilicsearch
+        - localhost:7700
+        - 
+        - 
+        - True
+
+      * - mongodb
+        - 
+        - 127.0.0.1
+        - 21017
+        - True
+
+      * - mysql_server
+        - 
+        - 127.0.0.1
+        - 3306
+        - True
+
+      * - postgresql
+        - 
+        - 127.0.0.1
+        - 5432
+        - True
+
+      * - redis_server
+        - 
+        - 127.0.0.1
+        - 6379
+        - False
+
+      * - solr
+        - localhost:8983
+        - 
+        - 
+        - True
+
+      * - sqlite
+        - 
+        - 
+        - 
+        - True

docs/admin/engines/recoll.rst

@@ -39,7 +39,7 @@ Example
 Scenario:
 
 #. Recoll indexes a local filesystem mounted in ``/export/documents/reference``,
-#. the Recoll search inteface can be reached at https://recoll.example.org/ and
+#. the Recoll search interface can be reached at https://recoll.example.org/ and
 #. the contents of this filesystem can be reached though https://download.example.org/reference
 
 .. code:: yaml

docs/admin/index.rst

@@ -19,5 +19,9 @@ Administrator documentation
    filtron
    morty
    engines
+   private-engines
+   command-engine
+   indexer-engines
+   no-sql-engines
    plugins
    buildhosts

docs/admin/indexer-engines.rst

@@ -0,0 +1,89 @@
+==================
+Search in indexers
+==================
+
+Searx supports three popular indexer search engines:
+
+* Elasticsearch
+* Meilisearch
+* Solr
+
+Elasticsearch
+=============
+
+Make sure that the Elasticsearch user has access to the index you are querying.
+If you are not using TLS during your connection, set ``enable_http`` to ``True``.
+
+.. code:: yaml
+
+  - name : elasticsearch
+    shortcut : es
+    engine : elasticsearch
+    base_url : http://localhost:9200
+    username : elastic
+    password : changeme
+    index : my-index
+    query_type : match
+    enable_http : True
+
+Available settings
+------------------
+
+* ``base_url``: URL of Elasticsearch instance. By default it is set to ``http://localhost:9200``.
+* ``index``: Name of the index to query. Required.
+* ``query_type``: Elasticsearch query method to use. Available: ``match``,
+  ``simple_query_string``, ``term``, ``terms``, ``custom``.
+* ``custom_query_json``: If you selected ``custom`` for ``query_type``, you must
+  provide the JSON payload in this option.
+* ``username``: Username in Elasticsearch
+* ``password``: Password for the Elasticsearch user
+
+Meilisearch
+===========
+
+If you are not using TLS during connection, set ``enable_http`` to ``True``.
+
+.. code:: yaml
+
+  - name : meilisearch
+    engine : meilisearch
+    shortcut: mes
+    base_url : http://localhost:7700
+    index : my-index
+    enable_http: True
+
+Available settings
+------------------
+
+* ``base_url``: URL of the Meilisearch instance. By default it is set to http://localhost:7700
+* ``index``: Name of the index to query. Required.
+* ``auth_key``: Key required for authentication.
+* ``facet_filters``: List of facets to search in.
+
+Solr
+====
+
+If you are not using TLS during connection, set ``enable_http`` to ``True``.
+
+.. code:: yaml
+
+  - name : solr
+    engine : solr
+    shortcut : slr
+    base_url : http://localhost:8983
+    collection : my-collection
+    sort : asc
+    enable_http : True
+
+Available settings
+------------------
+
+* ``base_url``: URL of the Meilisearch instance. By default it is set to http://localhost:8983
+* ``collection``: Name of the collection to query. Required.
+* ``sort``: Sorting of the results. Available: ``asc``, ``desc``.
+* ``rows``: Maximum number of results from a query. Default value: 10.
+* ``field_list``: List of fields returned from the query.
+* ``default_fields``: Default fields to query.
+* ``query_fields``: List of fields with a boost factor. The bigger the boost
+  factor of a field, the more important the field is in the query. Example:
+  ``qf="field1^2.3 field2"``

docs/admin/installation-docker.rst

@@ -51,7 +51,7 @@ It's also possible to build searx from the embedded Dockerfile.
 
    git clone https://github.com/searx/searx.git
    cd searx
-   make docker
+   make docker.build
 
 
 Public instance

docs/admin/installation-searx.rst

@@ -61,7 +61,7 @@ from the login (*~/.profile*):
 
 .. tip::
 
-   Open a second terminal for the configuration tasks and left the ``(searx)$``
+   Open a second terminal for the configuration tasks and leave the ``(searx)$``
    terminal open for the tasks below.
 
 

docs/admin/installation-uwsgi.rst

@@ -94,8 +94,8 @@ My experience is, that this command is a bit buggy.
 
 .. _uwsgi configuration:
 
-Alltogether
-===========
+All together
+============
 
 Create the configuration ini-file according to your distribution (see below) and
 restart the uwsgi application.

docs/admin/installation.rst

@@ -39,13 +39,18 @@ install from ``root``, take into account that the scripts are creating a
 these new created users do need read access to the clone of searx, which is not
 the case if you clone into a folder below ``/root``.
 
-
 .. code:: bash
 
    $ cd ~/Downloads
    $ git clone https://github.com/searx/searx searx
    $ cd searx
 
+.. sidebar:: further read
+
+   - :ref:`toolboxing`
+   - :ref:`update searx`
+   - :ref:`inspect searx`
+
 **Install** :ref:`searx service <searx.sh>`
 
 This installs searx as described in :ref:`installation basic`.

docs/admin/no-sql-engines.rst

@@ -0,0 +1,170 @@
+===========================
+Query SQL and NoSQL servers
+===========================
+
+SQL
+===
+
+SQL servers are traditional databases with predefined data schema. Furthermore,
+modern versions also support BLOB data.
+
+You can search in the following servers:
+
+* `PostgreSQL`_
+* `MySQL`_
+* `SQLite`_
+
+The configuration of the new database engines are similar. You must put a valid
+SELECT SQL query in ``query_str``. At the moment you can only bind at most
+one parameter in your query.
+
+Do not include LIMIT or OFFSET in your SQL query as the engines
+rely on these keywords during paging.
+
+PostgreSQL
+----------
+
+Required PyPi package: ``psychopg2``
+
+You can find an example configuration below:
+
+.. code:: yaml
+
+  - name : postgresql
+    engine : postgresql
+    database : my_database
+    username : searx
+    password : password
+    query_str : 'SELECT * from my_table WHERE my_column = %(query)s'
+    shortcut : psql
+
+
+Available options
+~~~~~~~~~~~~~~~~~
+* ``host``: IP address of the host running PostgreSQL. By default it is ``127.0.0.1``.
+* ``port``: Port number PostgreSQL is listening on. By default it is ``5432``.
+* ``database``: Name of the database you are connecting to.
+* ``username``: Name of the user connecting to the database.
+* ``password``: Password of the database user.
+* ``query_str``: Query string to run. Keywords like ``LIMIT`` and ``OFFSET`` are not allowed. Required.
+* ``limit``: Number of returned results per page. By default it is 10.
+
+MySQL
+-----
+
+Required PyPi package: ``mysql-connector-python``
+
+This is an example configuration for quering a MySQL server:
+
+.. code:: yaml
+
+  - name : mysql
+    engine : mysql_server
+    database : my_database
+    username : searx
+    password : password
+    limit : 5
+    query_str : 'SELECT * from my_table WHERE my_column=%(query)s'
+    shortcut : mysql
+
+
+Available options
+~~~~~~~~~~~~~~~~~
+* ``host``: IP address of the host running MySQL. By default it is ``127.0.0.1``.
+* ``port``: Port number MySQL is listening on. By default it is ``3306``.
+* ``database``: Name of the database you are connecting to.
+* ``auth_plugin``: Authentication plugin to use. By default it is ``caching_sha2_password``.
+* ``username``: Name of the user connecting to the database.
+* ``password``: Password of the database user.
+* ``query_str``: Query string to run. Keywords like ``LIMIT`` and ``OFFSET`` are not allowed. Required.
+* ``limit``: Number of returned results per page. By default it is 10.
+
+SQLite
+------
+
+You can read from your database ``my_database`` using this example configuration:
+
+.. code:: yaml
+
+  - name : sqlite
+    engine : sqlite
+    shortcut: sq
+    database : my_database
+    query_str : 'SELECT * FROM my_table WHERE my_column=:query'
+
+
+Available options
+~~~~~~~~~~~~~~~~~
+* ``database``: Name of the database you are connecting to.
+* ``query_str``: Query string to run. Keywords like ``LIMIT`` and ``OFFSET`` are not allowed. Required.
+* ``limit``: Number of returned results per page. By default it is 10.
+
+NoSQL
+=====
+
+NoSQL data stores are used for storing arbitrary data without first defining their
+structure. To query the supported servers, you must install their drivers using PyPi.
+
+You can search in the following servers:
+
+* `Redis`_
+* `MongoDB`_
+
+Redis
+-----
+
+Reqired PyPi package: ``redis``
+
+Example configuration:
+
+.. code:: yaml
+
+  - name : mystore
+    engine : redis_server
+    exact_match_only : True
+    host : 127.0.0.1
+    port : 6379
+    password : secret-password
+    db : 0
+    shortcut : rds
+    enable_http : True
+
+Available options
+~~~~~~~~~~~~~~~~~
+
+* ``host``: IP address of the host running Redis. By default it is ``127.0.0.1``.
+* ``port``: Port number Redis is listening on. By default it is ``6379``.
+* ``password``: Password if required by Redis.
+* ``db``: Number of the database you are connecting to.
+* ``exact_match_only``: Enable if you need exact matching. By default it is ``True``.
+
+
+MongoDB
+-------
+
+Required PyPi package: ``pymongo``
+
+Below is an example configuration for using a MongoDB collection:
+
+.. code:: yaml
+
+  - name : mymongo
+    engine : mongodb
+    shortcut : icm
+    host : '127.0.0.1'
+    port : 27017
+    database : personal
+    collection : income
+    key : month
+    enable_http: True
+
+
+Available options
+~~~~~~~~~~~~~~~~~
+
+* ``host``: IP address of the host running MongoDB. By default it is ``127.0.0.1``.
+* ``port``: Port number MongoDB is listening on. By default it is ``27017``.
+* ``password``: Password if required by Redis.
+* ``database``: Name of the database you are connecting to.
+* ``collection``: Name of the collection you want to search in.
+* ``exact_match_only``: Enable if you need exact matching. By default it is ``True``.

docs/admin/private-engines.rst

@@ -0,0 +1,44 @@
+=============================
+How to create private engines
+=============================
+
+If you are running your public searx instance, you might want to restrict access
+to some engines. Maybe you are afraid of bots might abusing the engine. Or the
+engine might return private results you do not want to share with strangers.
+
+Server side configuration
+=========================
+
+You can make any engine private by setting a list of tokens in your settings.yml
+file. In the following example, we set two different tokens that provide access
+to the engine.
+
+.. code:: yaml
+
+    - name: my-private-google
+      engine: google
+      shortcut: pgo
+      tokens: ['my-secret-token-1', 'my-secret-token-2']
+
+
+To access the private engine, you must distribute the tokens to your searx
+users. It is up to you how you let them know what the access token is you
+created.
+
+Client side configuration
+=========================
+
+As a searx instance user, you can add any number of access tokens on the
+Preferences page. You have to set a comma separated lists of strings in "Engine
+tokens" input, then save your new preferences.
+
+.. image:: prefernces-private.png
+    :width: 600px
+    :align: center
+    :alt: location of token textarea
+
+Once the Preferences page is loaded again, you can see the information of the
+private engines you got access to. If you cannot see the expected engines in the
+engines list, double check your token. If there is no issue with the token,
+contact your instance administrator.
+

docs/admin/settings.rst

@@ -129,7 +129,7 @@ Global Settings
    outgoing: # communication with search engines
        request_timeout : 2.0        # default timeout in seconds, can be override by engine
        # max_request_timeout: 10.0  # the maximum timeout in seconds
-       useragent_suffix : ""        # informations like an email address to the administrator
+       useragent_suffix : ""        # information like an email address to the administrator
        pool_connections : 100       # Number of different hosts
        pool_maxsize : 10            # Number of simultaneous requests by host
    # uncomment below section if you want to use a proxy

docs/admin/update-searx.rst

@@ -4,20 +4,56 @@
 How to update
 =============
 
+How to update depends on the :ref:`installation` method.  If you have used the
+:ref:`installation scripts`, use ``update`` command from the scripts.
+
+**Update** :ref:`searx service <searx.sh>`
+
 .. code:: sh
 
-    sudo -H -u searx -i
-    (searx)$ git stash
-    (searx)$ git pull origin master
-    (searx)$ git stash apply
-    (searx)$ ./manage.sh update_packages
+    sudo -H ./utils/searx.sh update searx
 
-Restart uwsgi:
+**Update** :ref:`filtron reverse proxy <filtron.sh>`
 
-.. tabs::
+.. code:: sh
 
-   .. group-tab:: Ubuntu / debian
+    sudo -H ./utils/filtron.sh update filtron
 
-      .. code:: sh
+**Update** :ref:`result proxy <morty.sh>`
+
+.. code:: bash
+
+    sudo -H ./utils/morty.sh update morty
+
+.. _inspect searx:
+
+======================
+How to inspect & debug
+======================
+
+.. sidebar:: further read
+
+   - :ref:`toolboxing`
+   - :ref:`Makefile`
+
+How to debug depends on the :ref:`installation` method.  If you have used the
+:ref:`installation scripts`, use ``inspect`` command from the scripts.
+
+**Inspect** :ref:`searx service <searx.sh>`
+
+.. code:: sh
+
+    sudo -H ./utils/searx.sh inspect service
+
+**Inspect** :ref:`filtron reverse proxy <filtron.sh>`
+
+.. code:: sh
+
+    sudo -H ./utils/filtron.sh inspect service
+
+**Inspect** :ref:`result proxy <morty.sh>`
+
+.. code:: bash
+
+    sudo -H ./utils/morty.sh inspect service
 
-         sudo -H systemctl restart uwsgi

docs/blog/documentation-offline-engines.rst

@@ -0,0 +1,48 @@
+=================================
+Private searx project is finished
+=================================
+
+We are officially finished with the Private searx project. The goal was to
+extend searx capabilities beyond just searching on the Internet. We added
+support for offline engines. These engines do not connect to the Internet,
+they find results locally.
+
+As some of the offline engines run commands on the searx host, we added an
+option to protect any engine by making them private. Private engines can only be
+accessed using a token.
+
+After searx was prepared to run offline queries we added numerous new engines:
+
+1. Command line engine
+2. MySQL
+3. PostgreSQL
+4. SQLite
+5. Redis
+6. MongoDB
+
+We also added new engines that communicate over HTTP, but you might want to keep
+them private:
+
+1. Elasticsearch
+2. Meilisearch
+3. Solr
+
+The last step was to document this work. We added new tutorials on creating
+command engines, making engines private and also adding a custom result template
+to your own engines.
+
+Acknowledgement
+===============
+
+The project was sponsored by `Search and Discovery Fund`_ of `NLnet
+Foundation`_. We would like to thank the NLnet for not only the funds, but the
+conversations and their ideas. They were truly invested and passionate about
+supporting searx.
+
+.. _Search and Discovery Fund: https://nlnet.nl/discovery
+.. _NLnet Foundation: https://nlnet.nl/
+
+
+| Happy hacking.
+| kvch // 2022.09.30 23:15
+

docs/blog/index.rst

@@ -3,12 +3,16 @@ Blog
 ====
 
 .. toctree::
-   :maxdepth: 2
-   :caption: Contents
+   :titlesonly:
+   :reversed:
 
-   lxcdev-202006
    python3
    admin
    intro-offline
    private-engines
+   lxcdev-202006
    command-line-engines
+   search-indexer-engines
+   sql-engines
+   search-database-engines
+   documentation-offline-engines

docs/blog/lxcdev-202006.rst

@@ -31,7 +31,7 @@ might fail in some aspects we should not overlook.
 
     The environment in which we run all our development processes matters!
 
-The :ref:`makefile` and the :ref:`make pyenv` encapsulate a lot for us, but they
+The :ref:`makefile` and the :ref:`make install` encapsulate a lot for us, but they
 do not have access to all prerequisites.  For example, there may have
 dependencies on packages that are installed on the developer's desktop, but
 usually are not preinstalled on a server or client system.  Another examples
@@ -207,7 +207,7 @@ debug services from filtron and morty analogous use:
 Another point we have to notice is that each service (:ref:`searx <searx.sh>`,
 :ref:`filtron <filtron.sh>` and :ref:`morty <morty.sh>`) runs under dedicated
 system user account with the same name (compare :ref:`create searx user`).  To
-get a shell from theses accounts, simply call one of the scripts:
+get a shell from these accounts, simply call one of the scripts:
 
 .. tabs::
 
@@ -311,7 +311,7 @@ of the container:
 
 Now we can develop as usual in the working tree of our desktop system.  Every
 time the software was changed, you have to restart the searx service (in the
-conatiner):
+container):
 
 .. tabs::
 
@@ -356,7 +356,7 @@ daily usage:
      .. code:: sh
 
         $ sudo -H ./utils/lxc.sh cmd searx-archlinux \
-          make docs
+          make docs.html
 
 .. _blog-lxcdev-202006 abstract:
 
@@ -370,7 +370,7 @@ We build up a fully functional searx suite in a archlinux container:
    $ sudo -H ./utils/lxc.sh install suite searx-archlinux
 
 To access HTTP from the desktop we installed nginx for the services inside the
-conatiner:
+container:
 
 .. tabs::
 
@@ -407,7 +407,7 @@ To get remarks from the suite of the archlinux container we can use:
         ...
         [searx-archlinux]  INFO:  (eth0) filtron:    http://10.174.184.156:4004/ http://10.174.184.156/searx
         [searx-archlinux]  INFO:  (eth0) morty:      http://10.174.184.156:3000/
-        [searx-archlinux]  INFO:  (eth0) docs-live:  http://10.174.184.156:8080/
+        [searx-archlinux]  INFO:  (eth0) docs.live:  http://10.174.184.156:8080/
         [searx-archlinux]  INFO:  (eth0) IPv6:       http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65]
         ...
 

docs/blog/search-database-engines.rst

@@ -0,0 +1,95 @@
+===============================
+Query more of your NoSQL stores
+===============================
+
+From now on, searx lets you to query your NoSQL data stores:
+
+* `Redis`_
+* `MongoDB`_
+
+The reference configuration of the engines are included ``settings.yml`` just commented out,
+as you have to set various options and install dependencies before using them.
+
+By default, the engines use ``key-value`` template for displaying results.
+If you are not satisfied with the original result layout,
+you can use your owm template by placing the template under
+``searx/templates/{theme_name}/result_templates/{template_name}`` and setting
+``result_template`` attribute to ``{template_name}``.
+
+Furthermore, if you do not want to expose these engines on a public instance, you can
+still add them and limit the access by setting ``tokens`` as described in the `blog post about
+private engines`_.
+
+Configuring searx to use the stores
+===================================
+
+NoSQL data stores are used for storing arbitrary data without first defining their
+structure.
+
+Redis
+-----
+
+Required package: ``redis``
+
+Redis is a key value based data store usually stored in memory. 
+
+Select a database to search in and set its index in the option ``db``. You can
+either look for exact matches or use partial keywords to find what you are looking for
+by configuring ``exact_match_only``.
+
+In this example you can search for exact matches in your first database:
+
+.. code:: yaml
+
+  - name : mystore
+    engine : redis_server
+    exact_match_only : True
+    host : 127.0.0.1
+    port : 6379
+    password : secret-password
+    db : 0
+    shortcut : rds
+    enable_http : True
+
+
+MongoDB
+-------
+
+Required package: ``pymongo``
+
+MongoDB is a document based database program that handles JSON like data. 
+
+In order to query MongoDB, you have to select a ``database`` and a ``collection``. Furthermore,
+you have to select a ``key`` that is going to be searched. MongoDB also supports the option ``exact_match_only``, so configure it
+as you wish.
+
+Above is an example configuration for using a MongoDB collection:
+
+.. code:: yaml
+
+  - name : mymongo
+    engine : mongodb
+    shortcut : md
+    host : '127.0.0.1'
+    port : 27017
+    database : personal
+    collection : income
+    key : month
+    enable_http: True
+
+
+Acknowledgement
+===============
+
+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .
+
+.. _Redis: https://redis.io/
+.. _MongoDB: https://mongodb.com/
+.. _blog post about private engines: private-engines.html#private-engines
+.. _Search and Discovery Fund: https://nlnet.nl/discovery
+.. _NLnet Foundation: https://nlnet.nl/
+
+
+| Happy hacking.
+| kvch // 2021.07.13 23:16
+

docs/blog/search-indexer-engines.rst

@@ -0,0 +1,114 @@
+===============================
+Query your local search engines
+===============================
+
+From now on, searx lets you to query your locally running search engines. The following
+ones are supported now:
+
+* `Elasticsearch`_
+* `Meilisearch`_
+* `Solr`_
+
+All of the engines above are added to ``settings.yml`` just commented out, as you have to
+``base_url`` for all them.
+
+Please note that if you are not using HTTPS to access these engines, you have to enable
+HTTP requests by setting ``enable_http`` to ``True``.
+
+Furthermore, if you do not want to expose these engines on a public instance, you can
+still add them and limit the access by setting ``tokens`` as described in the `blog post about
+private engines`_.
+
+Configuring searx for search engines
+====================================
+
+Each search engine is powerful, capable of full-text search.
+
+Elasticsearch
+-------------
+
+Elasticsearch supports numerous ways to query the data it is storing. At the moment
+the engine supports the most popular search methods: ``match``, ``simple_query_string``, ``term`` and ``terms``.
+
+If none of the methods fit your use case, you can select ``custom`` query type and provide the JSON payload
+searx has to submit to Elasticsearch in ``custom_query_json``.
+
+The following is an example configuration for an Elasticsearch instance with authentication
+configured to read from ``my-index`` index.
+
+.. code:: yaml
+
+  - name : elasticsearch
+    shortcut : es
+    engine : elasticsearch
+    base_url : http://localhost:9200
+    username : elastic
+    password : changeme
+    index : my-index
+    query_type : match
+    enable_http : True
+
+
+Meilisearch
+-----------
+
+This search engine is aimed at individuals and small companies. It is designed for
+small-scale (less than 10 million documents) data collections. E.g. it is great for storing
+web pages you have visited and searching in the contents later.
+
+The engine supports faceted search, so you can search in a subset of documents of the collection.
+Furthermore, you can search in Meilisearch instances that require authentication by setting ``auth_token``.
+
+Here is a simple example to query a Meilisearch instance:
+
+.. code:: yaml
+
+  - name : meilisearch
+    engine : meilisearch
+    shortcut: mes
+    base_url : http://localhost:7700
+    index : my-index
+    enable_http: True
+
+
+Solr
+----
+
+Solr is a popular search engine based on Lucene, just like Elasticsearch.
+But instead of searching in indices, you can search in collections.
+
+This is an example configuration for searching in the collection ``my-collection`` and get
+the results in ascending order.
+
+.. code:: yaml
+
+  - name : solr
+    engine : solr
+    shortcut : slr
+    base_url : http://localhost:8983
+    collection : my-collection
+    sort : asc
+    enable_http : True
+
+
+Next steps
+==========
+
+The next step is to add support for various SQL databases.
+
+Acknowledgement
+===============
+
+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .
+
+.. _blog post about private engines: private-engines.html#private-engines
+.. _Elasticsearch: https://www.elastic.co/elasticsearch/
+.. _Meilisearch: https://www.meilisearch.com/
+.. _Solr: https://solr.apache.org/
+.. _Search and Discovery Fund: https://nlnet.nl/discovery
+.. _NLnet Foundation: https://nlnet.nl/
+
+
+| Happy hacking.
+| kvch // 2021.04.07 23:16
+

docs/blog/sql-engines.rst

@@ -0,0 +1,117 @@
+=================
+Query SQL servers 
+=================
+
+Now you can query SQL servers using searx. The following ones are supported:
+
+* `PostgreSQL`_
+* `MySQL`_
+* `SQLite`_
+
+All of the engines above are added to ``settings.yml`` just commented out, as you have to
+set the required attributes for the engines, e.g. ``database``. By default, the engines use
+``key-value`` template for displaying results. If you are not satisfied with the original result layout,
+you can use your owm template by placing the template under
+``searx/templates/{theme_name}/result_templates/{template_name}`` and setting
+``result_template`` attribute to ``{template_name}``.
+
+As mentioned in previous blog posts, if you do not wish to expose these engines on a
+public instance, you can still add them and limit the access by setting ``tokens``
+as described in the `blog post about private engines`_.
+
+Configure the engines
+=====================
+
+The configuration of the new database engines are similar. You must put a valid
+SELECT SQL query in ``query_str``. At the moment you can only bind at most
+one parameter in your query. By setting the attribute ``limit`` you can
+define how many results you want from the SQL server. Basically, it
+is the same as the LIMIT keyword in SQL.
+
+Please, do not include LIMIT or OFFSET in your SQL query as the engines
+rely on these keywords during paging. If you want to configure the number of returned results
+use the option ``limit``.
+
+PostgreSQL
+----------
+
+PostgreSQL is a powerful and robust open source database.
+
+Before configuring the PostgreSQL engine, you must install the dependency ``psychopg2``.
+
+You can find an example configuration below:
+
+.. code:: yaml
+
+  - name : postgresql
+    engine : postgresql
+    database : my_database
+    username : searx
+    password : password
+    query_str : 'SELECT * from my_table WHERE my_column = %(query)s'
+    shortcut : psql
+
+
+MySQL
+-----
+
+MySQL is said to be the most popular open source database. 
+
+Before enabling MySQL engine, you must install the package ``mysql-connector-python``.
+
+The authentication plugin is configurable by setting ``auth_plugin`` in the attributes.
+By default it is set to ``caching_sha2_password``.
+
+This is an example configuration for querying a MySQL server:
+
+.. code:: yaml
+
+  - name : mysql
+    engine : mysql_server
+    database : my_database
+    username : searx
+    password : password
+    limit : 5
+    query_str : 'SELECT * from my_table WHERE my_column=%(query)s'
+    shortcut : mysql
+
+
+SQLite
+------
+
+SQLite is a small, fast and reliable SQL database engine. It does not require
+any extra dependency.
+
+You can read from your database ``my_database`` using this example configuration:
+
+.. code:: yaml
+
+  - name : sqlite
+    engine : sqlite
+    shortcut: sq
+    database : my_database
+    query_str : 'SELECT * FROM my_table WHERE my_column=:query'
+
+
+Next steps
+==========
+
+The next step is to add support for more data stores, e.g. Redis and MongoDB.
+
+Acknowledgement
+===============
+
+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .
+
+.. _PostgreSQL: https://www.postgresql.org/
+.. _MySQL: https://www.mysql.com/
+.. _SQLite: https://www.sqlite.org/index.html
+.. _blog post about private engines: private-engines.html#private-engines
+.. _Search and Discovery Fund: https://nlnet.nl/discovery
+.. _NLnet Foundation: https://nlnet.nl/
+
+
+| Happy hacking.
+| kvch // 2021.05.23 23:16
+
+

docs/build-templates/morty.rst

@@ -32,7 +32,7 @@
        (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local
        (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_TAR}\" \\
                    \"${GO_PKG_URL}\"
-       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf \"${GO_TAR}\"
+       (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_TAR}\"
        (${SERVICE_USER}) $ which go
        ${SERVICE_HOME}/local/go/bin/go
 

docs/conf.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
 
 import  sys, os
-from sphinx_build_tools import load_sphinx_config
 from pallets_sphinx_themes import ProjectLink
 
 from searx import brand
@@ -10,7 +10,7 @@ from searx.version import VERSION_STRING
 # Project --------------------------------------------------------------
 
 project = u'searx'
-copyright = u'2015-2020, Adam Tauber, Noémi Ványi'
+copyright = u'2015-2022, Adam Tauber, Noémi Ványi'
 author = u'Adam Tauber'
 release, version = VERSION_STRING, VERSION_STRING
 highlight_language = 'none'
@@ -38,26 +38,26 @@ jinja_contexts = {
 extlinks = {}
 
 # upstream links
-extlinks['wiki'] = ('https://github.com/searx/searx/wiki/%s', ' ')
-extlinks['pull'] = ('https://github.com/searx/searx/pull/%s', 'PR ')
+extlinks['wiki'] = ('https://github.com/searx/searx/wiki/%s', '%s')
+extlinks['pull'] = ('https://github.com/searx/searx/pull/%s', 'PR %s')
 
 # links to custom brand
-extlinks['origin'] = (brand.GIT_URL + '/blob/' + brand.GIT_BRANCH + '/%s', 'git://')
-extlinks['patch'] = (brand.GIT_URL + '/commit/%s', '#')
-extlinks['search'] = (brand.SEARX_URL + '/%s', '#')
-extlinks['docs'] = (brand.DOCS_URL + '/%s', 'docs: ')
-extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: ')
-extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '')
+extlinks['origin'] = (brand.GIT_URL + '/blob/' + brand.GIT_BRANCH + '/%s', 'Origin: %s')
+extlinks['patch'] = (brand.GIT_URL + '/commit/%s', 'path %s')
+extlinks['search'] = (brand.SEARX_URL + '/%s', 'URL: %s')
+extlinks['docs'] = (brand.DOCS_URL + '/%s', 'docs: %s')
+extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: %s')
+extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', 'man: %s')
 #extlinks['role'] = (
 #    'https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-%s', '')
 extlinks['duref'] = (
-    'https://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '')
+    'https://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '%s')
 extlinks['durole'] = (
-    'https://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '')
+    'https://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '%s')
 extlinks['dudir'] =  (
-    'https://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '')
+    'https://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '%s')
 extlinks['ctan'] =  (
-    'https://ctan.org/pkg/%s', 'CTAN: ')
+    'https://ctan.org/pkg/%s', 'CTAN: %s')
 
 extensions = [
     'sphinx.ext.imgmath',
@@ -67,7 +67,7 @@ extensions = [
     "sphinx.ext.intersphinx",
     "pallets_sphinx_themes",
     "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
-    "sphinxcontrib.jinja",  # https://github.com/tardyp/sphinx-jinja
+    "sphinx_jinja",  # https://github.com/tardyp/sphinx-jinja
     "sphinxcontrib.programoutput",  # https://github.com/NextThought/sphinxcontrib-programoutput
     'linuxdoc.kernel_include',  # Implementation of the 'kernel-include' reST-directive.
     'linuxdoc.rstFlatTable',    # Implementation of the 'flat-table' reST-directive.
@@ -101,12 +101,11 @@ imgmath_font_size = 14
 
 html_theme_options = {"index_sidebar_logo": True}
 html_context = {"project_links": [] }
+html_context["project_links"].append(ProjectLink("Blog", brand.DOCS_URL + "/blog/index.html"))
 if brand.GIT_URL:
     html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))
 if brand.WIKI_URL:
     html_context["project_links"].append(ProjectLink("Wiki", brand.WIKI_URL))
-if brand.PUBLIC_INSTANCES:
-    html_context["project_links"].append(ProjectLink("Public instances", brand.PUBLIC_INSTANCES))
 if brand.TWITTER_URL:
     html_context["project_links"].append(ProjectLink("Twitter", brand.TWITTER_URL))
 if brand.ISSUE_URL:
@@ -128,9 +127,3 @@ html_show_sourcelink = False
 latex_documents = [
     (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")
 ]
-
-# ------------------------------------------------------------------------------
-# Since loadConfig overwrites settings from the global namespace, it has to be
-# the last statement in the conf.py file
-# ------------------------------------------------------------------------------
-load_sphinx_config(globals())

docs/dev/contribution_guide.rst

@@ -132,11 +132,11 @@ Here is an example which makes a complete rebuild:
 
 .. code:: sh
 
-   $ make docs-clean docs
+   $ make docs.clean docs.html
    ...
    The HTML pages are in dist/docs.
 
-.. _make docs-live:
+.. _make docs.live:
 
 live build
 ----------
@@ -144,19 +144,19 @@ live build
 .. _sphinx-autobuild:
    https://github.com/executablebooks/sphinx-autobuild/blob/master/README.md
 
-.. sidebar:: docs-clean
+.. sidebar:: docs.clean
 
    It is recommended to assert a complete rebuild before deploying (use
-   ``docs-clean``).
+   ``docs.clean``).
 
 Live build is like WYSIWYG.  If you want to edit the documentation, its
-recommended to use.  The Makefile target ``docs-live`` builds the docs, opens
+recommended to use.  The Makefile target ``docs.live`` builds the docs, opens
 URL in your favorite browser and rebuilds every time a reST file has been
 changed.
 
 .. code:: sh
 
-   $ make docs-live
+   $ make docs.live
    ...
    The HTML pages are in dist/docs.
    ... Serving on http://0.0.0.0:8000
@@ -169,7 +169,7 @@ argument.  E.g to find and use a free port, use:
 
 .. code:: sh
 
-   $ SPHINXOPTS="--port 0" make docs-live
+   $ SPHINXOPTS="--port 0" make docs.live
    ...
    ... Serving on http://0.0.0.0:50593
    ...
@@ -180,21 +180,10 @@ argument.  E.g to find and use a free port, use:
 deploy on github.io
 -------------------
 
-To deploy documentation at :docs:`github.io <.>` use Makefile target
-:ref:`make gh-pages`, which will builds the documentation, clones searx into a sub
-folder ``gh-pages``, cleans it, copies the doc build into and runs all the
-needed git add, commit and push:
+To deploy documentation at :docs:`github.io <.>` use Makefile target :ref:`make
+docs.gh-pages`, which builds the documentation and runs all the needed git add,
+commit and push:
 
 .. code:: sh
 
-   $ make docs-clean gh-pages
-   ...
-   SPHINX    docs --> file://<...>/dist/docs
-   The HTML pages are in dist/docs.
-   ...
-   Cloning into 'gh-pages' ...
-   ...
-   cd gh-pages; git checkout gh-pages >/dev/null
-   Switched to a new branch 'gh-pages'
-   ...
-   doc available at --> https://searx.github.io/searx
+   $ make docs.clean docs.gh-pages

docs/dev/engine_overview.rst

@@ -41,10 +41,10 @@ engine file
 argument                type        information
 ======================= =========== ========================================================
 categories              list        pages, in which the engine is working
-paging                  boolean     support multible pages
+paging                  boolean     support multiple pages
 time_range_support      boolean     support search time range
 engine_type             str         ``online`` by default, other possibles values are 
-                                    ``offline``, ``online_dictionnary``, ``online_currency``
+                                    ``offline``, ``online_dictionary``, ``online_currency``
 ======================= =========== ========================================================
 
 .. _engine settings:
@@ -132,7 +132,7 @@ language               str            specific language code like ``'en_US'``, o
 ====================== ============== ========================================================================
 
 
-If the ``engine_type`` is ``online_dictionnary```, in addition to the ``online`` arguments:
+If the ``engine_type`` is ``online_dictionary```, in addition to the ``online`` arguments:
 
 ====================== ============ ========================================================================
 argument               type         default-value, information
@@ -159,7 +159,7 @@ parsed arguments
 ----------------
 
 The function ``def request(query, params):`` always returns the ``params``
-variable.  Inside searx, the following paramters can be used to specify a search
+variable.  Inside searx, the following parameters can be used to specify a search
 request:
 
 =================== =========== ==========================================================================
@@ -171,7 +171,7 @@ headers             set         HTTP header information
 data                set         HTTP data information
 cookies             set         HTTP cookies
 verify              bool        Performing SSL-Validity check
-allow_redirects     bool        Follow redirects
+follow_redirects     bool        Follow redirects
 max_redirects       int         maximum redirects, hard limit
 soft_max_redirects  int         maximum redirects, soft limit. Record an error but don't stop the engine
 raise_for_httperror bool        True by default: raise an exception if the HTTP code of response is >= 300

docs/dev/makefile.rst

@@ -1,33 +1,33 @@
 .. _makefile:
 
-================
-Makefile Targets
-================
+========
+Makefile
+========
 
 .. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction
 
 .. sidebar:: build environment
 
-   Before looking deeper at the targets, first read about :ref:`make pyenv`.
+   Before looking deeper at the targets, first read about :ref:`make
+   install`.
 
    To install system requirements follow :ref:`buildhosts`.
 
-With the aim to simplify development cycles, started with :pull:`1756` a
-``Makefile`` based boilerplate was added.  If you are not familiar with
-Makefiles, we recommend to read gnu-make_ introduction.
+All relevant build tasks are implemented in :origin:`manage.sh` and for CI or
+IDE integration a small ``Makefile`` wrapper is available.  If you are not
+familiar with Makefiles, we recommend to read gnu-make_ introduction.
 
 The usage is simple, just type ``make {target-name}`` to *build* a target.
 Calling the ``help`` target gives a first overview (``make help``):
 
 .. program-output:: bash -c "cd ..; make --no-print-directory help"
 
-
 .. contents:: Contents
    :depth: 2
    :local:
    :backlinks: entry
 
-.. _make pyenv:
+.. _make install:
 
 Python environment
 ==================
@@ -36,31 +36,42 @@ Python environment
 
    ``source ./local/py3/bin/activate``
 
-With Makefile we do no longer need to build up the virtualenv manually (as
-described in the :ref:`devquickstart` guide).  Jump into your git working tree
-and release a ``make pyenv``:
-
-.. code:: sh
+We do no longer need to build up the virtualenv manually.  Jump into your git
+working tree and release a ``make install`` to get a virtualenv with a
+*developer install* of searx (:origin:`setup.py`). ::
 
    $ cd ~/searx-clone
-   $ make pyenv
-   PYENV     usage: source ./local/py3/bin/activate
+   $ make install
+   PYENV     [virtualenv] installing ./requirements*.txt into local/py3
    ...
+   PYENV     OK
+   PYENV     [install] pip install -e 'searx[test]'
+   ...
+   Successfully installed argparse-1.4.0 searx
+   BUILDENV  INFO:searx:load the default settings from ./searx/settings.yml
+   BUILDENV  INFO:searx:Initialisation done
+   BUILDENV  build utils/brand.env
 
-With target ``pyenv`` a development environment (aka virtualenv) was build up in
-``./local/py3/``.  To make a *developer install* of searx (:origin:`setup.py`)
-into this environment, use make target ``install``:
-
-.. code:: sh
+If you release ``make install`` multiple times the installation will only
+rebuild if the sha256 sum of the *requirement files* fails.  With other words:
+the check fails if you edit the requirements listed in
+:origin:`requirements-dev.txt` and :origin:`requirements.txt`). ::
 
    $ make install
-   PYENV     usage: source ./local/py3/bin/activate
-   PYENV     using virtualenv from ./local/py3
-   PYENV     install .
-
-You have never to think about intermediate targets like ``pyenv`` or
-``install``, the ``Makefile`` chains them as requisites.  Just run your main
-target.
+   PYENV     OK
+   PYENV     [virtualenv] requirements.sha256 failed
+             [virtualenv] - 6cea6eb6def9e14a18bf32f8a3e...  ./requirements-dev.txt
+             [virtualenv] - 471efef6c73558e391c3adb35f4...  ./requirements.txt
+   ...
+   PYENV     [virtualenv] installing ./requirements*.txt into local/py3
+   ...
+   PYENV     OK
+   PYENV     [install] pip install -e 'searx[test]'
+   ...
+   Successfully installed argparse-1.4.0 searx
+   BUILDENV  INFO:searx:load the default settings from ./searx/settings.yml
+   BUILDENV  INFO:searx:Initialisation done
+   BUILDENV  build utils/brand.env
 
 .. sidebar:: drop environment
 
@@ -68,10 +79,7 @@ target.
    <make clean>` first.
 
 If you think, something goes wrong with your ./local environment or you change
-the :origin:`setup.py` file (or the requirements listed in
-:origin:`requirements-dev.txt` and :origin:`requirements.txt`), you have to call
-:ref:`make clean`.
-
+the :origin:`setup.py` file, you have to call :ref:`make clean`.
 
 .. _make run:
 
@@ -81,77 +89,44 @@ the :origin:`setup.py` file (or the requirements listed in
 To get up a running a developer instance simply call ``make run``.  This enables
 *debug* option in :origin:`searx/settings.yml`, starts a ``./searx/webapp.py``
 instance, disables *debug* option again and opens the URL in your favorite WEB
-browser (:man:`xdg-open`):
+browser (:man:`xdg-open`)::
 
-.. code:: sh
-
-  $ make run
-  PYENV     usage: source ./local/py3/bin/activate
-  PYENV     install .
-  ./local/py3/bin/python ./searx/webapp.py
-  ...
-  INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit)
-  ...
+   $ make run
+   PYENV     OK
+   SEARX_DEBUG=1 ./manage.sh pyenv.cmd python ./searx/webapp.py
+   ...
+   INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit)
 
 .. _make clean:
 
 ``make clean``
 ==============
 
-Drop all intermediate files, all builds, but keep sources untouched.  Includes
-target ``pyclean`` which drops ./local environment.  Before calling ``make
-clean`` stop all processes using :ref:`make pyenv`.
-
-.. code:: sh
+Drop all intermediate files, all builds, but keep sources untouched.  Before
+calling ``make clean`` stop all processes using :ref:`make install`. ::
 
    $ make clean
-   CLEAN     pyclean
-   CLEAN     clean
+   CLEAN     pyenv
+   PYENV     [virtualenv] drop ./local/py3
+   CLEAN     docs -- ./build/docs ./dist/docs
+   CLEAN     locally installed npm dependencies
+   CLEAN     test stuff
+   CLEAN     common files
 
 .. _make docs:
 
-``make docs docs-live docs-clean``
-==================================
+``make docs docs.autobuild docs.clean``
+=======================================
 
-We describe the usage of the ``doc*`` targets in the :ref:`How to contribute /
+We describe the usage of the ``doc.*`` targets in the :ref:`How to contribute /
 Documentation <contrib docs>` section.  If you want to edit the documentation
-read our :ref:`make docs-live` section.  If you are working in your own brand,
+read our :ref:`make docs.live` section.  If you are working in your own brand,
 adjust your :ref:`settings global`.
 
-.. _make books:
+.. _make docs.gh-pages:
 
-``make books/{name}.html books/{name}.pdf``
-===========================================
-
-.. _intersphinx: https://www.sphinx-doc.org/en/stable/ext/intersphinx.html
-.. _XeTeX: https://tug.org/xetex/
-
-.. sidebar:: info
-
-   To build PDF a XeTeX_ is needed, see :ref:`buildhosts`.
-
-
-The ``books/{name}.*`` targets are building *books*.  A *book* is a
-sub-directory containing a ``conf.py`` file.  One example is the user handbook
-which can deployed separately (:origin:`docs/user/conf.py`).  Such ``conf.py``
-do inherit from :origin:`docs/conf.py` and overwrite values to fit *book's*
-needs.
-
-With the help of Intersphinx_ (:ref:`reST smart ref`) the links to searx’s
-documentation outside of the book will be bound by the object inventory of
-``DOCS_URL``.  Take into account that URLs will be picked from the inventary at
-documentation's build time.
-
-Use ``make docs-help`` to see which books available:
-
-.. program-output:: bash -c "cd ..; make --no-print-directory docs-help"
-   :ellipsis: 0,-6
-
-
-.. _make gh-pages:
-
-``make gh-pages``
-=================
+``make docs.gh-pages``
+======================
 
 To deploy on github.io first adjust your :ref:`settings global`.  For any
 further read :ref:`deploy on github.io`.
@@ -161,37 +136,66 @@ further read :ref:`deploy on github.io`.
 ``make test``
 =============
 
-Runs a series of tests: ``test.pep8``, ``test.unit``, ``test.robot`` and does
-additional :ref:`pylint checks <make pylint>`.  You can run tests selective,
-e.g.:
-
-.. code:: sh
+Runs a series of tests: :ref:`make test.pylint`, ``test.pep8``, ``test.unit``
+and ``test.robot``.  You can run tests selective, e.g.::
 
   $ make test.pep8 test.unit test.sh
-  . ./local/py3/bin/activate; ./manage.sh pep8_check
-  [!] Running pep8 check
-  . ./local/py3/bin/activate; ./manage.sh unit_tests
-  [!] Running unit tests
+  TEST      test.pep8 OK
+  ...
+  TEST      test.unit OK
+  ...
+  TEST      test.sh OK
 
-.. _make pylint:
+.. _make test.sh:
 
-``make pylint``
-===============
+``make test.sh``
+================
+
+:ref:`sh lint` / if you have changed some bash scripting run this test before
+commit.
+
+.. _make test.pylint:
+
+``make test.pylint``
+====================
 
 .. _Pylint: https://www.pylint.org/
 
-Before commiting its recommend to do some (more) linting.  Pylint_ is known as
-one of the best source-code, bug and quality checker for the Python programming
-language.  Pylint_ is not yet a quality gate within our searx project (like
-:ref:`test.pep8 <make test>` it is), but Pylint_ can help to improve code
-quality anyway.  The pylint profile we use at searx project is found in
-project's root folder :origin:`.pylintrc`.
+Pylint_ is known as one of the best source-code, bug and quality checker for the
+Python programming language.  The pylint profile we use at searx project is
+found in project's root folder :origin:`.pylintrc`.
 
-Code quality is a ongoing process.  Don't try to fix all messages from Pylint,
-run Pylint and check if your changed lines are bringing up new messages.  If so,
-fix it.  By this, code quality gets incremental better and if there comes the
-day, the linting is balanced out, we might decide to add Pylint as a quality
-gate.
+.. _make search.checker:
+
+``search.checker.{engine name}``
+================================
+
+To check all engines::
+
+    make search.checker
+
+To check a engine with whitespace in the name like *google news* replace space
+by underline::
+
+    make search.checker.google_news
+
+To see HTTP requests and more use SEARX_DEBUG::
+
+    make SEARX_DEBUG=1 search.checker.google_news
+
+.. _3xx: https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_redirection
+
+To filter out HTTP redirects (3xx_)::
+
+    make SEARX_DEBUG=1 search.checker.google_news | grep -A1 "HTTP/1.1\" 3[0-9][0-9]"
+    ...
+    Engine google news                   Checking
+    https://news.google.com:443 "GET /search?q=life&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
+    https://news.google.com:443 "GET /search?q=life&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
+    --
+    https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
+    https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
+    --
 
 
 ``make pybuild``
@@ -200,9 +204,7 @@ gate.
 .. _PyPi: https://pypi.org/
 .. _twine: https://twine.readthedocs.io/en/latest/
 
-Build Python packages in ``./dist/py``.
-
-.. code:: sh
+Build Python packages in ``./dist/py``::
 
   $ make pybuild
   ...
@@ -210,9 +212,11 @@ Build Python packages in ``./dist/py``.
   running sdist
   running egg_info
   ...
-  $ ls  ./dist/py/
-  searx-0.15.0-py3-none-any.whl  searx-0.15.0.tar.gz
+  running bdist_wheel
 
-To upload packages to PyPi_, there is also a ``upload-pypi`` target.  It needs
-twine_ to be installed.  Since you are not the owner of :pypi:`searx` you will
-never need the latter.
+  $ ls  ./dist
+  searx-0.18.0-py3-none-any.whl  searx-0.18.0.tar.gz
+
+To upload packages to PyPi_, there is also a ``pypi.upload`` target (to test use
+``pypi.upload.test``).  Since you are not the owner of :pypi:`searx` you will
+never need to upload.

docs/dev/reST.rst

@@ -15,8 +15,8 @@ generated and deployed at :docs:`github.io <.>`.  For build prerequisites read
 :ref:`docs build`.
 
 The source files of Searx's documentation are located at :origin:`docs`.  Sphinx
-assumes source files to be encoded in UTF-8 by defaul.  Run :ref:`make docs-live
-<make docs-live>` to build HTML while editing.
+assumes source files to be encoded in UTF-8 by default.  Run :ref:`make docs.live
+<make docs.live>` to build HTML while editing.
 
 .. sidebar:: Further reading
 
@@ -227,13 +227,13 @@ To refer anchors use the `ref role`_ markup:
 
 .. code:: reST
 
-   Visit chapter :ref:`reST anchor`.  Or set hyperlink text manualy :ref:`foo
+   Visit chapter :ref:`reST anchor`.  Or set hyperlink text manually :ref:`foo
    bar <reST anchor>`.
 
 .. admonition:: ``:ref:`` role
    :class: rst-example
 
-   Visist chapter :ref:`reST anchor`.  Or set hyperlink text manualy :ref:`foo
+   Visist chapter :ref:`reST anchor`.  Or set hyperlink text manually :ref:`foo
    bar <reST anchor>`.
 
 .. _reST ordinary ref:
@@ -494,8 +494,8 @@ Figures & Images
    is flexible.  To get best results in the generated output format, install
    ImageMagick_ and Graphviz_.
 
-Searx's sphinx setup includes: :ref:`linuxdoc:kfigure`.  Scaleable here means;
-scaleable in sense of the build process.  Normally in absence of a converter
+Searx's sphinx setup includes: :ref:`linuxdoc:kfigure`.  Scalable here means;
+scalable in sense of the build process.  Normally in absence of a converter
 tool, the build process will break.  From the authors POV it’s annoying to care
 about the build process when handling with images, especially since he has no
 access to the build process.  With :ref:`linuxdoc:kfigure` the build process
@@ -503,7 +503,7 @@ continues and scales output quality in dependence of installed image processors.
 
 If you want to add an image, you should use the ``kernel-figure`` (inheritance
 of :dudir:`figure`) and ``kernel-image`` (inheritance of :dudir:`image`)
-directives.  E.g. to insert a figure with a scaleable image format use SVG
+directives.  E.g. to insert a figure with a scalable image format use SVG
 (:ref:`svg image example`):
 
 .. code:: reST
@@ -1185,7 +1185,7 @@ and *targets* (e.g. a ref to :ref:`row 2 of table's body <row body 2>`).
         - cell 4.4
 
       * - row 5
-        - cell 5.1 with automatic span to rigth end
+        - cell 5.1 with automatic span to right end
 
       * - row 6
         - cell 6.1
@@ -1237,7 +1237,7 @@ and *targets* (e.g. a ref to :ref:`row 2 of table's body <row body 2>`).
         - cell 4.4
 
       * - row 5
-        - cell 5.1 with automatic span to rigth end
+        - cell 5.1 with automatic span to right end
 
       * - row 6
         - cell 6.1
@@ -1276,13 +1276,12 @@ Templating
 
 .. sidebar:: Build environment
 
-   All *generic-doc* tasks are running in the :ref:`build environment <make
-   pyenv>`.
+   All *generic-doc* tasks are running in the :ref:`make install`.
 
 Templating is suitable for documentation which is created generic at the build
-time.  The sphinx-jinja_ extension evaluates jinja_ templates in the :ref:`build
-environment <make pyenv>` (with searx modules installed).  We use this e.g. to
-build chapter: :ref:`engines generic`.  Below the jinja directive from the
+time.  The sphinx-jinja_ extension evaluates jinja_ templates in the :ref:`make
+install` (with searx modules installed).  We use this e.g. to build chapter:
+:ref:`engines generic`.  Below the jinja directive from the
 :origin:`docs/admin/engines.rst` is shown:
 
 .. literalinclude:: ../admin/engines.rst

docs/index.rst

@@ -8,9 +8,6 @@ Searx is a free internet metasearch engine which aggregates results from more
 than 70 search services.  Users are neither tracked nor profiled.  Additionally,
 searx can be used over Tor for online anonymity.
 
-Get started with searx by using one of the Searx-instances_.  If you don't trust
-anyone, you can set up your own, see :ref:`installation`.
-
 .. sidebar::  Features
 
    - Self hosted
@@ -33,5 +30,3 @@ anyone, you can set up your own, see :ref:`installation`.
    searx_extra/index
    utils/index
    blog/index
-
-.. _Searx-instances: https://searx.space

docs/user/conf.py

@@ -1,21 +0,0 @@
-# -*- coding: utf-8; mode: python -*-
-"""Configuration for the Searx user handbook
-"""
-project   = 'Searx User-HB'
-version   = release = VERSION_STRING
-
-intersphinx_mapping['searx'] = (brand.DOCS_URL, None)
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    ('index'                       # startdocname
-     , 'searx-user-hb.tex'         # targetname
-     , ''                          # take title from .rst
-     , author                      # author
-     , 'howto'                     # documentclass
-     , False                       # toctree_only
-    ),
-]
-

docs/user/search_syntax.rst

@@ -17,7 +17,7 @@ Prefix: ``:``
 Prefix: ``?``
   to add engines and categories to the currently selected categories
 
-Abbrevations of the engines and languages are also accepted.  Engine/category
+Abbreviations of the engines and languages are also accepted.  Engine/category
 modifiers are chainable and inclusive (e.g. with :search:`!it !ddg !wp qwer
 <?q=%21it%20%21ddg%20%21wp%20qwer>` search in IT category **and** duckduckgo
 **and** wikipedia for ``qwer``).

docs/utils/index.rst

@@ -1,9 +1,9 @@
 .. _searx_utils:
 .. _toolboxing:
 
-========================================
-Tooling box ``utils`` for administrators
-========================================
+===================
+Admin's tooling box
+===================
 
 In the folder :origin:`utils/` we maintain some tools useful for administrators.
 

docs/utils/lxc.sh.rst

@@ -119,15 +119,15 @@ of coffee).::
 
 To build (live) documentation inside a archlinux_ container::
 
-  sudo -H ./utils/lxc.sh cmd searx-archlinux make docs-clean docs-live
+  sudo -H ./utils/lxc.sh cmd searx-archlinux make docs.clean docs.live
   ...
   [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080
 
 To get IP of the container and the port number *live docs* is listening::
 
-  $ sudo ./utils/lxc.sh show suite | grep docs-live
+  $ sudo ./utils/lxc.sh show suite | grep docs.live
   ...
-  [searx-archlinux]  INFO:  (eth0) docs-live:  http://n.n.n.12:8080/
+  [searx-archlinux]  INFO:  (eth0) docs.live:  http://n.n.n.12:8080/
 
 
 .. _lxc.sh help:

manage

@@ -0,0 +1,498 @@
+#!/usr/bin/env bash
+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# shellcheck disable=SC2031
+
+# shellcheck source=utils/lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
+# shellcheck source=utils/brand.env
+source "${REPO_ROOT}/utils/brand.env"
+source_dot_config
+
+# config
+
+PY_SETUP_EXTRAS='[test]'
+NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"
+GECKODRIVER_VERSION="v0.30.0"
+# SPHINXOPTS=
+
+# These py files are linted by test.pylint(), all other files are linted by
+# test.pep8()
+PYLINT_FILES=(
+    searx/preferences.py
+    searx/testing.py
+    searx/engines/gigablast.py
+    searx/engines/deviantart.py
+    searx/engines/digg.py
+    searx/engines/google.py
+    searx/engines/google_news.py
+    searx/engines/google_videos.py
+    searx/engines/google_images.py
+    searx/engines/mediathekviewweb.py
+    searx/engines/meilisearch.py
+    searx/engines/solidtorrents.py
+    searx/engines/solr.py
+    searx/engines/sqlite.py
+    searx/engines/springer.py
+    searx/engines/google_scholar.py
+    searx/engines/yahoo_news.py
+    searx/engines/apkmirror.py
+    searx/engines/core.py
+    searx_extra/update/update_external_bangs.py
+)
+
+PYLINT_SEARX_DISABLE_OPTION="\
+I,C,R,\
+W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
+E1136"
+PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
+PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
+
+help() {
+    cat <<EOF
+buildenv
+  rebuild ./utils/brand.env
+babel.compile
+  pybabel compile ./searx/translations
+data.*
+  all       : update searx/languages.py and ./data/*
+  languages : update searx/data/engines_languages.json & searx/languages.py
+  useragents: update searx/data/useragents.json with the most recent versions of Firefox.
+docs.*
+  html      : build HTML documentation
+  live      : autobuild HTML documentation while editing
+  gh-pages  : deploy on gh-pages branch
+  prebuild  : build reST include files (./${DOCS_BUILD}/includes)
+  clean     : clean documentation build
+docker
+  build     : build docker image
+  push      : build and push docker image
+gecko.driver
+  download & install geckodriver if not already installed (required for
+  robot_tests)
+node.*
+  env       : download & install npm dependencies locally
+  clean     : drop npm installations
+py.*
+  build     : Build python packages at ./${PYDIST}
+  clean     : delete virtualenv and intermediate py files
+pyenv.* :
+  install   : developer install of searx into virtualenv
+  uninstall : uninstall developer installation
+  cmd ...   : run command ... in virtualenv
+  OK        : test if virtualenv is OK
+pypi.upload:
+  Upload python packages to PyPi (to test use pypi.upload.test)
+test.* :
+  pylint    : lint PYLINT_FILES, searx/engines, searx & tests
+  pep8      : pycodestyle (pep8) for all files except PYLINT_FILES
+  unit      : run unit tests
+  coverage  : run unit tests with coverage
+  robot     : run robot test
+  clean     : clean intermediate test stuff
+themes.* :
+  all       : build all themes
+  oscar     : build oscar theme
+  simple    : build simple theme
+EOF
+}
+
+
+if [ "$VERBOSE" = "1" ]; then
+    SPHINX_VERBOSE="-v"
+    PYLINT_VERBOSE="-v"
+fi
+
+# needed by sphinx-docs
+export DOCS_BUILD
+
+buildenv() {
+    SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1
+    return "${PIPESTATUS[0]}"
+}
+
+babel.compile() {
+    build_msg BABEL compile
+    pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
+    dump_return $?
+}
+
+
+data.all() {
+    data.languages
+    data.useragents
+    build_msg DATA "update searx/data/ahmia_blacklist.txt"
+    pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py
+    build_msg DATA "update searx/data/wikidata_units.json"
+    pyenv.cmd python searx_extra/update/update_wikidata_units.py
+    build_msg DATA "update searx/data/currencies.json"
+    pyenv.cmd python searx_extra/update/update_currencies.py
+}
+
+
+data.languages() {
+    (   set -e
+        build_msg ENGINES "fetch languages .."
+        pyenv.cmd python searx_extra/update/update_languages.py
+        build_msg ENGINES "update update searx/languages.py"
+        build_msg DATA "update searx/data/engines_languages.json"
+    )
+    dump_return $?
+}
+
+data.useragents() {
+    build_msg DATA "update searx/data/useragents.json"
+    pyenv.cmd python searx_extra/update/update_firefox_version.py
+    dump_return $?
+}
+
+docs.prebuild() {
+    build_msg DOCS "build ${DOCS_BUILD}/includes"
+    (
+        set -e
+        [ "$VERBOSE" = "1" ] && set -x
+        mkdir -p "${DOCS_BUILD}/includes"
+        ./utils/searx.sh doc   | cat > "${DOCS_BUILD}/includes/searx.rst"
+        ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
+        ./utils/morty.sh doc   | cat > "${DOCS_BUILD}/includes/morty.rst"
+    )
+    dump_return $?
+}
+
+docker.push() {
+    docker.build push
+}
+
+# shellcheck disable=SC2119
+docker.build() {
+    pyenv.install
+
+    build_msg DOCKER build
+    # run installation in a subprocess and activate pyenv
+
+    # See https://www.shellcheck.net/wiki/SC1001 and others ..
+    # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
+    (   set -e
+        # shellcheck source=/dev/null
+        source "${PY_ENV_BIN}/activate"
+
+        # Check if it is a git repository
+        if [ ! -d .git ]; then
+	    die 1 "This is not Git repository"
+        fi
+        if [ ! -x "$(which git)" ]; then
+	    die 1 "git is not installed"
+        fi
+
+        if ! git remote get-url origin 2> /dev/null; then
+	    die 1 "there is no remote origin"
+        fi
+
+        # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
+        # awk to remove the "v" and the "g"
+        SEARX_GIT_VERSION=$(git describe --tags | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); }  print}')
+
+        # add the suffix "-dirty" if the repository has uncommitted change
+        # /!\ HACK for searx/searx: ignore utils/brand.env
+        git update-index -q --refresh
+        if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
+	    SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
+        fi
+
+        # Get the last git commit id, will be added to the Searx version (see Dockerfile)
+        VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
+        build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
+
+        # Check consistency between the git tag and the searx/version.py file
+        # /! HACK : parse Python file with bash /!
+        # otherwise it is not possible build the docker image without all Python
+        # dependencies ( version.py loads __init__.py )
+        # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
+        SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
+        if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
+	    err_msg "git tag:          $SEARX_GIT_VERSION"
+	    err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
+	    die 1 "Inconsistency between the last git tag and the searx/version.py file"
+        fi
+
+        # define the docker image name
+        GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
+        SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"
+
+        # build Docker image
+        build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
+        docker build \
+         --build-arg GIT_URL="${GIT_URL}" \
+         --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
+         --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
+         --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+         --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
+         --build-arg LABEL_VCS_URL="${GIT_URL}" \
+	 --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
+	 --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
+         -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
+
+        if [ "$1" = "push" ]; then
+	    docker push "${SEARX_IMAGE_NAME}:latest"
+	    docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
+        fi
+    )
+    dump_return $?
+}
+
+# shellcheck disable=SC2119
+gecko.driver() {
+    pyenv.install
+
+    build_msg INSTALL "gecko.driver"
+    # run installation in a subprocess and activate pyenv
+    (   set -e
+        # shellcheck source=/dev/null
+        source "${PY_ENV_BIN}/activate"
+
+        # TODO : check the current geckodriver version
+        geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
+        set +e
+        if [ -z "$NOTFOUND" ]; then
+            build_msg INSTALL "geckodriver already installed"
+            return
+        fi
+        PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
+        case "$PLATFORM" in
+            "linux 32bit" | "linux2 32bit") ARCH="linux32";;
+            "linux 64bit" | "linux2 64bit") ARCH="linux64";;
+            "windows 32 bit") ARCH="win32";;
+            "windows 64 bit") ARCH="win64";;
+            "mac 64bit") ARCH="macos";;
+        esac
+        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
+
+        build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
+
+        FILE="$(mktemp)"
+        wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
+        rm -- "$FILE"
+        chmod 755 -- "${PY_ENV_BIN}/geckodriver"
+    )
+    dump_return $?
+}
+
+node.env() {
+    local err=0
+    pushd "${REPO_ROOT}" &> /dev/null
+    # shellcheck disable=SC2230
+    which npm &> /dev/null || die 1 'node.env - npm is not found!'
+
+    (   set -e
+
+        build_msg INSTALL "npm install $NPM_PACKAGES"
+        # shellcheck disable=SC2086
+        npm install $NPM_PACKAGES
+
+        cd "${REPO_ROOT}/searx/static/themes/oscar"
+        build_msg INSTALL "($(pwd)) npm install"
+        npm install
+
+        build_msg INSTALL "($(pwd)) npm install"
+        cd "${REPO_ROOT}/searx/static/themes/simple"
+        npm install
+    )
+    err=$?
+    popd &> /dev/null
+    dump_return "$err"
+}
+
+node.clean() {
+
+    build_msg CLEAN "locally installed npm dependencies"
+    rm -rf \
+       ./node_modules  \
+       ./package-lock.json \
+       ./searx/static/themes/oscar/package-lock.json \
+       ./searx/static/themes/oscar/node_modules \
+       ./searx/static/themes/simple/package-lock.json \
+       ./searx/static/themes/simple/node_modules
+    dump_return $?
+}
+
+py.build() {
+    build_msg BUILD "[pylint] python package ${PYDIST}"
+    pyenv.cmd python setup.py \
+              sdist -d "${PYDIST}" \
+              bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
+}
+
+py.clean() {
+    build_msg CLEAN pyenv
+    (   set -e
+        pyenv.drop
+        [ "$VERBOSE" = "1" ] && set -x
+        rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
+        find . -name '*.pyc' -exec rm -f {} +
+        find . -name '*.pyo' -exec rm -f {} +
+        find . -name __pycache__ -exec rm -rf {} +
+    )
+}
+
+pyenv.check() {
+    cat  <<EOF
+import yaml
+print('import yaml --> OK')
+EOF
+}
+
+pyenv.install() {
+
+    if ! pyenv.OK; then
+        py.clean > /dev/null
+    fi
+    if pyenv.install.OK > /dev/null; then
+        return 0
+    fi
+    (   set -e
+        pyenv
+    	build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
+    	"${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
+        buildenv
+    ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"
+}
+
+pyenv.uninstall() {
+    build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
+    pyenv.cmd python setup.py develop --uninstall 2>&1 \
+        | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+
+}
+
+pypi.upload() {
+    py.clean
+    py.build
+    # https://github.com/pypa/twine
+    pyenv.cmd twine upload "${PYDIST}"/*
+}
+
+pypi.upload.test() {
+    py.clean
+    py.build
+    pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
+}
+
+test.pylint() {
+    # shellcheck disable=SC2086
+    (   set -e
+        build_msg TEST "[pylint] \$PYLINT_FILES"
+        pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            "${PYLINT_FILES[@]}"
+
+        build_msg TEST "[pylint] searx/engines"
+        pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
+            --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
+            searx/engines
+
+        build_msg TEST "[pylint] searx tests"
+        pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
+            --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
+	    --ignore=searx/engines \
+	    tests
+    )
+    dump_return $?
+}
+
+test.pep8() {
+    build_msg TEST 'pycodestyle (formerly pep8)'
+    local _exclude=""
+    printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
+    pyenv.cmd pycodestyle \
+              --exclude="searx/static, searx/languages.py, $_exclude " \
+              --max-line-length=120 \
+              --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
+              searx tests
+    dump_return $?
+}
+
+test.unit() {
+    build_msg TEST 'tests/unit'
+    pyenv.cmd python -m nose2 -s tests/unit
+    dump_return $?
+}
+
+test.coverage() {
+    build_msg TEST 'unit test coverage'
+    (   set -e
+        pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
+        pyenv.cmd coverage report
+        pyenv.cmd coverage html
+    )
+    dump_return $?
+}
+
+test.robot() {
+    build_msg TEST 'robot'
+    gecko.driver
+    PYTHONPATH=. pyenv.cmd python searx/testing.py robot
+    dump_return $?
+}
+
+test.clean() {
+    build_msg CLEAN  "test stuff"
+    rm -rf geckodriver.log .coverage coverage/
+    dump_return $?
+}
+
+themes.all() {
+    (   set -e
+        node.env
+        themes.oscar
+        themes.simple
+    )
+    dump_return $?
+}
+
+themes.oscar() {
+    local gruntfile=searx/static/themes/oscar/gruntfile.js
+    build_msg GRUNT "${gruntfile}"
+    PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
+    dump_return $?
+}
+
+themes.simple() {
+    local gruntfile=searx/static/themes/simple/gruntfile.js
+    build_msg GRUNT "${gruntfile}"
+    PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
+    dump_return $?
+}
+
+# shellcheck disable=SC2119
+main() {
+
+    local _type
+    local cmd="$1"; shift
+
+    if [ "$cmd" == "" ]; then
+        help
+        err_msg "missing command"
+        return 42
+    fi
+
+    case "$cmd" in
+        --getenv) var="$1"; echo "${!var}";;
+        --help) help;;
+        --*)
+            help
+            err_msg "unknown option $cmd"
+            return 42
+            ;;
+        *)
+            _type="$(type -t "$cmd")"
+            if [ "$_type" != 'function' ]; then
+                err_msg "unknown command $1 / use --help"
+                return 42
+            else
+                "$cmd" "$@"
+            fi
+            ;;
+    esac
+}
+
+main "$@"

manage.sh

@@ -1,205 +0,0 @@
-#!/bin/sh
-
-export LANG=C
-
-BASE_DIR="$(dirname -- "`readlink -f -- "$0"`")"
-
-cd -- "$BASE_DIR"
-set -e
-
-# subshell
-PYTHONPATH="$BASE_DIR"
-SEARX_DIR="$BASE_DIR/searx"
-ACTION="$1"
-
-. "${BASE_DIR}/utils/brand.env"
-
-#
-# Python
-#
-
-update_packages() {
-    pip install --upgrade pip
-    pip install --upgrade setuptools
-    pip install -Ur "$BASE_DIR/requirements.txt"
-}
-
-update_dev_packages() {
-    update_packages
-    pip install -Ur "$BASE_DIR/requirements-dev.txt"
-}
-
-install_geckodriver() {
-    echo '[!] Checking geckodriver'
-    # TODO : check the current geckodriver version
-    set -e
-    geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
-    set +e
-    if [ -z "$NOTFOUND" ]; then
-        return
-    fi
-    GECKODRIVER_VERSION="v0.28.0"
-    PLATFORM="`python3 -c "import platform; print(platform.system().lower(), platform.architecture()[0])"`"
-    case "$PLATFORM" in
-        "linux 32bit" | "linux2 32bit") ARCH="linux32";;
-        "linux 64bit" | "linux2 64bit") ARCH="linux64";;
-        "windows 32 bit") ARCH="win32";;
-        "windows 64 bit") ARCH="win64";;
-        "mac 64bit") ARCH="macos";;
-    esac
-    GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
-
-    if [ -z "$1" ]; then
-        if [ -z "$VIRTUAL_ENV" ]; then
-            printf "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n  %s" "$GECKODRIVER_URL"
-            exit
-        else
-            GECKODRIVER_DIR="$VIRTUAL_ENV/bin"
-        fi
-    else
-        GECKODRIVER_DIR="$1"
-        mkdir -p -- "$GECKODRIVER_DIR"
-    fi
-
-    printf "Installing %s/geckodriver from\n  %s" "$GECKODRIVER_DIR" "$GECKODRIVER_URL"
-
-    FILE="`mktemp`"
-    wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "$GECKODRIVER_DIR" -f "$FILE" geckodriver
-    rm -- "$FILE"
-    chmod 777 -- "$GECKODRIVER_DIR/geckodriver"
-}
-
-locales() {
-    pybabel compile -d "$SEARX_DIR/translations"
-}
-
-
-#
-# Web
-#
-
-npm_path_setup() {
-    which npm || (printf 'Error: npm is not found\n'; exit 1)
-    export PATH="$(npm bin)":$PATH
-}
-
-npm_packages() {
-    npm_path_setup
-
-    echo '[!] install NPM packages'
-    cd -- "$BASE_DIR"
-    npm install less@2.7 less-plugin-clean-css grunt-cli
-
-    echo '[!] install NPM packages for oscar theme'
-    cd -- "$BASE_DIR/searx/static/themes/oscar"
-    npm install
-
-    echo '[!] install NPM packages for simple theme'
-    cd -- "$BASE_DIR/searx/static/themes/simple"
-    npm install
-}
-
-docker_build() {
-    # Check if it is a git repository
-    if [ ! -d .git ]; then
-	echo "This is not Git repository"
-	exit 1
-    fi
-
-    if [ ! -x "$(which git)" ]; then
-	echo "git is not installed"
-	exit 1
-    fi
-
-    if [ ! git remote get-url origin 2> /dev/null ]; then
-	echo "there is no remote origin"
-	exit 1
-    fi
-
-    # This is a git repository
-
-    # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
-    # awk to remove the "v" and the "g"
-    SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); }  print}')
-
-    # add the suffix "-dirty" if the repository has uncommited change
-    # /!\ HACK for searx/searx: ignore utils/brand.env
-    git update-index -q --refresh
-    if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
-	SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
-    fi
-
-    # Get the last git commit id, will be added to the Searx version (see Dockerfile)
-    VERSION_GITCOMMIT=$(echo $SEARX_GIT_VERSION | cut -d- -f2-4)
-    echo "Last commit : $VERSION_GITCOMMIT"
-
-    # Check consistency between the git tag and the searx/version.py file
-    # /!\ HACK : parse Python file with bash /!\
-    # otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py )
-    # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
-    SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
-    if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then
-	echo "Inconsistency between the last git tag and the searx/version.py file"
-	echo "git tag:          $SEARX_GIT_VERSION"
-	echo "searx/version.py: $SEARX_PYTHON_VERSION"
-	exit 1
-    fi
-
-    # define the docker image name
-    GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
-    SEARX_IMAGE_NAME="${GITHUB_USER:-searx}/searx"
-
-    # build Docker image
-    echo "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
-    sudo docker build \
-         --build-arg GIT_URL="${GIT_URL}" \
-         --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
-         --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
-         --build-arg LABEL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
-         --build-arg LABEL_VCS_REF=$(git rev-parse HEAD) \
-         --build-arg LABEL_VCS_URL="${GIT_URL}" \
-	 --build-arg TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- searx/settings.yml) \
-	 --build-arg TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini) \
-         -t ${SEARX_IMAGE_NAME}:latest -t ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} .
-
-    if [ "$1" = "push" ]; then
-	sudo docker push ${SEARX_IMAGE_NAME}:latest
-	sudo docker push ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}
-    fi
-}
-
-#
-# Help
-#
-
-help() {
-    [ -z "$1" ] || printf 'Error: %s\n' "$1"
-    echo "Searx manage.sh help
-
-Commands
-========
-    help                 - This text
-
-    Build requirements
-    ------------------
-    update_packages      - Check & update production dependency changes
-    update_dev_packages  - Check & update development and production dependency changes
-    install_geckodriver  - Download & install geckodriver if not already installed (required for robot_tests)
-    npm_packages         - Download & install npm dependencies
-
-    Build
-    -----
-    locales              - Compile locales
-
-Environment:
-    GIT_URL:          ${GIT_URL}
-    ISSUE_URL:        ${ISSUE_URL}
-    SEARX_URL:        ${SEARX_URL}
-    DOCS_URL:         ${DOCS_URL}
-    PUBLIC_INSTANCES: ${PUBLIC_INSTANCES}
-"
-}
-
-[ "$(command -V "$ACTION" | grep ' function$')" = "" ] \
-    && help "action not found" \
-    || "$ACTION" "$2"

requirements-dev.txt

@@ -1,17 +1,19 @@
-mock==4.0.3
-nose2[coverage_plugin]==0.10.0
+mock==5.0.1
+nose2[coverage_plugin]==0.12.0
 cov-core==1.15.0
-pycodestyle==2.6.0
-pylint==2.7.2
-splinter==0.14.0
-transifex-client==0.14.2
-selenium==3.141.0
-twine==3.4.1
-Pallets-Sphinx-Themes==1.2.3
-Sphinx==3.5.2
-sphinx-issues==1.2.0
-sphinx-jinja==1.1.1
-sphinx-tabs==2.1.0
-sphinxcontrib-programoutput==0.16
-sphinx-autobuild==2020.9.1
-linuxdoc==20210324
+pycodestyle==2.10.0
+pylint==2.15.9
+splinter==0.19.0
+transifex-client==0.14.3; python_version < '3.10'
+transifex-client==0.12.5; python_version == '3.10'
+selenium==4.8.3
+twine==4.0.2
+Pallets-Sphinx-Themes==2.0.3
+docutils==0.18
+Sphinx==5.3.0
+sphinx-issues==3.0.1
+sphinx-jinja==2.0.2
+sphinx-tabs==3.4.1
+sphinxcontrib-programoutput==0.17
+sphinx-autobuild==2021.3.14
+linuxdoc==20221127

requirements.txt

@@ -1,12 +1,13 @@
-certifi==2020.12.05
-babel==2.9.0
+Brotli==1.0.9
+babel==2.11.0
+certifi==2022.12.7
 flask-babel==2.0.0
-flask==1.1.2
-idna==2.10
-jinja2==2.11.3
-lxml==4.6.3
-pygments==2.8.0
-python-dateutil==2.8.1
-pyyaml==5.4.1
-requests[socks]==2.25.1
-langdetect==1.0.8
+flask==2.2.2
+jinja2==3.1.2
+langdetect==1.0.9
+lxml==4.9.2
+pygments==2.12.0
+python-dateutil==2.8.2
+pyyaml==6.0
+requests[socks]==2.28.2
+setproctitle==1.3.2

searx/__init__.py

@@ -19,23 +19,33 @@ import logging
 import searx.settings_loader
 from os import environ
 from os.path import realpath, dirname, join, abspath, isfile
+from sys import exit
 
+from searx.exceptions import SearxSettingsException
 
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
 static_path = abspath(join(dirname(__file__), 'static'))
-settings, settings_load_message = searx.settings_loader.load_settings()
+
+settings, settings_outgoing = {}, ''
+try:
+    settings, settings_load_message = searx.settings_loader.load_settings()
+except SearxSettingsException as e:
+    logger = logging.getLogger('searx')
+    logger.error('Failed to load settings file: {}'.format(str(e)))
+    exit(1)
+
 
 if settings['ui']['static_path']:
     static_path = settings['ui']['static_path']
 
 '''
 enable debug if
-the environnement variable SEARX_DEBUG is 1 or true
+the environment variable SEARX_DEBUG is 1 or true
 (whatever the value in settings.yml)
 or general.debug=True in settings.yml
 disable debug if
-the environnement variable SEARX_DEBUG is 0 or false
+the environment variable SEARX_DEBUG is 0 or false
 (whatever the value in settings.yml)
 or general.debug=False in settings.yml
 '''

searx/autocomplete.py

@@ -34,6 +34,22 @@ def get(*args, **kwargs):
     return http_get(*args, **kwargs)
 
 
+def brave(query, lang):
+    # brave search autocompleter
+    url = 'https://search.brave.com/api/suggest?{query}'
+
+    resp = get(url.format(query=urlencode({'q': query})))
+
+    results = []
+
+    if resp.ok:
+        data = loads(resp.text)
+        for suggestion in data[1]:
+            results.append(suggestion)
+
+    return results
+
+
 def dbpedia(query, lang):
     # dbpedia autocompleter, no HTTPS
     autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'
@@ -76,12 +92,11 @@ def google(query, lang):
 
 def startpage(query, lang):
     # startpage autocompleter
-    url = 'https://startpage.com/do/suggest?{query}'
-
-    resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')
-    if len(resp) > 1:
-        return resp
-    return []
+    lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')
+    url = 'https://startpage.com/suggestions?{query}'
+    resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))
+    data = resp.json()
+    return [e['text'] for e in data.get('suggestions', []) if 'text' in e]
 
 
 def swisscows(query, lang):
@@ -119,7 +134,8 @@ def wikipedia(query, lang):
     return []
 
 
-backends = {'dbpedia': dbpedia,
+backends = {'brave': brave,
+            'dbpedia': dbpedia,
             'duckduckgo': duckduckgo,
             'google': google,
             'startpage': startpage,

searx/data/__init__.py

@@ -1,25 +1,50 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""This module holds the *data* created by::
+
+  make data.all
+
+"""
+
+__all__ = [
+    'ENGINES_LANGUAGES',
+    'CURRENCIES',
+    'USER_AGENTS',
+    'EXTERNAL_URLS',
+    'WIKIDATA_UNITS',
+    'EXTERNAL_BANGS',
+    'OSM_KEYS_TAGS',
+    'ahmia_blacklist_loader',
+]
+
 import json
 from pathlib import Path
 
-
-__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'EXTERNAL_URLS', 'WIKIDATA_UNITS', 'EXTERNAL_BANGS',
-            'bangs_loader', 'ahmia_blacklist_loader']
 data_dir = Path(__file__).parent
 
 
-def load(filename):
-    with open(data_dir / filename, encoding='utf-8') as fd:
-        return json.load(fd)
+def _load(filename):
+    with open(data_dir / filename, encoding='utf-8') as f:
+        return json.load(f)
 
 
 def ahmia_blacklist_loader():
-    with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as fd:
-        return fd.read().split()
+    """Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion
+    names.  The MD5 values are fetched by::
+
+      searx_extra/update/update_ahmia_blacklist.py
+
+    This function is used by :py:mod:`searx.plugins.ahmia_filter`.
+
+    """
+    with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as f:
+        return f.read().split()
 
 
-ENGINES_LANGUAGES = load('engines_languages.json')
-CURRENCIES = load('currencies.json')
-USER_AGENTS = load('useragents.json')
-EXTERNAL_URLS = load('external_urls.json')
-WIKIDATA_UNITS = load('wikidata_units.json')
-EXTERNAL_BANGS = load('external_bangs.json')
+ENGINES_LANGUAGES = _load('engines_languages.json')
+CURRENCIES = _load('currencies.json')
+USER_AGENTS = _load('useragents.json')
+EXTERNAL_URLS = _load('external_urls.json')
+WIKIDATA_UNITS = _load('wikidata_units.json')
+EXTERNAL_BANGS = _load('external_bangs.json')
+OSM_KEYS_TAGS = _load('osm_keys_tags.json')

searx/data/engines_languages.json

@@ -24570,13 +24570,17 @@
     "en-CA",
     "en-ID",
     "en-IE",
+    "en-IL",
     "en-IN",
     "en-MY",
     "en-NZ",
     "en-PH",
+    "en-PK",
     "en-SG",
+    "en-TH",
     "en-UK",
     "en-US",
+    "en-VN",
     "en-ZA",
     "es-AR",
     "es-CL",
@@ -24591,17 +24595,13 @@
     "fr-CA",
     "fr-CH",
     "fr-FR",
-    "he-IL",
     "hr-HR",
     "hu-HU",
-    "id-ID",
-    "it-CH",
     "it-IT",
     "jp-JP",
     "kr-KR",
     "lt-LT",
     "lv-LV",
-    "ms-MY",
     "nl-BE",
     "nl-NL",
     "no-NO",
@@ -24613,12 +24613,10 @@
     "sk-SK",
     "sl-SL",
     "sv-SE",
-    "th-TH",
-    "tl-PH",
     "tr-TR",
     "tzh-HK",
     "tzh-TW",
-    "vi-VN",
+    "uk-UA",
     "wt-WT",
     "zh-CN"
   ],
@@ -24637,13 +24635,17 @@
     "en-CA",
     "en-ID",
     "en-IE",
+    "en-IL",
     "en-IN",
     "en-MY",
     "en-NZ",
     "en-PH",
+    "en-PK",
     "en-SG",
+    "en-TH",
     "en-UK",
     "en-US",
+    "en-VN",
     "en-ZA",
     "es-AR",
     "es-CL",
@@ -24658,17 +24660,13 @@
     "fr-CA",
     "fr-CH",
     "fr-FR",
-    "he-IL",
     "hr-HR",
     "hu-HU",
-    "id-ID",
-    "it-CH",
     "it-IT",
     "jp-JP",
     "kr-KR",
     "lt-LT",
     "lv-LV",
-    "ms-MY",
     "nl-BE",
     "nl-NL",
     "no-NO",
@@ -24680,12 +24678,10 @@
     "sk-SK",
     "sl-SL",
     "sv-SE",
-    "th-TH",
-    "tl-PH",
     "tr-TR",
     "tzh-HK",
     "tzh-TW",
-    "vi-VN",
+    "uk-UA",
     "wt-WT",
     "zh-CN"
   ],
@@ -24704,13 +24700,17 @@
     "en-CA",
     "en-ID",
     "en-IE",
+    "en-IL",
     "en-IN",
     "en-MY",
     "en-NZ",
     "en-PH",
+    "en-PK",
     "en-SG",
+    "en-TH",
     "en-UK",
     "en-US",
+    "en-VN",
     "en-ZA",
     "es-AR",
     "es-CL",
@@ -24725,17 +24725,13 @@
     "fr-CA",
     "fr-CH",
     "fr-FR",
-    "he-IL",
     "hr-HR",
     "hu-HU",
-    "id-ID",
-    "it-CH",
     "it-IT",
     "jp-JP",
     "kr-KR",
     "lt-LT",
     "lv-LV",
-    "ms-MY",
     "nl-BE",
     "nl-NL",
     "no-NO",
@@ -24747,12 +24743,10 @@
     "sk-SK",
     "sl-SL",
     "sv-SE",
-    "th-TH",
-    "tl-PH",
     "tr-TR",
     "tzh-HK",
     "tzh-TW",
-    "vi-VN",
+    "uk-UA",
     "wt-WT",
     "zh-CN"
   ],
@@ -25650,480 +25644,178 @@
     "zh",
     "zu"
   ],
-  "qwant": {
-    "bg-BG": {
-      "name": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438 \u0435\u0437\u0438\u043a"
-    },
-    "br-FR": {
-      "name": "Brezhoneg"
-    },
-    "ca-AD": {
-      "name": "Catal\u00e0"
-    },
-    "ca-ES": {
-      "name": "Catal\u00e0"
-    },
-    "ca-FR": {
-      "name": "Catal\u00e0"
-    },
-    "co-FR": {
-      "name": "Corsu"
-    },
-    "cs-CZ": {
-      "name": "\u010cesky"
-    },
-    "cy-GB": {
-      "name": "Welsh"
-    },
-    "da-DK": {
-      "name": "Dansk"
-    },
-    "de-AT": {
-      "name": "Deutsch"
-    },
-    "de-CH": {
-      "name": "Deutsch"
-    },
-    "de-DE": {
-      "name": "Deutsch"
-    },
-    "el-GR": {
-      "name": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"
-    },
-    "en-AU": {
-      "name": "English"
-    },
-    "en-CA": {
-      "name": "English"
-    },
-    "en-GB": {
-      "name": "English"
-    },
-    "en-IE": {
-      "name": "English"
-    },
-    "en-IN": {
-      "name": "English"
-    },
-    "en-MY": {
-      "name": "English"
-    },
-    "en-NZ": {
-      "name": "English"
-    },
-    "en-US": {
-      "name": "English"
-    },
-    "es-AD": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-AR": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-CL": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-ES": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-MX": {
-      "name": "Espa\u00f1ol"
-    },
-    "et-EE": {
-      "name": "Eesti keel"
-    },
-    "eu-ES": {
-      "name": "Euskara"
-    },
-    "eu-FR": {
-      "name": "Euskara"
-    },
-    "fi-FI": {
-      "name": "Suomen kieli"
-    },
-    "fr-AD": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-BE": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CA": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CH": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-FR": {
-      "name": "Fran\u00e7ais"
-    },
-    "gd-GB": {
-      "name": "Scottish"
-    },
-    "hu-HU": {
-      "name": "magyar"
-    },
-    "it-CH": {
-      "name": "Italiano"
-    },
-    "it-IT": {
-      "name": "Italiano"
-    },
-    "ko-KR": {
-      "name": "\ud55c\uad6d\uc5b4"
-    },
-    "nb-NO": {
-      "name": "Norsk"
-    },
-    "nl-BE": {
-      "name": "Nederlands"
-    },
-    "nl-NL": {
-      "name": "Nederlands"
-    },
-    "pl-PL": {
-      "name": "Polski"
-    },
-    "pt-AD": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-BR": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-PT": {
-      "name": "Portugu\u00eas"
-    },
-    "ro-RO": {
-      "name": "Rom\u00e2n\u0103"
-    },
-    "sv-SE": {
-      "name": "Svenska"
-    },
-    "th-TH": {
-      "name": "\u0e44\u0e17\u0e22"
-    },
-    "zh-CN": {
-      "name": "\u4e2d\u6587"
-    },
-    "zh-HK": {
-      "name": "\u4e2d\u6587"
-    }
-  },
-  "qwant images": {
-    "bg-BG": {
-      "name": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438 \u0435\u0437\u0438\u043a"
-    },
-    "br-FR": {
-      "name": "Brezhoneg"
-    },
-    "ca-AD": {
-      "name": "Catal\u00e0"
-    },
-    "ca-ES": {
-      "name": "Catal\u00e0"
-    },
-    "ca-FR": {
-      "name": "Catal\u00e0"
-    },
-    "co-FR": {
-      "name": "Corsu"
-    },
-    "cs-CZ": {
-      "name": "\u010cesky"
-    },
-    "cy-GB": {
-      "name": "Welsh"
-    },
-    "da-DK": {
-      "name": "Dansk"
-    },
-    "de-AT": {
-      "name": "Deutsch"
-    },
-    "de-CH": {
-      "name": "Deutsch"
-    },
-    "de-DE": {
-      "name": "Deutsch"
-    },
-    "el-GR": {
-      "name": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"
-    },
-    "en-AU": {
-      "name": "English"
-    },
-    "en-CA": {
-      "name": "English"
-    },
-    "en-GB": {
-      "name": "English"
-    },
-    "en-IE": {
-      "name": "English"
-    },
-    "en-IN": {
-      "name": "English"
-    },
-    "en-MY": {
-      "name": "English"
-    },
-    "en-NZ": {
-      "name": "English"
-    },
-    "en-US": {
-      "name": "English"
-    },
-    "es-AD": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-AR": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-CL": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-ES": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-MX": {
-      "name": "Espa\u00f1ol"
-    },
-    "et-EE": {
-      "name": "Eesti keel"
-    },
-    "eu-ES": {
-      "name": "Euskara"
-    },
-    "eu-FR": {
-      "name": "Euskara"
-    },
-    "fi-FI": {
-      "name": "Suomen kieli"
-    },
-    "fr-AD": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-BE": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CA": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CH": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-FR": {
-      "name": "Fran\u00e7ais"
-    },
-    "gd-GB": {
-      "name": "Scottish"
-    },
-    "hu-HU": {
-      "name": "magyar"
-    },
-    "it-CH": {
-      "name": "Italiano"
-    },
-    "it-IT": {
-      "name": "Italiano"
-    },
-    "ko-KR": {
-      "name": "\ud55c\uad6d\uc5b4"
-    },
-    "nb-NO": {
-      "name": "Norsk"
-    },
-    "nl-BE": {
-      "name": "Nederlands"
-    },
-    "nl-NL": {
-      "name": "Nederlands"
-    },
-    "pl-PL": {
-      "name": "Polski"
-    },
-    "pt-AD": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-BR": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-PT": {
-      "name": "Portugu\u00eas"
-    },
-    "ro-RO": {
-      "name": "Rom\u00e2n\u0103"
-    },
-    "sv-SE": {
-      "name": "Svenska"
-    },
-    "th-TH": {
-      "name": "\u0e44\u0e17\u0e22"
-    },
-    "zh-CN": {
-      "name": "\u4e2d\u6587"
-    },
-    "zh-HK": {
-      "name": "\u4e2d\u6587"
-    }
-  },
-  "qwant news": {
-    "bg-BG": {
-      "name": "\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438 \u0435\u0437\u0438\u043a"
-    },
-    "br-FR": {
-      "name": "Brezhoneg"
-    },
-    "ca-AD": {
-      "name": "Catal\u00e0"
-    },
-    "ca-ES": {
-      "name": "Catal\u00e0"
-    },
-    "ca-FR": {
-      "name": "Catal\u00e0"
-    },
-    "co-FR": {
-      "name": "Corsu"
-    },
-    "cs-CZ": {
-      "name": "\u010cesky"
-    },
-    "cy-GB": {
-      "name": "Welsh"
-    },
-    "da-DK": {
-      "name": "Dansk"
-    },
-    "de-AT": {
-      "name": "Deutsch"
-    },
-    "de-CH": {
-      "name": "Deutsch"
-    },
-    "de-DE": {
-      "name": "Deutsch"
-    },
-    "el-GR": {
-      "name": "\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"
-    },
-    "en-AU": {
-      "name": "English"
-    },
-    "en-CA": {
-      "name": "English"
-    },
-    "en-GB": {
-      "name": "English"
-    },
-    "en-IE": {
-      "name": "English"
-    },
-    "en-IN": {
-      "name": "English"
-    },
-    "en-MY": {
-      "name": "English"
-    },
-    "en-NZ": {
-      "name": "English"
-    },
-    "en-US": {
-      "name": "English"
-    },
-    "es-AD": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-AR": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-CL": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-ES": {
-      "name": "Espa\u00f1ol"
-    },
-    "es-MX": {
-      "name": "Espa\u00f1ol"
-    },
-    "et-EE": {
-      "name": "Eesti keel"
-    },
-    "eu-ES": {
-      "name": "Euskara"
-    },
-    "eu-FR": {
-      "name": "Euskara"
-    },
-    "fi-FI": {
-      "name": "Suomen kieli"
-    },
-    "fr-AD": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-BE": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CA": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-CH": {
-      "name": "Fran\u00e7ais"
-    },
-    "fr-FR": {
-      "name": "Fran\u00e7ais"
-    },
-    "gd-GB": {
-      "name": "Scottish"
-    },
-    "hu-HU": {
-      "name": "magyar"
-    },
-    "it-CH": {
-      "name": "Italiano"
-    },
-    "it-IT": {
-      "name": "Italiano"
-    },
-    "ko-KR": {
-      "name": "\ud55c\uad6d\uc5b4"
-    },
-    "nb-NO": {
-      "name": "Norsk"
-    },
-    "nl-BE": {
-      "name": "Nederlands"
-    },
-    "nl-NL": {
-      "name": "Nederlands"
-    },
-    "pl-PL": {
-      "name": "Polski"
-    },
-    "pt-AD": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-BR": {
-      "name": "Portugu\u00eas"
-    },
-    "pt-PT": {
-      "name": "Portugu\u00eas"
-    },
-    "ro-RO": {
-      "name": "Rom\u00e2n\u0103"
-    },
-    "sv-SE": {
-      "name": "Svenska"
-    },
-    "th-TH": {
-      "name": "\u0e44\u0e17\u0e22"
-    },
-    "zh-CN": {
-      "name": "\u4e2d\u6587"
-    },
-    "zh-HK": {
-      "name": "\u4e2d\u6587"
-    }
-  },
+  "qwant": [
+    "bg-BG",
+    "ca-ES",
+    "cs-CZ",
+    "da-DK",
+    "de-AT",
+    "de-CH",
+    "de-DE",
+    "el-GR",
+    "en-AU",
+    "en-CA",
+    "en-GB",
+    "en-IE",
+    "en-IN",
+    "en-MY",
+    "en-NZ",
+    "en-US",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et-EE",
+    "fi-FI",
+    "fr-BE",
+    "fr-CA",
+    "fr-CH",
+    "fr-FR",
+    "hu-HU",
+    "it-CH",
+    "it-IT",
+    "ko-KR",
+    "nb-NO",
+    "nl-BE",
+    "nl-NL",
+    "pl-PL",
+    "pt-BR",
+    "pt-PT",
+    "ro-RO",
+    "sv-SE",
+    "th-TH",
+    "zh-CN",
+    "zh-HK"
+  ],
+  "qwant images": [
+    "bg-BG",
+    "ca-ES",
+    "cs-CZ",
+    "da-DK",
+    "de-AT",
+    "de-CH",
+    "de-DE",
+    "el-GR",
+    "en-AU",
+    "en-CA",
+    "en-GB",
+    "en-IE",
+    "en-IN",
+    "en-MY",
+    "en-NZ",
+    "en-US",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et-EE",
+    "fi-FI",
+    "fr-BE",
+    "fr-CA",
+    "fr-CH",
+    "fr-FR",
+    "hu-HU",
+    "it-CH",
+    "it-IT",
+    "ko-KR",
+    "nb-NO",
+    "nl-BE",
+    "nl-NL",
+    "pl-PL",
+    "pt-BR",
+    "pt-PT",
+    "ro-RO",
+    "sv-SE",
+    "th-TH",
+    "zh-CN",
+    "zh-HK"
+  ],
+  "qwant news": [
+    "bg-BG",
+    "ca-ES",
+    "cs-CZ",
+    "da-DK",
+    "de-AT",
+    "de-CH",
+    "de-DE",
+    "el-GR",
+    "en-AU",
+    "en-CA",
+    "en-GB",
+    "en-IE",
+    "en-IN",
+    "en-MY",
+    "en-NZ",
+    "en-US",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et-EE",
+    "fi-FI",
+    "fr-BE",
+    "fr-CA",
+    "fr-CH",
+    "fr-FR",
+    "hu-HU",
+    "it-CH",
+    "it-IT",
+    "ko-KR",
+    "nb-NO",
+    "nl-BE",
+    "nl-NL",
+    "pl-PL",
+    "pt-BR",
+    "pt-PT",
+    "ro-RO",
+    "sv-SE",
+    "th-TH",
+    "zh-CN",
+    "zh-HK"
+  ],
+  "qwant videos": [
+    "bg-BG",
+    "ca-ES",
+    "cs-CZ",
+    "da-DK",
+    "de-AT",
+    "de-CH",
+    "de-DE",
+    "el-GR",
+    "en-AU",
+    "en-CA",
+    "en-GB",
+    "en-IE",
+    "en-IN",
+    "en-MY",
+    "en-NZ",
+    "en-US",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et-EE",
+    "fi-FI",
+    "fr-BE",
+    "fr-CA",
+    "fr-CH",
+    "fr-FR",
+    "hu-HU",
+    "it-CH",
+    "it-IT",
+    "ko-KR",
+    "nb-NO",
+    "nl-BE",
+    "nl-NL",
+    "pl-PL",
+    "pt-BR",
+    "pt-PT",
+    "ro-RO",
+    "sv-SE",
+    "th-TH",
+    "zh-CN",
+    "zh-HK"
+  ],
   "startpage": {
     "af": {
       "alias": "afrikaans"
@@ -26998,7 +26690,7 @@
     },
     "lij": {
       "english_name": "Ligurian",
-      "name": "L\u00edguru"
+      "name": "L\u00ecgure"
     },
     "lld": {
       "english_name": "Ladin",
@@ -27428,6 +27120,10 @@
       "english_name": "Tamil",
       "name": "\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"
     },
+    "tay": {
+      "english_name": "Tayal",
+      "name": "Tayal"
+    },
     "tcy": {
       "english_name": "Tulu",
       "name": "\u0ca4\u0cc1\u0cb3\u0cc1"
@@ -27476,6 +27172,10 @@
       "english_name": "Turkish",
       "name": "T\u00fcrk\u00e7e"
     },
+    "trv": {
+      "english_name": "Seediq",
+      "name": "Taroko"
+    },
     "ts": {
       "english_name": "Tsonga",
       "name": "Xitsonga"
@@ -28232,7 +27932,7 @@
     },
     "lij": {
       "english_name": "Ligurian",
-      "name": "L\u00edguru"
+      "name": "L\u00ecgure"
     },
     "lld": {
       "english_name": "Ladin",
@@ -28662,6 +28362,10 @@
       "english_name": "Tamil",
       "name": "\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"
     },
+    "tay": {
+      "english_name": "Tayal",
+      "name": "Tayal"
+    },
     "tcy": {
       "english_name": "Tulu",
       "name": "\u0ca4\u0cc1\u0cb3\u0cc1"
@@ -28710,6 +28414,10 @@
       "english_name": "Turkish",
       "name": "T\u00fcrk\u00e7e"
     },
+    "trv": {
+      "english_name": "Seediq",
+      "name": "Taroko"
+    },
     "ts": {
       "english_name": "Tsonga",
       "name": "Xitsonga"
@@ -28874,41 +28582,7 @@
     "sv",
     "th",
     "tr",
-    "zh-CHS",
-    "zh-CHT"
-  ],
-  "yahoo news": [
-    "ar",
-    "bg",
-    "cs",
-    "da",
-    "de",
-    "el",
-    "en",
-    "es",
-    "et",
-    "fi",
-    "fr",
-    "he",
-    "hr",
-    "hu",
-    "it",
-    "ja",
-    "ko",
-    "lt",
-    "lv",
-    "nl",
-    "no",
-    "pl",
-    "pt",
-    "ro",
-    "ru",
-    "sk",
-    "sl",
-    "sv",
-    "th",
-    "tr",
-    "zh-CHS",
-    "zh-CHT"
+    "zh_chs",
+    "zh_cht"
   ]
 }

searx/data/useragents.json

@@ -1,9 +1,9 @@
 {
     "versions": [
-        "86.0",
-        "85.0.2",
-        "85.0.1",
-        "85.0"
+        "111.0.1",
+        "111.0",
+        "110.0.1",
+        "110.0"
     ],
     "os": [
         "Windows NT 10.0; WOW64",

searx/data/wikidata_units.json

@@ -4,6 +4,8 @@
     "Q100149279": "°We",
     "Q100995": "lb",
     "Q101194838": "GHz/V",
+    "Q101427873": "pk (US)",
+    "Q101427917": "pk (UK)",
     "Q101463141": "ym²",
     "Q101463237": "zm²",
     "Q101463321": "am²",
@@ -19,12 +21,8 @@
     "Q101464875": "Ym²",
     "Q101515060": "g/J",
     "Q101875087": "cd/cm²",
-    "Q101877596": "g/ml",
     "Q101879174": "dm/s",
     "Q102068844": "cm⁻³",
-    "Q102129339": "min⁻¹",
-    "Q102129428": "h⁻¹",
-    "Q102129592": "d⁻¹",
     "Q102130673": "ym/s",
     "Q102130674": "zm/s",
     "Q102130677": "am/s",
@@ -61,7 +59,6 @@
     "Q102130773": "Em/s²",
     "Q102130775": "Zm/s²",
     "Q102130777": "Ym/s²",
-    "Q102178883": "dm³/h",
     "Q1022113": "cm³",
     "Q102573": "Bq",
     "Q103246": "Sv",
@@ -71,6 +68,9 @@
     "Q1040427": "hs",
     "Q104117265": "Bi",
     "Q1042866": "Zib",
+    "Q104628312": "vars",
+    "Q104629193": "m/Es²",
+    "Q104816263": "kg/dm³",
     "Q104907398": "μN m",
     "Q104907399": "mN m",
     "Q1050958": "inHg",
@@ -79,28 +79,166 @@
     "Q1054140": "Mm",
     "Q10543042": "Ym",
     "Q105519288": "B SPL",
-    "Q105687125": "eV⁻¹ m⁻³",
     "Q1057069": "hg",
+    "Q105761866": "mV/K",
+    "Q105816142": "tsp",
+    "Q105816269": "cup (US)",
+    "Q105840138": "BU",
     "Q1063756": "rad/s",
     "Q1063786": "in²",
     "Q1065153": "mrad",
+    "Q106589841": "MHz/T",
+    "Q106611785": "g/dm³",
+    "Q106611903": "J/g",
     "Q1066138": "Ps",
+    "Q106617289": "GBq/kg",
+    "Q106617513": "mGy/s",
+    "Q106617558": "mSv/s",
+    "Q106617559": "μSv/s",
+    "Q106617560": "nSv/s",
+    "Q106617579": "kBq/kg",
+    "Q106617585": "MBq/kg",
+    "Q106623539": "Mg/m³",
+    "Q106623548": "μg/m³",
+    "Q106623562": "kg/cm³",
+    "Q106623580": "kg/kmol",
+    "Q106623615": "kg cm²",
+    "Q106623620": "kg mm²",
+    "Q106639792": "dm³/kg",
+    "Q106645176": "dz",
+    "Q106645210": "mg/g",
+    "Q106645232": "g/mm",
+    "Q106645237": "kg/mm",
+    "Q106645241": "mg/m",
+    "Q106645245": "kg/km",
+    "Q106645257": "MN m",
+    "Q106645261": "kN m",
+    "Q106645290": "dN m",
     "Q1067722": "Fg",
+    "Q106777906": "μS/m",
+    "Q106777917": "S/cm",
+    "Q106777923": "mS/cm",
+    "Q106777925": "MS/m",
+    "Q106777933": "kS/m",
+    "Q106777943": "nS/m",
+    "Q106777952": "nS/cm",
+    "Q106777957": "pS/m",
+    "Q106808096": "kJ/g",
+    "Q106808114": "GC/m³",
+    "Q106808120": "C/mm³",
+    "Q106808129": "MC/m³",
+    "Q106808138": "C/cm³",
+    "Q106808144": "kC/m³",
+    "Q106808151": "mC/m³",
+    "Q106808156": "μC/m³",
+    "Q106808167": "MC/m²",
+    "Q106808174": "C/mm²",
+    "Q106808180": "C/cm²",
+    "Q106808187": "kC/m²",
+    "Q106808194": "mC/m²",
+    "Q106808200": "μC/m²",
+    "Q106808221": "mC/kg",
+    "Q106885926": "N/mm",
+    "Q106886632": "hbar",
+    "Q106886776": "N/mm²",
     "Q1069725": "p.",
+    "Q106998070": "cm³/m³",
+    "Q106998079": "dm³/m³",
+    "Q107028522": "μg/hg",
+    "Q107028836": "MK⁻¹",
+    "Q107133620": "mg/m²",
+    "Q107133637": "g/cm²",
+    "Q107133667": "mg/cm²",
+    "Q107133676": "g/m²",
+    "Q107164998": "cd mm²/m²",
+    "Q107210119": "g/s",
+    "Q107210344": "mg/s",
+    "Q107226391": "cm⁻¹",
     "Q1072404": "K",
+    "Q107244316": "mm⁻¹",
+    "Q107244475": "dm⁻¹",
+    "Q107244557": "km⁻¹",
+    "Q107256474": "l.",
+    "Q107299747": "kvar",
+    "Q107299751": "Mvar",
+    "Q107313731": "μg/kg",
+    "Q107313738": "ng/kg",
+    "Q107313800": "cm³/s",
+    "Q107325155": "kWb/m",
+    "Q107325173": "Wb/mm",
+    "Q107361007": "kJ/s",
+    "Q107361082": "MJ/m³",
+    "Q107361092": "J/cm²",
+    "Q107361171": "mW/m²",
+    "Q107361180": "μW/m²",
+    "Q107361187": "pW/m²",
+    "Q107378499": "kN/m²",
+    "Q107410680": "cN m",
+    "Q107410689": "N cm",
+    "Q107410785": "g/mm²",
+    "Q107410801": "g/(cm s)",
+    "Q107410895": "kJ/hg",
+    "Q107440604": "W/cm²",
+    "Q107440662": "mmol/kg",
+    "Q107440685": "mmol/g",
+    "Q107440698": "kmol/kg",
+    "Q107440839": "g/g",
+    "Q107440910": "g/hg",
+    "Q107441004": "mg/hg",
+    "Q107460729": "mm³/mm³",
+    "Q107460790": "kg/GJ",
+    "Q107460866": "g/m",
+    "Q107460882": "mg/km",
+    "Q107461064": "MJ/m²",
+    "Q107461092": "g/(m² s)",
+    "Q107461119": "μg/(m² s)",
+    "Q107461139": "μg/J",
+    "Q107461146": "g/MJ",
+    "Q107538710": "μH/m",
+    "Q107538724": "nH/m",
+    "Q107970291": "mol/dm³",
     "Q1084321": "Tb/s",
     "Q1086691": "fg",
+    "Q108730765": "kW a",
+    "Q108888186": "eV/c²",
+    "Q108888198": "keV/c²",
+    "Q108888206": "MeV/c²",
+    "Q108888224": "GeV/c²",
     "Q1091257": "tex",
     "Q1092296": "a",
+    "Q110143852": "Ω cm",
+    "Q110143896": "cm³/g",
     "Q1104069": "$",
     "Q11061003": "μm²",
     "Q11061005": "nm²",
+    "Q110742003": "dppx",
     "Q1131660": "st",
     "Q1137675": "cr",
+    "Q114002440": "𒄀",
+    "Q114002534": "𒃻",
+    "Q114002639": "𒈨𒊑",
+    "Q114002796": "𒂆",
+    "Q114002930": "𒀺",
+    "Q114002955": "𒀹𒃷",
+    "Q114002974": "𒃷",
     "Q1140444": "Zb",
     "Q1140577": "Yb",
+    "Q114589269": "A",
     "Q1152074": "Pb",
     "Q1152323": "Tb",
+    "Q115277430": "QB",
+    "Q115280832": "RB",
+    "Q115359862": "qg",
+    "Q115359863": "rg",
+    "Q115359865": "Rg",
+    "Q115359866": "Qg",
+    "Q115359910": "Rm",
+    "Q115533751": "rm",
+    "Q115533764": "qm",
+    "Q115533776": "Qm",
+    "Q116432446": "ᵐ",
+    "Q116432563": "ˢ",
+    "Q116443090": "ʰ",
     "Q1165799": "mil",
     "Q11776930": "Mg",
     "Q11830636": "psf",
@@ -114,16 +252,19 @@
     "Q11982288": "Zm³",
     "Q11982289": "Tm³",
     "Q12011178": "Zs",
+    "Q12034595": "oz (ap.)",
     "Q1204894": "Gib",
     "Q12257695": "Eb/s",
     "Q12257696": "EB/s",
     "Q12261466": "kB/s",
+    "Q12263659": "mgal",
     "Q12265780": "Pb/s",
     "Q12265783": "PB/s",
     "Q12269121": "Yb/s",
     "Q12269122": "YB/s",
     "Q12269308": "Zb/s",
     "Q12269309": "ZB/s",
+    "Q1238720": "vols.",
     "Q1247300": "cm H₂O",
     "Q12714022": "cwt",
     "Q12789864": "GeV",
@@ -133,6 +274,7 @@
     "Q130964": "cal",
     "Q131255": "F",
     "Q13147228": "g/cm³",
+    "Q131723": "₿",
     "Q1322380": "Ts",
     "Q1323615": "oz t",
     "Q132643": "kr",
@@ -152,6 +294,7 @@
     "Q1396128": "F",
     "Q1413142": "Gb",
     "Q14158377": "A_P",
+    "Q1427899": "U",
     "Q14623803": "MDa",
     "Q14623804": "kDa",
     "Q1472674": "S",
@@ -162,8 +305,6 @@
     "Q14914907": "th",
     "Q14916719": "Gpc",
     "Q14923662": "Pm³",
-    "Q1501273": "HU",
-    "Q1511773": "LSd",
     "Q15120301": "l atm",
     "Q1542309": "xu",
     "Q1545979": "ft³",
@@ -171,8 +312,8 @@
     "Q15551713": "Sh",
     "Q1569733": "St",
     "Q15784325": "apc",
-    "Q160680": "Br",
     "Q160857": "hp",
+    "Q162525": "°E",
     "Q1628990": "hph",
     "Q163343": "T",
     "Q163354": "H",
@@ -185,7 +326,6 @@
     "Q17255465": "v_P",
     "Q173117": "R$",
     "Q1741429": "kpm",
-    "Q174467": "Lm",
     "Q174728": "cm",
     "Q174789": "mm",
     "Q175821": "μm",
@@ -198,6 +338,7 @@
     "Q177974": "atm",
     "Q178506": "bbl",
     "Q178674": "nm",
+    "Q1790908": "mi3",
     "Q1793863": "sn",
     "Q179836": "lx",
     "Q180154": "km/h",
@@ -208,14 +349,13 @@
     "Q182429": "m/s",
     "Q1826195": "dl",
     "Q18413919": "cm/s",
-    "Q184172": "F",
     "Q185078": "a",
     "Q185153": "erg",
     "Q185648": "Torr",
     "Q185759": "span",
     "Q1872619": "zs",
-    "Q189097": "₧",
     "Q190095": "Gy",
+    "Q19017495": "mm²",
     "Q190951": "S$",
     "Q191118": "t",
     "Q1913097": "fg",
@@ -229,6 +369,7 @@
     "Q194339": "B$",
     "Q1970718": "mam",
     "Q1972579": "pdl",
+    "Q19877834": "cd-ft",
     "Q199462": "LE",
     "Q199471": "Afs",
     "Q200323": "dm",
@@ -239,6 +380,7 @@
     "Q2029519": "hl",
     "Q203567": "₦",
     "Q2042279": "m H₂O",
+    "Q204992": "L.",
     "Q2051195": "GWh",
     "Q2055118": "ppb",
     "Q2064166": "fc",
@@ -250,6 +392,7 @@
     "Q208528": "gon",
     "Q208634": "kat",
     "Q208788": "fm",
+    "Q2090348": "Kib/s",
     "Q209351": "b",
     "Q209426": "′",
     "Q21006887": "ppm",
@@ -259,14 +402,13 @@
     "Q21061369": "g/kg",
     "Q21062777": "MPa",
     "Q21064807": "kPa",
-    "Q21064845": "mol/L",
     "Q21075844": "ml/l",
     "Q21077820": "mg/m³",
     "Q21091747": "mg/kg",
     "Q211256": "mi/h",
     "Q21154419": "PD",
     "Q211580": "BTU (th)",
-    "Q212120": "A h",
+    "Q212120": "A⋅h",
     "Q213005": "G$",
     "Q2140397": "in³",
     "Q214377": "ell",
@@ -295,18 +437,17 @@
     "Q2269250": "kb/s",
     "Q2282891": "μl",
     "Q2282906": "ng",
+    "Q22934083": "nC",
     "Q229354": "Ci",
     "Q232291": "mi²",
     "Q2332346": "ml",
     "Q235729": "y (365 days)",
-    "Q23808021": "oz (ap.)",
     "Q23823681": "TW",
     "Q23925410": "gal (UK)",
     "Q23925413": "gal (US)",
     "Q23931040": "dam²",
     "Q23931103": "nmi²",
     "Q240468": "syr£",
-    "Q2414435": "$b.",
     "Q242988": "Lib$",
     "Q2438073": "ag",
     "Q2448803": "mV",
@@ -318,6 +459,7 @@
     "Q249439": "q_P",
     "Q2518569": "nSv",
     "Q253276": "mi",
+    "Q254532": "deg²",
     "Q25472681": "GB/s",
     "Q25472693": "TB/s",
     "Q25499149": "oct",
@@ -328,6 +470,7 @@
     "Q260126": "rem",
     "Q2612219": "Pg",
     "Q261247": "ct",
+    "Q26162546": "mm²/s",
     "Q2619500": "foe",
     "Q2636421": "nH",
     "Q2637946": "dal",
@@ -348,6 +491,7 @@
     "Q2756030": "pF",
     "Q2757753": "PW h",
     "Q2762458": "ys",
+    "Q2784622": "T",
     "Q27864215": "μW h",
     "Q2793566": "GV",
     "Q27949241": "R",
@@ -381,6 +525,7 @@
     "Q3013059": "ka",
     "Q304479": "tr",
     "Q305896": "DPI",
+    "Q3095010": "γ",
     "Q31889818": "ppq",
     "Q3194304": "kb",
     "Q3207456": "mW",
@@ -420,7 +565,7 @@
     "Q3773454": "Mpc",
     "Q3815076": "Kib",
     "Q3833309": "£",
-    "Q3858002": "mA h",
+    "Q3858002": "mA⋅h",
     "Q3867152": "ft/s²",
     "Q389062": "Tib",
     "Q3902688": "pl",
@@ -445,7 +590,6 @@
     "Q4041686": "in H20",
     "Q4068266": "Ʒ",
     "Q4176683": "aC",
-    "Q420266": "fl oz",
     "Q42319606": "people/m²",
     "Q4243638": "km³",
     "Q4456994": "mF",
@@ -458,6 +602,7 @@
     "Q4861171": "H",
     "Q494083": "fur",
     "Q4989854": "kJ",
+    "Q4992853": "kt",
     "Q500515": "Gal",
     "Q5042194": "£",
     "Q50808017": "kg m²",
@@ -481,6 +626,8 @@
     "Q53393868": "GJ",
     "Q53393886": "PJ",
     "Q53393890": "EJ",
+    "Q53393893": "ZJ",
+    "Q53393898": "YJ",
     "Q53448786": "yHz",
     "Q53448790": "zHz",
     "Q53448794": "fHz",
@@ -494,6 +641,7 @@
     "Q53448826": "hHz",
     "Q53448828": "yJ",
     "Q53448832": "zJ",
+    "Q53448835": "fJ",
     "Q53448842": "pJ",
     "Q53448844": "nJ",
     "Q53448847": "μJ",
@@ -556,6 +704,7 @@
     "Q53951982": "Mt",
     "Q53952048": "kt",
     "Q54006645": "ZWb",
+    "Q54081354": "ZT",
     "Q54081925": "ZSv",
     "Q54082468": "ZS",
     "Q54083144": "ZΩ",
@@ -580,8 +729,6 @@
     "Q56157046": "nmol",
     "Q56157048": "pmol",
     "Q56160603": "fmol",
-    "Q56302633": "UM",
-    "Q56317116": "mgal",
     "Q56317622": "Q_P",
     "Q56318907": "kbar",
     "Q56349362": "Bs.S",
@@ -618,7 +765,7 @@
     "Q651336": "M_J",
     "Q6517513": "dag",
     "Q667419": "UK t",
-    "Q681996": "M⊕",
+    "Q681996": "M🜨",
     "Q685662": "p_P",
     "Q686163": "$",
     "Q68725821": "°Rø",
@@ -661,7 +808,6 @@
     "Q70444514": "Ymol",
     "Q70444609": "Pmol",
     "Q712226": "km²",
-    "Q717310": "Mg",
     "Q72081071": "MeV",
     "Q723733": "ms",
     "Q730251": "ft·lbf",
@@ -670,12 +816,13 @@
     "Q7350781": "Mb/s",
     "Q7398951": "PPI",
     "Q743895": "bpm",
-    "Q748716": "ft/s",
-    "Q750178": "‱",
+    "Q748716": "fps",
     "Q752079": "RT",
     "Q752197": "kJ/mol",
+    "Q7574000": "sp",
     "Q7672057": "TU",
     "Q777017": "dBm",
+    "Q780456": "Td",
     "Q78754556": "rot",
     "Q78756901": "r",
     "Q78757683": "windings",
@@ -726,6 +873,7 @@
     "Q87262709": "kΩ",
     "Q87416053": "MΩ",
     "Q88296091": "tsp",
+    "Q89187604": "bbl (US)",
     "Q89473028": "bu (UK)",
     "Q89662131": "pt (UK)",
     "Q901492": "ph",
@@ -739,9 +887,9 @@
     "Q914151": "P_P",
     "Q915169": "F_P",
     "Q93318": "M",
+    "Q933427": "B",
     "Q93678895": "gill (US)",
     "Q93679498": "gill (UK)",
-    "Q940052": "q",
     "Q94076025": "dalm",
     "Q94076717": "dakat",
     "Q942092": "BWI$",
@@ -863,7 +1011,6 @@
     "Q95379580": "hC",
     "Q95379588": "dC",
     "Q95379596": "EC",
-    "Q95445986": "nC",
     "Q95446327": "pC",
     "Q95446670": "fC",
     "Q95447079": "zC",
@@ -1043,9 +1190,6 @@
     "Q97143849": "Y°C",
     "Q97143851": "a°C",
     "Q98492214": "den",
-    "Q98538634": "eV/m²",
-    "Q98635536": "eV/m",
-    "Q98642859": "eV m²/kg",
     "Q98793302": "qt (UK)",
     "Q98793408": "liq qt (US)",
     "Q98793687": "dry qt (US)",
@@ -1065,7 +1209,6 @@
     "Q11582": "L",
     "Q12129": "pc",
     "Q12438": "N",
-    "Q16068": "DM",
     "Q1811": "AU",
     "Q20764": "Ma",
     "Q2101": "e",
@@ -1103,5 +1246,6 @@
     "Q573": "d",
     "Q577": "a",
     "Q7727": "min",
-    "Q8799": "B"
+    "Q8799": "B",
+    "Q8805": "bit"
 }

searx/engines/__init__.py

@@ -110,8 +110,8 @@ def load_engine(engine_data):
             sys.exit(1)
 
     # assign supported languages from json file
-    if engine_data['name'] in ENGINES_LANGUAGES:
-        setattr(engine, 'supported_languages', ENGINES_LANGUAGES[engine_data['name']])
+    if engine_data['engine'] in ENGINES_LANGUAGES:
+        setattr(engine, 'supported_languages', ENGINES_LANGUAGES[engine_data['engine']])
 
     # find custom aliases for non standard language codes
     if hasattr(engine, 'supported_languages'):
@@ -142,7 +142,7 @@ def load_engine(engine_data):
 
     engine.stats = {
         'sent_search_count': 0,  # sent search
-        'search_count': 0,  # succesful search
+        'search_count': 0,  # successful search
         'result_count': 0,
         'engine_time': 0,
         'engine_time_count': 0,
@@ -171,7 +171,7 @@ def load_engine(engine_data):
         categories.setdefault(category_name, []).append(engine)
 
     if engine.shortcut in engine_shortcuts:
-        logger.error('Engine config error: ambigious shortcut: {0}'.format(engine.shortcut))
+        logger.error('Engine config error: ambiguous shortcut: {0}'.format(engine.shortcut))
         sys.exit(1)
 
     engine_shortcuts[engine.shortcut] = engine.name
@@ -277,7 +277,7 @@ def get_engines_stats(preferences):
 
 
 def load_engines(engine_list):
-    global engines, engine_shortcuts
+    global engines, engine_shortcuts  # pylint: disable=global-variable-not-assigned
     engines.clear()
     engine_shortcuts.clear()
     for engine_data in engine_list:

searx/engines/ahmia.py

@@ -9,7 +9,7 @@ from searx.engines.xpath import extract_url, extract_text, eval_xpath_list, eval
 
 # about
 about = {
-    "website": 'http://msydqstlz2kzerdg.onion',
+    "website": 'http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion',
     "wikidata_id": 'Q18693938',
     "official_api_documentation": None,
     "use_official_api": False,
@@ -23,7 +23,7 @@ paging = True
 page_size = 10
 
 # search url
-search_url = 'http://msydqstlz2kzerdg.onion/search/?{query}'
+search_url = 'http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion/search/?{query}'
 time_range_support = True
 time_range_dict = {'day': 1,
                    'week': 7,

searx/engines/bandcamp.py

@@ -0,0 +1,73 @@
+"""
+Bandcamp (Music)
+
+@website     https://bandcamp.com/
+@provide-api no
+@results     HTML
+@parse       url, title, content, publishedDate, embedded, thumbnail
+"""
+
+from urllib.parse import urlencode, urlparse, parse_qs
+from dateutil.parser import parse as dateparse
+from lxml import html
+from searx.utils import extract_text
+
+categories = ['music']
+paging = True
+
+base_url = "https://bandcamp.com/"
+search_string = search_string = 'search?{query}&page={page}'
+embedded_url = '''<iframe width="100%" height="166"
+    scrolling="no" frameborder="no"
+    data-src="https://bandcamp.com/EmbeddedPlayer/{type}={result_id}/size=large/bgcol=ffffff/linkcol=0687f5/tracklist=false/artwork=small/transparent=true/"
+></iframe>'''
+
+
+def request(query, params):
+    '''pre-request callback
+    params<dict>:
+      method  : POST/GET
+      headers : {}
+      data    : {} # if method == POST
+      url     : ''
+      category: 'search category'
+      pageno  : 1 # number of the requested page
+    '''
+
+    search_path = search_string.format(
+        query=urlencode({'q': query}),
+        page=params['pageno'])
+
+    params['url'] = base_url + search_path
+
+    return params
+
+
+def response(resp):
+    '''post-response callback
+    resp: requests response object
+    '''
+    results = []
+    tree = html.fromstring(resp.text)
+    search_results = tree.xpath('//li[contains(@class, "searchresult")]')
+    for result in search_results:
+        link = result.xpath('.//div[@class="itemurl"]/a')[0]
+        result_id = parse_qs(urlparse(link.get('href')).query)["search_item_id"][0]
+        title = result.xpath('.//div[@class="heading"]/a/text()')
+        date = dateparse(result.xpath('//div[@class="released"]/text()')[0].replace("released ", ""))
+        content = result.xpath('.//div[@class="subhead"]/text()')
+        new_result = {
+            "url": extract_text(link),
+            "title": extract_text(title),
+            "content": extract_text(content),
+            "publishedDate": date,
+        }
+        thumbnail = result.xpath('.//div[@class="art"]/img/@src')
+        if thumbnail:
+            new_result['thumbnail'] = thumbnail[0]
+        if "album" in result.classes:
+            new_result["embedded"] = embedded_url.format(type='album', result_id=result_id)
+        elif "track" in result.classes:
+            new_result["embedded"] = embedded_url.format(type='track', result_id=result_id)
+        results.append(new_result)
+    return results

searx/engines/bing.py

@@ -52,6 +52,7 @@ def request(query, params):
         offset=offset)
 
     params['url'] = base_url + search_path
+    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
 
     return params
 
@@ -66,11 +67,13 @@ def response(resp):
     for result in eval_xpath(dom, '//div[@class="sa_cc"]'):
         link = eval_xpath(result, './/h3/a')[0]
         url = link.attrib.get('href')
+        pretty_url = extract_text(eval_xpath(result, './/cite'))
         title = extract_text(link)
         content = extract_text(eval_xpath(result, './/p'))
 
         # append result
         results.append({'url': url,
+                        'pretty_url': pretty_url,
                         'title': title,
                         'content': content})
 
@@ -78,11 +81,13 @@ def response(resp):
     for result in eval_xpath(dom, '//li[@class="b_algo"]'):
         link = eval_xpath(result, './/h2/a')[0]
         url = link.attrib.get('href')
+        pretty_url = extract_text(eval_xpath(result, './/cite'))
         title = extract_text(link)
         content = extract_text(eval_xpath(result, './/p'))
 
         # append result
         results.append({'url': url,
+                        'pretty_url': pretty_url,
                         'title': title,
                         'content': content})
 

searx/engines/bing_videos.py

@@ -70,7 +70,7 @@ def request(query, params):
     if params['time_range'] in time_range_dict:
         params['url'] += time_range_string.format(interval=time_range_dict[params['time_range']])
 
-    # bing videos did not like "older" versions < 70.0.1 when selectin other
+    # bing videos did not like "older" versions < 70.0.1 when selecting other
     # languages then 'en' .. very strange ?!?!
     params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:73.0.1) Gecko/20100101 Firefox/73.0.1'
 

searx/engines/command.py

@@ -35,7 +35,7 @@ def init(engine_settings):
     if 'command' not in engine_settings:
         raise ValueError('engine command : missing configuration key: command')
 
-    global command, working_dir, result_template, delimiter, parse_regex, timeout, environment_variables
+    global command, working_dir, delimiter, parse_regex, environment_variables
 
     command = engine_settings['command']
 

searx/engines/core.py

@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""CORE (science)
+
+"""
+# pylint: disable=missing-function-docstring
+
+from json import loads
+from datetime import datetime
+from urllib.parse import urlencode
+
+from searx import logger
+from searx.exceptions import SearxEngineAPIException
+
+logger = logger.getChild('CORE engine')
+
+about = {
+    "website": 'https://core.ac.uk',
+    "wikidata_id": None,
+    "official_api_documentation": 'https://core.ac.uk/documentation/api/',
+    "use_official_api": True,
+    "require_api_key": True,
+    "results": 'JSON',
+}
+
+categories = ['science']
+paging = True
+nb_per_page = 10
+
+api_key = 'unset'
+
+logger = logger.getChild('CORE engine')
+
+base_url = 'https://core.ac.uk:443/api-v2/search/'
+search_string = '{query}?page={page}&pageSize={nb_per_page}&apiKey={apikey}'
+
+def request(query, params):
+
+    if api_key == 'unset':
+        raise SearxEngineAPIException('missing CORE API key')
+
+    search_path = search_string.format(
+        query = urlencode({'q': query}),
+        nb_per_page = nb_per_page,
+        page = params['pageno'],
+        apikey = api_key,
+    )
+    params['url'] = base_url + search_path
+
+    logger.debug("query_url --> %s", params['url'])
+    return params
+
+def response(resp):
+    results = []
+    json_data = loads(resp.text)
+
+    for result in json_data['data']:
+
+        source = result['_source']
+        time = source['publishedDate'] or source['depositedDate']
+        if time :
+            date = datetime.fromtimestamp(time / 1000)
+        else:
+            date = None
+
+        metadata = []
+        if source['publisher'] and len(source['publisher']) > 3:
+            metadata.append(source['publisher'])
+        if source['topics']:
+            metadata.append(source['topics'][0])
+        if source['doi']:
+            metadata.append(source['doi'])
+        metadata = ' / '.join(metadata)
+
+        results.append({
+            'url': source['urls'][0].replace('http://', 'https://', 1),
+            'title': source['title'],
+            'content': source['description'],
+            'publishedDate': date,
+            'metadata' : metadata,
+        })
+
+    return results

searx/engines/dictzone.py

@@ -17,7 +17,7 @@ about = {
     "results": 'HTML',
 }
 
-engine_type = 'online_dictionnary'
+engine_type = 'online_dictionary'
 categories = ['general']
 url = 'https://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'
 weight = 100
@@ -27,9 +27,7 @@ https_support = True
 
 
 def request(query, params):
-    params['url'] = url.format(from_lang=params['from_lang'][2],
-                               to_lang=params['to_lang'][2],
-                               query=params['query'])
+    params['url'] = url.format(from_lang=params['from_lang'][2], to_lang=params['to_lang'][2], query=params['query'])
 
     return params
 
@@ -51,10 +49,12 @@ def response(resp):
             if t.strip():
                 to_results.append(to_result.text_content())
 
-        results.append({
-            'url': urljoin(resp.url, '?%d' % k),
-            'title': from_result.text_content(),
-            'content': '; '.join(to_results)
-        })
+        results.append(
+            {
+                'url': urljoin(str(resp.url), '?%d' % k),
+                'title': from_result.text_content(),
+                'content': '; '.join(to_results),
+            }
+        )
 
     return results

searx/engines/digg.py

@@ -4,11 +4,11 @@
 """
 # pylint: disable=missing-function-docstring
 
-from json import loads
 from urllib.parse import urlencode
 from datetime import datetime
 
 from lxml import html
+from searx.utils import eval_xpath, extract_text
 
 # about
 about = {
@@ -24,46 +24,45 @@ about = {
 categories = ['news', 'social media']
 paging = True
 base_url = 'https://digg.com'
+results_per_page = 10
 
 # search-url
 search_url = base_url + (
-    '/api/search/'
+    '/search'
     '?{query}'
-    '&from={position}'
-    '&size=20'
-    '&format=html'
+    '&size={size}'
+    '&offset={offset}'
 )
 
 def request(query, params):
-    offset = (params['pageno'] - 1) * 20
+    offset = (params['pageno'] - 1) * results_per_page + 1
     params['url'] = search_url.format(
         query = urlencode({'q': query}),
-        position = offset,
+        size = results_per_page,
+        offset = offset,
     )
     return params
 
 def response(resp):
     results = []
 
-    # parse results
-    for result in loads(resp.text)['mapped']:
+    dom = html.fromstring(resp.text)
 
-        # strip html tags and superfluous quotation marks from content
-        content = html.document_fromstring(
-            result['excerpt']
-        ).text_content()
+    results_list = eval_xpath(dom, '//section[contains(@class, "search-results")]')
 
-        # 'created': {'ISO': '2020-10-16T14:09:55Z', ...}
-        published = datetime.strptime(
-            result['created']['ISO'], '%Y-%m-%dT%H:%M:%SZ'
-        )
-        results.append({
-            'url': result['url'],
-            'title': result['title'],
-            'content' : content,
-            'template': 'videos.html',
-            'publishedDate': published,
-            'thumbnail': result['images']['thumbImage'],
-        })
+    for result in results_list:
+
+        titles = eval_xpath(result, '//article//header//h2')
+        contents = eval_xpath(result, '//article//p')
+        urls = eval_xpath(result, '//header/a/@href')
+        published_dates = eval_xpath(result, '//article/div/div/time/@datetime')
+
+        for (title, content, url, published_date) in zip(titles, contents, urls, published_dates):
+            results.append({
+                'url': url,
+                'publishedDate': datetime.strptime(published_date, '%Y-%m-%dT%H:%M:%SZ'),
+                'title': extract_text(title),
+                'content' : extract_text(content),
+            })
 
     return results

searx/engines/duckduckgo.py

@@ -1,16 +1,24 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- DuckDuckGo (Web)
+# lint: pylint
+"""DuckDuckGo Lite
 """
 
-from lxml.html import fromstring
 from json import loads
-from searx.utils import extract_text, match_language, eval_xpath, dict_subset
+
+from lxml.html import fromstring
+
+from searx.utils import (
+    dict_subset,
+    eval_xpath,
+    eval_xpath_getindex,
+    extract_text,
+    match_language,
+)
 from searx.poolrequests import get
 
 # about
 about = {
-    "website": 'https://duckduckgo.com/',
+    "website": 'https://lite.duckduckgo.com/lite/',
     "wikidata_id": 'Q12805',
     "official_api_documentation": 'https://duckduckgo.com/api',
     "use_official_api": False,
@@ -19,9 +27,9 @@ about = {
 }
 
 # engine dependent config
-categories = ['general']
-paging = False
-supported_languages_url = 'https://duckduckgo.com/util/u172.js'
+categories = ['general', 'web']
+paging = True
+supported_languages_url = 'https://duckduckgo.com/util/u588.js'
 time_range_support = True
 
 language_aliases = {
@@ -31,23 +39,14 @@ language_aliases = {
     'ko': 'kr-KR',
     'sl-SI': 'sl-SL',
     'zh-TW': 'tzh-TW',
-    'zh-HK': 'tzh-HK'
+    'zh-HK': 'tzh-HK',
 }
 
-# search-url
-url = 'https://html.duckduckgo.com/html'
-url_ping = 'https://duckduckgo.com/t/sl_h'
-time_range_dict = {'day': 'd',
-                   'week': 'w',
-                   'month': 'm',
-                   'year': 'y'}
+time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
 
-# specific xpath variables
-result_xpath = '//div[@class="result results_links results_links_deep web-result "]'  # noqa
-url_xpath = './/a[@class="result__a"]/@href'
-title_xpath = './/a[@class="result__a"]'
-content_xpath = './/a[@class="result__snippet"]'
-correction_xpath = '//div[@id="did_you_mean"]//a'
+# search-url
+url = 'https://lite.duckduckgo.com/lite/'
+url_ping = 'https://duckduckgo.com/t/sl_l'
 
 
 # match query's language to a region code that duckduckgo will accept
@@ -63,63 +62,113 @@ def get_region_code(lang, lang_list=None):
 
 
 def request(query, params):
-    if params['time_range'] is not None and params['time_range'] not in time_range_dict:
-        return params
 
     params['url'] = url
     params['method'] = 'POST'
+
     params['data']['q'] = query
-    params['data']['b'] = ''
+
+    # The API is not documented, so we do some reverse engineering and emulate
+    # what https://lite.duckduckgo.com/lite/ does when you press "next Page"
+    # link again and again ..
+
+    params['headers']['Content-Type'] = 'application/x-www-form-urlencoded'
+    params['headers']['Origin'] = 'https://lite.duckduckgo.com'
+    params['headers']['Referer'] = 'https://lite.duckduckgo.com/'
+    params['headers']['User-Agent'] = 'Mozilla/5.0'
+
+    # initial page does not have an offset
+    if params['pageno'] == 2:
+        # second page does have an offset of 30
+        offset = (params['pageno'] - 1) * 30
+        params['data']['s'] = offset
+        params['data']['dc'] = offset + 1
+
+    elif params['pageno'] > 2:
+        # third and following pages do have an offset of 30 + n*50
+        offset = 30 + (params['pageno'] - 2) * 50
+        params['data']['s'] = offset
+        params['data']['dc'] = offset + 1
+
+    # initial page does not have additional data in the input form
+    if params['pageno'] > 1:
+        # request the second page (and more pages) needs 'o' and 'api' arguments
+        params['data']['o'] = 'json'
+        params['data']['api'] = 'd.js'
+
+    # initial page does not have additional data in the input form
+    if params['pageno'] > 2:
+        # request the third page (and more pages) some more arguments
+        params['data']['nextParams'] = ''
+        params['data']['v'] = ''
+        params['data']['vqd'] = ''
 
     region_code = get_region_code(params['language'], supported_languages)
     if region_code:
         params['data']['kl'] = region_code
         params['cookies']['kl'] = region_code
 
+    params['data']['df'] = ''
     if params['time_range'] in time_range_dict:
         params['data']['df'] = time_range_dict[params['time_range']]
+        params['cookies']['df'] = time_range_dict[params['time_range']]
 
-    params['allow_redirects'] = False
     return params
 
 
 # get response from search-request
 def response(resp):
-    if resp.status_code == 303:
-        return []
 
-    # ping
     headers_ping = dict_subset(resp.request.headers, ['User-Agent', 'Accept-Encoding', 'Accept', 'Cookie'])
     get(url_ping, headers=headers_ping)
 
-    # parse the response
+    if resp.status_code == 303:
+        return []
+
     results = []
     doc = fromstring(resp.text)
-    for i, r in enumerate(eval_xpath(doc, result_xpath)):
-        if i >= 30:
-            break
-        try:
-            res_url = eval_xpath(r, url_xpath)[-1]
-        except:
+
+    result_table = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table')
+    if not len(result_table) >= 3:
+        # no more results
+        return []
+    result_table = result_table[2]
+
+    tr_rows = eval_xpath(result_table, './/tr')
+
+    # In the last <tr> is the form of the 'previous/next page' links
+    tr_rows = tr_rows[:-1]
+
+    len_tr_rows = len(tr_rows)
+    offset = 0
+
+    while len_tr_rows >= offset + 4:
+
+        # assemble table rows we need to scrap
+        tr_title = tr_rows[offset]
+        tr_content = tr_rows[offset + 1]
+        offset += 4
+
+        # ignore sponsored Adds <tr class="result-sponsored">
+        if tr_content.get('class') == 'result-sponsored':
             continue
 
-        if not res_url:
+        a_tag = eval_xpath_getindex(tr_title, './/td//a[@class="result-link"]', 0, None)
+        if a_tag is None:
             continue
 
-        title = extract_text(eval_xpath(r, title_xpath))
-        content = extract_text(eval_xpath(r, content_xpath))
+        td_content = eval_xpath_getindex(tr_content, './/td[@class="result-snippet"]', 0, None)
+        if td_content is None:
+            continue
 
-        # append result
-        results.append({'title': title,
-                        'content': content,
-                        'url': res_url})
+        results.append(
+            {
+                'title': a_tag.text_content(),
+                'content': extract_text(td_content),
+                'url': a_tag.get('href'),
+            }
+        )
 
-    # parse correction
-    for correction in eval_xpath(doc, correction_xpath):
-        # append correction
-        results.append({'correction': extract_text(correction)})
-
-    # return results
     return results
 
 
@@ -129,7 +178,7 @@ def _fetch_supported_languages(resp):
     # response is a js file with regions as an embedded object
     response_page = resp.text
     response_page = response_page[response_page.find('regions:{') + 8:]
-    response_page = response_page[:response_page.find('}') + 1]
+    response_page = response_page[: response_page.find('}') + 1]
 
     regions_json = loads(response_page)
     supported_languages = map((lambda x: x[3:] + '-' + x[:2].upper()), regions_json.keys())

searx/engines/duckduckgo_definitions.py

@@ -80,7 +80,7 @@ def response(resp):
     # * book / performing art / film / television  / media franchise / concert tour / playwright
     # * prepared food
     # * website / software / os / programming language / file format / software engineer
-    # * compagny
+    # * company
 
     content = ''
     heading = search_res.get('Heading', '')

searx/engines/emojipedia.py

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ Emojipedia
+"""
+
+from urllib.parse import urlencode
+from lxml import html
+
+from searx import logger
+from searx.utils import (
+    eval_xpath_list,
+    eval_xpath_getindex,
+    extract_text,
+)
+
+logger = logger.getChild('Emojipedia engine')
+
+about = {
+    "website": 'https://emojipedia.org',
+    "wikidata_id": None,
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
+categories = []
+paging = False
+time_range_support = False
+
+base_url = 'https://emojipedia.org'
+search_url = base_url + '/search/?{query}'
+
+
+def request(query, params):
+    params['url'] = search_url.format(
+        query=urlencode({'q': query}),
+    )
+    return params
+
+
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    for result in eval_xpath_list(dom, "//ol[@class='search-results']/li"):
+
+        extracted_desc = extract_text(eval_xpath_getindex(result, './/p', 0))
+
+        if 'No results found.' in extracted_desc:
+            break
+
+        link = eval_xpath_getindex(result, './/h2/a', 0)
+
+        url = base_url + link.attrib.get('href')
+        title = extract_text(link)
+        content = extracted_desc
+
+        res = {
+            'url': url,
+            'title': title,
+            'content': content
+        }
+
+        results.append(res)
+
+    return results

searx/engines/gigablast.py

@@ -5,9 +5,9 @@
 # pylint: disable=missing-function-docstring, invalid-name
 
 import re
-from json import loads
+from json import loads, JSONDecodeError
 from urllib.parse import urlencode
-# from searx import logger
+from searx.exceptions import SearxEngineResponseException
 from searx.poolrequests import get
 
 # about
@@ -22,6 +22,9 @@ about = {
 
 # engine dependent config
 categories = ['general']
+collections = 'main'
+search_type = ''
+fast = 0
 # gigablast's pagination is totally damaged, don't use it
 paging = False
 safesearch = True
@@ -34,6 +37,8 @@ base_url = 'https://gigablast.com'
 extra_param = ''
 extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'
 
+_wait_for_results_msg = 'Loading results takes too long. Please enable fast option in gigablast engine.'
+
 def parse_extra_param(text):
 
     # example:
@@ -54,7 +59,6 @@ def parse_extra_param(text):
         if re_var is not None and re_var.search(line):
             extra_param += re_var.search(line).group(1)
             break
-    # logger.debug('gigablast extra_param="%s"', extra_param)
 
 def init(engine_settings=None):  # pylint: disable=unused-argument
     parse_extra_param(get(base_url + extra_param_path).text)
@@ -65,14 +69,17 @@ def request(query, params):  # pylint: disable=unused-argument
 
     # see API http://www.gigablast.com/api.html#/search
     # Take into account, that the API has some quirks ..
+    query_args = {
+        'c': collections,
+        'format': 'json',
+        'q': query,
+        'dr': 1 ,
+        'showgoodimages': 0,
+        'fast': fast,
+    }
 
-    query_args = dict(
-        c = 'main'
-        , format = 'json'
-        , q = query
-        , dr = 1
-        , showgoodimages = 0
-    )
+    if search_type != '':
+        query_args['searchtype'] = search_type
 
     if params['language'] and params['language'] != 'all':
         query_args['qlangcountry'] = params['language']
@@ -90,9 +97,13 @@ def request(query, params):  # pylint: disable=unused-argument
 def response(resp):
     results = []
 
-    response_json = loads(resp.text)
+    try:
+        response_json = loads(resp.text)
+    except JSONDecodeError as e:
+        if 'Waiting for results' in resp.text:
+            raise SearxEngineResponseException(message=_wait_for_results_msg)  # pylint: disable=raise-missing-from
+        raise e
 
-    # logger.debug('gigablast returns %s results', len(response_json['results']))
 
     for result in response_json['results']:
         # see "Example JSON Output (&format=json)"

searx/engines/github.py

@@ -40,7 +40,7 @@ def response(resp):
 
     search_res = loads(resp.text)
 
-    # check if items are recieved
+    # check if items are received
     if 'items' not in search_res:
         return []
 

searx/engines/google.py

@@ -8,9 +8,10 @@ Definitions`_.
    https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
 """
 
-# pylint: disable=invalid-name, missing-function-docstring
+# pylint: disable=invalid-name, missing-function-docstring, too-many-branches
 
 from urllib.parse import urlencode, urlparse
+from random import random
 from lxml import html
 from searx import logger
 from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
@@ -33,6 +34,7 @@ categories = ['general']
 paging = True
 time_range_support = True
 safesearch = True
+use_mobile_ui = False
 supported_languages_url = 'https://www.google.com/preferences?#languages'
 
 # based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests
@@ -107,21 +109,15 @@ filter_mapping = {
 # specific xpath variables
 # ------------------------
 
-# google results are grouped into <div class="g" ../>
-results_xpath = '//div[@class="g"]'
+results_xpath = '//div[contains(@class, "MjjYud")]'
+title_xpath = './/h3[1]'
+href_xpath = './/a/@href'
+content_xpath = './/div[@data-sncf]'
+results_xpath_mobile_ui = '//div[contains(@class, "g ")]'
 
 # google *sections* are no usual *results*, we ignore them
 g_section_with_header = './g-section-with-header'
 
-# the title is a h3 tag relative to the result group
-title_xpath = './/h3[1]'
-
-# in the result group there is <div class="yuRUbf" ../> it's first child is a <a
-# href=...>
-href_xpath = './/div[@class="yuRUbf"]//a/@href'
-
-# in the result group there is <div class="IsZvec" ../> containing he *content*
-content_xpath = './/div[@class="IsZvec"]'
 
 # Suggestions are links placed in a *card-section*, we extract only the text
 # from the links not the links itself.
@@ -132,11 +128,12 @@ suggestion_xpath = '//div[contains(@class, "card-section")]//a'
 spelling_suggestion_xpath = '//div[@class="med"]/p/a'
 
 
-def get_lang_info(params, lang_list, custom_aliases):
+def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
     ret_val = {}
 
     _lang = params['language']
-    if _lang.lower() == 'all':
+    _any_language = _lang.lower() == 'all'
+    if _any_language:
         _lang = 'en-US'
 
     language = match_language(_lang, lang_list, custom_aliases)
@@ -158,31 +155,36 @@ def get_lang_info(params, lang_list, custom_aliases):
     # the combination (en-US, en-EN, de-DE, de-AU, fr-FR, fr-FR)
     lang_country = '%s-%s' % (language, country)
 
-    # Accept-Language: fr-CH, fr;q=0.8, en;q=0.6, *;q=0.5
-    ret_val['Accept-Language'] = ','.join([
-        lang_country,
-        language + ';q=0.8,',
-        'en;q=0.6',
-        '*;q=0.5',
-    ])
-
     # subdomain
     ret_val['subdomain']  = 'www.' + google_domains.get(country.upper(), 'google.com')
 
+    ret_val['params'] = {}
+    ret_val['headers'] = {}
+
+    if _any_language and supported_any_language:
+        # based on whoogle
+        ret_val['params']['source'] = 'lnt'
+    else:
+        # Accept-Language: fr-CH, fr;q=0.8, en;q=0.6, *;q=0.5
+        ret_val['headers']['Accept-Language'] = ','.join([
+            lang_country,
+            language + ';q=0.8,',
+            'en;q=0.6',
+            '*;q=0.5',
+        ])
+
+        # lr parameter:
+        #   https://developers.google.com/custom-search/docs/xml_results#lrsp
+        # Language Collection Values:
+        #   https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections
+        ret_val['params']['lr'] = "lang_" + lang_country if lang_country in lang_list else language
+
+    ret_val['params']['hl'] = lang_country if lang_country in lang_list else language
+
     # hl parameter:
     #   https://developers.google.com/custom-search/docs/xml_results#hlsp The
     # Interface Language:
     #   https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
-
-    ret_val['hl'] = lang_list.get(lang_country, language)
-
-    # lr parameter:
-    #   https://developers.google.com/custom-search/docs/xml_results#lrsp
-    # Language Collection Values:
-    #   https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections
-
-    ret_val['lr'] = "lang_" + lang_list.get(lang_country, language)
-
     return ret_val
 
 def detect_google_sorry(resp):
@@ -198,17 +200,26 @@ def request(query, params):
 
     lang_info = get_lang_info(
         # pylint: disable=undefined-variable
-        params, supported_languages, language_aliases
+        params, supported_languages, language_aliases, True
     )
 
+    additional_parameters = {}
+    if use_mobile_ui:
+        additional_parameters = {
+            'asearch': 'arc',
+            'async': 'use_ac:true,_fmt:html',
+        }
+
     # https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
     query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
         'q': query,
-        'hl': lang_info['hl'],
-        'lr': lang_info['lr'],
+        **lang_info['params'],
         'ie': "utf8",
         'oe': "utf8",
         'start': offset,
+        'filter': '0',
+        'ucbcb': 1,
+        **additional_parameters,
     })
 
     if params['time_range'] in time_range_dict:
@@ -219,11 +230,15 @@ def request(query, params):
     logger.debug("query_url --> %s", query_url)
     params['url'] = query_url
 
-    logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
-    params['headers']['Accept-Language'] = lang_info['Accept-Language']
-    params['headers']['Accept'] = (
-        'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
-    )
+    logger.debug("HTTP header Accept-Language --> %s", lang_info.get('Accept-Language'))
+    params['cookies']['CONSENT'] = "PENDING+" + str(random()*100)
+    params['headers'].update(lang_info['headers'])
+    if use_mobile_ui:
+        params['headers']['Accept'] = '*/*'
+    else:
+        params['headers']['Accept'] = (
+            'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+        )
 
     return params
 
@@ -239,52 +254,62 @@ def response(resp):
     dom = html.fromstring(resp.text)
 
     # results --> answer
-    answer = eval_xpath(dom, '//div[contains(@class, "LGOjhe")]//text()')
-    if answer:
-        results.append({'answer': ' '.join(answer)})
+    answer_list = eval_xpath(dom, '//div[contains(@class, "LGOjhe")]')
+    if answer_list:
+        answer_list = [_.xpath("normalize-space()") for _ in answer_list]
+        results.append({'answer': ' '.join(answer_list)})
     else:
-        logger.debug("did not found 'answer'")
+        logger.debug("did not find 'answer'")
 
     # results --> number_of_results
-        try:
-            _txt = eval_xpath_getindex(dom, '//div[@id="result-stats"]//text()', 0)
-            _digit = ''.join([n for n in _txt if n.isdigit()])
-            number_of_results = int(_digit)
-            results.append({'number_of_results': number_of_results})
-        except Exception as e:  # pylint: disable=broad-except
-            logger.debug("did not 'number_of_results'")
-            logger.error(e, exc_info=True)
+        if not use_mobile_ui:
+            try:
+                _txt = eval_xpath_getindex(dom, '//div[@id="result-stats"]//text()', 0)
+                _digit = ''.join([n for n in _txt if n.isdigit()])
+                number_of_results = int(_digit)
+                results.append({'number_of_results': number_of_results})
+            except Exception as e:  # pylint: disable=broad-except
+                logger.debug("did not 'number_of_results'")
+                logger.error(e, exc_info=True)
 
     # parse results
-    for result in eval_xpath_list(dom, results_xpath):
+
+    _results_xpath = results_xpath
+    if use_mobile_ui:
+        _results_xpath = results_xpath_mobile_ui
+
+    for result in eval_xpath_list(dom, _results_xpath):
 
         # google *sections*
         if extract_text(eval_xpath(result, g_section_with_header)):
-            logger.debug("ingoring <g-section-with-header>")
+            logger.debug("ignoring <g-section-with-header>")
             continue
 
         try:
             title_tag = eval_xpath_getindex(result, title_xpath, 0, default=None)
             if title_tag is None:
                 # this not one of the common google results *section*
-                logger.debug('ingoring <div class="g" ../> section: missing title')
+                logger.debug('ingoring item from the result_xpath list: missing title')
                 continue
             title = extract_text(title_tag)
             url = eval_xpath_getindex(result, href_xpath, 0, None)
             if url is None:
                 continue
             content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True)
+            if content is None:
+                logger.debug('ingoring item from the result_xpath list: missing content of title "%s"', title)
+                continue
+
+            logger.debug('add link to results: %s', title)
+
             results.append({
                 'url': url,
                 'title': title,
                 'content': content
             })
+
         except Exception as e:  # pylint: disable=broad-except
             logger.error(e, exc_info=True)
-            # from lxml import etree
-            # logger.debug(etree.tostring(result, pretty_print=True))
-            # import pdb
-            # pdb.set_trace()
             continue
 
     # parse suggestion

searx/engines/google_images.py

@@ -70,7 +70,7 @@ filter_mapping = {
 def scrap_out_thumbs(dom):
     """Scrap out thumbnail data from <script> tags.
     """
-    ret_val = dict()
+    ret_val = {}
     for script in eval_xpath(dom, '//script[contains(., "_setImgSrc(")]'):
         _script = script.text
         # _setImgSrc('0','data:image\/jpeg;base64,\/9j\/4AAQSkZJR ....');
@@ -88,7 +88,7 @@ def scrap_img_by_id(script, data_id):
     img_url = ''
     _script = script.split('\n')
     for i, line in enumerate(_script):
-        if 'gstatic.com/images' in line and data_id in line:
+        if 'gstatic.com/images' in line and data_id in line and i + 1 < len(_script):
             url_line = _script[i + 1]
             img_url = url_line.split('"')[1]
             img_url = unquote(img_url.replace(r'\u00', r'%'))
@@ -100,16 +100,16 @@ def request(query, params):
 
     lang_info = get_lang_info(
         # pylint: disable=undefined-variable
-        params, supported_languages, language_aliases
+        params, supported_languages, language_aliases, False
     )
 
     query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
         'q': query,
         'tbm': "isch",
-        'hl': lang_info['hl'],
-        'lr': lang_info['lr'],
+        **lang_info['params'],
         'ie': "utf8",
         'oe': "utf8",
+        'ucbcd': 1,
         'num': 30,
     })
 
@@ -121,8 +121,9 @@ def request(query, params):
     logger.debug("query_url --> %s", query_url)
     params['url'] = query_url
 
-    logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
-    params['headers']['Accept-Language'] = lang_info['Accept-Language']
+    logger.debug("HTTP header Accept-Language --> %s", lang_info.get('Accept-Language'))
+    params['cookies']['CONSENT'] = "YES+"
+    params['headers'].update(lang_info['headers'])
     params['headers']['Accept'] = (
         'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
     )

searx/engines/google_news.py

@@ -2,7 +2,7 @@
 """Google (News)
 
 For detailed description of the *REST-full* API see: `Query Parameter
-Definitions`_.  Not all parameters can be appied:
+Definitions`_.  Not all parameters can be applied:
 
 - num_ : the number of search results is ignored
 - save_ : is ignored / Google-News results are always *SafeSearch*
@@ -21,6 +21,7 @@ import binascii
 import re
 from urllib.parse import urlencode
 from base64 import b64decode
+from random import random
 from lxml import html
 
 from searx import logger
@@ -82,7 +83,7 @@ def request(query, params):
 
     lang_info = get_lang_info(
         # pylint: disable=undefined-variable
-        params, supported_languages, language_aliases
+        params, supported_languages, language_aliases, False
     )
 
     # google news has only one domain
@@ -91,8 +92,8 @@ def request(query, params):
     ceid = "%s:%s" % (lang_info['country'], lang_info['language'])
 
     # google news redirects en to en-US
-    if lang_info['hl'] == 'en':
-        lang_info['hl'] = 'en-US'
+    if lang_info['params']['hl'] == 'en':
+        lang_info['params']['hl'] = 'en-US'
 
     # Very special to google-news compared to other google engines, the time
     # range is included in the search term.
@@ -101,21 +102,23 @@ def request(query, params):
 
     query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
         'q': query,
-        'hl': lang_info['hl'],
-        'lr': lang_info['lr'],
+        **lang_info['params'],
         'ie': "utf8",
         'oe': "utf8",
+        'ucbcb': 1,
         'gl': lang_info['country'],
     }) + ('&ceid=%s' % ceid)  # ceid includes a ':' character which must not be urlencoded
 
     logger.debug("query_url --> %s", query_url)
     params['url'] = query_url
 
-    logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
-    params['headers']['Accept-Language'] = lang_info['Accept-Language']
+    logger.debug("HTTP header Accept-Language --> %s", lang_info.get('Accept-Language'))
+
+    params['cookies']['CONSENT'] = "PENDING+" + str(random()*100)
+    params['headers'].update(lang_info['headers'])
     params['headers']['Accept'] = (
         'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
-        )
+    )
 
     return params
 
@@ -152,7 +155,7 @@ def response(resp):
                 padding = (4 -(len(jslog) % 4)) * "="
                 jslog = b64decode(jslog + padding)
             except binascii.Error:
-                # URL cant be read, skip this result
+                # URL can't be read, skip this result
                 continue
 
             # now we have : b'[null, ... null,"https://www.cnn.com/.../index.html"]'

searx/engines/google_play_apps.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+  Google Play Apps
+"""
+
+from urllib.parse import urlencode
+from lxml import html
+from searx.utils import (
+    eval_xpath,
+    extract_url,
+    extract_text,
+    eval_xpath_list,
+    eval_xpath_getindex,
+)
+
+about = {
+    "website": "https://play.google.com/",
+    "wikidata_id": "Q79576",
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "HTML",
+}
+
+categories = ["files", "apps"]
+search_url = "https://play.google.com/store/search?{query}&c=apps&ucbcb=1"
+
+
+def request(query, params):
+    params["url"] = search_url.format(query=urlencode({"q": query}))
+    params['cookies']['CONSENT'] = "YES+"
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    if eval_xpath(dom, '//div[@class="v6DsQb"]'):
+        return []
+
+    spot = eval_xpath_getindex(dom, '//div[@class="ipRz4"]', 0, None)
+    if spot is not None:
+        url = extract_url(eval_xpath(spot, './a[@class="Qfxief"]/@href'), search_url)
+        title = extract_text(eval_xpath(spot, './/div[@class="vWM94c"]'))
+        content = extract_text(eval_xpath(spot, './/div[@class="LbQbAe"]'))
+        img = extract_text(eval_xpath(spot, './/img[@class="T75of bzqKMd"]/@src'))
+
+        results.append({"url": url, "title": title, "content": content, "img_src": img})
+
+    more = eval_xpath_list(dom, '//c-wiz[@jsrenderer="RBsfwb"]//div[@role="listitem"]', min_len=1)
+    for result in more:
+        url = extract_url(eval_xpath(result, ".//a/@href"), search_url)
+        title = extract_text(eval_xpath(result, './/span[@class="DdYX5"]'))
+        content = extract_text(eval_xpath(result, './/span[@class="wMUdtb"]'))
+        img = extract_text(
+            eval_xpath(
+                result,
+                './/img[@class="T75of stzEZd" or @class="T75of etjhNc Q8CSx "]/@src',
+            )
+        )
+
+        results.append({"url": url, "title": title, "content": content, "img_src": img})
+
+    for suggestion in eval_xpath_list(dom, '//c-wiz[@jsrenderer="qyd4Kb"]//div[@class="ULeU3b neq64b"]'):
+        results.append({"suggestion": extract_text(eval_xpath(suggestion, './/div[@class="Epkrse "]'))})
+
+    return results

searx/engines/google_scholar.py

@@ -12,6 +12,7 @@ Definitions`_.
 
 from urllib.parse import urlencode
 from datetime import datetime
+from random import random
 from lxml import html
 from searx import logger
 
@@ -80,27 +81,27 @@ def request(query, params):
 
         # params, {}, language_aliases
 
-        params, supported_languages, language_aliases
+        params, supported_languages, language_aliases, False
     )
     # subdomain is: scholar.google.xy
     lang_info['subdomain'] = lang_info['subdomain'].replace("www.", "scholar.")
 
-    query_url = 'https://'+ lang_info['subdomain'] + '/scholar' + "?" + urlencode({
-        'q':  query,
-        'hl': lang_info['hl'],
-        'lr': lang_info['lr'],
-        'ie': "utf8",
-        'oe':  "utf8",
-        'start' : offset,
-    })
+    query_url = (
+        'https://'
+        + lang_info['subdomain']
+        + '/scholar'
+        + "?"
+        + urlencode({'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'start': offset, 'ucbcb': 1})
+    )
 
     query_url += time_range_url(params)
 
     logger.debug("query_url --> %s", query_url)
     params['url'] = query_url
 
-    logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
-    params['headers']['Accept-Language'] = lang_info['Accept-Language']
+    logger.debug("HTTP header Accept-Language --> %s", lang_info.get('Accept-Language'))
+    params['cookies']['CONSENT'] = "PENDING+" + str(random()*100)
+    params['headers'].update(lang_info['headers'])
     params['headers']['Accept'] = (
         'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
     )

searx/engines/google_videos.py

@@ -2,7 +2,7 @@
 """Google (Video)
 
 For detailed description of the *REST-full* API see: `Query Parameter
-Definitions`_.  Not all parameters can be appied.
+Definitions`_.  Not all parameters can be applied.
 
 .. _admonition:: Content-Security-Policy (CSP)
 
@@ -22,6 +22,7 @@ Definitions`_.  Not all parameters can be appied.
 
 import re
 from urllib.parse import urlencode
+from random import random
 from lxml import html
 
 from searx import logger
@@ -84,7 +85,7 @@ def _re(regexpr):
 def scrap_out_thumbs(dom):
     """Scrap out thumbnail data from <script> tags.
     """
-    ret_val = dict()
+    ret_val = {}
     thumb_name = 'vidthumb'
 
     for script in eval_xpath_list(dom, '//script[contains(., "_setImagesSrc")]'):
@@ -118,14 +119,14 @@ def request(query, params):
 
     lang_info = get_lang_info(
         # pylint: disable=undefined-variable
-        params, supported_languages, language_aliases
+        params, supported_languages, language_aliases, False
     )
 
     query_url = 'https://' + lang_info['subdomain'] + '/search' + "?" + urlencode({
         'q':   query,
         'tbm': "vid",
-        'hl': lang_info['hl'],
-        'lr': lang_info['lr'],
+        **lang_info['params'],
+        'ucbcb': 1,
         'ie': "utf8",
         'oe': "utf8",
     })
@@ -138,8 +139,9 @@ def request(query, params):
     logger.debug("query_url --> %s", query_url)
     params['url'] = query_url
 
-    logger.debug("HTTP header Accept-Language --> %s", lang_info['Accept-Language'])
-    params['headers']['Accept-Language'] = lang_info['Accept-Language']
+    logger.debug("HTTP header Accept-Language --> %s", lang_info.get('Accept-Language'))
+    params['cookies']['CONSENT'] = "PENDING+" + str(random()*100)
+    params['headers'].update(lang_info['headers'])
     params['headers']['Accept'] = (
         'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
         )
@@ -161,7 +163,7 @@ def response(resp):
 
         # google *sections*
         if extract_text(eval_xpath(result, g_section_with_header)):
-            logger.debug("ingoring <g-section-with-header>")
+            logger.debug("ignoring <g-section-with-header>")
             continue
 
         title = extract_text(eval_xpath_getindex(result, title_xpath, 0))

searx/engines/imdb.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+"""
+IMDB - Internet Movie Database
+Retrieves results from a basic search
+Advanced search options are not supported
+"""
+
+import json
+
+about = {
+    "website": 'https://imdb.com/',
+    "wikidata_id": 'Q37312',
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
+categories = ['general']
+paging = False
+base_url = 'https://imdb.com/{category}/{id}'
+suggestion_url = "https://v2.sg.media-imdb.com/suggestion/{letter}/{query}.json"
+search_categories = {
+    "nm": "name",
+    "tt": "title",
+    "kw": "keyword",
+    "co": "company",
+    "ep": "episode"
+}
+
+
+def request(query, params):
+    query = query.replace(" ", "_").lower()
+    params['url'] = suggestion_url.format(letter=query[0], query=query)
+    return params
+
+
+def response(resp):
+    suggestions = json.loads(resp.text)
+    results = []
+    for entry in suggestions['d']:
+        content = ""
+        if entry['id'][:2] in search_categories:
+            href = base_url.format(category=search_categories[entry['id'][:2]], id=entry['id'])
+        if 'y' in entry:
+            content += str(entry['y']) + " - "
+        if 's' in entry:
+            content += entry['s']
+        results.append({
+            "title": entry['l'],
+            "url": href,
+            "content": content
+        })
+    return results

searx/engines/invidious.py

@@ -10,9 +10,9 @@ import random
 
 # about
 about = {
-    "website": 'https://instances.invidio.us/',
+    "website": 'https://api.invidious.io/',
     "wikidata_id": 'Q79343316',
-    "official_api_documentation": 'https://github.com/omarroth/invidious/wiki/API',
+    "official_api_documentation": 'https://github.com/iv-org/documentation/blob/master/API.md',
     "use_official_api": True,
     "require_api_key": False,
     "results": 'JSON',

searx/engines/ipfs_search.py

@@ -0,0 +1,93 @@
+from urllib.parse import urlencode
+import json
+import re
+from datetime import datetime
+
+# config
+categories = ['general', 'images', 'music', 'videos']
+paging = True
+time_range_support = True
+
+# url
+base_url = 'https://api.ipfs-search.com/v1/'
+search_string = 'search?{query} first-seen:{time_range} metadata.Content-Type:({mime_type})&page={page} '
+
+
+mime_types_map = {
+    'general': "*",
+    'images': 'image*',
+    'music': 'audio*',
+    'videos': 'video*'
+}
+
+time_range_map = {'day': '[ now-24h\/h TO *]',
+                  'week': '[ now\/h-7d TO *]',
+                  'month': '[ now\/d-30d TO *]',
+                  'year': '[ now\/d-1y TO *]'}
+
+ipfs_url = 'https://gateway.ipfs.io/ipfs/{hash}'
+
+
+def request(query, params):
+    mime_type = mime_types_map.get(params['category'], '*')
+    time_range = time_range_map.get(params['time_range'], '*')
+    search_path = search_string.format(
+        query=urlencode({'q': query}),
+        time_range=time_range,
+        page=params['pageno'],
+        mime_type=mime_type)
+
+    params['url'] = base_url + search_path
+
+    return params
+
+
+def clean_html(text):
+    if not text:
+        return ""
+    return str(re.sub(re.compile('<.*?>'), '', text))
+
+
+def create_base_result(record):
+    url = ipfs_url.format(hash=record.get('hash'))
+    title = clean_html(record.get('title'))
+    published_date = datetime.strptime(record.get('first-seen'), '%Y-%m-%dT%H:%M:%SZ')
+    return {'url': url,
+            'title': title,
+            'publishedDate': published_date}
+
+
+def create_text_result(record):
+    result = create_base_result(record)
+    description = clean_html(record.get('description'))
+    result['description'] = description
+    return result
+
+
+def create_image_result(record):
+    result = create_base_result(record)
+    result['img_src'] = result['url']
+    result['template'] = 'images.html'
+    return result
+
+
+def create_video_result(record):
+    result = create_base_result(record)
+    result['thumbnail'] = ''
+    result['template'] = 'videos.html'
+    return result
+
+
+def response(resp):
+    api_results = json.loads(resp.text)
+    results = []
+    for result in api_results.get('hits', []):
+        mime_type = result.get('mimetype', 'text/plain')
+
+        if mime_type.startswith('image'):
+            results.append(create_image_result(result))
+        elif mime_type.startswith('video'):
+            results.append(create_video_result(result))
+        else:
+            results.append(create_text_result(result))
+    return results

searx/engines/lingva.py

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""Lingva (alternative Google Translate frontend)"""
+
+from json import loads
+
+about = {
+    "website": 'https://lingva.ml',
+    "wikidata_id": None,
+    "official_api_documentation": 'https://github.com/thedaviddelta/lingva-translate#public-apis',
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": 'JSON',
+}
+
+engine_type = 'online_dictionary'
+categories = ['general']
+
+url = "https://lingva.ml"
+search_url = "{url}/api/v1/{from_lang}/{to_lang}/{query}"
+
+
+def request(_query, params):
+    params['url'] = search_url.format(
+        url=url, from_lang=params['from_lang'][1], to_lang=params['to_lang'][1], query=params['query']
+    )
+    return params
+
+
+def response(resp):
+    results = []
+
+    result = loads(resp.text)
+    info = result["info"]
+    from_to_prefix = "%s-%s " % (resp.search_params['from_lang'][1], resp.search_params['to_lang'][1])
+
+    if "typo" in info:
+        results.append({"suggestion": from_to_prefix + info["typo"]})
+
+    if 'definitions' in info:  # pylint: disable=too-many-nested-blocks
+        for definition in info['definitions']:
+            if 'list' in definition:
+                for item in definition['list']:
+                    if 'synonyms' in item:
+                        for synonym in item['synonyms']:
+                            results.append({"suggestion": from_to_prefix + synonym})
+
+    infobox = ""
+
+    for translation in info["extraTranslations"]:
+        infobox += f"<b>{translation['type']}</b>"
+
+        for word in translation["list"]:
+            infobox += f"<dl><dt>{word['word']}</dt>"
+
+            for meaning in word["meanings"]:
+                infobox += f"<dd>{meaning}</dd>"
+
+            infobox += "</dl>"
+
+    results.append(
+        {
+            'infobox': result["translation"],
+            'content': infobox,
+        }
+    )
+
+    return results

searx/engines/meilisearch.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ Meilisearch
+"""
+
+# pylint: disable=global-statement, missing-function-docstring
+
+from json import loads, dumps
+
+
+base_url = 'http://localhost:7700'
+index = ''
+auth_key = ''
+facet_filters = []
+_search_url = ''
+result_template = 'key-value.html'
+categories = ['general']
+paging = True
+
+
+def init(_):
+    if index == '':
+        raise ValueError('index cannot be empty')
+
+    global _search_url
+    _search_url = base_url + '/indexes/' + index + '/search'
+
+
+def request(query, params):
+    if auth_key != '':
+        params['headers']['X-Meili-API-Key'] = auth_key
+
+    params['headers']['Content-Type'] = 'application/json'
+    params['url'] = _search_url
+    params['method'] = 'POST'
+
+    data = {
+        'q': query,
+        'offset': 10 * (params['pageno'] - 1),
+        'limit': 10,
+    }
+    if len(facet_filters) > 0:
+        data['facetFilters'] = facet_filters
+
+    params['data'] = dumps(data)
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    resp_json = loads(resp.text)
+    for result in resp_json['hits']:
+        r = {key: str(value) for key, value in result.items()}
+        r['template'] = result_template
+        results.append(r)
+
+    return results

searx/engines/mongodb.py

@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ MongoDB engine (Offline)
+"""
+
+# pylint: disable=missing-function-docstring
+# pylint: disable=import-error
+
+import re
+from pymongo import MongoClient
+
+engine_type = 'offline'
+paging = True
+# mongodb connection variables
+host = '127.0.0.1'
+port = 27017
+username = ''
+password = ''
+database = None
+collection = None
+key = None
+# engine specific variables
+results_per_page = 20
+exact_match_only = False
+result_template = 'key-value.html'
+
+_client = None
+
+
+def init(_):
+    connect()
+
+
+def connect():
+    global _client
+    kwargs = {
+        'port': port,
+    }
+    if username:
+        kwargs['username'] = username
+    if password:
+        kwargs['password'] = password
+    _client = MongoClient(host, **kwargs)[database][collection]
+
+
+def search(query, params):
+    ret = []
+    if exact_match_only:
+        q = {'$eq': query}
+    else:
+        q = {'$regex': re.compile('.*{0}.*'.format(re.escape(query)), re.I | re.M)}
+    results = _client.find({key: q})\
+        .skip((params['pageno'] - 1) * results_per_page)\
+        .limit(results_per_page)
+    ret.append({'number_of_results': results.count()})
+    for r in results:
+        del r['_id']
+        r = {str(k): str(v) for k, v in r.items()}
+        r['template'] = result_template
+        ret.append(r)
+    return ret

searx/engines/mysql_server.py

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ MySQL database (Offline)
+"""
+
+# error is ignored because the admin has to
+# install it manually to use the engine
+# pylint: disable=import-error
+
+import mysql.connector
+
+
+engine_type = 'offline'
+auth_plugin = 'caching_sha2_password'
+host = "127.0.0.1"
+port = 3306
+database = ""
+username = ""
+password = ""
+query_str = ""
+limit = 10
+paging = True
+result_template = 'key-value.html'
+_connection = None
+
+
+def init(engine_settings):
+    if 'query_str' not in engine_settings:
+        raise ValueError('query_str cannot be empty')
+
+    if not engine_settings['query_str'].lower().startswith('select '):
+        raise ValueError('only SELECT query is supported')
+
+    global _connection
+    _connection = mysql.connector.connect(
+        database=database,
+        user=username,
+        password=password,
+        host=host,
+        port=port,
+        auth_plugin=auth_plugin,
+    )
+
+
+def search(query, params):
+    query_params = {'query': query}
+    query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
+
+    with _connection.cursor() as cur:
+        cur.execute(query_to_run, query_params)
+
+        return _fetch_results(cur)
+
+
+def _fetch_results(cur):
+    results = []
+    for res in cur:
+        result = dict(zip(cur.column_names, map(str, res)))
+        result['template'] = result_template
+        results.append(result)
+
+    return results

searx/engines/not_evil.py

@@ -1,67 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- not Evil (Onions)
-"""
-
-from urllib.parse import urlencode
-from lxml import html
-from searx.engines.xpath import extract_text
-
-# about
-about = {
-    "website": 'http://hss3uro2hsxfogfq.onion',
-    "wikidata_id": None,
-    "official_api_documentation": 'http://hss3uro2hsxfogfq.onion/api.htm',
-    "use_official_api": False,
-    "require_api_key": False,
-    "results": 'HTML',
-}
-
-# engine dependent config
-categories = ['onions']
-paging = True
-page_size = 20
-
-# search-url
-base_url = 'http://hss3uro2hsxfogfq.onion/'
-search_url = 'index.php?{query}&hostLimit=20&start={pageno}&numRows={page_size}'
-
-# specific xpath variables
-results_xpath = '//*[@id="content"]/div/p'
-url_xpath = './span[1]'
-title_xpath = './a[1]'
-content_xpath = './text()'
-
-
-# do search-request
-def request(query, params):
-    offset = (params['pageno'] - 1) * page_size
-
-    params['url'] = base_url + search_url.format(pageno=offset,
-                                                 query=urlencode({'q': query}),
-                                                 page_size=page_size)
-
-    return params
-
-
-# get response from search-request
-def response(resp):
-    results = []
-
-    # needed because otherwise requests guesses wrong encoding
-    resp.encoding = 'utf8'
-    dom = html.fromstring(resp.text)
-
-    # parse results
-    for result in dom.xpath(results_xpath):
-        url = extract_text(result.xpath(url_xpath)[0])
-        title = extract_text(result.xpath(title_xpath)[0])
-        content = extract_text(result.xpath(content_xpath))
-
-        # append result
-        results.append({'url': url,
-                        'title': title,
-                        'content': content,
-                        'is_onion': True})
-
-    return results

searx/engines/omnom.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ Omnom (General)
+"""
+
+from json import loads
+from urllib.parse import urlencode
+
+# about
+about = {
+    "website": 'https://github.com/asciimoo/omnom',
+    "wikidata_id": None,
+    "official_api_documentation": 'http://your.omnom.host/api',
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": 'JSON',
+}
+
+# engine dependent config
+categories = ['general']
+paging = True
+
+# search-url
+base_url = None
+search_path = 'bookmarks?{query}&pageno={pageno}&format=json'
+bookmark_path = 'bookmark?id='
+
+
+# do search-request
+def request(query, params):
+    params['url'] = base_url +\
+        search_path.format(query=urlencode({'query': query}),
+                           pageno=params['pageno'])
+
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    json = loads(resp.text)
+
+    # parse results
+    for r in json.get('Bookmarks', {}):
+        content = r['url']
+        if r.get('notes'):
+            content += ' - ' + r['notes']
+        results.append({
+            'title': r['title'],
+            'content': content,
+            'url': base_url + bookmark_path + str(r['id']),
+        })
+
+    # return results
+    return results

searx/engines/onesearch.py

@@ -0,0 +1,57 @@
+
+"""Onesearch
+"""
+
+from lxml.html import fromstring
+
+import re
+
+from searx.utils import (
+    eval_xpath,
+    extract_text,
+)
+
+from urllib.parse import unquote
+
+# about
+about = {
+    "website": 'https://www.onesearch.com/',
+    "wikidata_id": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
+# engine dependent config
+categories = ['general']
+paging = True
+
+# search-url
+URL = 'https://www.onesearch.com/yhs/search;?p=%s&b=%d'
+
+
+def request(query, params):
+    starting_from = (params['pageno'] * 10) - 9
+    params['url'] = URL % (query, starting_from)
+    return params
+
+
+# get response from search-request
+def response(resp):
+
+    results = []
+    doc = fromstring(resp.text)
+
+    titles_tags = eval_xpath(doc, '//div[contains(@class, "algo")]//h3[contains(@class, "title")]')
+    contents = eval_xpath(doc, '//div[contains(@class, "algo")]/div[contains(@class, "compText")]/p')
+    onesearch_urls = eval_xpath(doc, '//div[contains(@class, "algo")]//h3[contains(@class, "title")]/a/@href')
+
+    for title_tag, content, onesearch_url in zip(titles_tags, contents, onesearch_urls):
+        matches = re.search(r'RU=(.*?)\/', onesearch_url)
+        results.append({
+            'title': title_tag.text_content(),
+            'content': extract_text(content),
+            'url': unquote(matches.group(1)),
+        })
+
+    return results

searx/engines/openstreetmap.py

@@ -1,12 +1,22 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""OpenStreetMap (Map)
+
 """
- OpenStreetMap (Map)
-"""
+# pylint: disable=missing-function-docstring
 
 import re
 from json import loads
+from urllib.parse import urlencode
+from functools import partial
+
 from flask_babel import gettext
 
+from searx.data import OSM_KEYS_TAGS, CURRENCIES
+from searx.utils import searx_useragent
+from searx.external_urls import get_external_url
+from searx.engines.wikidata import send_wikidata_query, sparql_string_escape
+
 # about
 about = {
     "website": 'https://www.openstreetmap.org/',
@@ -20,29 +30,129 @@ about = {
 # engine dependent config
 categories = ['map']
 paging = False
+language_support = True
 
 # search-url
 base_url = 'https://nominatim.openstreetmap.org/'
-search_string = 'search/{query}?format=json&polygon_geojson=1&addressdetails=1'
-result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
+search_string = 'search?{query}&polygon_geojson=1&format=jsonv2&addressdetails=1&extratags=1&dedupe=1'
+result_id_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
+result_lat_lon_url = 'https://www.openstreetmap.org/?mlat={lat}&mlon={lon}&zoom={zoom}&layers=M'
 
-route_url = 'https://graphhopper.com/maps/?point={}&point={}&locale=en-US&vehicle=car&weighting=fastest&turn_costs=true&use_miles=false&layer=Omniscale'  # noqa
+route_url = 'https://graphhopper.com/maps/?point={}&point={}&locale=en-US&vehicle=car&weighting=fastest&turn_costs=true&use_miles=false&layer=Omniscale'  # NOQA
 route_re = re.compile('(?:from )?(.+) to (.+)')
 
+wikidata_image_sparql = """
+select ?item ?itemLabel ?image ?sign ?symbol ?website ?wikipediaName
+where {
+  values ?item { %WIKIDATA_IDS% }
+  OPTIONAL { ?item wdt:P18|wdt:P8517|wdt:P4291|wdt:P5252|wdt:P3451|wdt:P4640|wdt:P5775|wdt:P2716|wdt:P1801|wdt:P4896 ?image }
+  OPTIONAL { ?item wdt:P1766|wdt:P8505|wdt:P8667 ?sign }
+  OPTIONAL { ?item wdt:P41|wdt:P94|wdt:P154|wdt:P158|wdt:P2910|wdt:P4004|wdt:P5962|wdt:P8972 ?symbol }
+  OPTIONAL { ?item wdt:P856 ?website }
+  SERVICE wikibase:label {
+    bd:serviceParam wikibase:language "%LANGUAGE%,en".
+    ?item rdfs:label ?itemLabel .
+  }
+  OPTIONAL {
+    ?wikipediaUrl schema:about ?item;
+                  schema:isPartOf/wikibase:wikiGroup "wikipedia";
+                  schema:name ?wikipediaName;
+                  schema:inLanguage "%LANGUAGE%" .
+  }
+}
+ORDER by ?item
+"""  # NOQA
+
+
+# key value that are link: mapping functions
+# 'mapillary': P1947
+# but https://github.com/kartaview/openstreetcam.org/issues/60
+# but https://taginfo.openstreetmap.org/keys/kartaview ...
+def value_to_https_link(value):
+    http = 'http://'
+    if value.startswith(http):
+        value = 'https://' + value[len(http):]
+    return (value, value)
+
+
+def value_to_website_link(value):
+    value = value.split(';')[0]
+    return (value, value)
+
+
+def value_wikipedia_link(value):
+    value = value.split(':', 1)
+    return ('https://{0}.wikipedia.org/wiki/{1}'.format(*value), '{1} ({0})'.format(*value))
+
+
+def value_with_prefix(prefix, value):
+    return (prefix + value, value)
+
+
+VALUE_TO_LINK = {
+    'website': value_to_website_link,
+    'contact:website': value_to_website_link,
+    'email': partial(value_with_prefix, 'mailto:'),
+    'contact:email': partial(value_with_prefix, 'mailto:'),
+    'contact:phone': partial(value_with_prefix, 'tel:'),
+    'phone': partial(value_with_prefix, 'tel:'),
+    'fax': partial(value_with_prefix, 'fax:'),
+    'contact:fax': partial(value_with_prefix, 'fax:'),
+    'contact:mastodon': value_to_https_link,
+    'facebook': value_to_https_link,
+    'contact:facebook': value_to_https_link,
+    'contact:foursquare': value_to_https_link,
+    'contact:instagram': value_to_https_link,
+    'contact:linkedin': value_to_https_link,
+    'contact:pinterest': value_to_https_link,
+    'contact:telegram': value_to_https_link,
+    'contact:tripadvisor': value_to_https_link,
+    'contact:twitter': value_to_https_link,
+    'contact:yelp': value_to_https_link,
+    'contact:youtube': value_to_https_link,
+    'contact:webcam': value_to_website_link,
+    'wikipedia': value_wikipedia_link,
+    'wikidata': partial(value_with_prefix, 'https://wikidata.org/wiki/'),
+    'brand:wikidata': partial(value_with_prefix, 'https://wikidata.org/wiki/'),
+}
+KEY_ORDER = [
+    'cuisine',
+    'organic',
+    'delivery',
+    'delivery:covid19',
+    'opening_hours',
+    'opening_hours:covid19',
+    'fee',
+    'payment:*',
+    'currency:*',
+    'outdoor_seating',
+    'bench',
+    'wheelchair',
+    'level',
+    'building:levels',
+    'bin',
+    'public_transport',
+    'internet_access:ssid',
+]
+KEY_RANKS = {k: i for i, k in enumerate(KEY_ORDER)}
+
 
-# do search-request
 def request(query, params):
-
-    params['url'] = base_url + search_string.format(query=query)
+    """do search-request"""
+    params['url'] = base_url + search_string.format(query=urlencode({'q': query}))
     params['route'] = route_re.match(query)
+    params['headers']['User-Agent'] = searx_useragent()
 
+    accept_language = 'en' if params['language'] == 'all' else params['language']
+    params['headers']['Accept-Language'] = accept_language
     return params
 
 
-# get response from search-request
 def response(resp):
+    """get response from search-request"""
     results = []
-    json = loads(resp.text)
+    nominatim_json = loads(resp.text)
+    user_language = resp.search_params['language']
 
     if resp.search_params['route']:
         results.append({
@@ -50,62 +160,287 @@ def response(resp):
             'url': route_url.format(*resp.search_params['route'].groups()),
         })
 
-    # parse results
-    for r in json:
-        if 'display_name' not in r:
+    fetch_wikidata(nominatim_json, user_language)
+
+    for result in nominatim_json:
+        title, address = get_title_address(result)
+
+        # ignore result without title
+        if not title:
             continue
 
-        title = r['display_name'] or ''
-        osm_type = r.get('osm_type', r.get('type'))
-        url = result_base_url.format(osm_type=osm_type,
-                                     osm_id=r['osm_id'])
+        url, osm, geojson = get_url_osm_geojson(result)
+        img_src = get_img_src(result)
+        links, link_keys = get_links(result, user_language)
+        data = get_data(result, user_language, link_keys)
 
-        osm = {'type': osm_type,
-               'id': r['osm_id']}
+        results.append({
+            'template': 'map.html',
+            'title': title,
+            'address': address,
+            'address_label': get_key_label('addr', user_language),
+            'url': url,
+            'osm': osm,
+            'geojson': geojson,
+            'img_src': img_src,
+            'links': links,
+            'data': data,
+            'type': get_tag_label(
+                result.get('category'), result.get('type', ''), user_language
+            ),
+            'type_icon': result.get('icon'),
+            'content': '',
+            'longitude': result['lon'],
+            'latitude': result['lat'],
+            'boundingbox': result['boundingbox'],
+        })
 
-        geojson = r.get('geojson')
-
-        # if no geojson is found and osm_type is a node, add geojson Point
-        if not geojson and osm_type == 'node':
-            geojson = {'type': 'Point', 'coordinates': [r['lon'], r['lat']]}
-
-        address_raw = r.get('address')
-        address = {}
-
-        # get name
-        if r['class'] == 'amenity' or\
-           r['class'] == 'shop' or\
-           r['class'] == 'tourism' or\
-           r['class'] == 'leisure':
-            if address_raw.get('address29'):
-                address = {'name': address_raw.get('address29')}
-            else:
-                address = {'name': address_raw.get(r['type'])}
-
-        # add rest of adressdata, if something is already found
-        if address.get('name'):
-            address.update({'house_number': address_raw.get('house_number'),
-                           'road': address_raw.get('road'),
-                           'locality': address_raw.get('city',
-                                       address_raw.get('town',          # noqa
-                                       address_raw.get('village'))),    # noqa
-                           'postcode': address_raw.get('postcode'),
-                           'country': address_raw.get('country'),
-                           'country_code': address_raw.get('country_code')})
-        else:
-            address = None
-
-        # append result
-        results.append({'template': 'map.html',
-                        'title': title,
-                        'content': '',
-                        'longitude': r['lon'],
-                        'latitude': r['lat'],
-                        'boundingbox': r['boundingbox'],
-                        'geojson': geojson,
-                        'address': address,
-                        'osm': osm,
-                        'url': url})
-
-    # return results
     return results
+
+
+def get_wikipedia_image(raw_value):
+    if not raw_value:
+        return None
+    return get_external_url('wikimedia_image', raw_value)
+
+
+def fetch_wikidata(nominatim_json, user_language):
+    """Update nominatim_json using the result of an unique to wikidata
+
+    For result in nominatim_json:
+        If result['extratags']['wikidata'] or r['extratags']['wikidata link']:
+            Set result['wikidata'] to { 'image': ..., 'image_sign':..., 'image_symbal':... }
+            Set result['extratags']['wikipedia'] if not defined
+            Set result['extratags']['contact:website'] if not defined
+    """
+    wikidata_ids = []
+    wd_to_results = {}
+    for result in nominatim_json:
+        e = result.get("extratags")
+        if e:
+            # ignore brand:wikidata
+            wd_id = e.get("wikidata", e.get("wikidata link"))
+            if wd_id and wd_id not in wikidata_ids:
+                wikidata_ids.append("wd:" + wd_id)
+                wd_to_results.setdefault(wd_id, []).append(result)
+
+    if wikidata_ids:
+        user_language = 'en' if user_language == 'all' else user_language.split('-')[0]
+        wikidata_ids_str = " ".join(wikidata_ids)
+        query = wikidata_image_sparql.replace('%WIKIDATA_IDS%', sparql_string_escape(wikidata_ids_str)).replace(
+            '%LANGUAGE%', sparql_string_escape(user_language)
+        )
+        wikidata_json = send_wikidata_query(query)
+        for wd_result in wikidata_json.get('results', {}).get('bindings', {}):
+            wd_id = wd_result['item']['value'].replace('http://www.wikidata.org/entity/', '')
+            for result in wd_to_results.get(wd_id, []):
+                result['wikidata'] = {
+                    'itemLabel': wd_result['itemLabel']['value'],
+                    'image': get_wikipedia_image(wd_result.get('image', {}).get('value')),
+                    'image_sign': get_wikipedia_image(wd_result.get('sign', {}).get('value')),
+                    'image_symbol': get_wikipedia_image(wd_result.get('symbol', {}).get('value')),
+                }
+                # overwrite wikipedia link
+                wikipedia_name = wd_result.get('wikipediaName', {}).get('value')
+                if wikipedia_name:
+                    result['extratags']['wikipedia'] = user_language + ':' + wikipedia_name
+                # get website if not already defined
+                website = wd_result.get('website', {}).get('value')
+                if (
+                    website
+                    and not result['extratags'].get('contact:website')
+                    and not result['extratags'].get('website')
+                ):
+                    result['extratags']['contact:website'] = website
+
+
+def get_title_address(result):
+    """Return title and address
+
+    title may be None
+    """
+    address_raw = result.get('address')
+    address_name = None
+    address = {}
+
+    # get name
+    if (
+        result['category'] == 'amenity'
+        or result['category'] == 'shop'
+        or result['category'] == 'tourism'
+        or result['category'] == 'leisure'
+    ):
+        if address_raw.get('address29'):
+            # https://github.com/osm-search/Nominatim/issues/1662
+            address_name = address_raw.get('address29')
+        else:
+            address_name = address_raw.get(result['category'])
+    elif result['type'] in address_raw:
+        address_name = address_raw.get(result['type'])
+
+    # add rest of adressdata, if something is already found
+    if address_name:
+        title = address_name
+        address.update(
+            {
+                'name': address_name,
+                'house_number': address_raw.get('house_number'),
+                'road': address_raw.get('road'),
+                'locality': address_raw.get(
+                    'city', address_raw.get('town', address_raw.get('village'))  # noqa
+                ),  # noqa
+                'postcode': address_raw.get('postcode'),
+                'country': address_raw.get('country'),
+                'country_code': address_raw.get('country_code'),
+            }
+        )
+    else:
+        title = result.get('display_name')
+
+    return title, address
+
+
+def get_url_osm_geojson(result):
+    """Get url, osm and geojson
+    """
+    osm_type = result.get('osm_type', result.get('type'))
+    if 'osm_id' not in result:
+        # see https://github.com/osm-search/Nominatim/issues/1521
+        # query example: "EC1M 5RF London"
+        url = result_lat_lon_url.format(lat=result['lat'], lon=result['lon'], zoom=12)
+        osm = {}
+    else:
+        url = result_id_url.format(osm_type=osm_type, osm_id=result['osm_id'])
+        osm = {'type': osm_type, 'id': result['osm_id']}
+
+    geojson = result.get('geojson')
+    # if no geojson is found and osm_type is a node, add geojson Point
+    if not geojson and osm_type == 'node':
+        geojson = {'type': 'Point', 'coordinates': [result['lon'], result['lat']]}
+
+    return url, osm, geojson
+
+
+def get_img_src(result):
+    """Get image URL from either wikidata or r['extratags']"""
+    # wikidata
+    img_src = None
+    if 'wikidata' in result:
+        img_src = result['wikidata']['image']
+        if not img_src:
+            img_src = result['wikidata']['image_symbol']
+        if not img_src:
+            img_src = result['wikidata']['image_sign']
+
+    # img_src
+    if not img_src and result.get('extratags', {}).get('image'):
+        img_src = result['extratags']['image']
+        del result['extratags']['image']
+    if not img_src and result.get('extratags', {}).get('wikimedia_commons'):
+        img_src = get_external_url('wikimedia_image', result['extratags']['wikimedia_commons'])
+        del result['extratags']['wikimedia_commons']
+
+    return img_src
+
+
+def get_links(result, user_language):
+    """Return links from result['extratags']"""
+    links = []
+    link_keys = set()
+    for k, mapping_function in VALUE_TO_LINK.items():
+        raw_value = result['extratags'].get(k)
+        if raw_value:
+            url, url_label = mapping_function(raw_value)
+            if url.startswith('https://wikidata.org'):
+                url_label = result.get('wikidata', {}).get('itemLabel') or url_label
+            links.append({
+                'label': get_key_label(k, user_language),
+                'url': url,
+                'url_label': url_label,
+            })
+            link_keys.add(k)
+    return links, link_keys
+
+
+def get_data(result, user_language, ignore_keys):
+    """Return key, value of result['extratags']
+
+    Must be call after get_links
+
+    Note: the values are not translated
+    """
+    data = []
+    for k, v in result['extratags'].items():
+        if k in ignore_keys:
+            continue
+        if get_key_rank(k) is None:
+            continue
+        k_label = get_key_label(k, user_language)
+        if k_label:
+            data.append({
+                'label': k_label,
+                'key': k,
+                'value': v,
+            })
+    data.sort(key=lambda entry: (get_key_rank(entry['key']), entry['label']))
+    return data
+
+
+def get_key_rank(k):
+    """Get OSM key rank
+
+    The rank defines in which order the key are displayed in the HTML result
+    """
+    key_rank = KEY_RANKS.get(k)
+    if key_rank is None:
+        # "payment:*" in KEY_ORDER matches "payment:cash", "payment:debit card", etc...
+        key_rank = KEY_RANKS.get(k.split(':')[0] + ':*')
+    return key_rank
+
+
+def get_label(labels, lang):
+    """Get label from labels in OSM_KEYS_TAGS
+
+    in OSM_KEYS_TAGS, labels have key == '*'
+    """
+    tag_label = labels.get(lang.lower())
+    if tag_label is None:
+        # example: if 'zh-hk' is not found, check 'zh'
+        tag_label = labels.get(lang.split('-')[0])
+    if tag_label is None and lang != 'en':
+        # example: if 'zh' is not found, check 'en'
+        tag_label = labels.get('en')
+    if tag_label is None and len(labels.values()) > 0:
+        # example: if still not found, use the first entry
+        tag_label = labels.values()[0]
+    return tag_label
+
+
+def get_tag_label(tag_category, tag_name, lang):
+    """Get tag label from OSM_KEYS_TAGS"""
+    tag_name = '' if tag_name is None else tag_name
+    tag_labels = OSM_KEYS_TAGS['tags'].get(tag_category, {}).get(tag_name, {})
+    return get_label(tag_labels, lang)
+
+
+def get_key_label(key_name, lang):
+    """Get key label from OSM_KEYS_TAGS"""
+    if key_name.startswith('currency:'):
+        # currency:EUR --> get the name from the CURRENCIES variable
+        # see https://wiki.openstreetmap.org/wiki/Key%3Acurrency
+        # and for exampe https://taginfo.openstreetmap.org/keys/currency:EUR#values
+        # but there is also currency=EUR (currently not handled)
+        # https://taginfo.openstreetmap.org/keys/currency#values
+        currency = key_name.split(':')
+        if len(currency) > 1:
+            o = CURRENCIES['iso4217'].get(currency)
+            if o:
+                return get_label(o, lang).lower()
+            return currency
+
+    labels = OSM_KEYS_TAGS['keys']
+    for k in key_name.split(':') + ['*']:
+        labels = labels.get(k)
+        if labels is None:
+            return None
+    return get_label(labels, lang)

searx/engines/openverse.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """
 
- Creative Commons search engine (Images)
+ Openverse (formerly known as: Creative Commons search engine) [Images]
 
 """
 
@@ -23,8 +23,8 @@ categories = ['images']
 paging = True
 nb_per_page = 20
 
-base_url = 'https://api.creativecommons.engineering/v1/images?'
-search_string = '&page={page}&page_size={nb_per_page}&format=json&{query}'
+base_url = 'https://api.openverse.engineering/v1/images/'
+search_string = '?page={page}&page_size={nb_per_page}&format=json&{query}'
 
 
 def request(query, params):

searx/engines/photon.py

@@ -72,7 +72,7 @@ def response(resp):
         elif properties.get('osm_type') == 'R':
             osm_type = 'relation'
         else:
-            # continue if invalide osm-type
+            # continue if invalid osm-type
             continue
 
         url = result_base_url.format(osm_type=osm_type,

searx/engines/postgresql.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ PostgreSQL database (Offline)
+"""
+
+# error is ignored because the admin has to
+# install it manually to use the engine
+# pylint: disable=import-error
+
+import psycopg2
+
+engine_type = 'offline'
+host = "127.0.0.1"
+port = "5432"
+database = ""
+username = ""
+password = ""
+query_str = ""
+limit = 10
+paging = True
+result_template = 'key-value.html'
+_connection = None
+
+
+def init(engine_settings):
+    if 'query_str' not in engine_settings:
+        raise ValueError('query_str cannot be empty')
+
+    if not engine_settings['query_str'].lower().startswith('select '):
+        raise ValueError('only SELECT query is supported')
+
+    global _connection
+    _connection = psycopg2.connect(
+        database=database,
+        user=username,
+        password=password,
+        host=host,
+        port=port,
+    )
+
+
+def search(query, params):
+    query_params = {'query': query}
+    query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
+
+    with _connection:
+        with _connection.cursor() as cur:
+            cur.execute(query_to_run, query_params)
+
+            return _fetch_results(cur)
+
+
+def _fetch_results(cur):
+    results = []
+    titles = []
+
+    try:
+        titles = [column_desc.name for column_desc in cur.description]
+
+        for res in cur:
+            result = dict(zip(titles, map(str, res)))
+            result['template'] = result_template
+            results.append(result)
+
+    # no results to fetch
+    except psycopg2.ProgrammingError:
+        pass
+
+    return results

searx/engines/prowlarr.py

@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+
+ Prowlarr search (Files)
+
+"""
+
+from json import loads
+from urllib.parse import urlencode
+from searx.exceptions import SearxEngineAPIException
+
+categories = ''
+
+paging = False
+api_key = ''
+indexer_ids = ''
+search_type = 'search'
+search_categories = ''
+base_url = ''
+
+
+def request(query, params):
+    if not base_url:
+        raise SearxEngineAPIException('missing prowlarr base url')
+
+    if not api_key:
+        raise SearxEngineAPIException('missing prowlarr API key')
+
+    query_args = {
+        'query': query,
+        'apikey': api_key,
+        'type': search_type
+    }
+
+    if indexer_ids:
+        query_args['indexerIds'] = indexer_ids
+
+    if search_categories:
+        query_args['categories'] = search_categories
+
+    params['url'] = base_url + urlencode(query_args)
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    json_data = loads(resp.text)
+
+    for result in json_data:
+
+        new_result = {
+            'title': result['title'],
+            'url': result['infoUrl'],
+            'template': 'torrent.html'
+        }
+
+        if 'files' in result:
+            new_result['files'] = result['files']
+
+        if 'size' in result:
+            new_result['filesize'] = result['size']
+
+        if 'seeders' in result:
+            new_result['seed'] = result['seeders']
+
+        if 'leechers' in result:
+            new_result['leech'] = result['leechers']
+
+        if 'downloadUrl' in result:
+            new_result['torrentfile'] = result['downloadUrl']
+
+        # magnet link *may* be in guid, but it may be also identical to infoUrl
+        if 'guid' in result and isinstance(result['guid'], str) and result['guid'].startswith('magnet'):
+            new_result['magnetlink'] = result['guid']
+
+        results.append(new_result)
+
+    return results

searx/engines/qwant.py

@@ -1,15 +1,41 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- Qwant (Web, Images, News, Social)
+# lint: pylint
+"""Qwant (Web, News, Images, Videos)
+
+This engine uses the Qwant API (https://api.qwant.com/v3). The API is
+undocumented but can be reverse engineered by reading the network log of
+https://www.qwant.com/ queries.
+
+This implementation is used by different qwant engines in the settings.yml::
+
+  - name: qwant
+    categories: general
+    ...
+  - name: qwant news
+    categories: news
+    ...
+  - name: qwant images
+    categories: images
+    ...
+  - name: qwant videos
+    categories: videos
+    ...
+
 """
 
-from datetime import datetime
+from datetime import (
+    datetime,
+    timedelta,
+)
 from json import loads
 from urllib.parse import urlencode
-from searx.utils import html_to_text, match_language
-from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
+from flask_babel import gettext
+
+from searx.utils import match_language
+from searx.exceptions import SearxEngineAPIException
 from searx.raise_for_httperror import raise_for_httperror
 
+
 # about
 about = {
     "website": 'https://www.qwant.com/',
@@ -23,100 +49,191 @@ about = {
 # engine dependent config
 categories = []
 paging = True
-supported_languages_url = 'https://qwant.com/region'
+supported_languages_url = about['website']
 
-category_to_keyword = {'general': 'web',
-                       'images': 'images',
-                       'news': 'news'}
+category_to_keyword = {
+    'general': 'web',
+    'news': 'news',
+    'images': 'images',
+    'videos': 'videos',
+}
 
 # search-url
-url = 'https://api.qwant.com/api/search/{keyword}?count=10&offset={offset}&f=&{query}&t={keyword}&uiv=4'
+url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}'
 
 
-# do search-request
 def request(query, params):
-    offset = (params['pageno'] - 1) * 10
+    """Qwant search request"""
+    keyword = category_to_keyword[categories[0]]
+    count = 10  # web: count must be equal to 10
 
-    if categories[0] and categories[0] in category_to_keyword:
-
-        params['url'] = url.format(keyword=category_to_keyword[categories[0]],
-                                   query=urlencode({'q': query}),
-                                   offset=offset)
+    if keyword == 'images':
+        count = 50
+        offset = (params['pageno'] - 1) * count
+        # count + offset must be lower than 250
+        offset = min(offset, 199)
     else:
-        params['url'] = url.format(keyword='web',
-                                   query=urlencode({'q': query}),
-                                   offset=offset)
+        offset = (params['pageno'] - 1) * count
+        # count + offset must be lower than 50
+        offset = min(offset, 40)
+
+    params['url'] = url.format(
+        keyword=keyword,
+        query=urlencode({'q': query}),
+        offset=offset,
+        count=count,
+    )
 
     # add language tag
-    if params['language'] != 'all':
-        language = match_language(params['language'], supported_languages, language_aliases)
-        params['url'] += '&locale=' + language.replace('-', '_').lower()
+    if params['language'] == 'all':
+        params['url'] += '&locale=en_US'
+    else:
+        language = match_language(
+            params['language'],
+            supported_languages,
+            language_aliases,
+        )
+        params['url'] += '&locale=' + language.replace('-', '_')
 
-    params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'
     params['raise_for_httperror'] = False
     return params
 
 
-# get response from search-request
 def response(resp):
+    """Get response from Qwant's search request"""
+    # pylint: disable=too-many-locals, too-many-branches, too-many-statements
+
+    keyword = category_to_keyword[categories[0]]
     results = []
 
-    # According to https://www.qwant.com/js/app.js
-    if resp.status_code == 429:
-        raise SearxEngineCaptchaException()
+    # load JSON result
+    search_results = loads(resp.text)
+    data = search_results.get('data', {})
+
+    # check for an API error
+    if search_results.get('status') != 'success':
+        msg = ",".join(
+            data.get(
+                'message',
+                [
+                    'unknown',
+                ],
+            )
+        )
+        raise SearxEngineAPIException('API error::' + msg)
 
     # raise for other errors
     raise_for_httperror(resp)
 
-    # load JSON result
-    search_results = loads(resp.text)
-
-    # check for an API error
-    if search_results.get('status') != 'success':
-        raise SearxEngineAPIException('API error ' + str(search_results.get('error', '')))
+    if keyword == 'web':
+        # The WEB query contains a list named 'mainline'.  This list can contain
+        # different result types (e.g. mainline[0]['type'] returns type of the
+        # result items in mainline[0]['items']
+        mainline = data.get('result', {}).get('items', {}).get('mainline', {})
+    else:
+        # Queries on News, Images and Videos do not have a list named 'mainline'
+        # in the response.  The result items are directly in the list
+        # result['items'].
+        mainline = data.get('result', {}).get('items', [])
+        mainline = [
+            {'type': keyword, 'items': mainline},
+        ]
 
     # return empty array if there are no results
-    if 'data' not in search_results:
+    if not mainline:
         return []
 
-    data = search_results.get('data', {})
+    for row in mainline:
 
-    res = data.get('result', {})
+        mainline_type = row.get('type', 'web')
+        if mainline_type != keyword:
+            continue
 
-    # parse results
-    for result in res.get('items', {}):
+        if mainline_type == 'ads':
+            # ignore adds
+            continue
 
-        title = html_to_text(result['title'])
-        res_url = result['url']
-        content = html_to_text(result['desc'])
+        mainline_items = row.get('items', [])
+        for item in mainline_items:
 
-        if category_to_keyword.get(categories[0], '') == 'web':
-            results.append({'title': title,
-                            'content': content,
-                            'url': res_url})
+            title = item.get('title', None)
+            res_url = item.get('url', None)
 
-        elif category_to_keyword.get(categories[0], '') == 'images':
-            thumbnail_src = result['thumbnail']
-            img_src = result['media']
-            results.append({'template': 'images.html',
-                            'url': res_url,
-                            'title': title,
-                            'content': '',
-                            'thumbnail_src': thumbnail_src,
-                            'img_src': img_src})
+            if mainline_type == 'web':
+                content = item['desc']
+                results.append(
+                    {
+                        'title': title,
+                        'url': res_url,
+                        'content': content,
+                    }
+                )
 
-        elif category_to_keyword.get(categories[0], '') == 'news':
-            published_date = datetime.fromtimestamp(result['date'], None)
-            media = result.get('media', [])
-            if len(media) > 0:
-                img_src = media[0].get('pict', {}).get('url', None)
-            else:
+            elif mainline_type == 'news':
+
+                pub_date = item['date']
+                if pub_date is not None:
+                    pub_date = datetime.fromtimestamp(pub_date)
+                news_media = item.get('media', [])
                 img_src = None
-            results.append({'url': res_url,
-                            'title': title,
-                            'publishedDate': published_date,
-                            'content': content,
-                            'img_src': img_src})
+                if news_media:
+                    img_src = news_media[0].get('pict', {}).get('url', None)
+                results.append(
+                    {
+                        'title': title,
+                        'url': res_url,
+                        'publishedDate': pub_date,
+                        'img_src': img_src,
+                    }
+                )
+
+            elif mainline_type == 'images':
+                thumbnail = item['thumbnail']
+                img_src = item['media']
+                results.append(
+                    {
+                        'title': title,
+                        'url': res_url,
+                        'template': 'images.html',
+                        'thumbnail_src': thumbnail,
+                        'img_src': img_src,
+                    }
+                )
+
+            elif mainline_type == 'videos':
+                # some videos do not have a description: while qwant-video
+                # returns an empty string, such video from a qwant-web query
+                # miss the 'desc' key.
+                d, s, c = item.get('desc'), item.get('source'), item.get('channel')
+                content_parts = []
+                if d:
+                    content_parts.append(d)
+                if s:
+                    content_parts.append("%s: %s " % (gettext("Source"), s))
+                if c:
+                    content_parts.append("%s: %s " % (gettext("Channel"), c))
+                content = ' // '.join(content_parts)
+                length = item['duration']
+                if length is not None:
+                    length = timedelta(milliseconds=length)
+                pub_date = item['date']
+                if pub_date is not None:
+                    pub_date = datetime.fromtimestamp(pub_date)
+                thumbnail = item['thumbnail']
+                # from some locations (DE and others?) the s2 link do
+                # response a 'Please wait ..' but does not deliver the thumbnail
+                thumbnail = thumbnail.replace('https://s2.qwant.com', 'https://s1.qwant.com', 1)
+                results.append(
+                    {
+                        'title': title,
+                        'url': res_url,
+                        'content': content,
+                        'publishedDate': pub_date,
+                        'thumbnail': thumbnail,
+                        'template': 'videos.html',
+                        'length': length,
+                    }
+                )
 
     return results
 
@@ -125,15 +242,15 @@ def response(resp):
 def _fetch_supported_languages(resp):
     # list of regions is embedded in page as a js object
     response_text = resp.text
-    response_text = response_text[response_text.find('regionalisation'):]
-    response_text = response_text[response_text.find('{'):response_text.find(');')]
+    response_text = response_text[response_text.find('INITIAL_PROPS'):]
+    response_text = response_text[response_text.find('{'): response_text.find('</script>')]
 
     regions_json = loads(response_text)
 
-    supported_languages = {}
-    for lang in regions_json['languages'].values():
-        for country in lang['countries']:
-            lang_code = "{lang}-{country}".format(lang=lang['code'], country=country)
-            supported_languages[lang_code] = {'name': lang['name']}
+    supported_languages = []
+    for country, langs in regions_json['locales'].items():
+        for lang in langs['langs']:
+            lang_code = "{lang}-{country}".format(lang=lang, country=country)
+            supported_languages.append(lang_code)
 
     return supported_languages

searx/engines/redis_server.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+ Redis engine (Offline)
+"""
+
+# pylint: disable=missing-function-docstring
+# pylint: disable=import-error
+
+import redis
+
+engine_type = 'offline'
+# redis connection variables
+host = '127.0.0.1'
+port = 6379
+password = ''
+db = 0
+# engine specific variables
+paging = False
+result_template = 'key-value.html'
+exact_match_only = True
+
+_redis_client = None
+
+
+def init(_):
+    connect()
+
+
+def connect():
+    global _redis_client
+    _redis_client = redis.StrictRedis(
+        host=host,
+        port=port,
+        db=db,
+        password=password or None,
+        decode_responses=True,
+    )
+
+
+def search(query, params):
+    if not exact_match_only:
+        return search_keys(query)
+    ret = _redis_client.hgetall(query)
+    if ret:
+        ret['template'] = result_template
+        return [ret]
+    if ' ' in query:
+        qset, rest = query.split(' ', 1)
+        ret = []
+        for res in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)):
+            ret.append({res[0]: res[1], 'template': result_template})
+        return ret
+    return []
+
+
+def search_keys(query):
+    ret = []
+    for key in _redis_client.scan_iter(match='*{}*'.format(query)):
+        key_type = _redis_client.type(key)
+        res = None
+        if key_type == 'hash':
+            res = _redis_client.hgetall(key)
+        elif key_type == 'list':
+            res = dict(enumerate(_redis_client.lrange(key, 0, -1)))
+        if res:
+            res['template'] = result_template
+            res['redis_key'] = key
+            ret.append(res)
+    return ret

searx/engines/rumble.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """
- Wikipedia (Web
+ Rumble (Videos)
 """
 from urllib.parse import urlencode
 from lxml import html

searx/engines/semantic_scholar.py

@@ -1,12 +1,23 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- Semantic Scholar (Science)
+# lint: pylint
+"""Semantic Scholar (Science)
 """
 
 from json import dumps, loads
+from datetime import datetime
 
 
+about = {
+    "website": 'https://www.semanticscholar.org/',
+    "wikidata_id": 'Q22908627',
+    "official_api_documentation": 'https://api.semanticscholar.org/',
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": 'JSON',
+}
+paging = True
 search_url = 'https://www.semanticscholar.org/api/1/search'
+paper_url = 'https://www.semanticscholar.org/paper'
 
 
 def request(query, params):
@@ -32,11 +43,37 @@ def request(query, params):
 def response(resp):
     res = loads(resp.text)
     results = []
+
     for result in res['results']:
-        results.append({
-            'url': result['primaryPaperLink']['url'],
-            'title': result['title']['text'],
-            'content': result['paperAbstractTruncated']
-        })
+        item = {}
+        metadata = []
+
+        url = result.get('primaryPaperLink', {}).get('url')
+        if not url and result.get('links'):
+            url = result.get('links')[0]
+        if not url:
+            alternatePaperLinks = result.get('alternatePaperLinks')
+            if alternatePaperLinks:
+                url = alternatePaperLinks[0].get('url')
+        if not url:
+            url = paper_url + '/%s' % result['id']
+
+        item['url'] = url
+
+        item['title'] = result['title']['text']
+        item['content'] = result['paperAbstract']['text']
+
+        metadata = result.get('fieldsOfStudy') or []
+        venue = result.get('venue', {}).get('text')
+        if venue:
+            metadata.append(venue)
+        if metadata:
+            item['metadata'] = ', '.join(metadata)
+
+        pubDate = result.get('pubDate')
+        if pubDate:
+            item['publishedDate'] = datetime.strptime(pubDate, "%Y-%m-%d")
+
+        results.append(item)
 
     return results

searx/engines/seznam.py

@@ -54,14 +54,14 @@ def response(resp):
 
     dom = html.fromstring(resp.content.decode())
     for result_element in eval_xpath_list(dom, '//div[@data-dot="results"]/div'):
-        result_data = eval_xpath_getindex(result_element, './/div[contains(@class, "Result")]', 0, default=None)
+        result_data = eval_xpath_getindex(result_element, './/div[contains(@class, "bec586")]', 0, default=None)
         if result_data is None:
             continue
         title_element = eval_xpath_getindex(result_element, './/h3/a', 0)
         results.append({
             'url': title_element.get('href'),
             'title': extract_text(title_element),
-            'content': extract_text(eval_xpath(result_data, './/p[@class="Result-description"]')),
+            'content': extract_text(eval_xpath(result_data, './/div[@class="_3eded7"]')),
         })
 
     return results

searx/engines/sjp.py

@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""Słownik Języka Polskiego
+
+Dictionary of the polish language from PWN (sjp.pwn)
+"""
+
+from lxml.html import fromstring
+from searx import logger
+from searx.utils import extract_text
+from searx.raise_for_httperror import raise_for_httperror
+
+logger = logger.getChild('sjp engine')
+
+# about
+about = {
+    "website": 'https://sjp.pwn.pl',
+    "wikidata_id": 'Q55117369',
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
+categories = ['general']
+paging = False
+
+URL = 'https://sjp.pwn.pl'
+SEARCH_URL = URL + '/szukaj/{query}.html'
+
+word_xpath = '//div[@class="query"]'
+dict_xpath = ['//div[@class="wyniki sjp-so-wyniki sjp-so-anchor"]',
+              '//div[@class="wyniki sjp-wyniki sjp-anchor"]',
+              '//div[@class="wyniki sjp-doroszewski-wyniki sjp-doroszewski-anchor"]']
+
+
+def request(query, params):
+    params['url'] = SEARCH_URL.format(query=query)
+    logger.debug(f"query_url --> {params['url']}")
+    return params
+
+
+def response(resp):
+    results = []
+
+    raise_for_httperror(resp)
+    dom = fromstring(resp.text)
+    word = extract_text(dom.xpath(word_xpath))
+
+    definitions = []
+
+    for dict_src in dict_xpath:
+        for src in dom.xpath(dict_src):
+            src_text = extract_text(src.xpath('.//span[@class="entry-head-title"]/text()')).strip()
+
+            src_defs = []
+            for def_item in src.xpath('.//div[contains(@class, "ribbon-element")]'):
+                if def_item.xpath('./div[@class="znacz"]'):
+                    sub_defs = []
+                    for def_sub_item in def_item.xpath('./div[@class="znacz"]'):
+                        def_sub_text = extract_text(def_sub_item).lstrip('0123456789. ')
+                        sub_defs.append(def_sub_text)
+                    src_defs.append((word, sub_defs))
+                else:
+                    def_text = extract_text(def_item).strip()
+                    def_link = def_item.xpath('./span/a/@href')
+                    if 'doroszewski' in def_link[0]:
+                        def_text = f"<a href='{def_link[0]}'>{def_text}</a>"
+                    src_defs.append((def_text, ''))
+
+            definitions.append((src_text, src_defs))
+
+    if not definitions:
+        return results
+
+    infobox = ''
+    for src in definitions:
+        infobox += f"<div><small>{src[0]}</small>"
+        infobox += "<ul>"
+        for (def_text, sub_def) in src[1]:
+            infobox += f"<li>{def_text}</li>"
+            if sub_def:
+                infobox += "<ol>"
+                for sub_def_text in sub_def:
+                    infobox += f"<li>{sub_def_text}</li>"
+                infobox += "</ol>"
+        infobox += "</ul></div>"
+
+    results.append({
+        'infobox': word,
+        'content': infobox,
+    })
+
+    return results

searx/engines/solidtorrents.py

@@ -45,7 +45,7 @@ def response(resp):
             'seed': result["swarm"]["seeders"],
             'leech': result["swarm"]["leechers"],
             'title': result["title"],
-            'link': "https://solidtorrents.net/view/" + result["_id"],
+            'url': "https://solidtorrents.net/view/" + result["_id"],
             'filesize': result["size"],
             'magnetlink': result["magnet"],
             'template': "torrent.html",