diff --git a/Makefile b/Makefile index b69202ba..f35b86c4 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,9 @@ PYOBJECTS = searx DOC = docs PY_SETUP_EXTRAS ?= \[test\] +PYDIST=./dist/py +PYBUILD=./build/py + include utils/makefile.include include utils/makefile.python include utils/makefile.sphinx @@ -23,6 +26,7 @@ help: @echo ' install - developer install (./local)' @echo ' uninstall - uninstall (./local)' @echo ' gh-pages - build docs & deploy on gh-pages branch' + @echo ' clean - drop builds and environments' @echo '' @$(MAKE) -s -f utils/makefile.include make-help @echo '' diff --git a/docs/_themes/searx/static/searx.css b/docs/_themes/searx/static/searx.css index 10f5b4ed..d6a664f0 100644 --- a/docs/_themes/searx/static/searx.css +++ b/docs/_themes/searx/static/searx.css @@ -24,7 +24,107 @@ p.sidebar-title, .sidebar p { margin: 6pt; } -.sidebar li { +.sidebar li, +.hlist li { list-style-type: disclosure-closed; } + +/* admonitions +*/ + +div.admonition, div.topic { + background-color: #fafafa; + margin: 8px 0px; + padding: 1em; + border-radius: 3pt 0 0 3pt; + border-top: none; + border-right: none; + border-bottom: none; + border-left: 5pt solid #ccc; +} + +p.admonition-title:after { + content: none; +} + +.admonition.hint { border-color: #416dc0b0; } +.admonition.note { border-color: #6c856cb0; } +.admonition.tip { border-color: #85c5c2b0; } +.admonition.attention { border-color: #ecec97b0; } +.admonition.caution { border-color: #a6c677b0; } +.admonition.danger { border-color: #d46262b0; } +.admonition.important { border-color: #dfa3a3b0; } +.admonition.error { border-color: red; } +.admonition.warning { border-color: darkred; } + +.admonition.admonition-generic-admonition-title { + border-color: #416dc0b0; +} + + +/* admonitions with (rendered) reST markup examples (:class: rst-example) + * + * .. admonition:: title of the example + * :class: rst-example + * .... +*/ + +div.rst-example { + background-color: inherit; + margin: 0; + border-top: none; + border-right: 1px solid #ccc; + border-bottom: none; + border-left: none; + border-radius: none; + padding: 0; +} + +div.rst-example > p.admonition-title { + font-family: Sans Serif; + font-style: italic; + font-size: 0.8em; + display: block; + border-bottom: 1px solid #ccc; + padding: 0.5em 1em; + text-align: right; +} + +/* code block in figures + */ + +div.highlight pre { + text-align: left; +} + +/* Table theme +*/ + +thead, tfoot { + background-color: #fff; +} + +th:hover, td:hover { + background-color: #ffc; +} + +thead th, tfoot th, tfoot td, tbody th { + background-color: #fffaef; +} + +tbody tr:nth-child(odd) { + background-color: #fff; +} + +tbody tr:nth-child(even) { + background-color: #fafafa; +} + +caption { + font-family: Sans Serif; + padding: 0.5em; + margin: 0.5em 0 0.5em 0; + caption-side: top; + text-align: left; +} diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot new file mode 100644 index 00000000..a46c96de --- /dev/null +++ b/docs/admin/arch_public.dot @@ -0,0 +1,33 @@ +digraph G { + + node [style=filled, shape=box, fillcolor="#ffffcc", fontname="Sans"]; + edge [fontname="Sans"]; + + browser [label="Browser", shape=Mdiamond]; + rp [label="Reverse Proxy", href="url to configure reverse proxy"]; + filtron [label="Filtron", href="https://github.com/asciimoo/filtron"]; + morty [label="Morty", href="https://github.com/asciimoo/morty"]; + static [label="Static files", href="url to configure static files"]; + uwsgi [label="uwsgi", href="url to configure uwsgi"] + searx1 [label="Searx #1"]; + searx2 [label="Searx #2"]; + searx3 [label="Searx #3"]; + searx4 [label="Searx #4"]; + + browser -> rp [label="HTTPS"] + + subgraph cluster_searx { + label = "Searx instance" fontname="Sans"; + bgcolor="#fafafa"; + { rank=same; static rp }; + rp -> morty [label="optional: images and HTML pages proxy"]; + rp -> static [label="optional: reverse proxy serves directly static files"]; + rp -> filtron [label="HTTP"]; + filtron -> uwsgi [label="HTTP"]; + uwsgi -> searx1; + uwsgi -> searx2; + uwsgi -> searx3; + uwsgi -> searx4; + } + +} diff --git a/docs/admin/architecture.rst b/docs/admin/architecture.rst new file mode 100644 index 00000000..7064a294 --- /dev/null +++ b/docs/admin/architecture.rst @@ -0,0 +1,24 @@ +.. _architecture: + +============ +Architecture +============ + +.. sidebar:: Needs work! + + This article needs some work / Searx is a collaborative effort. If you have + any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see + :ref:`how to contribute`. + +Herein you will find some hints and suggestions about typical architectures of +searx infrastructures. + +We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`. +It shows a *reference* setup for public searx instances. + +.. _arch public: + +.. kernel-figure:: arch_public.dot + :alt: arch_public.dot + + Reference architecture of a public searx setup. diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst new file mode 100644 index 00000000..5260da03 --- /dev/null +++ b/docs/admin/buildhosts.rst @@ -0,0 +1,103 @@ +.. _buildhosts: + +========== +Buildhosts +========== + +.. sidebar:: This article needs some work + + If you have any contribution send us your :pull:`PR <../pulls>`, see + :ref:`how to contribute`. + +To get best results from build, its recommend to install additional packages +on build hosts. + +.. _docs build: + +Build docs +========== + +.. _Graphviz: https://graphviz.gitlab.io +.. _ImageMagick: https://www.imagemagick.org +.. _XeTeX: https://tug.org/xetex/ +.. _dvisvgm: https://dvisvgm.de/ + +.. sidebar:: Sphinx build needs + + - ImageMagick_ + - Graphviz_ + - XeTeX_ + - dvisvgm_ + +Most of the sphinx requirements are installed from :origin:`setup.py` and the +docs can be build from scratch with ``make docs``. For better math and image +processing additional packages are needed. The XeTeX_ needed not only for PDF +creation, its also needed for :ref:`math` when HTML output is build. + +To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered +as images (``sphinx.ext.imgmath`` extension). If your docs build (``make +docs``) shows warnings like this:: + + WARNING: dot(1) not found, for better output quality install \ + graphviz from http://www.graphviz.org + .. + WARNING: LaTeX command 'latex' cannot be run (needed for math \ + display), check the imgmath_latex setting + +you need to install additional packages on your build host, to get better HTML +output. + +.. _system requirements: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo apt install graphviz imagemagick texlive-xetex librsvg2-bin + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo pacman -S graphviz imagemagick texlive-bin extra/librsvg + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo dnf install graphviz graphviz-gd texlive-xetex-bin librsvg2-tools + + +For PDF output you also need: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + $ sudo apt texlive-latex-recommended texlive-extra-utils ttf-dejavu + + .. group-tab:: Arch Linux + + .. code:: sh + + $ sudo pacman -S texlive-core texlive-latexextra ttf-dejavu + + .. group-tab:: Fedora / RHEL + + .. code:: sh + + $ sudo dnf install \ + texlive-collection-fontsrecommended texlive-collection-latex \ + dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts + +.. _system requirements END: + +.. literalinclude:: ../conf.py + :language: python + :start-after: # sphinx.ext.imgmath setup + :end-before: # sphinx.ext.imgmath setup END + diff --git a/docs/admin/engines.rst b/docs/admin/engines.rst new file mode 100644 index 00000000..4b78c8c2 --- /dev/null +++ b/docs/admin/engines.rst @@ -0,0 +1,71 @@ +.. _engines generic: + +======= +Engines +======= + +.. sidebar:: Further reading .. + + - :ref:`settings engine` + - :ref:`engine settings` + - :ref:`engine file` + +============= =========== ==================== ============ +:ref:`engine settings` :ref:`engine file` +------------------------- --------------------------------- +Name (cfg) Categories +------------------------- --------------------------------- +Engine .. Paging support **P** +------------------------- -------------------- ------------ +Shortcut **S** Language support **L** +Timeout **TO** Time range support **TR** +Disabled **D** Offline **O** +------------- ----------- -------------------- ------------ +Safe search **SS** +------------- ----------- --------------------------------- +Weigth **W** +------------- ----------- --------------------------------- +Disabled **D** +============= =========== ================================= + +Configuration defaults (at built time): + +.. _configured engines: + +.. jinja:: webapp + + .. flat-table:: Engines configured at built time (defaults) + :header-rows: 1 + :stub-columns: 2 + + * - Name (cfg) + - S + - Engine + - TO + - Categories + - P + - L + - SS + - D + - TR + - O + - W + - D + + {% for name, mod in engines.items() %} + + * - {{name}} + - !{{mod.shortcut}} + - {{mod.__name__}} + - {{mod.timeout}} + - {{", ".join(mod.categories)}} + - {{(mod.paging and "y") or ""}} + - {{(mod.language_support and "y") or ""}} + - {{(mod.safesearch and "y") or ""}} + - {{(mod.disabled and "y") or ""}} + - {{(mod.time_range_support and "y") or ""}} + - {{(mod.offline and "y") or ""}} + - {{mod.weight or 1 }} + - {{(mod.disabled and "y") or ""}} + + {% endfor %} diff --git a/docs/admin/index.rst b/docs/admin/index.rst index f3a99576..b3c7f511 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -6,6 +6,11 @@ Administrator documentation :maxdepth: 1 installation + settings api + architecture filtron morty + engines + plugins + buildhosts diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst index 239ce070..15800fc0 100644 --- a/docs/admin/installation.rst +++ b/docs/admin/installation.rst @@ -114,6 +114,9 @@ content: # Module to import module = searx.webapp + # Support running the module from a webserver subdirectory. + route-run = fixpathinfo: + # Virtualenv and python path virtualenv = /usr/local/searx/searx-ve/ pythonpath = /usr/local/searx/ @@ -151,7 +154,10 @@ content: server { listen 80; server_name searx.example.com; - root /usr/local/searx; + root /usr/local/searx/searx; + + location /static { + } location / { include uwsgi_params; @@ -180,14 +186,13 @@ Add this configuration in the server config file .. code:: nginx - location = /searx { rewrite ^ /searx/; } - location /searx { - try_files $uri @searx; + location /searx/static { + alias /usr/local/searx/searx/static; } - location @searx { + + location /searx { uwsgi_param SCRIPT_NAME /searx; include uwsgi_params; - uwsgi_modifier1 30; uwsgi_pass unix:/run/uwsgi/app/searx/socket; } @@ -197,6 +202,10 @@ in case of single-user or low-traffic instances.) .. code:: nginx + location /searx/static { + alias /usr/local/searx/searx/static; + } + location /searx { proxy_pass http://127.0.0.1:8888; proxy_set_header Host $host; @@ -338,4 +347,3 @@ References * How to: `Setup searx in a couple of hours with a free SSL certificate `__ - diff --git a/docs/admin/plugins.rst b/docs/admin/plugins.rst new file mode 100644 index 00000000..4ed9066f --- /dev/null +++ b/docs/admin/plugins.rst @@ -0,0 +1,39 @@ +.. _plugins generic: + +=============== +Plugins builtin +=============== + +.. sidebar:: Further reading .. + + - :ref:`dev plugin` + +Configuration defaults (at built time): + +:DO: Default on + +.. _configured plugins: + +.. jinja:: webapp + + .. flat-table:: Plugins configured at built time (defaults) + :header-rows: 1 + :stub-columns: 1 + :widths: 3 1 9 + + * - Name + - DO + - Description + + JS & CSS dependencies + + {% for plgin in plugins %} + + * - {{plgin.name}} + - {{(plgin.default_on and "y") or ""}} + - {{plgin.description}} + + {% for dep in (plgin.js_dependencies + plgin.css_dependencies) %} + | ``{{dep}}`` {% endfor %} + + {% endfor %} diff --git a/docs/admin/settings.rst b/docs/admin/settings.rst new file mode 100644 index 00000000..0bfdcc6c --- /dev/null +++ b/docs/admin/settings.rst @@ -0,0 +1,181 @@ +.. _settings.yml: + +================ +``settings.yml`` +================ + +.. sidebar:: Further reading .. + + - :ref:`search API` + +This page describe the options possibilities of the settings.yml file. + +.. _settings global: + +Global Settings +=============== + +.. code:: yaml + + server: + port : 8888 + secret_key : "ultrasecretkey" # change this! + debug : False # debug mode, only for development + request_timeout : 2.0 # seconds + base_url : False # set custom base_url (or False) + themes_path : "" # custom ui themes path + default_theme : oscar # ui theme + useragent_suffix : "" # suffix of searx_useragent, could contain + # informations like admins email address + image_proxy : False # proxying image results through searx + default_locale : "" # default interface locale + + # uncomment below section if you want to use a proxy + + #outgoing_proxies : + # http : http://127.0.0.1:8080 + # https: http://127.0.0.1:8080 + + # uncomment below section only if you have more than one network interface + # which can be the source of outgoing search requests + + #source_ips: + # - 1.1.1.1 + # - 1.1.1.2 + + locales: + en : English + de : Deutsch + he : Hebrew + hu : Magyar + fr : Français + es : Español + it : Italiano + nl : Nederlands + ja : 日本語 (Japanese) + tr : Türkçe + ru : Russian + ro : Romanian + + +``port`` : + Port number of the searx web application if you run it directly using ``python + searx/webapp.py``. Doesn't apply to searx running on Apache or Nginx. + +``secret_key`` : + Used for cryptography purpose. + +``debug`` : + Allow a more detailed log if you run searx directly. Display *detailed* error + messages in the browser too, so this must be deactivated in production. + +``request_timeout`` : + Global timeout of the requests made to others engines in seconds. A bigger + timeout will allow to wait for answers from slow engines, but in consequence + will slow searx reactivity (the result page may take the time specified in the + timeout to load) + +``base_url`` : + The base URL where searx is deployed. Used to create correct inbound links. + +``themes_path`` : + Path to where the themes are located. If you didn't develop anything, leave it + blank. + +``default_theme`` : + Name of the theme you want to use by default on you searx instance. + +``useragent_suffix`` : + Suffix to the user-agent searx uses to send requests to others engines. If an + engine wish to block you, a contact info here may be useful to avoid that. + +``image_proxy`` : + Allow your instance of searx of being able to proxy images. Uses memory space. + +``default_locale`` : + Aearx interface language. If blank, the locale is detected by using the + browser language. If it doesn't work, or you are deploying a language + specific instance of searx, a locale can be defined using an ISO language + code, like ``fr``, ``en``, ``de``. + +.. _requests proxies: http://docs.python-requests.org/en/latest/user/advanced/#proxies +.. _PR SOCKS support: https://github.com/kennethreitz/requests/pull/478 + +``outgoing_proxies`` : + Define a proxy you wish to use, see `requests proxies`_. SOCKS proxies are + not supported / see `PR SOCKS support`. + +``source_ips`` : + If you use multiple nework interfaces, define from which IP the requests must + be made. + +``locales`` : + Locales codes and their names. Available translations of searx interface. + + +.. _settings engine: + +Engine settings +=============== + +.. sidebar:: Further reading .. + + - :ref:`engines-dev` + +.. code:: yaml + + - name : bing + engine : bing + shortcut : bi + base_url : 'https://{language}.wikipedia.org/' + categories : general + timeout : 3.0 + api_key : 'apikey' + disabled : True + language : en_US + +``name`` : + Name that will be used accross searx to define this engine. In settings, on + the result page... + +``engine`` : + Name of the python file used to handle requests and responses to and from this + search engine. + +``shortcut`` : + Code used to execute bang requests (in this case using ``!bi`` or ``?bi``) + +``base_url`` : optional + Part of the URL that should be stable accross every request. Can be useful to + use multiple sites using only one engine, or updating the site URL without + touching at the code. + +``categories`` : optional + Define in which categories this engine will be active. Most of the time, it is + defined in the code of the engine, but in a few cases it is useful, like when + describing multiple search engine using the same code. + +``timeout`` : optional + Timeout of the search with the current search engine. **Be careful, it will + modify the global timeout of searx.** + +``api_key`` : optional + In a few cases, using an API needs the use of a secret key. How to obtain them + is described in the file. + +``disabled`` : optional + To disable by default the engine, but not deleting it. It will allow the user + to manually activate it in the settings. + +``language`` : optional + If you want to use another language for a specific engine, you can define it + by using the full ISO code of language and country, like ``fr_FR``, ``en_US``, + ``de_DE``. + +``weigth`` : default ``1`` + Weighting of the results of this engine. + +.. note:: + + A few more options are possible, but they are pretty specific to some + engines, and so won't be described here. diff --git a/docs/conf.py b/docs/conf.py index c0bd246a..af255e23 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,11 +14,18 @@ project = u'searx' copyright = u'2015-2019, Adam Tauber, Noémi Ványi' author = u'Adam Tauber' release, version = VERSION_STRING, VERSION_STRING +highlight_language = 'none' # General -------------------------------------------------------------- master_doc = "index" source_suffix = '.rst' +numfig = True + +from searx import webapp +jinja_contexts = { + 'webapp': dict(**webapp.__dict__) +} # usage:: lorem :patch:`f373169` ipsum extlinks = {} @@ -32,21 +39,40 @@ extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://') extlinks['patch'] = (GIT_URL + '/commit/%s', '#') extlinks['search'] = (SEARX_URL + '/%s', '#') extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ') +extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: ') +extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '') +#extlinks['role'] = ( +# 'https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-%s', '') +extlinks['duref'] = ( + 'http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '') +extlinks['durole'] = ( + 'http://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '') +extlinks['dudir'] = ( + 'http://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '') +extlinks['ctan'] = ( + 'https://ctan.org/pkg/%s', 'CTAN: ') extensions = [ + 'sphinx.ext.imgmath', 'sphinx.ext.extlinks', 'sphinx.ext.viewcode', "sphinx.ext.autodoc", "sphinx.ext.intersphinx", "pallets_sphinx_themes", "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst + "sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja + 'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive. + 'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling. + "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs ] intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), - # "flask": ("https://flask.palletsprojects.com/", None), + "flask": ("https://flask.palletsprojects.com/", None), # "werkzeug": ("https://werkzeug.palletsprojects.com/", None), - # "jinja": ("https://jinja.palletsprojects.com/", None), + "jinja": ("https://jinja.palletsprojects.com/", None), + "linuxdoc" : ("https://return42.github.io/linuxdoc/", None), + "sphinx" : ("https://www.sphinx-doc.org/en/master/", None), } issues_github_path = "asciimoo/searx" @@ -54,16 +80,21 @@ issues_github_path = "asciimoo/searx" # HTML ----------------------------------------------------------------- sys.path.append(os.path.abspath('_themes')) - html_theme_path = ['_themes'] html_theme = "searx" +# sphinx.ext.imgmath setup +html_math_renderer = 'imgmath' +imgmath_image_format = 'svg' +imgmath_font_size = 14 +# sphinx.ext.imgmath setup END + html_theme_options = {"index_sidebar_logo": True} html_context = { "project_links": [ ProjectLink("Source", GIT_URL), ProjectLink("Wiki", "https://github.com/asciimoo/searx/wiki"), - ProjectLink("Public instances", "https://github.com/asciimoo/searx/wiki/Searx-instances"), + ProjectLink("Public instances", "https://asciimoo.github.io/searx/user/public_instances.html"), ProjectLink("Twitter", "https://twitter.com/Searx_engine"), ] } diff --git a/docs/dev/contribution_guide.rst b/docs/dev/contribution_guide.rst index 064f28e6..459dfb44 100644 --- a/docs/dev/contribution_guide.rst +++ b/docs/dev/contribution_guide.rst @@ -1,3 +1,5 @@ +.. _how to contribute: + ================= How to contribute ================= @@ -48,7 +50,20 @@ Code ==== .. _PEP8: https://www.python.org/dev/peps/pep-0008/ +.. _Conventional Commits: https://www.conventionalcommits.org/ +.. _Git Commit Good Practice: https://wiki.openstack.org/wiki/GitCommitMessages +.. _Structural split of changes: + https://wiki.openstack.org/wiki/GitCommitMessages#Structural_split_of_changes +.. _gitmoji: https://gitmoji.carloscuesta.me/ +.. _Semantic PR: https://github.com/zeke/semantic-pull-requests +.. sidebar:: Create good commits! + + - `Structural split of changes`_ + - `Conventional Commits`_ + - `Git Commit Good Practice`_ + - some like to use: gitmoji_ + - not yet active: `Semantic PR`_ In order to submit a patch, please follow the steps below: @@ -57,6 +72,9 @@ In order to submit a patch, please follow the steps below: - PEP8_ standards apply, except the convention of line length - Maximum line length is 120 characters +- The cardinal rule for creating good commits is to ensure there is only one + *logical change* per commit / read `Structural split of changes`_ + - Check if your code breaks existing tests. If so, update the tests or fix your code. @@ -64,6 +82,16 @@ In order to submit a patch, please follow the steps below: - Add yourself to the :origin:`AUTHORS.rst` file. +- Choose meaning full commit messages, read `Conventional Commits`_ + + .. code:: + + [optional scope]: + + [optional body] + + [optional footer(s)] + - Create a pull request. For more help on getting started with searx development, see :ref:`devquickstart`. @@ -79,6 +107,8 @@ Translation currently takes place on :ref:`transifex `. Please, do not update translation files in the repo. +.. _contrib docs: + Documentation ============= @@ -91,7 +121,7 @@ Documentation The documentation is built using Sphinx_. So in order to be able to generate the required files, you have to install it on your system. Much easier, use -Makefile our targets. +our :ref:`makefile`. Here is an example which makes a complete rebuild: @@ -101,6 +131,7 @@ Here is an example which makes a complete rebuild: ... The HTML pages are in dist/docs. +.. _make docs-live: live build ---------- @@ -110,9 +141,10 @@ live build It is recommended to assert a complete rebuild before deploying (use ``docs-clean``). -Live build is like WYSIWYG, If you want to edit the documentation, its -recommended to use. The Makefile target ``docs-live`` builds the docs, opens URL -in your favorite browser and rebuilds every time a reST file has been changed. +Live build is like WYSIWYG. If you want to edit the documentation, its +recommended to use. The Makefile target ``docs-live`` builds the docs, opens +URL in your favorite browser and rebuilds every time a reST file has been +changed. .. code:: sh @@ -123,12 +155,13 @@ in your favorite browser and rebuilds every time a reST file has been changed. ... Start watching changes +.. _deploy on github.io: deploy on github.io ------------------- To deploy documentation at :docs:`github.io <.>` use Makefile target -``gh-pages``, which will builds the documentation, clones searx into a sub +:ref:`make gh-pages`, which will builds the documentation, clones searx into a sub folder ``gh-pages``, cleans it, copies the doc build into and runs all the needed git add, commit and push: diff --git a/docs/dev/csv_table.txt b/docs/dev/csv_table.txt new file mode 100644 index 00000000..8a145413 --- /dev/null +++ b/docs/dev/csv_table.txt @@ -0,0 +1,6 @@ +stub col row 1, column, "loremLorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy +eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam +voluptua." +stub col row 1, "At vero eos et accusam et justo duo dolores et ea rebum. Stet clita +kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.", column +stub col row 1, column, column diff --git a/docs/dev/engine_overview.rst b/docs/dev/engine_overview.rst index 92405dc6..449c837a 100644 --- a/docs/dev/engine_overview.rst +++ b/docs/dev/engine_overview.rst @@ -29,6 +29,7 @@ the ones in the engine file. It does not matter if an option is stored in the engine file or in the settings. However, the standard way is the following: +.. _engine file: engine file ----------- @@ -43,6 +44,7 @@ time_range_support boolean support search time range offline boolean engine runs offline ======================= =========== =========================================== +.. _engine settings: settings.yml ------------ diff --git a/docs/dev/hello.dot b/docs/dev/hello.dot new file mode 100644 index 00000000..504621df --- /dev/null +++ b/docs/dev/hello.dot @@ -0,0 +1,3 @@ +graph G { + Hello -- World +} diff --git a/docs/dev/index.rst b/docs/dev/index.rst index 8e18066c..cb913a82 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -11,3 +11,5 @@ Developer documentation search_api plugins translation + makefile + reST diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst new file mode 100644 index 00000000..f5957001 --- /dev/null +++ b/docs/dev/makefile.rst @@ -0,0 +1,221 @@ +.. _makefile: + +================ +Makefile Targets +================ + +.. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction + +.. sidebar:: build environment + + Before looking deeper at the targets, first read about :ref:`makefile setup` + and :ref:`make pyenv`. + +With the aim to simplify development cycles, started with :pull:`1756` a +``Makefile`` based boilerplate was added. If you are not familiar with +Makefiles, we recommend to read gnu-make_ introduction. + +The usage is simple, just type ``make {target-name}`` to *build* a target. +Calling the ``help`` target gives a first overview:: + + $ make help + test - run developer tests + docs - build documentation + docs-live - autobuild HTML documentation while editing + run - run developer instance + install - developer install (./local) + uninstall - uninstall (./local) + gh-pages - build docs & deploy on gh-pages branch + clean - drop builds and environments + ... + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + + +.. _makefile setup: + +Setup +===== + +.. _git stash: https://git-scm.com/docs/git-stash + +The main setup is done in the :origin:`Makefile`:: + + export GIT_URL=https://github.com/asciimoo/searx + export SEARX_URL=https://searx.me + export DOCS_URL=https://asciimoo.github.io/searx + +.. sidebar:: fork & upstream + + Commit changes in your (local) branch, fork or whatever, but do not push them + upstream / `git stash`_ is your friend. + +:GIT_URL: Changes this, to point to your searx fork. + +:SEARX_URL: Changes this, to point to your searx instance. + +:DOCS_URL: If you host your own (branded) documentation, change this URL. + +.. _make pyenv: + +Python environment +================== + +.. sidebar:: activate environment + + ``source ./local/py3/bin/activate`` + +With Makefile we do no longer need to build up the virualenv manually (as +described in the :ref:`devquickstart` guide). Jump into your git working tree +and release a ``make pyenv``: + +.. code:: sh + + $ cd ~/searx-clone + $ make pyenv + PYENV usage: source ./local/py3/bin/activate + ... + +With target ``pyenv`` a development environment (aka virtualenv) was build up in +``./local/py3/``. To make a *developer install* of searx (:origin:`setup.py`) +into this environment, use make target ``install``: + +.. code:: sh + + $ make install + PYENV usage: source ./local/py3/bin/activate + PYENV using virtualenv from ./local/py3 + PYENV install . + +You have never to think about intermediate targets like ``pyenv`` or +``install``, the ``Makefile`` chains them as requisites. Just run your main +target. + +.. sidebar:: drop environment + + To get rid of the existing environment before re-build use :ref:`clean target + ` first. + +If you think, something goes wrong with your ./local environment or you change +the :origin:`setup.py` file (or the requirements listed in +:origin:`requirements-dev.txt` and :origin:`requirements.txt`), you have to call +:ref:`make clean`. + + +.. _make run: + +``make run`` +============ + +To get up a running a developer instance simply call ``make run``. This enables +*debug* option in :origin:`searx/settings.yml`, starts a ``./searx/webapp.py`` +instance, disables *debug* option again and opens the URL in your favorite WEB +browser (:man:`xdg-open`): + +.. code:: sh + + $ make run + PYENV usage: source ./local/py3/bin/activate + PYENV install . + ./local/py3/bin/python ./searx/webapp.py + ... + INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit) + ... + +.. _make clean: + +``make clean`` +============== + +Drop all intermediate files, all builds, but keep sources untouched. Includes +target ``pyclean`` which drops ./local environment. Before calling ``make +clean`` stop all processes using :ref:`make pyenv`. + +.. code:: sh + + $ make clean + CLEAN pyclean + CLEAN clean + +.. _make docs: + +``make docs docs-live docs-clean`` +================================== + +We describe the usage of the ``doc*`` targets in the :ref:`How to contribute / +Documentation ` section. If you want to edit the documentation +read our :ref:`make docs-live` section. If you are working in your own brand, +adjust your :ref:`Makefile setup `. + + +.. _make gh-pages: + +``make gh-pages`` +================= + +To deploy on github.io first adjust your :ref:`Makefile setup `. For any further read :ref:`deploy on github.io`. + +.. _make test: + +``make test`` +============= + +Runs a series of tests: ``test.pep8``, ``test.unit``, ``test.robot`` and does +additional :ref:`pylint checks `. You can run tests selective, +e.g.: + +.. code:: sh + + $ make test.pep8 test.unit + . ./local/py3/bin/activate; ./manage.sh pep8_check + [!] Running pep8 check + . ./local/py3/bin/activate; ./manage.sh unit_tests + [!] Running unit tests + +.. _make pylint: + +``make pylint`` +=============== + +.. _Pylint: https://www.pylint.org/ + +Before commiting its recommend to do some (more) linting. Pylint_ is known as +one of the best source-code, bug and quality checker for the Python programming +language. Pylint_ is not yet a quality gate within our searx project (like +:ref:`test.pep8 ` it is), but Pylint_ can help to improve code +quality anyway. The pylint profile we use at searx project is found in +project's root folder :origin:`.pylintrc`. + +Code quality is a ongoing process. Don't try to fix all messages from Pylint, +run Pylint and check if your changed lines are bringing up new messages. If so, +fix it. By this, code quality gets incremental better and if there comes the +day, the linting is balanced out, we might decide to add Pylint as a quality +gate. + + +``make pybuild`` +================ + +.. _PyPi: https://pypi.org/ +.. _twine: https://twine.readthedocs.io/en/latest/ + +Build Python packages in ``./dist/py``. + +.. code:: sh + + $ make pybuild + ... + BUILD pybuild + running sdist + running egg_info + ... + $ ls ./dist/py/ + searx-0.15.0-py3-none-any.whl searx-0.15.0.tar.gz + +To upload packages to PyPi_, there is also a ``upload-pypi`` target. It needs +twine_ to be installed. Since you are not the owner of :pypi:`searx` you will +never need the latter. diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index e97bbeb4..2bf44f18 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -1,7 +1,13 @@ +.. _dev plugin: + ======= Plugins ======= +.. sidebar:: Further reading .. + + - :ref:`plugins generic` + Plugins can extend or replace functionality of various components of searx. Example plugin diff --git a/docs/dev/quickstart.rst b/docs/dev/quickstart.rst index a4a37a26..e40772b3 100644 --- a/docs/dev/quickstart.rst +++ b/docs/dev/quickstart.rst @@ -4,15 +4,23 @@ Development Quickstart ====================== +.. sidebar:: :ref:`makefile` + + For additional developer purpose there are :ref:`makefile`. + This quickstart guide gets your environment set up with searx. Furthermore, it gives a short introduction to the ``manage.sh`` script. How to setup your development environment ========================================= +.. sidebar:: :ref:`make pyenv ` + + Alternatively use the :ref:`make pyenv`. + First, clone the source code of searx to the desired folder. In this case the source is cloned to ``~/myprojects/searx``. Then create and activate the -searx-ve virtualenv and install the required packages using manage.sh. +searx-ve virtualenv and install the required packages using ``manage.sh``. .. code:: sh @@ -27,6 +35,10 @@ searx-ve virtualenv and install the required packages using manage.sh. How to run tests ================ +.. sidebar:: :ref:`make test.unit ` + + Alternatively use the ``test.pep8``, ``test.unit``, ``test.robot`` targets. + Tests can be run using the ``manage.sh`` script. Following tests and checks are available: @@ -41,7 +53,8 @@ For example unit tests are run with the command below: ./manage.sh unit_tests -For further test options, please consult the help of the ``manage.sh`` script. +For further test options, please consult the help of the ``manage.sh`` script or +read :ref:`make test`. How to compile styles and javascript @@ -97,6 +110,11 @@ After installing grunt, the files can be built using the following command: Tips for debugging/development ============================== +.. sidebar:: :ref:`make run` + + Makefile target ``run`` already enables debug option for your developer + session / see :ref:`make run`. + Turn on debug logging Whether you are working on a new engine or trying to eliminate a bug, it is always a good idea to turn on debug logging. When debug logging is enabled a @@ -104,6 +122,10 @@ Turn on debug logging message. It can be turned on by setting ``debug: False`` to ``debug: True`` in :origin:`settings.yml `. +.. sidebar:: :ref:`make test` + + Alternatively use the :ref:`make test` targets. + Run ``./manage.sh tests`` before creating a PR. Failing build on Travis is common because of PEP8 checks. So a new commit must be created containing these format fixes. This phase can be skipped if diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst new file mode 100644 index 00000000..4dc1279f --- /dev/null +++ b/docs/dev/reST.rst @@ -0,0 +1,1428 @@ +.. _reST primer: + +=========== +reST primer +=========== + +.. sidebar:: KISS_ and readability_ + + Instead of defining more and more roles, we at searx encourage our + contributors to follow principles like KISS_ and readability_. + +We at searx are using reStructuredText (aka reST_) markup for all kind of +documentation, with the builders from the Sphinx_ project a HTML output is +generated and deployed at :docs:`github.io <.>`. For build prerequisites read +:ref:`docs build`. + +The source files of Searx's documentation are located at :origin:`docs`. Sphinx +assumes source files to be encoded in UTF-8 by defaul. Run :ref:`make docs-live +` to build HTML while editing. + +.. sidebar:: Further reading + + - Sphinx-Primer_ + - `Sphinx markup constructs`_ + - reST_, docutils_, `docutils FAQ`_ + - Sphinx_, `sphinx-doc FAQ`_ + - `sphinx config`_, doctree_ + - `sphinx cross references`_ + - linuxdoc_ + - intersphinx_ + - sphinx-jinja_ + - `Sphinx's autodoc`_ + - `Sphinx's Python domain`_, `Sphinx's C domain`_ + - SVG_, ImageMagick_ + - DOT_, `Graphviz's dot`_, Graphviz_ + + +.. contents:: Contents + :depth: 3 + :local: + :backlinks: entry + +Sphinx_ and reST_ have their place in the python ecosystem. Over that reST is +used in popular projects, e.g the Linux kernel documentation `[kernel doc]`_. + +.. _[kernel doc]: https://www.kernel.org/doc/html/latest/doc-guide/sphinx.html + +.. sidebar:: Content matters + + The readability_ of the reST sources has its value, therefore we recommend to + make sparse usage of reST markup / .. content matters! + +**reST** is a plaintext markup language, its markup is *mostly* intuitive and +you will not need to learn much to produce well formed articles with. I use the +word *mostly*: like everything in live, reST has its advantages and +disadvantages, some markups feel a bit grumpy (especially if you are used to +other plaintext markups). + +Soft skills +=========== + +Before going any deeper into the markup let's face on some **soft skills** a +trained author brings with, to reach a well feedback from readers: + +- Documentation is dedicated to an audience and answers questions from the + audience point of view. +- Don't detail things which are general knowledge from the audience point of + view. +- Limit the subject, use cross links for any further reading. + +To be more concrete what a *point of view* means. In the (:origin:`docs`) +folder we have three sections (and the *blog* folder), each dedicate to a +different group of audience. + +User's POV: :origin:`docs/user` + A typical user knows about search engines and might have heard about + meta crawlers and privacy. + +Admin's POV: :origin:`docs/admin` + A typical Admin knows about setting up services on a linux system, but he does + not know all the pros and cons of a searx setup. + +Developer's POV: :origin:`docs/dev` + Depending on the readability_ of code, a typical developer is able to read and + understand source code. Describe what a item aims to do (e.g. a function). + If the chronological order matters, describe it. Name the *out-of-limits + conditions* and all the side effects a external developer will not know. + +.. _reST inline markup: + +Basic inline markup +=================== + +.. sidebar:: Inline markup + + - :ref:`reST roles` + - :ref:`reST smart ref` + +Basic inline markup is done with asterisks and backquotes. If asterisks or +backquotes appear in running text and could be confused with inline markup +delimiters, they have to be escaped with a backslash (``\*pointer``). + +.. table:: basic inline markup + :widths: 4 3 7 + + ================================================ ==================== ======================== + description rendered markup + ================================================ ==================== ======================== + one asterisk for emphasis *italics* ``*italics*`` + two asterisks for strong emphasis **boldface** ``**boldface**`` + backquotes for code samples and literals ``foo()`` ````foo()```` + quote asterisks or backquotes \*foo is a pointer ``\*foo is a pointer`` + ================================================ ==================== ======================== + +.. _reST basic structure: + +Basic article structure +======================= + +The basic structure of an article makes use of heading adornments to markup +chapter, sections and subsections. + +.. _reST template: + +reST template +------------- + +reST template for an simple article: + +.. code:: reST + + .. _doc refname: + + ============== + Document title + ============== + + Lorem ipsum dolor sit amet, consectetur adipisici elit .. Further read + :ref:`chapter refname`. + + .. _chapter refname: + + Chapter + ======= + + Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut + aliquid ex ea commodi consequat ... + + .. _section refname: + + Section + ------- + + lorem .. + + .. _subsection refname: + + Subsection + ~~~~~~~~~~ + + lorem .. + + +Headings +-------- + +#. title - with overline for document title: + + .. code:: reST + + ============== + Document title + ============== + + +#. chapter - with anchor named ``anchor name``: + + .. code:: reST + + .. _anchor name: + + Chapter + ======= + +#. section + + .. code:: reST + + Section + ------- + +#. subsection + + .. code:: reST + + Subsection + ~~~~~~~~~~ + + + +Anchors & Links +=============== + +.. _reST anchor: + +Anchors +------- + +.. _ref role: + https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref + +To refer a point in the documentation a anchor is needed. The :ref:`reST +template ` shows an example where a chapter titled *"Chapters"* +gets an anchor named ``chapter title``. Another example from *this* document, +where the anchor named ``reST anchor``: + +.. code:: reST + + .. _reST anchor: + + Anchors + ------- + + To refer a point in the documentation a anchor is needed ... + +To refer anchors use the `ref role`_ markup: + +.. code:: reST + + Visit chapter :ref:`reST anchor`. Or set hyperlink text manualy :ref:`foo + bar `. + +.. admonition:: ``:ref:`` role + :class: rst-example + + Visist chapter :ref:`reST anchor`. Or set hyperlink text manualy :ref:`foo + bar `. + +.. _reST ordinary ref: + +Link ordinary URL +----------------- + +If you need to reference external URLs use *named* hyperlinks to maintain +readability of reST sources. Here is a example taken from *this* article: + +.. code:: reST + + .. _Sphinx Field Lists: + https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html + + With the *named* hyperlink `Sphinx Field Lists`_, the raw text is much more + readable. + + And this shows the alternative (less readable) hyperlink markup `Sphinx Field + Lists + `__. + +.. admonition:: Named hyperlink + :class: rst-example + + With the *named* hyperlink `Sphinx Field Lists`_, the raw text is much more + readable. + + And this shows the alternative (less readable) hyperlink markup `Sphinx Field + Lists + `__. + + +.. _reST smart ref: + +Smart refs +---------- + +With the power of sphinx.ext.extlinks_ and intersphinx_ referencing external +content becomes smart. + +.. table:: smart refs with sphinx.ext.extlinks_ and intersphinx_ + :widths: 4 3 7 + + ========================== ================================== ==================================== + refer ... rendered example markup + ========================== ================================== ==================================== + :rst:role:`rfc` :rfc:`822` ``:rfc:`822``` + :rst:role:`pep` :pep:`8` ``:pep:`8``` + sphinx.ext.extlinks_ + -------------------------------------------------------------------------------------------------- + project's wiki article :wiki:`Searx-instances` ``:wiki:`Searx-instances``` + to docs public URL :docs:`dev/reST.html` ``:docs:`dev/reST.html``` + files & folders origin :origin:`docs/dev/reST.rst` ``:origin:`docs/dev/reST.rst``` + pull request :pull:`1756` ``:pull:`1756``` + patch :patch:`af2cae6` ``:patch:`af2cae6``` + PyPi package :pypi:`searx` ``:pypi:`searx``` + manual page man :man:`bash` ``:man:`bash``` + intersphinx_ + -------------------------------------------------------------------------------------------------- + external anchor :ref:`python:and` ``:ref:`python:and``` + external doc anchor :doc:`jinja:templates` ``:doc:`jinja:templates``` + python code object :py:obj:`datetime.datetime` ``:py:obj:`datetime.datetime``` + flask code object :py:obj:`flask.Flask` ``:py:obj:`flask.Flask``` + ========================== ================================== ==================================== + + +Intersphinx is configured in :origin:`docs/conf.py`: + +.. code:: python + + intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), + "flask": ("https://flask.palletsprojects.com/", None), + "jinja": ("https://jinja.palletsprojects.com/", None), + "linuxdoc" : ("https://return42.github.io/linuxdoc/", None), + "sphinx" : ("https://www.sphinx-doc.org/en/master/", None), + } + + +To list all anchors of the inventory (e.g. ``python``) use: + +.. code:: sh + + $ python -m sphinx.ext.intersphinx https://docs.python.org/3/objects.inv + +Literal blocks +============== + +The simplest form of :duref:`literal-blocks` is a indented block introduced by +two colons (``::``). For highlighting use :dudir:`highlight` or :ref:`reST +code` directive. To include literals from external files use directive +:dudir:`literalinclude`. + +.. _reST literal: + +``::`` +------ + +.. code:: reST + + :: + + Literal block + + Lorem ipsum dolor:: + + Literal block + + Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy + eirmod tempor invidunt ut labore :: + + Literal block + +.. admonition:: Literal block + :class: rst-example + + :: + + Literal block + + Lorem ipsum dolor:: + + Literal block + + Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy + eirmod tempor invidunt ut labore :: + + Literal block + + +.. _reST code: + +``code-block`` +-------------- + +.. _pygments: https://pygments.org/languages/ + +.. sidebar:: Syntax highlighting + + is handled by pygments_. + +The :rst:dir:`code-block` directive is a variant of the :dudir:`code` directive +with additional options. To learn more about code literals visit +:ref:`sphinx:code-examples`. + +.. code-block:: reST + + The URL ``/stats`` handle is shown in :ref:`stats-handle` + + .. code-block:: Python + :caption: python code block + :name: stats-handle + + @app.route('/stats', methods=['GET']) + def stats(): + """Render engine statistics page.""" + stats = get_engines_stats() + return render( + 'stats.html' + , stats = stats ) + +.. code-block:: reST + +.. admonition:: Code block + :class: rst-example + + The URL ``/stats`` handle is shown in :ref:`stats-handle` + + .. code-block:: Python + :caption: python code block + :name: stats-handle + + @app.route('/stats', methods=['GET']) + def stats(): + """Render engine statistics page.""" + stats = get_engines_stats() + return render( + 'stats.html' + , stats = stats ) + +Unicode substitution +==================== + +The :dudir:`unicode directive ` converts Unicode +character codes (numerical values) to characters. This directive can only be +used within a substitution definition. + +.. code-block:: reST + + .. |copy| unicode:: 0xA9 .. copyright sign + .. |(TM)| unicode:: U+2122 + + Trademark |(TM)| and copyright |copy| glyphs. + +.. admonition:: Unicode + :class: rst-example + + .. |copy| unicode:: 0xA9 .. copyright sign + .. |(TM)| unicode:: U+2122 + + Trademark |(TM)| and copyright |copy| glyphs. + + +.. _reST roles: + +Roles +===== + +.. sidebar:: Further reading + + - `Sphinx Roles`_ + - :doc:`sphinx:usage/restructuredtext/domains` + + +A *custom interpreted text role* (:duref:`ref `) is an inline piece of +explicit markup. It signifies that that the enclosed text should be interpreted +in a specific way. + +The general markup is one of: + +.. code:: reST + + :rolename:`ref-name` + :rolename:`ref text ` + +.. table:: smart refs with sphinx.ext.extlinks_ and intersphinx_ + :widths: 4 3 7 + + ========================== ================================== ==================================== + role rendered example markup + ========================== ================================== ==================================== + :rst:role:`guilabel` :guilabel:`&Cancel` ``:guilabel:`&Cancel``` + :rst:role:`kbd` :kbd:`C-x C-f` ``:kbd:`C-x C-f``` + :rst:role:`menuselection` :menuselection:`Open --> File` ``:menuselection:`Open --> File``` + :rst:role:`download` :download:`this file ` ``:download:`this file ``` + math_ :math:`a^2 + b^2 = c^2` ``:math:`a^2 + b^2 = c^2``` + :rst:role:`ref` :ref:`svg image example` ``:ref:`svg image example``` + :rst:role:`command` :command:`ls -la` ``:command:`ls -la``` + :durole:`emphasis` :emphasis:`italic` ``:emphasis:`italic``` + :durole:`strong` :strong:`bold` ``:strong:`bold``` + :durole:`literal` :literal:`foo()` ``:literal:`foo()``` + :durole:`subscript` H\ :sub:`2`\ O ``H\ :sub:`2`\ O`` + :durole:`superscript` E = mc\ :sup:`2` ``E = mc\ :sup:`2``` + :durole:`title-reference` :title:`Time` ``:title:`Time``` + ========================== ================================== ==================================== + +Figures & Images +================ + +.. sidebar:: Image processing + + With the directives from :ref:`linuxdoc ` the build process + is flexible. To get best results in the generated output format, install + ImageMagick_ and Graphviz_. + +Searx's sphinx setup includes: :ref:`linuxdoc:kfigure`. Scaleable here means; +scaleable in sense of the build process. Normally in absence of a converter +tool, the build process will break. From the authors POV it’s annoying to care +about the build process when handling with images, especially since he has no +access to the build process. With :ref:`linuxdoc:kfigure` the build process +continues and scales output quality in dependence of installed image processors. + +If you want to add an image, you should use the ``kernel-figure`` (inheritance +of :dudir:`figure`) and ``kernel-image`` (inheritance of :dudir:`image`) +directives. E.g. to insert a figure with a scaleable image format use SVG +(:ref:`svg image example`): + +.. code:: reST + + .. _svg image example: + + .. kernel-figure:: svg_image.svg + :alt: SVG image example + + Simple SVG image + + To refer the figure, a caption block is needed: :ref:`svg image example`. + +.. _svg image example: + +.. kernel-figure:: svg_image.svg + :alt: SVG image example + + Simple SVG image. + +To refer the figure, a caption block is needed: :ref:`svg image example`. + +DOT files (aka Graphviz) +------------------------ + +With :ref:`linuxdoc:kernel-figure` reST support for **DOT** formatted files is +given. + +- `Graphviz's dot`_ +- DOT_ +- Graphviz_ + +A simple example is shown in :ref:`dot file example`: + +.. code:: reST + + .. _dot file example: + + .. kernel-figure:: hello.dot + :alt: hello world + + DOT's hello world example + +.. admonition:: hello.dot + :class: rst-example + + .. _dot file example: + + .. kernel-figure:: hello.dot + :alt: hello world + + DOT's hello world example + +``kernel-render`` DOT +--------------------- + +Embed *render* markups (or languages) like Graphviz's **DOT** is provided by the +:ref:`linuxdoc:kernel-render` directive. A simple example of embedded DOT_ is +shown in figure :ref:`dot render example`: + +.. code:: reST + + .. _dot render example: + + .. kernel-render:: DOT + :alt: digraph + :caption: Embedded DOT (Graphviz) code + + digraph foo { + "bar" -> "baz"; + } + + Attribute ``caption`` is needed, if you want to refer the figure: :ref:`dot + render example`. + +Please note :ref:`build tools `. If Graphviz_ is +installed, you will see an vector image. If not, the raw markup is inserted as +*literal-block*. + +.. admonition:: kernel-render DOT + :class: rst-example + + .. _dot render example: + + .. kernel-render:: DOT + :alt: digraph + :caption: Embedded DOT (Graphviz) code + + digraph foo { + "bar" -> "baz"; + } + + Attribute ``caption`` is needed, if you want to refer the figure: :ref:`dot + render example`. + +``kernel-render`` SVG +--------------------- + +A simple example of embedded SVG_ is shown in figure :ref:`svg render example`: + +.. code:: reST + + .. _svg render example: + + .. kernel-render:: SVG + :caption: Embedded **SVG** markup + :alt: so-nw-arrow +.. + + .. code:: xml + + + + + + + +.. admonition:: kernel-render SVG + :class: rst-example + + .. _svg render example: + + .. kernel-render:: SVG + :caption: Embedded **SVG** markup + :alt: so-nw-arrow + + + + + + + + + + +.. _reST lists: + +List markups +============ + +Bullet list +----------- + +List markup (:duref:`ref `) is simple: + +.. code:: reST + + - This is a bulleted list. + + 1. Nested lists are possible, but be aware that they must be separated from + the parent list items by blank line + 2. Second item of nested list + + - It has two items, the second + item uses two lines. + + #. This is a numbered list. + #. It has two items too. + +.. admonition:: bullet list + :class: rst-example + + - This is a bulleted list. + + 1. Nested lists are possible, but be aware that they must be separated from + the parent list items by blank line + 2. Second item of nested list + + - It has two items, the second + item uses two lines. + + #. This is a numbered list. + #. It has two items too. + + +Horizontal list +--------------- + +The :rst:dir:`.. hlist:: ` transforms a bullet list into a more compact +list. + +.. code:: reST + + .. hlist:: + + - first list item + - second list item + - third list item + ... + +.. admonition:: hlist + :class: rst-example + + .. hlist:: + + - first list item + - second list item + - third list item + - next list item + - next list item xxxx + - next list item yyyy + - next list item zzzz + + +Definition list +--------------- + +.. sidebar:: Note .. + + - the term cannot have more than one line of text + + - there is **no blank line between term and definition block** // this + distinguishes definition lists (:duref:`ref `) from block + quotes (:duref:`ref `). + +Each definition list (:duref:`ref `) item contains a term, +optional classifiers and a definition. A term is a simple one-line word or +phrase. Optional classifiers may follow the term on the same line, each after +an inline ' : ' (**space, colon, space**). A definition is a block indented +relative to the term, and may contain multiple paragraphs and other body +elements. There may be no blank line between a term line and a definition block +(*this distinguishes definition lists from block quotes*). Blank lines are +required before the first and after the last definition list item, but are +optional in-between. + +Definition lists are created as follows: + +.. code:: reST + + term 1 (up to a line of text) + Definition 1. + + See the typo : this line is not a term! + + And this is not term's definition. **There is a blank line** in between + the line above and this paragraph. That's why this paragraph is taken as + **block quote** (:duref:`ref `) and not as term's definition! + + term 2 + Definition 2, paragraph 1. + + Definition 2, paragraph 2. + + term 3 : classifier + Definition 3. + + term 4 : classifier one : classifier two + Definition 4. + +.. admonition:: definition list + :class: rst-example + + term 1 (up to a line of text) + Definition 1. + + See the typo : this line is not a term! + + And this is not term's definition. **There is a blank line** in between + the line above and this paragraph. That's why this paragraph is taken as + **block quote** (:duref:`ref `) and not as term's definition! + + + term 2 + Definition 2, paragraph 1. + + Definition 2, paragraph 2. + + term 3 : classifier + Definition 3. + + term 4 : classifier one : classifier two + + +Quoted paragraphs +----------------- + +Quoted paragraphs (:duref:`ref `) are created by just indenting +them more than the surrounding paragraphs. Line blocks (:duref:`ref +`) are a way of preserving line breaks: + +.. code:: reST + + normal paragraph ... + lorem ipsum. + + Quoted paragraph ... + lorem ipsum. + + | These lines are + | broken exactly like in + | the source file. + + +.. admonition:: Quoted paragraph and line block + :class: rst-example + + normal paragraph ... + lorem ipsum. + + Quoted paragraph ... + lorem ipsum. + + | These lines are + | broken exactly like in + | the source file. + + +.. _reST field list: + +Field Lists +----------- + +.. _Sphinx Field Lists: + https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html + +.. sidebar:: bibliographic fields + + First lines fields are bibliographic fields, see `Sphinx Field Lists`_. + +Field lists are used as part of an extension syntax, such as options for +directives, or database-like records meant for further processing. Field lists +are mappings from field names to field bodies. They marked up like this: + +.. code:: reST + + :fieldname: Field content + :foo: first paragraph in field foo + + second paragraph in field foo + + :bar: Field content + +.. admonition:: Field List + :class: rst-example + + :fieldname: Field content + :foo: first paragraph in field foo + + second paragraph in field foo + + :bar: Field content + + +They are commonly used in Python documentation: + +.. code:: python + + def my_function(my_arg, my_other_arg): + """A function just for me. + + :param my_arg: The first of my arguments. + :param my_other_arg: The second of my arguments. + + :returns: A message (just for me, of course). + """ + +Further list blocks +------------------- + +- field lists (:duref:`ref `, with caveats noted in + :ref:`reST field list`) +- option lists (:duref:`ref `) +- quoted literal blocks (:duref:`ref `) +- doctest blocks (:duref:`ref `) + + +Admonitions +=========== + +Sidebar +------- + +Sidebar is an eye catcher, often used for admonitions pointing further stuff or +site effects. Here is the source of the sidebar :ref:`on top of this page `. + +.. code:: reST + + .. sidebar:: KISS_ and readability_ + + Instead of defining more and more roles, we at searx encourage our + contributors to follow principles like KISS_ and readability_. + +Generic admonition +------------------ + +The generic :dudir:`admonition ` needs a title: + +.. code:: reST + + .. admonition:: generic admonition title + + lorem ipsum .. + + +.. admonition:: generic admonition title + + lorem ipsum .. + + +Specific admonitions +-------------------- + +Specific admonitions: :dudir:`hint`, :dudir:`note`, :dudir:`tip` :dudir:`attention`, +:dudir:`caution`, :dudir:`danger`, :dudir:`error`, , :dudir:`important`, and +:dudir:`warning` . + +.. code:: reST + + .. hint:: + + lorem ipsum .. + + .. note:: + + lorem ipsum .. + + .. warning:: + + lorem ipsum .. + + +.. hint:: + + lorem ipsum .. + +.. note:: + + lorem ipsum .. + +.. tip:: + + lorem ipsum .. + +.. attention:: + + lorem ipsum .. + +.. caution:: + + lorem ipsum .. + +.. danger:: + + lorem ipsum .. + +.. important:: + + lorem ipsum .. + +.. error:: + + lorem ipsum .. + +.. warning:: + + lorem ipsum .. + + +Tables +====== + +.. sidebar:: Nested tables + + Nested tables are ugly! Not all builder support nested tables, don't use + them! + +ASCII-art tables like :ref:`reST simple table` and :ref:`reST grid table` might +be comfortable for readers of the text-files, but they have huge disadvantages +in the creation and modifying. First, they are hard to edit. Think about +adding a row or a column to a ASCII-art table or adding a paragraph in a cell, +it is a nightmare on big tables. + + +.. sidebar:: List tables + + For meaningful patch and diff use :ref:`reST flat table`. + +Second the diff of modifying ASCII-art tables is not meaningful, e.g. widening a +cell generates a diff in which also changes are included, which are only +ascribable to the ASCII-art. Anyway, if you prefer ASCII-art for any reason, +here are some helpers: + +* `Emacs Table Mode`_ +* `Online Tables Generator`_ + +.. _reST simple table: + +Simple tables +------------- + +:duref:`Simple tables ` allow *colspan* but not *rowspan*. If +your table need some metadata (e.g. a title) you need to add the ``.. table:: +directive`` :dudir:`(ref) ` in front and place the table in its body: + +.. code:: reST + + .. table:: foo gate truth table + :widths: grid + :align: left + + ====== ====== ====== + Inputs Output + ------------- ------ + A B A or B + ====== ====== ====== + False + -------------------- + True + -------------------- + True False True + (foo) + ------ ------ ------ + False True + (foo) + ====== ============= + +.. admonition:: Simple ASCII table + :class: rst-example + + .. table:: foo gate truth table + :widths: grid + :align: left + + ====== ====== ====== + Inputs Output + ------------- ------ + A B A or B + ====== ====== ====== + False + -------------------- + True + -------------------- + True False True + (foo) + ------ ------ ------ + False True + (foo) + ====== ============= + + + +.. _reST grid table: + +Grid tables +----------- + +:duref:`Grid tables ` allow colspan *colspan* and *rowspan*: + +.. code:: reST + + .. table:: grid table example + :widths: 1 1 5 + + +------------+------------+-----------+ + | Header 1 | Header 2 | Header 3 | + +============+============+===========+ + | body row 1 | column 2 | column 3 | + +------------+------------+-----------+ + | body row 2 | Cells may span columns.| + +------------+------------+-----------+ + | body row 3 | Cells may | - Cells | + +------------+ span rows. | - contain | + | body row 4 | | - blocks. | + +------------+------------+-----------+ + +.. admonition:: ASCII grid table + :class: rst-example + + .. table:: grid table example + :widths: 1 1 5 + + +------------+------------+-----------+ + | Header 1 | Header 2 | Header 3 | + +============+============+===========+ + | body row 1 | column 2 | column 3 | + +------------+------------+-----------+ + | body row 2 | Cells may span columns.| + +------------+------------+-----------+ + | body row 3 | Cells may | - Cells | + +------------+ span rows. | - contain | + | body row 4 | | - blocks. | + +------------+------------+-----------+ + + +.. _reST flat table: + +flat-table +---------- + +The ``flat-table`` is a further developed variant of the :ref:`list tables +`. It is a double-stage list similar to the +:dudir:`list-table` with some additional features: + +column-span: ``cspan`` + with the role ``cspan`` a cell can be extended through additional columns + +row-span: ``rspan`` + with the role ``rspan`` a cell can be extended through additional rows + +auto-span: + spans rightmost cell of a table row over the missing cells on the right side + of that table-row. With Option ``:fill-cells:`` this behavior can changed + from *auto span* to *auto fill*, which automatically inserts (empty) cells + instead of spanning the last cell. + +options: + :header-rows: [int] count of header rows + :stub-columns: [int] count of stub columns + :widths: [[int] [int] ... ] widths of columns + :fill-cells: instead of auto-span missing cells, insert missing cells + +roles: + :cspan: [int] additional columns (*morecols*) + :rspan: [int] additional rows (*morerows*) + +The example below shows how to use this markup. The first level of the staged +list is the *table-row*. In the *table-row* there is only one markup allowed, +the list of the cells in this *table-row*. Exception are *comments* ( ``..`` ) +and *targets* (e.g. a ref to :ref:`row 2 of table's body `). + +.. code:: reST + + .. flat-table:: ``flat-table`` example + :header-rows: 2 + :stub-columns: 1 + :widths: 1 1 1 1 2 + + * - :rspan:`1` head / stub + - :cspan:`3` head 1.1-4 + + * - head 2.1 + - head 2.2 + - head 2.3 + - head 2.4 + + * .. row body 1 / this is a comment + + - row 1 + - :rspan:`2` cell 1-3.1 + - cell 1.2 + - cell 1.3 + - cell 1.4 + + * .. Comments and targets are allowed on *table-row* stage. + .. _`row body 2`: + + - row 2 + - cell 2.2 + - :rspan:`1` :cspan:`1` + cell 2.3 with a span over + + * col 3-4 & + * row 2-3 + + * - row 3 + - cell 3.2 + + * - row 4 + - cell 4.1 + - cell 4.2 + - cell 4.3 + - cell 4.4 + + * - row 5 + - cell 5.1 with automatic span to rigth end + + * - row 6 + - cell 6.1 + - .. + + +.. admonition:: List table + :class: rst-example + + .. flat-table:: ``flat-table`` example + :header-rows: 2 + :stub-columns: 1 + :widths: 1 1 1 1 2 + + * - :rspan:`1` head / stub + - :cspan:`3` head 1.1-4 + + * - head 2.1 + - head 2.2 + - head 2.3 + - head 2.4 + + * .. row body 1 / this is a comment + + - row 1 + - :rspan:`2` cell 1-3.1 + - cell 1.2 + - cell 1.3 + - cell 1.4 + + * .. Comments and targets are allowed on *table-row* stage. + .. _`row body 2`: + + - row 2 + - cell 2.2 + - :rspan:`1` :cspan:`1` + cell 2.3 with a span over + + * col 3-4 & + * row 2-3 + + * - row 3 + - cell 3.2 + + * - row 4 + - cell 4.1 + - cell 4.2 + - cell 4.3 + - cell 4.4 + + * - row 5 + - cell 5.1 with automatic span to rigth end + + * - row 6 + - cell 6.1 + - .. + + +CSV table +--------- + +CSV table might be the choice if you want to include CSV-data from a outstanding +(build) process into your documentation. + +.. code:: reST + + .. csv-table:: CSV table example + :header: .. , Column 1, Column 2 + :widths: 2 5 5 + :stub-columns: 1 + :file: csv_table.txt + +Content of file ``csv_table.txt``: + +.. literalinclude:: csv_table.txt + +.. admonition:: CSV table + :class: rst-example + + .. csv-table:: CSV table example + :header: .. , Column 1, Column 2 + :widths: 3 5 5 + :stub-columns: 1 + :file: csv_table.txt + +Templating +========== + +.. sidebar:: Build environment + + All *generic-doc* tasks are running in the :ref:`build environment `. + +Templating is suitable for documentation which is created generic at the build +time. The sphinx-jinja_ extension evaluates jinja_ templates in the :ref:`build +environment ` (with searx modules installed). We use this e.g. to +build chapter: :ref:`engines generic`. Below the jinja directive from the +:origin:`docs/admin/engines.rst` is shown: + +.. literalinclude:: ../admin/engines.rst + :language: reST + :start-after: .. _configured engines: + +The context for the template is selected in the line ``.. jinja:: webapp``. In +sphinx's build configuration (:origin:`docs/conf.py`) the ``webapp`` context +points to the name space of the python module: ``webapp``. + +.. code:: py + + from searx import webapp + jinja_contexts = { + 'webapp': dict(**webapp.__dict__) + } + + +Tabbed views +============ + +.. _sphinx-tabs: https://github.com/djungelorm/sphinx-tabs +.. _basic-tabs: https://github.com/djungelorm/sphinx-tabs#basic-tabs +.. _group-tabs: https://github.com/djungelorm/sphinx-tabs#group-tabs +.. _code-tabs: https://github.com/djungelorm/sphinx-tabs#code-tabs + +With `sphinx-tabs`_ extension we have *tabbed views*. To provide installation +instructions with one tab per distribution we use the `group-tabs`_ directive, +others are basic-tabs_ and code-tabs_. Below a *group-tab* example from +:ref:`docs build` is shown: + +.. literalinclude:: ../admin/buildhosts.rst + :language: reST + :start-after: .. _system requirements: + :end-before: .. _system requirements END: + + +.. _math: + +Math equations +============== + +.. _Mathematics: https://en.wikibooks.org/wiki/LaTeX/Mathematics +.. _amsmath user guide: + http://vesta.informatik.rwth-aachen.de/ftp/pub/mirror/ctan/macros/latex/required/amsmath/amsldoc.pdf + +.. sidebar:: About LaTeX + + - `amsmath user guide`_ + - Mathematics_ + - :ref:`docs build` + +The input language for mathematics is LaTeX markup using the :ctan:`amsmath` +package. + +To embed LaTeX markup in reST documents, use role :rst:role:`:math: ` for +inline and directive :rst:dir:`.. math:: ` for block markup. + +.. code:: reST + + In :math:numref:`schroedinger general` the time-dependent Schrödinger equation + is shown. + + .. math:: + :label: schroedinger general + + \mathrm{i}\hbar\dfrac{\partial}{\partial t} |\,\psi (t) \rangle = + \hat{H} |\,\psi (t) \rangle. + +.. admonition:: LaTeX math equation + :class: rst-example + + In :math:numref:`schroedinger general` the time-dependent Schrödinger equation + is shown. + + .. math:: + :label: schroedinger general + + \mathrm{i}\hbar\dfrac{\partial}{\partial t} |\,\psi (t) \rangle = + \hat{H} |\,\psi (t) \rangle. + + +The next example shows the difference of ``\tfrac`` (*textstyle*) and ``\dfrac`` +(*displaystyle*) used in a inline markup or another fraction. + +.. code:: reST + + ``\tfrac`` **inline example** :math:`\tfrac{\tfrac{1}{x}+\tfrac{1}{y}}{y-z}` + ``\dfrac`` **inline example** :math:`\dfrac{\dfrac{1}{x}+\dfrac{1}{y}}{y-z}` + +.. admonition:: Line spacing + :class: rst-example + + Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy + eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam + voluptua. ... + ``\tfrac`` **inline example** :math:`\tfrac{\tfrac{1}{x}+\tfrac{1}{y}}{y-z}` + At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd + gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. + + Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy + eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam + voluptua. ... + ``\tfrac`` **inline example** :math:`\dfrac{\dfrac{1}{x}+\dfrac{1}{y}}{y-z}` + At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd + gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. + + +.. _KISS: https://en.wikipedia.org/wiki/KISS_principle + +.. _readability: https://docs.python-guide.org/writing/style/ +.. _Sphinx-Primer: + http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html +.. _reST: https://docutils.sourceforge.io/rst.html +.. _Sphinx Roles: + https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html +.. _Sphinx: http://www.sphinx-doc.org +.. _`sphinx-doc FAQ`: http://www.sphinx-doc.org/en/stable/faq.html +.. _Sphinx markup constructs: + http://www.sphinx-doc.org/en/stable/markup/index.html +.. _`sphinx cross references`: + http://www.sphinx-doc.org/en/stable/markup/inline.html#cross-referencing-arbitrary-locations +.. _sphinx.ext.extlinks: + https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html +.. _intersphinx: http://www.sphinx-doc.org/en/stable/ext/intersphinx.html +.. _sphinx config: http://www.sphinx-doc.org/en/stable/config.html +.. _Sphinx's autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html +.. _Sphinx's Python domain: + http://www.sphinx-doc.org/en/stable/domains.html#the-python-domain +.. _Sphinx's C domain: + http://www.sphinx-doc.org/en/stable/domains.html#cross-referencing-c-constructs +.. _doctree: + http://www.sphinx-doc.org/en/master/extdev/tutorial.html?highlight=doctree#build-phases +.. _docutils: http://docutils.sourceforge.net/docs/index.html +.. _docutils FAQ: http://docutils.sourceforge.net/FAQ.html +.. _linuxdoc: https://return42.github.io/linuxdoc +.. _jinja: https://jinja.palletsprojects.com/ +.. _sphinx-jinja: https://github.com/tardyp/sphinx-jinja +.. _SVG: https://www.w3.org/TR/SVG11/expanded-toc.html +.. _DOT: https://graphviz.gitlab.io/_pages/doc/info/lang.html +.. _`Graphviz's dot`: https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf +.. _Graphviz: https://graphviz.gitlab.io +.. _ImageMagick: https://www.imagemagick.org + +.. _`Emacs Table Mode`: https://www.emacswiki.org/emacs/TableMode +.. _`Online Tables Generator`: http://www.tablesgenerator.com/text_tables +.. _`OASIS XML Exchange Table Model`: https://www.oasis-open.org/specs/tm9901.html diff --git a/docs/dev/search_api.rst b/docs/dev/search_api.rst index 158cab7c..922548ff 100644 --- a/docs/dev/search_api.rst +++ b/docs/dev/search_api.rst @@ -1,3 +1,5 @@ +.. _search API: + ========== Search API ========== @@ -14,6 +16,12 @@ Furthermore, two enpoints ``/`` and ``/search`` are available for querying. Parameters ========== +.. sidebar:: Further reading .. + + - :ref:`engines-dev` + - :ref:`settings.yml` + - :ref:`engines generic` + ``q`` : required The search query. This string is passed to external search services. Thus, searx supports syntax of each search service. For example, ``site:github.com @@ -26,48 +34,48 @@ Parameters ``categories`` : optional Comma separated list, specifies the active search categories -``engines``: optional +``engines`` : optional Comma separated list, specifies the active search engines. -``lang``: default ``all`` +``lang`` : default ``all`` Code of the language. -``pageno``: default ``1`` +``pageno`` : default ``1`` Search page number. -``time_range``: optional +``time_range`` : optional [ ``day``, ``month``, ``year`` ] Time range of search for engines which support it. See if an engine supports time range search in the preferences page of an instance. -``format``: optional +``format`` : optional [ ``json``, ``csv``, ``rss`` ] Output format of results. -``results_on_new_tab``: default ``0`` +``results_on_new_tab`` : default ``0`` [ ``0``, ``1`` ] Open search results on new tab. -``image_proxy``: default ``False`` +``image_proxy`` : default ``False`` [ ``True``, ``False`` ] Proxy image results through searx. -``autocomplete``: default *empty* +``autocomplete`` : default *empty* [ ``google``, ``dbpedia``, ``duckduckgo``, ``startpage``, ``wikipedia`` ] Service which completes words as you type. -``safesearch``: default ``None`` +``safesearch`` : default ``None`` [ ``0``, ``1``, ``None`` ] Filter search results of engines which support safe search. See if an engine supports safe search in the preferences page of an instance. -``theme``: default ``oscar`` +``theme`` : default ``oscar`` [ ``oscar``, ``simple``, ``legacy``, ``pix-art``, ``courgette`` ] Theme of instance. @@ -76,7 +84,7 @@ Parameters instance administrator deleted, created or renamed themes on his/her instance. See the available options in the preferences page of the instance. -``oscar-style``: default ``logicodev`` +``oscar-style`` : default ``logicodev`` [ ``pointhi``, ``logicodev`` ] Style of Oscar theme. It is only parsed if the theme of an instance is @@ -86,7 +94,7 @@ Parameters instance administrator deleted, created or renamed styles on his/her instance. See the available options in the preferences page of the instance. -``enabled_plugins``: optional +``enabled_plugins`` : optional List of enabled plugins. :default: ``HTTPS_rewrite``, ``Self_Informations``, @@ -104,9 +112,9 @@ Parameters ``Vim-like_hotkeys``, ``Self_Informations``, ``Tracker_URL_remover``, ``Search_on_category_select`` -``enabled_engines``: optional : *all* :origin:`engines ` +``enabled_engines`` : optional : *all* :origin:`engines ` List of enabled engines. -``disabled_engines``: optional : *all* :origin:`engines ` +``disabled_engines`` : optional : *all* :origin:`engines ` List of disabled engines. diff --git a/docs/dev/svg_image.svg b/docs/dev/svg_image.svg new file mode 100644 index 00000000..5405f85b --- /dev/null +++ b/docs/dev/svg_image.svg @@ -0,0 +1,10 @@ + + + + + + + + diff --git a/docs/user/index.rst b/docs/user/index.rst index b13aca21..329be3b1 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -5,5 +5,6 @@ User documentation .. toctree:: :maxdepth: 1 + public_instances search_syntax own-instance diff --git a/docs/user/public_instances.rst b/docs/user/public_instances.rst new file mode 100644 index 00000000..9665c58b --- /dev/null +++ b/docs/user/public_instances.rst @@ -0,0 +1,318 @@ +.. _public instances: + +.. + links has been ported from markdown to reST by:: + + regexpr: \[([^\]]*)\]\(([^)]*)\) + substitution: `\1 <\2>`__ + + +====================== +Public Searx instances +====================== + +.. _mailing list: mailto:searx-instances@autistici.org +.. _subscription page: https://www.autistici.org/mailman/listinfo/searx-instances + + +Useful information +================== + +* Up-to-date health report available on https://stats.searx.xyz [1]_, for onion + (tor) services: https://stats.searx.xyz/tor.html + +* Searx instances `mailing list`_ & `subscription page`_. + +* Some of the Searx instances have a CAcert SSL certificate. You can install the + missing root cert `from here `__. + +* To add your own Searx instance to this page send us your PR. A GitHub account + is required to send PR or add an issue. + +.. [1] Note that most of the instances with a A+ grade in CSP column in this + site are not fully functional - for example auto-completion may not work. + + +List of public Searx instances +============================== + +Meta-searx instances +==================== + +These are websites that source from other searx instances. These are useful if +you can't decide which Searx instance to use: + + +.. flat-table:: Meta-searx instances + :header-rows: 1 + :stub-columns: 0 + :widths: 2 1 2 4 4 + + * - clearnet host + - onion host + - issuer + - source selection method + - extra privacy features + + * - `Neocities `__ + - n/a + - Comodo (`Verification `__) + - Redirects users directly to a random selection of any known running + server after entering query. Requires + Javascript. `Changelog `__. + - Excludes servers with user tracking and analytics or are proxied through + Cloudflare. + + * - `Searxes `__ @Danwin + - onion v3 `hidden service + `__ + - Let's Encrypt (`Verification + `__) + - sources data from a randomly selected running server that satisfies + admin's quality standards which is used for post-processing + - filters out privacy-hostile websites (like CloudFlare) and either marks + them as such or folds them below the high ranking results. + + +Alive and running +================= + +**BEFORE EDITING**: Please add your Searx instance by respecting the alphabetic order. + +.. note:: + + Public instances listed here may yield less accurate results as they have + much higher traffic and consequently have a higher chance of being blocked by + search providers such as Google, Qwant, Bing, Startpage, etc. Hosting your + own instance or using an instance that isn't listed here may give you a more + consistent search experience. + +* `ai.deafpray.wtf/searx `__ - Issuer: Let's Encrypt `Verification `__ +* `bamboozle.it `__ - Issuer: Let's Encrypt `Verification `__ +* `bee.jaekr.dev `__ - Issuer: Let's Encrypt `Verification `__ +* `beezboo.com `__ - Issuer: Let's Encrypt `Verification `__ +* `burtrum.org/searx `__ - Issuer: Let's Encrypt `Verification `__ +* `darmarit.cloud/searx `__ - Issuer: Let's Encrypt `Verification `__ +* `dc.ax `__ - Issuer: Let's Encrypt `Verification `__ +* `dynabyte.ca `__ - Issuer: Let's Encrypt `Verification `__ +* `goso.ga `__ - Issuer: Let's Encrypt `Verification `__ +* `gruble.de `__ - Issuer: Let's Encrypt `Verification `__ +* `haku.ahmia.fi `__ - Issuer: Let's Encrypt `Verification `__ +* `haku.lelux.fi `__ - Issuer: Let's Encrypt `Verification `__ +* `huyo.me `__ - Issuer: Let's Encrypt `Verification `__ +* `jsearch.pw `__ - Issuer: Let's Encrypt `Verification `__ +* `le-dahut.com/searx `__ - Issuer: Let's Encrypt `Verification `__ +* `mijisou.com `__ - Issuer: Let's Encrypt `Verification `__ +* `null.media `__ - Issuer: Let's Encrypt `Verification `__ +* `openworlds.info `__ - Issuer: Let's Encrypt +* `perfectpixel.de/searx/ `__ - Issuer: LetsEncrypt `Verification `__ +* `ransack.i2p `__ - I2P eepsite, only accessible with `I2P `__ (`base32 address `__) +* `rapu.nz `__ - Issuer: Let's Encrypt `Verification `__ +* `roflcopter.fr `__ - Issuer: Let's Encrypt `Verification `__ +* `roteserver.de/searx `__ - Issuer: Let's Encrypt `Verification `__ +* `s.cmd.gg `__ - Issuer: Let's Encrypt `Verification `__ +* `search.activemail.de `__ - Issuer: Let's Encrypt `Verification `__ +* `search.anonymize.com `__ - Issuer: Let's Encrypt `Verification `__ +* `search.azkware.net `__ - Issuer: Let's Encrypt `Verification `__ +* `search.biboumail.fr `__ - Issuer: Let's Encrypt `Verification `__ +* `search.blankenberg.eu `__ - Issuer: Let's Encrypt `Verification `__ +* `search.d4networks.com `__ - Issuer: Let's Encrypt `Verification `__ +* `search.datensturm.net `__ - Issuer: Let's Encrypt `Verification `__ +* `search.disroot.org `__ - Issuer: Let's Encrypt `Verification `__ +* `search.ethibox.fr `__ - Issuer: Let's Encrypt `Verification `__ +* `search.fossdaily.xyz `__ - Issuer: Let's Encrypt `Verification `__ +* `search.galaxy.cat `__ - Issuer: Let's Encrypt `Verification `__ +* `search.gibberfish.org `__ (as `Hidden Service `__ or `Proxied through Tor `__) - Issuer: Let's Encrypt `Verification `__ +* `search.koehn.com `__ - Issuer: Let's Encrypt `Verification `__ +* `search.lgbtq.cool `__ - Issuer: Let's Encrypt `Verification `__ +* `search.mdosch.de `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `search.modalogi.com `__ - Issuer: Let's Encrypt `Verification `__ +* `search.moravit.com `__ - Issuer: Let's Encrypt `Verification `__ +* `search.nebulacentre.net `__ - Issuer: Let's Encrypt `Verification `__ +* `search.paulla.asso.fr `__ - Issuer: Let's Encrypt `Verification `__ +* `search.pifferi.info `__ - Issuer: Let's Encrypt `Verification `__ +* `search.poal.co `__ - Issuer: Let's Encrypt `Verification `__ +* `search.privacytools.io `__ - Issuer: Let's Encrypt `Verification `__ - Uses Matomo for user tracking and analytics +* `search.seds.nl `__ - Issuer: Let's Encrypt `Verification `__ +* `search.snopyta.org `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `search.spaeth.me `__ - Issuer: Let's Encrypt `Verification `__ +* `search.st8.at `__ - Issuer: Let's Encrypt `Verification `__ +* `search.stinpriza.org `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `search.sudo-i.net `__ - Issuer: Let's Encrypt `Verification `__ +* `search.tolstoevsky.ml `__ - Issuer: Let's Encrypt `Verification `__ +* `searchsin.com/searx `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.anongoth.pl `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.be `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.ca `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.canox.net `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.cybt.de `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.de `__ - Issuer: COMODO `Verification `__ +* `searx.decatec.de `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.devol.it `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.dnswarden.com `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.drakonix.net `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `searx.dresden.network `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.elukerio.org `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.everdot.org `__ - Issuer: Let's Encrypt `Verification `__ - Crawls using YaCy +* `searx.foo.li `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.fossencdi.org `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `searx.fr32k.de `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.good.one.pl `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `searx.gotrust.de `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `searx.hardwired.link `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.hlfh.space `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.info `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.itunix.eu `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.john-at-me.net `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.kvch.me `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.laquadrature.net `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `searx.lelux.fi `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.lhorn.de `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.li `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.libmail.eu `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.linux.pizza `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.lynnesbian.space `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.mastodontech.de `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.me `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `searx.mxchange.org `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.nakhan.net `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.nixnet.xyz `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `searx.nnto.net `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.openhoofd.nl `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.openpandora.org `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.operationtulip.com `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.orcadian.net `__ - Issuer: Comodo CA Limited `Verification `__ +* `searx.ouahpit.info `__ - Issuer: Let's Encrypt +* `searx.pofilo.fr `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.prvcy.eu `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `searx.pwoss.org `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.ro `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.ru `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.solusar.de `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.targaryen.house `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.tuxcloud.net `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.tyil.nl `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.wegeeks.win `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.win `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.xyz `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.zareldyn.net `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.zdechov.net `__ - Issuer: Let's Encrypt `Verification `__ +* `searxs.eu `__ - Issuer: Let's Encrypt `Verification `__ +* `seeks.hsbp.org `__ - Issuer: Let's Encrypt `Verification `__ - `PGP signed fingerprints of cert `__ +* `skyn3t.in/srx `__ - Issuer: Let's Encrypt | onion `hidden service `__ +* `spot.ecloud.global `__ - Issuer: Let's Encrypt `Verification `__ +* `srx.sx `__ - Issuer: Let's Encrypt `Verification `__ +* `stemy.me/searx `__ - Issuer: Let's Encrypt `Verification `__ +* `suche.dasnetzundich.de `__ - Issuer: Let's Encrypt `Verification `__ +* `suche.elaon.de `__ - Issuer: Let's Encrypt `Verification `__ +* `suche.xyzco456vwisukfg.onion `__ +* `suche.uferwerk.org `__ - Issuer: Let's Encrypt `Verification `__ +* `timdor.noip.me/searx `__ - Issuer: Let's Encrypt `Verification `__ +* `trovu.komun.org `__ - Issuer: Let's Encrypt `Verification `__ +* `unmonito.red `__ - Issuer: Let's Encrypt `Verification `__ +* `www.finden.tk `__ - Issuer: Let's Encrypt `Verification `__ +* `zoek.anchel.nl `__ - Issuer: Let's Encrypt `Verification `__ + + + +Running in exclusive private walled-gardens +=========================================== + +These instances run in walled-gardens that exclude some segment of the general +public (e.g. Tor users and users sharing IPs with many other users). Caution: +privacy is also compromised on these sites due to exposure of cleartext traffic +to a third party other than the website operator. + +* `intelme.com `__ - Issuer: Cloudflare `Verification `__ +* `search404.io `__ - Issuer: Cloudflare `Verification `__ +* `searx.com.au `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.lavatech.top `__ - Issuer: Cloudflare `Verification `__ +* `searchx.mobi `__ - Issuer: Cloudflare `Verification `__ +* `searx.org `__ - Issuer: Cloudflare `Verification `__ +* `searx.run `__ - Issuer: Cloudflare `Verification `__ +* `searx.world `__ - Issuer: Cloudflare `Verification `__ - Adds Amazon affiliate links + + +Running with an incorrect SSL certificate +========================================= + +* `listi.me `__ - Issuer: Let's Encrypt `Verification `__ +* `s.matejc.com `__ - Issuer: Let's Encrypt `Verification `__ +* `search.jollausers.de `__ - Incorrectly configured `SSL certificate `__ +* `search.paviro.de `__ - Issuer: LetsEncrypt `Verification `__ +* `searx.abenthung.it `__ - Issuer: Comodo CA Limited `Verification `__ +* `searx.coding4schoki.org `__ - Incorrectly configured `SSL Certificate `__ +* `searx.haxors.club `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.nulltime.net `__ (as `Hidden Service `__) - Issuer: Let's Encrypt `Verification `__ +* `searx.ch `__ - Issuer: Let's Encrypt `Verification `__ (cert clock problems) + + +Offline +======= + +* `a.searx.space `__ - Issuer: Let's Encrypt `Verification `__ (unstable, under construction). +* `anyonething.de `__ - (was found to have become a pastebin on or before 2019-03-01) Issuer: Comodo CA Limited (Warning: uses Cloudflare) `Verification `__ +* `h7jwxg5rakyfvikpi.onion `__ - available only as Tor Hidden Service (down on 2019-06-26) +* `hacktivis.me/searx `__ - (down) - Issuer: Let's Encrypt `Verification `__ +* `icebal.com `__ - (down) Issuer: Let's Encrypt +* `netrangler.host `__ - (down) - Issuer: Let's Encrypt `Verification `__ +* `opengo.nl `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `p9e.de `__ - (down - timeout) Issuer: Let's Encrypt `Verification `__ +* `rubri.co `__ - (down) Issuer: Let's Encrypt +* `s.bacafe.xyz `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `search.alecpap.com `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `search.blackit.de `__ - (down) Let's Encrypt `Verification `__ +* `search.deblan.org `__ (down) - Issuer: COMODO via GANDI `Verification `__ +* `search.homecomputing.fr `__ - (down) Issuer: CAcert `Verification `__ +* `search.jpope.org `__ - (down - timeout) Issuer: Let's Encrypt `Verification `__ +* `search.kakise.xyz `__ - down +* `search.kosebamse.com `__ - Issuer: LetsEncrypt `Verification `__ +* `search.kujiu.org `__ - (down) Issuer: Let's Encrypt +* `search.mailaender.coffee `__ - Issuer: Let's Encrypt `Verification `__ +* `search.matrix.ac `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `search.mypsc.ca `__ - Issuer: Let's Encrypt `Verification `__ +* `search.namedkitten.pw `__ - (SSL error) - Issuer: Let's Encrypt `Verification `__ +* `search.opentunisia.org `__ - Issuer: Let's Encrypt `Verification `__ +* `search.r3d007.com `__ - (down) Issuer: Let's Encrypt +* `search.static.lu `__ - (down) Issuer: StartCom `Verification `__ +* `search.teej.xyz `__ - (down) Issuer: LetsEncrypt `Verification `__ +* `search.wxzm.sx `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.4ray.co `__ - (no longer an instance, redirects to main page) Issuer: Let's Encrypt `Verification `__ +* `searx.32bitflo.at `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `searx.ahh.si `__ - (down) - Issuer: Let's Encrypt `Verification `__ +* `searx.angristan.xyz `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `searx.antirep.net `__ - (return a 502 HTTP error) Issuer: Let's Encrypt `Verification `__ +* `searx.aquilenet.fr `__ - (down - 429 HTTP error) Issuer: Let's Encrypt `Verification `__ +* `searx.at `__ - (return "request exception" at every search) Issuer: Let's Encrypt `Verification `__ +* `searx.cc `__ - (down on 2019-06-26) Issuer: Let's Encrypt `Verification `__ +* `searx.dk `__ - (down - 429 HTTP error) Issuer: Let's Encrypt `Verification `__ +* `searx.ehrmanns.ch `__ - (down) Issuer: Let's Encrypt +* `searx.glibre.net `__ - Issuer: Let's Encrypt `Verification `__ +* `searx.infini.fr `__ - (return a page stating that the website is not installed) Issuer: Let's Encrypt `Verification `__ +* `searx.jeanphilippemorvan.info `__ - (down) Issuer: StartCom `Verification `__ +* `searx.lhorn.de `__ - (redirect the Searx's github repository page) Issuer: Let's Encrypt `Verification `__ (only reachable from european countries) +* `searx.lvweb.host `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `searx.mrtino.eu `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `searx.netzspielplatz.de `__ - (error page about GDPR even when browsing it from USA and Asia) - Issuer: Let's Encrypt `Verification `__ +* `searx.new-admin.net `__ - (down) Issuer: Let's Encrypt +* `searx.nogafa.org `__ - (broken CSS) Issuer: Let's Encrypt `Verification `__ +* `searx.potato.hu `__ - (not a searx instance) - Issuer: Let's Encrypt `Verification `__ +* `searx.rubbeldiekatz.info `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `searx.s42.space `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `searx.salcay.hu `__ - (down - blank page) Issuer: Let's Encrypt `Verification `__ +* `searx.selea.se `__ - (Leads to default Apache page) Issuer: RapidSSL (HSTS preloaded, DNSSEC) `Verification `__ | `HSTS Preload `__ +* `searx.steinscraft.net `__ - (down) Issuer: Cloudflare +* `searx.techregion.de `__ - (domain expired) - Issuer: Let's Encrypt `Verification `__ +* `searx.tognella.com `__ - (down) Issuer: Cloudflare +* `searx.xi.ht `__ - (return a 502 HTTP error) Issuer: Let's Encrypt `Verification `__ +* `searxist.com `__ - (down) - Issuer: Let's Encrypt `Verification `__ +* `so.sb `__ - (down) - Issuer: Let's Encrypt `Verification `__ +* `srx.stdout.net `__ - Issuer: Let's Encrypt `Verification `__ +* `w6f7cgdm54cyvohcuhraaafhajctyj3ihenrovuxogoagrr5g43qmoid.onion `__ - Hidden Service +* `win8linux.nohost.me `__ - (down) Issuer: Let's Encrypt +* `wiznet.tech `__ - (down) - Issuer: Let's Encrypt `Verification `__ +* `www.mercurius.space `__ - (down) Issuer: Let's Encrypt +* `www.ready.pm `__ - Issuer: WoSign `Verification `__ +* `z.awsmppl.com `__ - (down) Issuer: Let's Encrypt `Verification `__ +* `zlsdzh.tk `__ - (down - 404 HTTP error) Issuer: TrustAsia Technologies, Inc. `Verification `__ * + diff --git a/requirements-dev.txt b/requirements-dev.txt index 0bdf2046..3e8f617a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,3 +11,6 @@ transifex-client==0.12.2 unittest2==1.1.0 zope.testrunner==4.5.1 selenium==3.141.0 +linuxdoc @ git+http://github.com/return42/linuxdoc.git +sphinx-jinja +sphinx-tabs diff --git a/searx/engines/bing.py b/searx/engines/bing.py index ed0b87db..b193f7c6 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -89,8 +89,7 @@ def response(resp): 'content': content}) try: - result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()')) - result_len_container = utils.to_string(result_len_container) + result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()')) if "-" in result_len_container: # Remove the part "from-to" for paginated request ... result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:] @@ -102,7 +101,7 @@ def response(resp): logger.debug('result error :\n%s', e) pass - if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len: + if result_len and _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len: return [] results.append({'number_of_results': result_len}) diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py index 198ac2cf..c8ee34f7 100644 --- a/searx/engines/flickr_noapi.py +++ b/searx/engines/flickr_noapi.py @@ -109,14 +109,22 @@ def response(resp): else: url = build_flickr_url(photo['ownerNsid'], photo['id']) - results.append({'url': url, - 'title': title, - 'img_src': img_src, - 'thumbnail_src': thumbnail_src, - 'content': content, - 'author': author, - 'source': source, - 'img_format': img_format, - 'template': 'images.html'}) + result = { + 'url': url, + 'img_src': img_src, + 'thumbnail_src': thumbnail_src, + 'source': source, + 'img_format': img_format, + 'template': 'images.html' + } + try: + result['author'] = author + result['title'] = title + result['content'] = content + except: + result['author'] = '' + result['title'] = '' + result['content'] = '' + results.append(result) return results diff --git a/searx/engines/ina.py b/searx/engines/ina.py index 37a05f09..ea509649 100644 --- a/searx/engines/ina.py +++ b/searx/engines/ina.py @@ -32,7 +32,7 @@ base_url = 'https://www.ina.fr' search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}' # specific xpath variables -results_xpath = '//div[contains(@class,"search-results--list")]/div[@class="media"]' +results_xpath = '//div[contains(@class,"search-results--list")]//div[@class="media-body"]' url_xpath = './/a/@href' title_xpath = './/h3[@class="h3--title media-heading"]' thumbnail_xpath = './/img/@src' @@ -65,8 +65,11 @@ def response(resp): videoid = result.xpath(url_xpath)[0] url = base_url + videoid title = p.unescape(extract_text(result.xpath(title_xpath))) - thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) - if thumbnail[0] == '/': + try: + thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) + except: + thumbnail = '' + if thumbnail and thumbnail[0] == '/': thumbnail = base_url + thumbnail d = extract_text(result.xpath(publishedDate_xpath)[0]) d = d.split('/') diff --git a/searx/engines/microsoft_academic.py b/searx/engines/microsoft_academic.py index 9387b08d..9bac0069 100644 --- a/searx/engines/microsoft_academic.py +++ b/searx/engines/microsoft_academic.py @@ -45,6 +45,8 @@ def request(query, params): def response(resp): results = [] response_data = loads(resp.text) + if not response_data: + return results for result in response_data['results']: url = _get_url(result) diff --git a/searx/engines/scanr_structures.py b/searx/engines/scanr_structures.py index 72fd2b3c..7208dcb7 100644 --- a/searx/engines/scanr_structures.py +++ b/searx/engines/scanr_structures.py @@ -29,7 +29,7 @@ def request(query, params): params['url'] = search_url params['method'] = 'POST' params['headers']['Content-type'] = "application/json" - params['data'] = dumps({"query": query, + params['data'] = dumps({"query": query.decode('utf-8'), "searchField": "ALL", "sortDirection": "ASC", "sortOrder": "RELEVANCY", diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py index aed756be..00c39570 100644 --- a/searx/engines/spotify.py +++ b/searx/engines/spotify.py @@ -12,10 +12,14 @@ from json import loads from searx.url_utils import urlencode +import requests +import base64 # engine dependent config categories = ['music'] paging = True +api_client_id = None +api_client_secret = None # search-url url = 'https://api.spotify.com/' @@ -31,6 +35,16 @@ def request(query, params): params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset) + r = requests.post( + 'https://accounts.spotify.com/api/token', + data={'grant_type': 'client_credentials'}, + headers={'Authorization': 'Basic ' + base64.b64encode( + "{}:{}".format(api_client_id, api_client_secret).encode('utf-8') + ).decode('utf-8')} + ) + j = loads(r.text) + params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))} + return params diff --git a/searx/settings.yml b/searx/settings.yml index e41b84c1..2777f9ca 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -79,9 +79,10 @@ engines: categories : science timeout : 4.0 - - name : base - engine : base - shortcut : bs +# tmp suspended: dh key too small +# - name : base +# engine : base +# shortcut : bs - name : wikipedia engine : wikipedia @@ -552,10 +553,11 @@ engines: timeout : 10.0 disabled : True - - name : scanr structures - shortcut: scs - engine : scanr_structures - disabled : True +# tmp suspended: bad certificate +# - name : scanr structures +# shortcut: scs +# engine : scanr_structures +# disabled : True - name : soundcloud engine : soundcloud @@ -598,9 +600,12 @@ engines: shortcut : se categories : science - - name : spotify - engine : spotify - shortcut : stf +# Spotify needs API credentials +# - name : spotify +# engine : spotify +# shortcut : stf +# api_client_id : ******* +# api_client_secret : ******* - name : startpage engine : startpage diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html index 9cf94269..43e3e26d 100644 --- a/searx/templates/oscar/results.html +++ b/searx/templates/oscar/results.html @@ -15,7 +15,68 @@ {% include 'oscar/search.html' %}
-
+ + +

{{ _('Search results') }}

{% if corrections -%} @@ -91,66 +152,5 @@ {% endif %} {% endif %}
- -
{% endblock %} diff --git a/utils/makefile.sphinx b/utils/makefile.sphinx index 5cbc5ebd..2c1922fc 100644 --- a/utils/makefile.sphinx +++ b/utils/makefile.sphinx @@ -198,6 +198,7 @@ $(GH_PAGES):: $(MAKE) docs [ -d "gh-pages/.git" ] || git clone $(GIT_URL) gh-pages -cd $(GH_PAGES); git checkout gh-pages >/dev/null + -cd $(GH_PAGES); git pull -cd $(GH_PAGES); ls -A | grep -v '.git$$' | xargs rm -rf cp -r $(DOCS_DIST)/* $(GH_PAGES)/ touch $(GH_PAGES)/.nojekyll