From 18805ee35296e6e34be59f37083190990f4afcbb Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 24 Jun 2020 15:15:15 +0200 Subject: [PATCH 1/2] [fix] markup of auto generated installation instruction (uWSGI part) Signed-off-by: Markus Heiser --- docs/admin/installation-uwsgi.rst | 11 +++--- utils/searx.sh | 66 +++++++++++++++++++------------ 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst index ac4c463b..7b482975 100644 --- a/docs/admin/installation-uwsgi.rst +++ b/docs/admin/installation-uwsgi.rst @@ -108,6 +108,7 @@ restart the uwsgi application. :start-after: START searx uwsgi-description ubuntu-20.04 :end-before: END searx uwsgi-description ubuntu-20.04 + .. hotfix: a bug group-tab need this comment .. group-tab:: Arch Linux @@ -115,6 +116,7 @@ restart the uwsgi application. :start-after: START searx uwsgi-description arch :end-before: END searx uwsgi-description arch + .. hotfix: a bug group-tab need this comment .. group-tab:: Fedora / RHEL @@ -128,22 +130,21 @@ restart the uwsgi application. .. group-tab:: Ubuntu / debian .. kernel-include:: $DOCS_BUILD/includes/searx.rst - :code: ini :start-after: START searx uwsgi-appini ubuntu-20.04 :end-before: END searx uwsgi-appini ubuntu-20.04 + .. hotfix: a bug group-tab need this comment + .. group-tab:: Arch Linux .. kernel-include:: $DOCS_BUILD/includes/searx.rst - :code: ini :start-after: START searx uwsgi-appini arch :end-before: END searx uwsgi-appini arch + .. hotfix: a bug group-tab need this comment + .. group-tab:: Fedora / RHEL .. kernel-include:: $DOCS_BUILD/includes/searx.rst - :code: ini :start-after: START searx uwsgi-appini fedora :end-before: END searx uwsgi-appini fedora - - diff --git a/utils/searx.sh b/utils/searx.sh index 4009fddb..886b38f8 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -816,47 +816,61 @@ rst-doc() { case $DIST_ID-$DIST_VERS in ubuntu-*|debian-*) cat < /usr/share/doc/uwsgi/README.Debian.gz -# For uWSGI debian uses the LSB init process, this might be changed -# one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 -create ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} -enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} ${uWSGI_APPS_ENABLED}/ -start: sudo -H service uwsgi start ${SEARX_UWSGI_APP%.*} -restart: sudo -H service uwsgi restart ${SEARX_UWSGI_APP%.*} -stop: sudo -H service uwsgi stop ${SEARX_UWSGI_APP%.*} -disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +.. code:: bash + + # init.d --> /usr/share/doc/uwsgi/README.Debian.gz + # For uWSGI debian uses the LSB init process, this might be changed + # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 + + create ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} + enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} ${uWSGI_APPS_ENABLED}/ + start: sudo -H service uwsgi start ${SEARX_UWSGI_APP%.*} + restart: sudo -H service uwsgi restart ${SEARX_UWSGI_APP%.*} + stop: sudo -H service uwsgi stop ${SEARX_UWSGI_APP%.*} + disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} + EOF ;; arch-*) cat < /usr/lib/systemd/system/uwsgi@.service -# For uWSGI archlinux uses systemd template units, see -# - http://0pointer.de/blog/projects/instances.html -# - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd -create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} -enable: sudo -H systemctl enable uwsgi@${SEARX_UWSGI_APP%.*} -start: sudo -H systemctl start uwsgi@${SEARX_UWSGI_APP%.*} -restart: sudo -H systemctl restart uwsgi@${SEARX_UWSGI_APP%.*} -stop: sudo -H systemctl stop uwsgi@${SEARX_UWSGI_APP%.*} -disable: sudo -H systemctl disable uwsgi@${SEARX_UWSGI_APP%.*} +.. code:: bash + + # systemd --> /usr/lib/systemd/system/uwsgi@.service + # For uWSGI archlinux uses systemd template units, see + # - http://0pointer.de/blog/projects/instances.html + # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd + + create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} + enable: sudo -H systemctl enable uwsgi@${SEARX_UWSGI_APP%.*} + start: sudo -H systemctl start uwsgi@${SEARX_UWSGI_APP%.*} + restart: sudo -H systemctl restart uwsgi@${SEARX_UWSGI_APP%.*} + stop: sudo -H systemctl stop uwsgi@${SEARX_UWSGI_APP%.*} + disable: sudo -H systemctl disable uwsgi@${SEARX_UWSGI_APP%.*} + EOF ;; fedora-*) cat < /usr/lib/systemd/system/uwsgi.service -# The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see -# - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html -create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} -restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} -disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +.. code:: bash + + # systemd --> /usr/lib/systemd/system/uwsgi.service + # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see + # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + + create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} + restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} + disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} + EOF ;; esac echo -e ".. END searx uwsgi-description $DIST_NAME" echo -e "\n.. START searx uwsgi-appini $DIST_NAME" - eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}")\"" + echo ".. code:: bash" + echo + eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}")\"" | prefix_stdout " " echo -e "\n.. END searx uwsgi-appini $DIST_NAME" ) From 9c71f620a7c29ea22e4f1743e733bd267cdb4b69 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 24 Jun 2020 17:49:38 +0200 Subject: [PATCH 2/2] [add blog post] Developing in Linux containers [202006] Signed-off-by: Markus Heiser --- docs/blog/index.rst | 1 + docs/blog/lxcdev-202006.rst | 413 ++++++++++++++++++++++++++++++++++++ docs/utils/lxc.sh.rst | 7 +- 3 files changed, 418 insertions(+), 3 deletions(-) create mode 100644 docs/blog/lxcdev-202006.rst diff --git a/docs/blog/index.rst b/docs/blog/index.rst index 5a8135fd..04d95350 100644 --- a/docs/blog/index.rst +++ b/docs/blog/index.rst @@ -6,6 +6,7 @@ Blog :maxdepth: 2 :caption: Contents + lxcdev-202006 python3 admin intro-offline diff --git a/docs/blog/lxcdev-202006.rst b/docs/blog/lxcdev-202006.rst new file mode 100644 index 00000000..f9ca3c2c --- /dev/null +++ b/docs/blog/lxcdev-202006.rst @@ -0,0 +1,413 @@ +.. _blog-lxcdev-202006: + +======================================= +Developing in Linux containers [202006] +======================================= + +.. _LXC: https://linuxcontainers.org/lxc/introduction/ + +.. sidebar:: Audience + + This blog post is written for experienced admins and developers / readers + should have a serious meaning about: *distributed*, *merge* and *linux + container*. + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +In PR :PR:`1803` we added a lot of scripts to Searx's boilerplate. In this blog +post I will show you, how you can make use of them in *distributed and +heterogeneous development cycles* (TL;DR; jump to the :ref:`blog-lxcdev-202006 +abstract`). + +Motivation +========== + +Normally in our development cycle, we edit the sources and run some test and/or +builds by using ``make`` before we commit. This cycle is simple and perfect but +might fail in some aspects we should not overlook. + + The environment in which we run all our development processes matters! + +The :ref:`makefile` and the :ref:`make pyenv` encapsulate a lot for us, but they +do not have access to all prerequisites. For example, there may have +dependencies on packages that are installed on the developer's desktop, but +usually are not preinstalled on a server or client system. Another examples +are; settings have been made to the software on the developer's host that would +never be set on a *production* system. + +*Linux Containers* (LXC_) are isolate environments and not to mix up on +developer's all the prerequisites of all the projects he contribute to, is +always a good choice. + +The scripts from PR :PR:`1803` can divide in those to install and maintain +software: + +- :ref:`searx.sh` +- :ref:`filtron.sh` +- :ref:`morty.sh` + +and the script :ref:`lxc.sh`, with we can scale our installation, maintenance or +even development tasks over a stack of containers, what we call: *Searx's lxc +suite*. + +Gentlemen, start your engines! +============================== + +.. _LXD: https://linuxcontainers.org/lxd/introduction/ +.. _archlinux: https://www.archlinux.org/ + +Before you can start with containers, you need to install and initiate LXD_ +once: + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ snap install lxd + $ lxd init --auto + +And you need to clone from origin or if you have your own fork, clone from your +fork: + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ cd ~/Downloads + $ git clone https://github.com/asciimoo/searx.git + $ cd searx + +The :ref:`lxc-searx.env` consists of several images, see ``export +LXC_SUITE=(...`` near by :origin:`utils/lxc-searx.env#L19`. For this blog post +we exercise on a archlinux_ image. The container of this image is named +``searx-archlinux``. Lets build the container, but be sure that this container +does not already exists, so first lets remove possible old one: + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ sudo -H ./utils/lxc.sh remove searx-archlinux + $ sudo -H ./utils/lxc.sh build searx-archlinux + +.. sidebar:: The ``searx-archlinux`` container + + is the base of all our exercises here. + +In this container we install all services :ref:`including searx, morty & filtron +` in once: + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ sudo -H ./utils/lxc.sh install suite searx-archlinux + +To proxy HTTP from filtron and morty in the container to the outside of the +container, install nginx into the container. Once for the bot blocker filtron: + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + ./utils/filtron.sh nginx install + ... + INFO: got 429 from http://10.174.184.156/searx + +and once for the content sanitizer (content proxy morty): + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + ./utils/morty.sh nginx install + ... + INFO: got 200 from http://10.174.184.156/morty/ + +.. sidebar:: Fully functional searx suite + + From here on you have a fully functional searx suite running with bot blocker + (filtron) and Web content sanitizer (content proxy morty) needed for a + *privacy protecting* search engine. + +On your system, the IP of your ``searx-archlinux`` container differs from +http://10.174.184.156/searx, just open the URL reported in your installation +protocol in your WEB browser from the desktop to test the instance from outside +of the container. + +In such a searx suite admins can maintain and access the debug log of the +different services quite easy. + +.. _working in containers: + +In containers, work as usual +============================ + +Usually you open a root-bash using ``sudo -H bash``. In case of LXC containers +open the root-bash in the container using ``./utils/lxc.sh cmd +searx-archlinux``: + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux bash + INFO: [searx-archlinux] bash + [root@searx-archlinux searx]# pwd + /share/searx + +The prompt ``[root@searx-archlinux ...]`` signals, that you are the root user in +the searx-container. To debug the running searx instance use: + +.. tabs:: + + .. group-tab:: root@searx-archlinux + + .. code:: sh + + $ ./utils/searx.sh inspect service + ... + use [CTRL-C] to stop monitoring the log + ... + +Back in the browser on your desktop open the service http://10.174.184.156/searx +and run your application tests while the debug log is shown in the terminal from +above. You can stop monitoring using ``CTRL-C``, this also disables the *"debug +option"* in searx's settings file and restarts the searx uwsgi application. To +debug services from filtron and morty analogous use: + +.. tabs:: + + .. group-tab:: root@searx-archlinux + + .. code:: sh + + $ ./utils/filtron.sh inspect service + $ ./utils/morty.sh inspect service + +Another point we have to notice is that each service (:ref:`searx `, +:ref:`filtron ` and :ref:`morty `) runs under dedicated +system user account with the same name (compare :ref:`create searx user`). To +get a shell from theses accounts, simply call one of the scripts: + +.. tabs:: + + .. group-tab:: root@searx-archlinux + + .. code:: sh + + $ ./utils/searx.sh shell + $ ./utils/filtron.sh shell + $ ./utils/morty.sh shell + +To get in touch, open a shell from the service user (searx@searx-archlinux): + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + ./utils/searx.sh shell + // exit with [CTRL-D] + (searx-pyenv) [searx@searx-archlinux ~]$ ... + +The prompt ``[searx@searx-archlinux]`` signals that you are logged in as system +user ``searx`` in the ``searx-archlinux`` container and the python *virtualenv* +``(searx-pyenv)`` environment is activated. + +.. tabs:: + + .. group-tab:: searx@searx-archlinux + + .. code:: sh + + (searx-pyenv) [searx@searx-archlinux ~]$ pwd + /usr/local/searx + + + +Wrap production into developer suite +==================================== + +In this section we will see how to change the *"Fully functional searx suite"* +from a LXC container (which is quite ready for production) into a developer +suite. For this, we have to keep an eye on the :ref:`installation basic`: + +- searx setup in: ``/etc/searx/settings.yml`` +- searx user's home: ``/usr/local/searx`` +- virtualenv in: ``/usr/local/searx/searx-pyenv`` +- searx software in: ``/usr/local/searx/searx-src`` + +The searx software is a clone of the ``GIT_URL`` (see :ref:`makefile setup`) and +the working tree is checked out from the ``GIT_BRANCH``. With the use of the +:ref:`searx.sh` the searx service was installed as :ref:`uWSGI application +`. To maintain this service, we can use ``systemctl`` (compare +:ref:`service architectures on distributions `). + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + systemctl stop uwsgi@searx + +With the command above, we stopped the searx uWSGI-App in the archlinux +container. + +The uWSGI-App for the archlinux dsitros is configured in +:origin:`utils/templates/etc/uwsgi/apps-archlinux/searx.ini`, from where at +least you should attend the settings of ``uid``, ``chdir``, ``env`` and +``http``:: + + env = SEARX_SETTINGS_PATH=/etc/searx/settings.yml + http = 127.0.0.1:8888 + + chdir = /usr/local/searx/searx-src/searx + virtualenv = /usr/local/searx/searx-pyenv + pythonpath = /usr/local/searx/searx-src + +If you have read the :ref:`"Good to know section" ` you remember, that +each container shares the root folder of the repository and the command +``utils/lxc.sh cmd`` handles relative path names **transparent**. To wrap the +searx installation into a developer one, we simple have to create a smylink to +the **transparent** reposetory from the desktop. Now lets replace the +repository at ``searx-src`` in the container with the working tree from outside +of the container: + +.. tabs:: + + .. group-tab:: container becomes a developer suite + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + ln -s /share/searx/ /usr/local/searx/searx-src + +Now we can develop as usual in the working tree of our desktop system. Every +time the software was changed, you have to restart the searx service (in the +conatiner): + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + systemctl restart uwsgi@searx + + +Remember: :ref:`working in containers` .. here are just some examples from my +daily usage: + +.. tabs:: + + .. group-tab:: desktop + + To *inspect* the searx instance (already described above): + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + ./utils/searx.sh inspect service + + Run :ref:`makefile`, e.g. to test inside the container: + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + make test + + To install all prerequisites needed for a :ref:`buildhosts`: + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + ./utils/searx.sh install buildhost + + To build the docs on a buildhost :ref:`buildhosts`: + + .. code:: sh + + $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + make docs + +.. _blog-lxcdev-202006 abstract: + +Abstract +======== + +We build up a fully functional searx suite in a archlinux container: + +.. code:: sh + + $ sudo -H ./utils/lxc.sh install suite searx-archlinux + +To access HTTP from the desktop we installed nginx for the services inside the +conatiner: + +.. tabs:: + + .. group-tab:: [root@searx-archlinux] + + .. code:: sh + + $ ./utils/filtron.sh nginx install + $ ./utils/morty.sh nginx install + +To wrap the suite into a developer one, we created a symbolic link to the +repository which is shared **transparent** from the desktop's file system into +the container : + +.. tabs:: + + .. group-tab:: [root@searx-archlinux] + + .. code:: sh + + $ mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old + $ ln -s /share/searx/ /usr/local/searx/searx-src + $ systemctl restart uwsgi@searx + +To get remarks from the suite of the archlinux container we can use: + +.. tabs:: + + .. group-tab:: desktop + + .. code:: sh + + $ sudo -H ./utils/lxc.sh show suite searx-archlinux + ... + [searx-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx + [searx-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/ + [searx-archlinux] INFO: (eth0) docs-live: http://10.174.184.156:8080/ + [searx-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65] + ... + diff --git a/docs/utils/lxc.sh.rst b/docs/utils/lxc.sh.rst index ae141287..114d0911 100644 --- a/docs/utils/lxc.sh.rst +++ b/docs/utils/lxc.sh.rst @@ -47,9 +47,9 @@ one**:: *Good to know ...* -Eeach container shares the root folder of the repository and the -command ``utils/lxc.sh cmd`` **handles relative path names transparent**, -compare output of:: +Each container shares the root folder of the repository and the command +``utils/lxc.sh cmd`` **handles relative path names transparent**, compare output +of:: $ sudo -H ./utils/lxc.sh cmd -- ls -la Makefile ... @@ -66,6 +66,7 @@ If there comes the time you want to **get rid off all** the containers and $ sudo -H ./utils/lxc.sh remove $ sudo -H ./utils/lxc.sh remove images +.. _lxc.sh install suite: Install suite =============