From f693149cded4f783380f8f02154bd9288b72cdd5 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 8 Apr 2020 18:38:36 +0200 Subject: [PATCH] Changes from the installation tests on (all) LXC containers. Tested and fixed HTTP & uWSGI installation on: ubu1604 ubu1804 ubu1910 ubu2004 fedora31 archlinux Signed-off-by: Markus Heiser --- docs/utils/index.rst | 16 +++- docs/utils/lxc.sh.rst | 6 +- utils/filtron.sh | 7 +- utils/lib.sh | 61 +++++++++++++- utils/lxc.sh | 1 - utils/morty.sh | 16 ++-- utils/searx.sh | 10 ++- .../etc/httpd/sites-available/morty.conf | 2 +- .../httpd/sites-available/searx.conf:filtron | 2 +- .../etc/uwsgi/apps-archlinux/searx.ini | 4 +- .../etc/uwsgi/apps-archlinux/searx.ini:socket | 80 +++++++++++++++++++ .../etc/uwsgi/apps-available/searx.ini | 4 +- .../etc/uwsgi/apps-available/searx.ini:socket | 79 ++++++++++++++++++ 13 files changed, 263 insertions(+), 25 deletions(-) create mode 100644 utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket create mode 100644 utils/templates/etc/uwsgi/apps-available/searx.ini:socket diff --git a/docs/utils/index.rst b/docs/utils/index.rst index 088586e4..13914af2 100644 --- a/docs/utils/index.rst +++ b/docs/utils/index.rst @@ -19,20 +19,28 @@ developers. .. _toolboxing common: -Common commands -=============== +Common commands & environment +============================= Scripts to maintain services often dispose of common commands and environments. -``shell``: +``shell`` : command Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for troubleshooting. -``inspect service``: +``inspect service`` : command Shows status and log of the service, most often you have a option to enable more verbose debug logs. Very helpful for debugging, but be careful not to enable debugging in a production environment! +``FORCE_TIMEOUT`` : environment + Sets timeout for interactive prompts. If you want to run a script in batch + job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a + reverse proxy for filtron on all containers of the :ref:`searx suite + ` use :: + + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install + .. _toolboxing setup: Tooling box setup diff --git a/docs/utils/lxc.sh.rst b/docs/utils/lxc.sh.rst index fcddfb34..44ab7903 100644 --- a/docs/utils/lxc.sh.rst +++ b/docs/utils/lxc.sh.rst @@ -88,7 +88,11 @@ WEB-Browser:: [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.18:4004/ [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.12:4004/ - +To install a reverse proxy for filtron and morty use:: + + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh apache install + Running commands ================ diff --git a/utils/filtron.sh b/utils/filtron.sh index 0fbf7478..d24b6c39 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -23,7 +23,6 @@ FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ [[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ FILTRON_ETC="/etc/filtron" - FILTRON_RULES="$FILTRON_ETC/rules.json" FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" @@ -447,7 +446,8 @@ This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SIT install_apache fi - echo + "${REPO_ROOT}/utils/searx.sh" install uwsgi + apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}" info_msg "testing public url .." @@ -465,11 +465,12 @@ This removes apache site ${APACHE_FILTRON_SITE}." ! apache_is_installed && err_msg "Apache is not installed." - if ! ask_yn "Do you really want to continue?"; then + if ! ask_yn "Do you really want to continue?" Yn; then return fi apache_remove_site "$APACHE_FILTRON_SITE" + } rst-doc() { diff --git a/utils/lib.sh b/utils/lib.sh index 2c0c179b..a032d046 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -389,7 +389,7 @@ install_template() { local chmod="${pos_args[4]-644}" info_msg "install (eval=$do_eval): ${dst}" - [[ -n $variant ]] && info_msg "variant: ${variant}" + [[ -n $variant ]] && info_msg "variant --> ${variant}" if [[ ! -f "${template_origin}" ]] ; then err_msg "${template_origin} does not exists" @@ -777,6 +777,7 @@ apache_dissable_site() { ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}" ;; esac + apache_reload } # uWSGI @@ -846,7 +847,7 @@ uWSGI_restart() { if uWSGI_app_available "${CONF}"; then systemctl restart "uwsgi@${CONF%.*}" else - info_msg "in systemd template mode: ${CONF} not installed (nothing to restart)" + info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)" fi ;; fedora-*) @@ -854,7 +855,7 @@ uWSGI_restart() { if uWSGI_app_enabled "${CONF}"; then touch "${uWSGI_APPS_ENABLED}/${CONF}" else - info_msg "in uWSGI emperor mode: ${CONF} not installed (nothing to restart)" + info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)" fi ;; *) @@ -864,6 +865,32 @@ uWSGI_restart() { esac } +uWSGI_prepare_app() { + + # usage: uWSGI_prepare_app + + local APP="${1%.*}" + if [[ -z $APP ]]; then + err_msg "uWSGI_prepare_app: missing arguments" + return 42 + fi + + case $DIST_ID-$DIST_VERS in + fedora-*) + # in emperor mode, the uwsgi user is the owner of the sockets + info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}" + mkdir -p "/run/uwsgi/app/${APP}" + chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}" + ;; + *) + info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}" + mkdir -p "/run/uwsgi/app/${APP}" + chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}" + ;; + esac +} + + uWSGI_app_available() { # usage: uWSGI_app_available local CONF="$1" @@ -888,6 +915,7 @@ uWSGI_install_app() { *) pos_args+=("$i");; esac done + uWSGI_prepare_app "${pos_args[1]}" mkdir -p "${uWSGI_APPS_AVAILABLE}" install_template "${template_opts[@]}" \ "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \ @@ -1281,3 +1309,30 @@ global_IPs(){ ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p' } + +primary_ip() { + + case $DIST_ID in + arch) + echo "$(ip -o addr show \ + | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \ + | head -n 1)" + ;; + *) echo "$(hostname -I | cut -d' ' -f1)" ;; + esac +} + +# URL +# --- + +url_replace_hostname(){ + + # usage: url_replace_hostname + + # to replace hostname by primary IP:: + # + # url_replace_hostname http://searx-ubu1604/morty $(primary_ip) + # http://10.246.86.250/morty + + echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|" +} diff --git a/utils/lxc.sh b/utils/lxc.sh index 6a26f80e..6a8dce94 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -438,7 +438,6 @@ lxc_cmd() { else info_msg "lxc $* $i" lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " - echo fi done } diff --git a/utils/morty.sh b/utils/morty.sh index 4de41199..34d792db 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -14,12 +14,16 @@ in_container && lxc_set_suite_env # config # ---------------------------------------------------------------------------- -PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" -PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" -PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}" -PUBLIC_URL_MORTY="$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}" - MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" +PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}" + +SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" +PUBLIC_URL_MORTY="$(echo "$SEARX_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}" +if in_container; then + # container hostnames do not have a DNS entry, use primary IP + PUBLIC_URL_MORTY="$(url_replace_hostname "$PUBLIC_URL_MORTY" "$(primary_ip)")" +fi + # shellcheck disable=SC2034 MORTY_TIMEOUT=5 @@ -425,7 +429,7 @@ This removes apache site ${APACHE_MORTY_SITE}." ! apache_is_installed && err_msg "Apache is not installed." - if ! ask_yn "Do you really want to continue?"; then + if ! ask_yn "Do you really want to continue?" Yn; then return fi diff --git a/utils/searx.sh b/utils/searx.sh index 6a8588c2..681b9a0e 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -748,6 +748,10 @@ excessively bot queries." apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP" + if ! service_is_available "${PUBLIC_URL}"; then err_msg "Public service at ${PUBLIC_URL} is not available!" fi @@ -762,11 +766,15 @@ This removes apache site ${APACHE_SEARX_SITE}." ! apache_is_installed && err_msg "Apache is not installed." - if ! ask_yn "Do you really want to continue?"; then + if ! ask_yn "Do you really want to continue?" Yn; then return fi apache_remove_site "${APACHE_SEARX_SITE}" + + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" } rst-doc() { diff --git a/utils/templates/etc/httpd/sites-available/morty.conf b/utils/templates/etc/httpd/sites-available/morty.conf index 326fcc75..daeb3635 100644 --- a/utils/templates/etc/httpd/sites-available/morty.conf +++ b/utils/templates/etc/httpd/sites-available/morty.conf @@ -2,7 +2,7 @@ LoadModule headers_module ${APACHE_MODULES}/mod_headers.so LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so -LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so +LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so #LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so # SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog diff --git a/utils/templates/etc/httpd/sites-available/searx.conf:filtron b/utils/templates/etc/httpd/sites-available/searx.conf:filtron index 11dd360b..379d47e2 100644 --- a/utils/templates/etc/httpd/sites-available/searx.conf:filtron +++ b/utils/templates/etc/httpd/sites-available/searx.conf:filtron @@ -2,7 +2,7 @@ LoadModule headers_module ${APACHE_MODULES}/mod_headers.so LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so -LoadModule proxy_module ${APACHE_MODULES}/mod_proxy_http.so +LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so #LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so # SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searx.ini b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini index 8d3349c5..51f659d0 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searx.ini +++ b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini @@ -74,7 +74,7 @@ http = ${SEARX_INTERNAL_HTTP} # # On some distributions you need to create the app folder for the sockets:: # -# mkdir -p /run/uwsgi/app/searx/socket -# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket +# mkdir -p /run/uwsgi/app/searx +# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx # # socket = /run/uwsgi/app/searx/socket \ No newline at end of file diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket new file mode 100644 index 00000000..eeabb371 --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket @@ -0,0 +1,80 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +logger = systemd +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +# http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +socket = /run/uwsgi/app/searx/socket \ No newline at end of file diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini b/utils/templates/etc/uwsgi/apps-available/searx.ini index 806f7443..9785d7cd 100644 --- a/utils/templates/etc/uwsgi/apps-available/searx.ini +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini @@ -73,7 +73,7 @@ http = ${SEARX_INTERNAL_HTTP} # # On some distributions you need to create the app folder for the sockets:: # -# mkdir -p /run/uwsgi/app/searx/socket -# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx/socket +# mkdir -p /run/uwsgi/app/searx +# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx # # socket = /run/uwsgi/app/searx/socket \ No newline at end of file diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini:socket b/utils/templates/etc/uwsgi/apps-available/searx.ini:socket new file mode 100644 index 00000000..88436e5e --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini:socket @@ -0,0 +1,79 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python3,http + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +# http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +socket = /run/uwsgi/app/searx/socket \ No newline at end of file