From fbe40001d35ea5cf74c92f35d76c642a480a5c9f Mon Sep 17 00:00:00 2001 From: Dalf Date: Sat, 29 Jun 2019 11:59:13 +0200 Subject: [PATCH] Update Docker image See #1561 , use uwsgi and Alpine Linux Volume: /var/log/uwsgi contains error log for 2 days (file uwsgi.log) /etc/searx contains the settings.yml and uwsgi.ini files. The docker image creates them if they don't exist. The two files can be modified after the first run. See below. Environement variables: MORTY_URL : external URL of Morty MORTY_KEY : base64 encoded key BASE_URL : external URL of Searx BIND_ADDRESS : internal HTTP port to listen to Labels : org.label-schema.schema.* Parameters: -h : display this help -d : will update the settings and quit immediately (settings.yml and uwsgi.ini) -f : always update the settings (previous version saved with suffix .old). without this parameter, the new settings are copied with suffix .new When the Docker image contains newer settings: - without -f parameter: the new versions are copied to /etc/searx/settings.yml.new and /etc/searx/uwsgi.ini.new. - with -f parameter: the old versions are renamed with .old suffix. The new version replaces /etc/searx/settings.yml and /etc/searx/uwsgi.ini Build using "./manage.sh docker_build", add "push" as parameter also push the Docker image. The script requires a git repository to work (it makes sure that the last git tag matches searx/version.py) "git describe" is used to create a meaningful version. Example : 0.15.0-90-49c5bcb4-dirty (dirty means that the docker image was made with uncommited changes). Use "docker inspect -f {{.Config.Labels.version}} searx" to get the version of an existing image. .dockerignore based on .gitignore .travis.yml: include docker stage --- .dockerignore | 41 ++++++++++ .gitignore | 1 + .travis.yml | 33 ++++++-- Dockerfile | 104 +++++++++++++++---------- dockerfiles/docker-entrypoint.sh | 128 +++++++++++++++++++++++++++++++ dockerfiles/uwsgi.ini | 33 ++++++++ manage.sh | 69 +++++++++++++++++ 7 files changed, 362 insertions(+), 47 deletions(-) create mode 100644 .dockerignore create mode 100755 dockerfiles/docker-entrypoint.sh create mode 100644 dockerfiles/uwsgi.ini diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..f4d03376 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,41 @@ +*~ +*/*~ +*/*/*~ +*/*/*/*~ +*/*/*/*/*~ + +# Git +.git +.gitignore + +# CI +.codeclimate.yml +.travis.yml +.taskcluster.yml + +# Byte-compiled / optimized / DLL files +__pycache__/ +*/__pycache__/ +*/*/__pycache__/ +*/*/*/__pycache__/ +*.py[cod] +*/*.py[cod] +*/*/*.py[cod] +*/*/*/*.py[cod] + +# to sync with .gitignore +.coverage +coverage/ +.installed.cfg +engines.cfg +env +searx-ve +robot_log.html +robot_output.xml +robot_report.html +test_basic/ +setup.cfg + +node_modules/ + +.tx/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3da4d038..db20da83 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +# to sync with .dockerignore .coverage coverage/ .installed.cfg diff --git a/.travis.yml b/.travis.yml index 08bcfaad..04654ac2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,14 @@ +language: python sudo: false cache: - pip - npm - directories: - $HOME/.cache/pip + addons: firefox: "latest" -language: python -python: - - "2.7" - - "3.5" - - "3.6" + before_install: - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" @@ -27,6 +25,31 @@ script: after_success: - ./manage.sh py_test_coverage - codecov + +stages: + - test + - name: docker + if: branch = master AND type != pull_request AND env(DOCKER_USERNAME) IS present + +jobs: + include: + - python: "2.7" + - python: "3.5" + - python: "3.6" + - stage: docker + python: "3.6" + git: + depth: false + services: + - docker + addons: [] + before_install: true + install: true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - ./manage.sh docker_build push + after_success: true + notifications: irc: channels: diff --git a/Dockerfile b/Dockerfile index 95e21813..03c4b76a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,57 +1,77 @@ -FROM alpine:3.8 -LABEL maintainer="searx " -LABEL description="A privacy-respecting, hackable metasearch engine." +FROM alpine:3.10 + +ARG VERSION_GITCOMMIT=unknow +ARG SEARX_GIT_VERSION=unknow + +ARG SEARX_GID=1000 +ARG SEARX_UID=1000 + +ARG TIMESTAMP_SETTINGS=0 +ARG TIMESTAMP_UWSGI=0 +ARG LABEL_VCS_REF= +ARG LABEL_VCS_URL= + +ENV BASE_URL= \ + MORTY_KEY= \ + MORTY_URL= +EXPOSE 8080 +VOLUME /etc/searx +VOLUME /var/log/uwsgi -ENV BASE_URL=False IMAGE_PROXY=False HTTP_PROXY_URL= HTTPS_PROXY_URL= -EXPOSE 8888 WORKDIR /usr/local/searx -CMD ["/sbin/tini","--","/usr/local/searx/run.sh"] -RUN adduser -D -h /usr/local/searx -s /bin/sh searx searx \ - && echo '#!/bin/sh' >> run.sh \ - && echo 'sed -i "s|base_url : False|base_url : $BASE_URL|g" searx/settings.yml' >> run.sh \ - && echo 'sed -i "s/image_proxy : False/image_proxy : $IMAGE_PROXY/g" searx/settings.yml' >> run.sh \ - && echo 'sed -i "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml' >> run.sh \ - && echo 'if [ -n "$HTTP_PROXY_URL" ] || [ -n "$HTTPS_PROXY_URL" ]; then' >> run.sh \ - && echo ' sed -i "s~^# proxies :~ proxies:\\n http: ${HTTP_PROXY_URL}\\n https: ${HTTPS_PROXY_URL}\\n~" searx/settings.yml' >> run.sh \ - && echo 'fi' >> run.sh \ - && echo 'python searx/webapp.py' >> run.sh \ - && chmod +x run.sh +RUN addgroup -g ${SEARX_GID} searx && \ + adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx COPY requirements.txt ./requirements.txt -RUN echo "@commuedge http://nl.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ - && apk -U add \ +RUN apk -U upgrade \ + && apk add -t build-dependencies \ build-base \ - python \ - python-dev \ - py-pip \ + py3-setuptools \ + python3-dev \ + libffi-dev \ + libxslt-dev \ + libxml2-dev \ + openssl-dev \ + tar \ + git \ + && apk add \ + ca-certificates \ + su-exec \ + python3 \ libxml2 \ - libxml2-dev \ libxslt \ - libxslt-dev \ - libffi-dev \ openssl \ - openssl-dev \ - ca-certificates \ - tini@commuedge \ - && pip install --upgrade pip \ - && pip install --no-cache -r requirements.txt \ - && apk del \ - build-base \ - python-dev \ - libffi-dev \ - openssl-dev \ - libxslt-dev \ - libxml2-dev \ - openssl-dev \ - ca-certificates \ + tini \ + uwsgi \ + uwsgi-python3 \ + && pip3 install --upgrade pip \ + && pip3 install --no-cache -r requirements.txt \ + && apk del build-dependencies \ && rm -f /var/cache/apk/* -COPY . . +COPY --chown=searx:searx . . -RUN chown -R searx:searx * +RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \ + touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \ + touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \ + if [ ! -z $VERSION_GITCOMMIT ]; then\ + echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \ + fi -USER searx +ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"] -RUN sed -i "s/127.0.0.1/0.0.0.0/g" searx/settings.yml +# Keep this argument at the end since it change each time +ARG LABEL_DATE= +LABEL maintainer="searx " \ + description="A privacy-respecting, hackable metasearch engine." \ + version="${SEARX_GIT_VERSION}" \ + org.label-schema.schema-version="1.0" \ + org.label-schema.name="searx" \ + org.label-schema.schema-version="${SEARX_GIT_VERSION}" \ + org.label-schema.url="${LABEL_VCS_URL}" \ + org.label-schema.vcs-ref=${LABEL_VCS_REF} \ + org.label-schema.vcs-url=${LABEL_VCS_URL} \ + org.label-schema.build-date="${LABEL_DATE}" \ + org.label-schema.usage="https://github.com/searx/searx-docker" diff --git a/dockerfiles/docker-entrypoint.sh b/dockerfiles/docker-entrypoint.sh new file mode 100755 index 00000000..60e26fd9 --- /dev/null +++ b/dockerfiles/docker-entrypoint.sh @@ -0,0 +1,128 @@ +#!/bin/sh + +export SEARX_VERSION=$(su searx -c 'python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)"') +printf 'searx version %s\n\n' "${SEARX_VERSION}" + +export UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini +export SEARX_SETTINGS_PATH=/etc/searx/settings.yml + +if [ -z "${BIND_ADDRESS}" ]; then + export BIND_ADDRESS=":8080" +fi + +# Parse command line +FORCE_CONF_UPDATE=0 +DRY_RUN=0 +while getopts "fdh" option +do + case $option in + f) + FORCE_CONF_UPDATE=1 + ;; + d) + DRY_RUN=1 + ;; + h) + printf "Command line:\n\n" + printf " -h Display this help\n" + printf " -d Dry run to update the configuration files.\n" + printf " -f Always update on the configuration files (existing files are renamed with the .old suffix)\n" + printf " Without this option, new configuration files are copied with the .new suffix\n" + printf "\nEnvironment variables:\n\n" + printf " BASE_URL settings.yml : server.base_url\n" + printf " MORTY_URL settings.yml : result_proxy.url\n" + printf " MORTY_KEY settings.yml : result_proxy.key\n" + printf " BIND_ADDRESS where uwsgi will accept HTTP request (format : host:port)\n" + exit 0 + esac +done + +# helpers to update the configuration files +patch_uwsgi_settings() { + CONF="$1" + + # Nothing +} + +patch_searx_settings() { + CONF="$1" + + # Make sure that there is trailing slash at the end of BASE_URL + # see http://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion + export BASE_URL="${BASE_URL%/}/" + + # update settings.yml + sed -i -e "s|base_url : False|base_url : ${BASE_URL}|g" \ + -e "s/ultrasecretkey/$(openssl rand -hex 32)/g" \ + "${CONF}" + + # Morty configuration + if [ ! -z "${MORTY_KEY}" -a ! -z "${MORTY_URL}" ]; then + sed -i -e "s/image_proxy : False/image_proxy : True/g" \ + "${CONF}" + cat >> "${CONF}" <<-EOF + +# Morty configuration +result_proxy: + url : ${MORTY_URL} + key : !!binary "${MORTY_KEY}" +EOF + fi +} + +update_conf() { + FORCE_CONF_UPDATE="$1" + CONF="$2" + NEW_CONF="${2}.new" + OLD_CONF="${2}.old" + REF_CONF="$3" + PATCH_REF_CONF="$4" + + if [ -f "${CONF}" ]; then + if [ "${REF_CONF}" -nt "${CONF}" ]; then + # There is a new version + if [ $FORCE_CONF_UPDATE ]; then + # Replace the current configuration + printf '⚠️ Automaticaly update %s to the new version\n' "${CONF}" + if [ ! -f "${OLD_CONF}" ]; then + printf 'The previous configuration is saved to %s\n' "${OLD_CONF}" + mv "${CONF}" "${OLD_CONF}" + fi + cp "${REF_CONF}" "${CONF}" + $PATCH_REF_CONF "${CONF}" + else + # Keep the current configuration + printf '⚠️ Check new version %s to make sure searx is working properly\n' "${NEW_CONF}" + cp "${REF_CONF}" "${NEW_CONF}" + $PATCH_REF_CONF "${NEW_CONF}" + fi + else + printf 'Use existing %s\n' "${CONF}" + fi + else + printf 'Create %s\n' "${CONF}" + cp "${REF_CONF}" "${CONF}" + $PATCH_REF_CONF "${CONF}" + fi +} + +# make sure there are uwsgi settings +update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searx/dockerfiles/uwsgi.ini" "patch_uwsgi_settings" + +# make sure there are searx settings +update_conf "${FORCE_CONF_UPDATE}" "${SEARX_SETTINGS_PATH}" "/usr/local/searx/searx/settings.yml" "patch_searx_settings" + +# dry run (to update configuration files, then inspect them) +if [ $DRY_RUN -eq 1 ]; then + printf 'Dry run\n' + exit +fi + +# +touch /var/run/uwsgi-logrotate +chown -R searx:searx /var/log/uwsgi /var/run/uwsgi-logrotate +unset MORTY_KEY + +# Start uwsgi +printf 'Listen on %s\n' "${BIND_ADDRESS}" +exec su-exec searx:searx uwsgi --master --http-socket "${BIND_ADDRESS}" "${UWSGI_SETTINGS_PATH}" diff --git a/dockerfiles/uwsgi.ini b/dockerfiles/uwsgi.ini new file mode 100644 index 00000000..fa2fd630 --- /dev/null +++ b/dockerfiles/uwsgi.ini @@ -0,0 +1,33 @@ +[uwsgi] +# Who will run the code +uid = searx +gid = searx + +# Number of workers (usually CPU count) +workers = 4 + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true +master = true +plugin = python3 +lazy-apps = true +enable-threads = true + +# Module to import +module = searx.webapp + +# Virtualenv and python path +pythonpath = /usr/local/searx/ +chdir = /usr/local/searx/searx/ + +# Disable logging for privacy +disable-logging=True + +# But keep errors for 2 days +touch-logrotate = /run/uwsgi-logrotate +unique-cron = 15 0 -1 -1 -1 { touch /run/uwsgi-logrotate } +log-backupname = /var/log/uwsgi/uwsgi.log.1 +logto = /var/log/uwsgi/uwsgi.log diff --git a/manage.sh b/manage.sh index c564827a..7279eaf3 100755 --- a/manage.sh +++ b/manage.sh @@ -158,6 +158,74 @@ grunt_build() { grunt --gruntfile "$SEARX_DIR/static/themes/simple/gruntfile.js" } +docker_build() { + # Check if it is a git repository + if [ ! -d .git ]; then + echo "This is not Git repository" + exit 1 + fi + + if [ ! -x "$(which git)" ]; then + echo "git is not installed" + exit 1 + fi + + if [ ! git remote get-url origin 2> /dev/null ]; then + echo "there is no remote origin" + exit 1 + fi + + # This is a git repository + + # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e) + # awk to remove the "v" and the "g" + SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); $3=substr($3, 2); print}') + + # add the suffix "-dirty" if the repository has uncommited change + git update-index -q --refresh + if [ ! -z "$(git diff-index --name-only HEAD --)" ]; then + SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty" + fi + + # Get the last git commit id, will be added to the Searx version (see Dockerfile) + VERSION_GITCOMMIT=$(echo $SEARX_GIT_VERSION | cut -d- -f2-4) + echo "Last commit : $VERSION_GITCOMMIT" + + # Check consistency between the git tag and the searx/version.py file + # /!\ HACK : parse Python file with bash /!\ + # otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py ) + # SEARX_PYTHON_VERSION=$(python -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)") + SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -) + if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then + echo "Inconsistency between the last git tag and the searx/version.py file" + echo "git tag: $SEARX_GIT_VERSION" + echo "searx/version.py: $SEARX_PYTHON_VERSION" + exit 1 + fi + + # define the docker image name + # /!\ HACK to get the user name /!\ + GITHUB_USER=$(git remote get-url origin | sed 's/.*github\.com\/\([^\/]*\).*/\1/') + SEARX_IMAGE_NAME="${GITHUB_USER:-searx}/searx" + + # build Docker image + echo "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" + sudo docker build \ + --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \ + --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ + --build-arg LABEL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --build-arg LABEL_VCS_REF=$(git rev-parse HEAD) \ + --build-arg LABEL_VCS_URL=$(git remote get-url origin) \ + --build-arg TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- searx/settings.yml) \ + --build-arg TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini) \ + -t ${SEARX_IMAGE_NAME}:latest -t ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} . + + if [ "$1" = "push" ]; then + sudo docker push ${SEARX_IMAGE_NAME}:latest + sudo docker push ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} + fi +} + # # Help # @@ -182,6 +250,7 @@ Commands locales - Compile locales styles - Build less files grunt_build - Build files for themes + docker_build - Build Docker image Tests -----