From d70c5a621a3afe8a6da2ee1daa241467e42c1b75 Mon Sep 17 00:00:00 2001
From: Alexandre Flament <alex@al-f.net>
Date: Fri, 29 Jan 2021 08:16:44 +0100
Subject: [PATCH 1/3] [mod] more robust make pyenv / make pyenvinstall

"make pyenv" ensures that ./local/py3/bin/python is an executable
---
 utils/makefile.python | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/utils/makefile.python b/utils/makefile.python
index 34543438..28701239 100644
--- a/utils/makefile.python
+++ b/utils/makefile.python
@@ -99,7 +99,8 @@ quiet_cmd_pyinstall = INSTALL   $2
 quiet_cmd_pyenvinstall = PYENV     install $2
       cmd_pyenvinstall = \
 	if ! cat $(PY_ENV)/requirements.sha256 2>/dev/null | sha256sum --check --status 2>/dev/null; then \
-		$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) ;\
+		rm -f $(PY_ENV)/requirements.sha256; \
+		$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) &&\
 		sha256sum requirements*.txt > $(PY_ENV)/requirements.sha256 ;\
 	else \
 		echo "PYENV     $2 already installed"; \
@@ -119,13 +120,13 @@ quiet_cmd_pyenvuninstall = PYENV     uninstall   $2
 # $2 path to folder where virtualenv take place
 quiet_cmd_virtualenv  = PYENV     usage: $ source ./$@/bin/activate
       cmd_virtualenv  = \
-	if [ ! -d "./$(PY_ENV)" ];then                                  \
-		$(PYTHON) -m venv $(VTENV_OPTS) $2;                     \
+	if [ -d "./$(PY_ENV)" -a -x "./$(PY_ENV_BIN)/python" ]; then \
+		echo "PYENV     using virtualenv from $2"; \
+	else \
+		$(PYTHON) -m venv $(VTENV_OPTS) $2; \
 		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel setuptools; \
-		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt;     \
-	else                                                            \
-		echo "PYENV     using virtualenv from $2";              \
-        fi
+		$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt; \
+	fi
 
 # $2 path to lint
 quiet_cmd_pylint      = LINT      $@

From 38b39ef0ae85b889b8a8401235c577c95845ed7a Mon Sep 17 00:00:00 2001
From: Markus Heiser <markus.heiser@darmarit.de>
Date: Sat, 30 Jan 2021 14:25:23 +0100
Subject: [PATCH 2/3] [fix] re-add 'pip-exe' target - partial revert 9b48ae47

Target pip-exe is a prerequisite of the targets:

  - pyinstall
  - pyuninstall

and was accidentally deleted in commit 9b48ae47.

HINT:
  do not confuse pyinstall with penvinstall

pyinstall & pyuninstall
    Installing into user's HOME using pip from OS,
    therefore the message is needed.

pyenvinstall & pyenvuninstall
    Installing into virtualenv (./local) using pip which is provided by
    prerequisite 'pyenv' in the virtualenv.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
---
 utils/makefile.python | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/utils/makefile.python b/utils/makefile.python
index 28701239..f4fd0219 100644
--- a/utils/makefile.python
+++ b/utils/makefile.python
@@ -87,6 +87,22 @@ python-exe:
 	@:
 endif
 
+msg-pip-exe:
+	@echo "\n  $(PIP) is required\n\n\
+  Make sure you have updated pip installed, grab it from\n\
+  https://pip.pypa.io or install it from your package\n\
+  manager. On debian based OS these requirements are\n\
+  installed by::\n\n\
+    sudo -H apt-get install python$(PY)-pip\n" | $(FMT)
+
+ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1)
+pip-exe: msg-pip-exe
+	$(error The '$(PIP)' command was not found)
+else
+pip-exe:
+	@:
+endif
+
 # ------------------------------------------------------------------------------
 # commands
 # ------------------------------------------------------------------------------

From 8c45f1149dd464317c8a11829b084f2d30037162 Mon Sep 17 00:00:00 2001
From: Markus Heiser <markus.heiser@darmarit.de>
Date: Mon, 1 Feb 2021 15:23:19 +0100
Subject: [PATCH 3/3] [hardening] github workflows - corrupted cache

aka: ensure that 'make test' works as expected

The cache contains a copy './local' which is - under some circumstance -
corrupted.  It is not possible to clear the cache [1] (see the top of the page).

Ensure that 'make test' works as expected [2] even if

- the python interpreter is missing
- the virtualenv exists but pyyaml is missing

To hardening when the workflow cache fails, this patch adds the new target
'travis.test' into the workflow.  This target probes to import a python module
'yaml'.  If this fails the virtualenv will be completely new build.

[1] https://github.com/actions/cache/issues/2#issuecomment-673493515
[2] https://github.com/searx/searx/pull/2517#discussion_r567240235

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
---
 .github/workflows/integration.yml | 2 +-
 Makefile                          | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 4f3b8e9c..39f3d341 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -34,7 +34,7 @@ jobs:
         make V=1 install
         make V=1 gecko.driver
     - name: Run tests
-      run: make V=1 test
+      run: make V=1 ci.test
     - name: Test coverage
       run: make V=1 test.coverage
     - name: Store coverage result
diff --git a/Makefile b/Makefile
index c52e93cd..d148581d 100644
--- a/Makefile
+++ b/Makefile
@@ -251,6 +251,11 @@ test.clean:
 # travis
 # ------
 
+PHONY += ci.test
+ci.test:
+	$(PY_ENV_BIN)/python -c "import yaml"  || make clean
+	$(MAKE) test
+
 travis.codecov:
 	$(Q)$(PY_ENV_BIN)/python -m pip install codecov