Motore di metasearch che rispetta la privacy, hackerabile / pronuncia səːks. https://devol.it/it/searx
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

505 lines
15 KiB

  1. #!/usr/bin/env bash
  2. # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
  3. # SPDX-License-Identifier: AGPL-3.0-or-later
  4. # shellcheck disable=SC2031
  5. # shellcheck source=utils/lib.sh
  6. source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
  7. # shellcheck source=utils/brand.env
  8. source "${REPO_ROOT}/utils/brand.env"
  9. source_dot_config
  10. # config
  11. PY_SETUP_EXTRAS='[test]'
  12. NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"
  13. GECKODRIVER_VERSION="v0.28.0"
  14. # SPHINXOPTS=
  15. # These py files are linted by test.pylint(), all other files are linted by
  16. # test.pep8()
  17. PYLINT_FILES=(
  18. searx/preferences.py
  19. searx/testing.py
  20. searx/engines/gigablast.py
  21. searx/engines/deviantart.py
  22. searx/engines/digg.py
  23. searx/engines/google.py
  24. searx/engines/google_news.py
  25. searx/engines/google_videos.py
  26. searx/engines/google_images.py
  27. searx/engines/mediathekviewweb.py
  28. searx/engines/meilisearch.py
  29. searx/engines/solidtorrents.py
  30. searx/engines/solr.py
  31. searx/engines/sqlite.py
  32. searx/engines/springer.py
  33. searx/engines/google_scholar.py
  34. searx/engines/yahoo_news.py
  35. searx/engines/apkmirror.py
  36. searx/engines/core.py
  37. searx_extra/update/update_external_bangs.py
  38. )
  39. PYLINT_SEARX_DISABLE_OPTION="\
  40. I,C,R,\
  41. W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
  42. E1136"
  43. PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
  44. PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
  45. help() {
  46. cat <<EOF
  47. buildenv
  48. rebuild ./utils/brand.env
  49. babel.compile
  50. pybabel compile ./searx/translations
  51. data.*
  52. all : update searx/languages.py and ./data/*
  53. languages : update searx/data/engines_languages.json & searx/languages.py
  54. useragents: update searx/data/useragents.json with the most recent versions of Firefox.
  55. docs.*
  56. html : build HTML documentation
  57. live : autobuild HTML documentation while editing
  58. gh-pages : deploy on gh-pages branch
  59. prebuild : build reST include files (./${DOCS_BUILD}/includes)
  60. clean : clean documentation build
  61. docker
  62. build : build docker image
  63. push : build and push docker image
  64. gecko.driver
  65. download & install geckodriver if not already installed (required for
  66. robot_tests)
  67. node.*
  68. env : download & install npm dependencies locally
  69. clean : drop npm installations
  70. py.*
  71. build : Build python packages at ./${PYDIST}
  72. clean : delete virtualenv and intermediate py files
  73. pyenv.* :
  74. install : developer install of searx into virtualenv
  75. uninstall : uninstall developer installation
  76. cmd ... : run command ... in virtualenv
  77. OK : test if virtualenv is OK
  78. pypi.upload:
  79. Upload python packages to PyPi (to test use pypi.upload.test)
  80. test.* :
  81. pylint : lint PYLINT_FILES, searx/engines, searx & tests
  82. pep8 : pycodestyle (pep8) for all files except PYLINT_FILES
  83. unit : run unit tests
  84. coverage : run unit tests with coverage
  85. robot : run robot test
  86. clean : clean intermediate test stuff
  87. themes.* :
  88. all : build all themes
  89. oscar : build oscar theme
  90. simple : build simple theme
  91. EOF
  92. }
  93. if [ "$VERBOSE" = "1" ]; then
  94. SPHINX_VERBOSE="-v"
  95. PYLINT_VERBOSE="-v"
  96. fi
  97. # needed by sphinx-docs
  98. export DOCS_BUILD
  99. buildenv() {
  100. SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
  101. | prefix_stdout "${_Blue}BUILDENV${_creset} "
  102. return "${PIPESTATUS[0]}"
  103. }
  104. babel.compile() {
  105. build_msg BABEL compile
  106. pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
  107. dump_return $?
  108. }
  109. data.all() {
  110. data.languages
  111. data.useragents
  112. build_msg DATA "update searx/data/ahmia_blacklist.txt"
  113. pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py
  114. build_msg DATA "update searx/data/wikidata_units.json"
  115. pyenv.cmd python searx_extra/update/update_wikidata_units.py
  116. build_msg DATA "update searx/data/currencies.json"
  117. pyenv.cmd python searx_extra/update/update_currencies.py
  118. }
  119. data.languages() {
  120. ( set -e
  121. build_msg ENGINES "fetch languages .."
  122. pyenv.cmd python searx_extra/update/update_languages.py
  123. build_msg ENGINES "update update searx/languages.py"
  124. build_msg DATA "update searx/data/engines_languages.json"
  125. )
  126. dump_return $?
  127. }
  128. data.useragents() {
  129. build_msg DATA "update searx/data/useragents.json"
  130. pyenv.cmd python searx_extra/update/update_firefox_version.py
  131. dump_return $?
  132. }
  133. docs.prebuild() {
  134. build_msg DOCS "build ${DOCS_BUILD}/includes"
  135. (
  136. set -e
  137. [ "$VERBOSE" = "1" ] && set -x
  138. mkdir -p "${DOCS_BUILD}/includes"
  139. ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
  140. ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
  141. ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
  142. )
  143. dump_return $?
  144. }
  145. docker.push() {
  146. docker.build push
  147. }
  148. # shellcheck disable=SC2119
  149. docker.build() {
  150. pyenv.install
  151. build_msg DOCKER build
  152. # run installation in a subprocess and activate pyenv
  153. # See https://www.shellcheck.net/wiki/SC1001 and others ..
  154. # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
  155. ( set -e
  156. # shellcheck source=/dev/null
  157. source "${PY_ENV_BIN}/activate"
  158. # Check if it is a git repository
  159. if [ ! -d .git ]; then
  160. die 1 "This is not Git repository"
  161. fi
  162. if [ ! -x "$(which git)" ]; then
  163. die 1 "git is not installed"
  164. fi
  165. if ! git remote get-url origin 2> /dev/null; then
  166. die 1 "there is no remote origin"
  167. fi
  168. # This is a git repository
  169. # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
  170. # awk to remove the "v" and the "g"
  171. SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')
  172. # add the suffix "-dirty" if the repository has uncommited change
  173. # /!\ HACK for searx/searx: ignore utils/brand.env
  174. git update-index -q --refresh
  175. if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
  176. SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
  177. fi
  178. # Get the last git commit id, will be added to the Searx version (see Dockerfile)
  179. VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
  180. build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
  181. # Check consistency between the git tag and the searx/version.py file
  182. # /! HACK : parse Python file with bash /!
  183. # otherwise it is not possible build the docker image without all Python
  184. # dependencies ( version.py loads __init__.py )
  185. # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
  186. SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
  187. if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
  188. err_msg "git tag: $SEARX_GIT_VERSION"
  189. err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
  190. die 1 "Inconsistency between the last git tag and the searx/version.py file"
  191. fi
  192. # define the docker image name
  193. GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
  194. SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"
  195. # build Docker image
  196. build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
  197. sudo docker build \
  198. --build-arg GIT_URL="${GIT_URL}" \
  199. --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
  200. --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
  201. --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
  202. --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
  203. --build-arg LABEL_VCS_URL="${GIT_URL}" \
  204. --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
  205. --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
  206. -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
  207. if [ "$1" = "push" ]; then
  208. sudo docker push "${SEARX_IMAGE_NAME}:latest"
  209. sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
  210. fi
  211. )
  212. dump_return $?
  213. }
  214. # shellcheck disable=SC2119
  215. gecko.driver() {
  216. pyenv.install
  217. build_msg INSTALL "gecko.driver"
  218. # run installation in a subprocess and activate pyenv
  219. ( set -e
  220. # shellcheck source=/dev/null
  221. source "${PY_ENV_BIN}/activate"
  222. # TODO : check the current geckodriver version
  223. geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
  224. set +e
  225. if [ -z "$NOTFOUND" ]; then
  226. build_msg INSTALL "geckodriver already installed"
  227. return
  228. fi
  229. PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
  230. case "$PLATFORM" in
  231. "linux 32bit" | "linux2 32bit") ARCH="linux32";;
  232. "linux 64bit" | "linux2 64bit") ARCH="linux64";;
  233. "windows 32 bit") ARCH="win32";;
  234. "windows 64 bit") ARCH="win64";;
  235. "mac 64bit") ARCH="macos";;
  236. esac
  237. GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
  238. build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
  239. FILE="$(mktemp)"
  240. wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
  241. rm -- "$FILE"
  242. chmod 755 -- "${PY_ENV_BIN}/geckodriver"
  243. )
  244. dump_return $?
  245. }
  246. node.env() {
  247. local err=0
  248. pushd "${REPO_ROOT}" &> /dev/null
  249. # shellcheck disable=SC2230
  250. which npm &> /dev/null || die 1 'node.env - npm is not found!'
  251. ( set -e
  252. # shellcheck disable=SC2030
  253. PATH="$(npm bin):$PATH"
  254. export PATH
  255. build_msg INSTALL "npm install $NPM_PACKAGES"
  256. # shellcheck disable=SC2086
  257. npm install $NPM_PACKAGES
  258. cd "${REPO_ROOT}/searx/static/themes/oscar"
  259. build_msg INSTALL "($(pwd)) npm install"
  260. npm install
  261. build_msg INSTALL "($(pwd)) npm install"
  262. cd "${REPO_ROOT}/searx/static/themes/simple"
  263. npm install
  264. )
  265. err=$?
  266. popd &> /dev/null
  267. dump_return "$err"
  268. }
  269. node.clean() {
  270. build_msg CLEAN "locally installed npm dependencies"
  271. rm -rf \
  272. ./node_modules \
  273. ./package-lock.json \
  274. ./searx/static/themes/oscar/package-lock.json \
  275. ./searx/static/themes/oscar/node_modules \
  276. ./searx/static/themes/simple/package-lock.json \
  277. ./searx/static/themes/simple/node_modules
  278. dump_return $?
  279. }
  280. py.build() {
  281. build_msg BUILD "[pylint] python package ${PYDIST}"
  282. pyenv.cmd python setup.py \
  283. sdist -d "${PYDIST}" \
  284. bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
  285. }
  286. py.clean() {
  287. build_msg CLEAN pyenv
  288. ( set -e
  289. pyenv.drop
  290. [ "$VERBOSE" = "1" ] && set -x
  291. rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
  292. find . -name '*.pyc' -exec rm -f {} +
  293. find . -name '*.pyo' -exec rm -f {} +
  294. find . -name __pycache__ -exec rm -rf {} +
  295. )
  296. }
  297. pyenv.check() {
  298. cat <<EOF
  299. import yaml
  300. print('import yaml --> OK')
  301. EOF
  302. }
  303. pyenv.install() {
  304. if ! pyenv.OK; then
  305. py.clean > /dev/null
  306. fi
  307. if pyenv.install.OK > /dev/null; then
  308. return 0
  309. fi
  310. ( set -e
  311. pyenv
  312. build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
  313. "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
  314. buildenv
  315. ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"
  316. }
  317. pyenv.uninstall() {
  318. build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
  319. pyenv.cmd python setup.py develop --uninstall 2>&1 \
  320. | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
  321. }
  322. pypi.upload() {
  323. py.clean
  324. py.build
  325. # https://github.com/pypa/twine
  326. pyenv.cmd twine upload "${PYDIST}"/*
  327. }
  328. pypi.upload.test() {
  329. py.clean
  330. py.build
  331. pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
  332. }
  333. test.pylint() {
  334. # shellcheck disable=SC2086
  335. ( set -e
  336. build_msg TEST "[pylint] \$PYLINT_FILES"
  337. pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
  338. "${PYLINT_FILES[@]}"
  339. build_msg TEST "[pylint] searx/engines"
  340. pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
  341. --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
  342. --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
  343. searx/engines
  344. build_msg TEST "[pylint] searx tests"
  345. pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
  346. --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
  347. --ignore=searx/engines \
  348. tests
  349. )
  350. dump_return $?
  351. }
  352. test.pep8() {
  353. build_msg TEST 'pycodestyle (formerly pep8)'
  354. local _exclude=""
  355. printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
  356. pyenv.cmd pycodestyle \
  357. --exclude="searx/static, searx/languages.py, $_exclude " \
  358. --max-line-length=120 \
  359. --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
  360. searx tests
  361. dump_return $?
  362. }
  363. test.unit() {
  364. build_msg TEST 'tests/unit'
  365. pyenv.cmd python -m nose2 -s tests/unit
  366. dump_return $?
  367. }
  368. test.coverage() {
  369. build_msg TEST 'unit test coverage'
  370. ( set -e
  371. pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
  372. pyenv.cmd coverage report
  373. pyenv.cmd coverage html
  374. )
  375. dump_return $?
  376. }
  377. test.robot() {
  378. build_msg TEST 'robot'
  379. gecko.driver
  380. PYTHONPATH=. pyenv.cmd python searx/testing.py robot
  381. dump_return $?
  382. }
  383. test.clean() {
  384. build_msg CLEAN "test stuff"
  385. rm -rf geckodriver.log .coverage coverage/
  386. dump_return $?
  387. }
  388. themes.all() {
  389. ( set -e
  390. node.env
  391. themes.oscar
  392. themes.simple
  393. )
  394. dump_return $?
  395. }
  396. themes.oscar() {
  397. local gruntfile=searx/static/themes/oscar/gruntfile.js
  398. build_msg GRUNT "${gruntfile}"
  399. PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
  400. dump_return $?
  401. }
  402. themes.simple() {
  403. local gruntfile=searx/static/themes/simple/gruntfile.js
  404. build_msg GRUNT "${gruntfile}"
  405. PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
  406. dump_return $?
  407. }
  408. # shellcheck disable=SC2119
  409. main() {
  410. local _type
  411. local cmd="$1"; shift
  412. if [ "$cmd" == "" ]; then
  413. help
  414. err_msg "missing command"
  415. return 42
  416. fi
  417. case "$cmd" in
  418. --getenv) var="$1"; echo "${!var}";;
  419. --help) help;;
  420. --*)
  421. help
  422. err_msg "unknown option $cmd"
  423. return 42
  424. ;;
  425. *)
  426. _type="$(type -t "$cmd")"
  427. if [ "$_type" != 'function' ]; then
  428. err_msg "unknown command $1 / use --help"
  429. return 42
  430. else
  431. "$cmd" "$@"
  432. fi
  433. ;;
  434. esac
  435. }
  436. main "$@"