mirror of https://github.com/searx/searx
[mod] checker : replace pycld3 by langdetect
pycld3 requires the native library cld3 langdetect is a pure python package
This commit is contained in:
parent
0495e15df4
commit
aa887eb375
|
@ -9,4 +9,4 @@ pygments==2.1.3
|
||||||
python-dateutil==2.8.1
|
python-dateutil==2.8.1
|
||||||
pyyaml==5.3.1
|
pyyaml==5.3.1
|
||||||
requests[socks]==2.25.1
|
requests[socks]==2.25.1
|
||||||
pycld3==0.20
|
langdetect==1.0.8
|
||||||
|
|
|
@ -9,7 +9,8 @@ from time import time
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import cld3
|
from langdetect import detect_langs
|
||||||
|
from langdetect.lang_detect_exception import LangDetectException
|
||||||
import requests.exceptions
|
import requests.exceptions
|
||||||
|
|
||||||
from searx import poolrequests, logger
|
from searx import poolrequests, logger
|
||||||
|
@ -181,10 +182,14 @@ class ResultContainerTests:
|
||||||
self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')
|
self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')
|
||||||
|
|
||||||
def _add_language(self, text: str) -> typing.Optional[str]:
|
def _add_language(self, text: str) -> typing.Optional[str]:
|
||||||
r = cld3.get_language(str(text)) # pylint: disable=E1101
|
try:
|
||||||
if r is not None and r.probability >= 0.98 and r.is_reliable:
|
r = detect_langs(str(text)) # pylint: disable=E1101
|
||||||
self.languages.add(r.language)
|
except LangDetectException:
|
||||||
self.test_results.add_language(r.language)
|
return None
|
||||||
|
|
||||||
|
if len(r) > 0 and r[0].prob > 0.95:
|
||||||
|
self.languages.add(r[0].lang)
|
||||||
|
self.test_results.add_language(r[0].lang)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _check_result(self, result):
|
def _check_result(self, result):
|
||||||
|
|
|
@ -46,7 +46,6 @@ SEARX_PACKAGES_debian="\
|
||||||
python3-dev python3-babel python3-venv
|
python3-dev python3-babel python3-venv
|
||||||
uwsgi uwsgi-plugin-python3
|
uwsgi uwsgi-plugin-python3
|
||||||
git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
|
git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
|
||||||
libprotobuf-dev protobuf-compiler
|
|
||||||
shellcheck"
|
shellcheck"
|
||||||
|
|
||||||
BUILD_PACKAGES_debian="\
|
BUILD_PACKAGES_debian="\
|
||||||
|
@ -59,7 +58,6 @@ SEARX_PACKAGES_arch="\
|
||||||
python python-pip python-lxml python-babel
|
python python-pip python-lxml python-babel
|
||||||
uwsgi uwsgi-plugin-python
|
uwsgi uwsgi-plugin-python
|
||||||
git base-devel libxml2
|
git base-devel libxml2
|
||||||
protobuf
|
|
||||||
shellcheck"
|
shellcheck"
|
||||||
|
|
||||||
BUILD_PACKAGES_arch="\
|
BUILD_PACKAGES_arch="\
|
||||||
|
@ -71,7 +69,7 @@ SEARX_PACKAGES_fedora="\
|
||||||
python python-pip python-lxml python-babel
|
python python-pip python-lxml python-babel
|
||||||
uwsgi uwsgi-plugin-python3
|
uwsgi uwsgi-plugin-python3
|
||||||
git @development-tools libxml2
|
git @development-tools libxml2
|
||||||
ShellCheck protobuf-compiler protobuf-devel"
|
ShellCheck"
|
||||||
|
|
||||||
BUILD_PACKAGES_fedora="\
|
BUILD_PACKAGES_fedora="\
|
||||||
firefox graphviz graphviz-gd ImageMagick librsvg2-tools
|
firefox graphviz graphviz-gd ImageMagick librsvg2-tools
|
||||||
|
@ -84,7 +82,7 @@ SEARX_PACKAGES_centos="\
|
||||||
python36 python36-pip python36-lxml python-babel
|
python36 python36-pip python36-lxml python-babel
|
||||||
uwsgi uwsgi-plugin-python3
|
uwsgi uwsgi-plugin-python3
|
||||||
git @development-tools libxml2
|
git @development-tools libxml2
|
||||||
ShellCheck protobuf-compiler protobuf-devel"
|
ShellCheck"
|
||||||
|
|
||||||
BUILD_PACKAGES_centos="\
|
BUILD_PACKAGES_centos="\
|
||||||
firefox graphviz graphviz-gd ImageMagick librsvg2-tools
|
firefox graphviz graphviz-gd ImageMagick librsvg2-tools
|
||||||
|
|
Loading…
Reference in New Issue