devol
/
searx
mirror of https://github.com/searx/searx
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #2602 from MarcAbonce/fix-bing-fetch-languages 

Fix fetch_languages for Bing
This commit is contained in:
Alexandre Flament 2021-03-01 09:06:37 +01:00 committed by GitHub
parent 5f6ac3afa2 d6681fd33b
commit f77983e174
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
searx/engines/__init__.py
View File

@ -27,7 +27,7 @@ from searx import settings
from searx import logger from searx import logger
from searx.data import ENGINES_LANGUAGES from searx.data import ENGINES_LANGUAGES
from searx.poolrequests import get, get_proxy_cycles from searx.poolrequests import get, get_proxy_cycles
from searx.utils import load_module, match_language, get_engine_from_settings from searx.utils import load_module, match_language, get_engine_from_settings, gen_useragent
logger = logger.getChild('engines') logger = logger.getChild('engines')
@ -131,8 +131,12 @@ def load_engine(engine_data):
# assign language fetching method if auxiliary method exists # assign language fetching method if auxiliary method exists
if hasattr(engine, '_fetch_supported_languages'): if hasattr(engine, '_fetch_supported_languages'):
headers = {
'User-Agent': gen_useragent(),
'Accept-Language': 'ja-JP,ja;q=0.8,en-US;q=0.5,en;q=0.3', # bing needs a non-English language
}
setattr(engine, 'fetch_supported_languages', setattr(engine, 'fetch_supported_languages',
lambda: engine._fetch_supported_languages(get(engine.supported_languages_url))) lambda: engine._fetch_supported_languages(get(engine.supported_languages_url, headers=headers)))
engine.stats = { engine.stats = {
'sent_search_count': 0, # sent search 'sent_search_count': 0, # sent search

2
searx/engines/wikipedia.py
View File

@ -106,6 +106,6 @@ def _fetch_supported_languages(resp):
articles = int(td[4].xpath('./a/b')[0].text.replace(',', '')) articles = int(td[4].xpath('./a/b')[0].text.replace(',', ''))
# exclude languages with too few articles # exclude languages with too few articles
if articles >= 100: if articles >= 100:
supported_languages[code] = {"name": name, "english_name": english_name, "articles": articles} supported_languages[code] = {"name": name, "english_name": english_name}
return supported_languages return supported_languages

3
utils/fetch_languages.py
View File

@ -2,8 +2,7 @@
# This script generates languages.py from intersecting each engine's supported languages. # This script generates languages.py from intersecting each engine's supported languages.
# #
# Output files (engines_languages.json and languages.py) # Output files: searx/data/engines_languages.json and searx/languages.py
# are written in current directory to avoid overwriting in case something goes wrong.
import json import json
from pathlib import Path from pathlib import Path