mirror of https://github.com/searx/searx
[mod] update currencies.json and fetch_currencies.py
use a sparql request on wikidata to get the list of currencies. currencies.json contains the translation for all supported searx languages. Supersede #993
This commit is contained in:
Alexandre Flament
parent
1be6ab2a91
commit
46ca32c3cc
|
|
@ -41,7 +41,7 @@ jobs:
|
||||||
python utils/fetch_languages.py
|
python utils/fetch_languages.py
|
||||||
python utils/fetch_ahmia_blacklist.py
|
python utils/fetch_ahmia_blacklist.py
|
||||||
python utils/fetch_wikidata_units.py
|
python utils/fetch_wikidata_units.py
|
||||||
# python utils/fetch_currencies.py
|
python utils/fetch_currencies.py
|
||||||
|
|
||||||
- name: Create Pull Request
|
- name: Create Pull Request
|
||||||
id: cpr
|
id: cpr
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -20,6 +20,8 @@ def name_to_iso4217(name):
|
||||||
global CURRENCIES
|
global CURRENCIES
|
||||||
name = normalize_name(name)
|
name = normalize_name(name)
|
||||||
currency = CURRENCIES['names'].get(name, [name])
|
currency = CURRENCIES['names'].get(name, [name])
|
||||||
|
if isinstance(currency, str):
|
||||||
|
return currency
|
||||||
return currency[0]
|
return currency[0]
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,163 +1,151 @@
|
||||||
# -*- coding: utf-8 -*-
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
|
||||||
import json
|
|
||||||
import re
|
import re
|
||||||
import unicodedata
|
import unicodedata
|
||||||
import string
|
import json
|
||||||
from urllib.parse import urlencode
|
|
||||||
from requests import get
|
|
||||||
|
|
||||||
languages = {'de', 'en', 'es', 'fr', 'hu', 'it', 'nl', 'jp'}
|
# set path
|
||||||
|
from sys import path
|
||||||
|
from os.path import realpath, dirname, join
|
||||||
|
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
||||||
|
|
||||||
url_template = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&{query}&props=labels%7Cdatatype%7Cclaims%7Caliases&languages=' + '|'.join(languages)
|
from searx import searx_dir, settings
|
||||||
url_wmflabs_template = 'http://wdq.wmflabs.org/api?q='
|
from searx.engines.wikidata import send_wikidata_query
|
||||||
url_wikidata_search_template = 'http://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'
|
|
||||||
|
|
||||||
wmflabs_queries = [
|
|
||||||
'CLAIM[31:8142]', # all devise
|
|
||||||
]
|
|
||||||
|
|
||||||
db = {
|
# ORDER BY (with all the query fields) is important to keep a deterministic result order
|
||||||
'iso4217': {
|
# so multiple invokation of this script doesn't change currencies.json
|
||||||
},
|
SARQL_REQUEST = """
|
||||||
'names': {
|
SELECT DISTINCT ?iso4217 ?unit ?unicode ?label ?alias WHERE {
|
||||||
}
|
?item wdt:P498 ?iso4217; rdfs:label ?label.
|
||||||
|
OPTIONAL { ?item skos:altLabel ?alias FILTER (LANG (?alias) = LANG(?label)). }
|
||||||
|
OPTIONAL { ?item wdt:P5061 ?unit. }
|
||||||
|
OPTIONAL { ?item wdt:P489 ?symbol.
|
||||||
|
?symbol wdt:P487 ?unicode. }
|
||||||
|
MINUS { ?item wdt:P582 ?end_data . } # Ignore monney with an end date
|
||||||
|
MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . } # Ignore "former entity" (obsolete currency)
|
||||||
|
FILTER(LANG(?label) IN (%LANGUAGES_SPARQL%)).
|
||||||
}
|
}
|
||||||
|
ORDER BY ?iso4217 ?unit ?unicode ?label ?alias
|
||||||
|
"""
|
||||||
|
|
||||||
|
# ORDER BY (with all the query fields) is important to keep a deterministic result order
|
||||||
|
# so multiple invokation of this script doesn't change currencies.json
|
||||||
|
SPARQL_WIKIPEDIA_NAMES_REQUEST = """
|
||||||
|
SELECT DISTINCT ?iso4217 ?article_name WHERE {
|
||||||
|
?item wdt:P498 ?iso4217 .
|
||||||
|
?article schema:about ?item ;
|
||||||
|
schema:name ?article_name ;
|
||||||
|
schema:isPartOf [ wikibase:wikiGroup "wikipedia" ]
|
||||||
|
MINUS { ?item wdt:P582 ?end_data . } # Ignore monney with an end date
|
||||||
|
MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . } # Ignore "former entity" (obsolete currency)
|
||||||
|
FILTER(LANG(?article_name) IN (%LANGUAGES_SPARQL%)).
|
||||||
|
}
|
||||||
|
ORDER BY ?iso4217 ?article_name
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
def remove_accents(data):
|
LANGUAGES = settings['locales'].keys()
|
||||||
return unicodedata.normalize('NFKD', data).lower()
|
LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))
|
||||||
|
|
||||||
|
|
||||||
def normalize_name(name):
|
def remove_accents(name):
|
||||||
return re.sub(' +', ' ', remove_accents(name.lower()).replace('-', ' '))
|
return unicodedata.normalize('NFKD', name).lower()
|
||||||
|
|
||||||
|
|
||||||
def add_currency_name(name, iso4217):
|
def remove_extra(name):
|
||||||
global db
|
for c in ('(', ':'):
|
||||||
|
if c in name:
|
||||||
|
name = name.split(c)[0].strip()
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_name(name):
|
||||||
|
name = re.sub(' +', ' ', remove_accents(name.lower()).replace('-', ' '))
|
||||||
|
name = remove_extra(name)
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
def add_currency_name(db, name, iso4217, normalize_name=True):
|
||||||
db_names = db['names']
|
db_names = db['names']
|
||||||
|
|
||||||
if not isinstance(iso4217, str):
|
if normalize_name:
|
||||||
print("problem", name, iso4217)
|
name = _normalize_name(name)
|
||||||
return
|
|
||||||
|
|
||||||
name = normalize_name(name)
|
iso4217_set = db_names.setdefault(name, [])
|
||||||
|
if iso4217 not in iso4217_set:
|
||||||
if name == '':
|
iso4217_set.insert(0, iso4217)
|
||||||
print("name empty", iso4217)
|
|
||||||
return
|
|
||||||
|
|
||||||
iso4217_set = db_names.get(name, None)
|
|
||||||
if iso4217_set is not None and iso4217 not in iso4217_set:
|
|
||||||
db_names[name].append(iso4217)
|
|
||||||
else:
|
|
||||||
db_names[name] = [iso4217]
|
|
||||||
|
|
||||||
|
|
||||||
def add_currency_label(label, iso4217, language):
|
def add_currency_label(db, label, iso4217, language):
|
||||||
global db
|
labels = db['iso4217'].setdefault(iso4217, {})
|
||||||
|
labels[language] = label
|
||||||
db['iso4217'][iso4217] = db['iso4217'].get(iso4217, {})
|
|
||||||
db['iso4217'][iso4217][language] = label
|
|
||||||
|
|
||||||
|
|
||||||
def get_property_value(data, name):
|
def wikidata_request_result_iterator(request):
|
||||||
prop = data.get('claims', {}).get(name, {})
|
result = send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
|
||||||
if len(prop) == 0:
|
if result is not None:
|
||||||
return None
|
for r in result['results']['bindings']:
|
||||||
|
yield r
|
||||||
value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '')
|
|
||||||
if value == '':
|
|
||||||
return None
|
|
||||||
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def parse_currency(data):
|
def fetch_db():
|
||||||
iso4217 = get_property_value(data, 'P498')
|
db = {
|
||||||
|
'names': {},
|
||||||
|
'iso4217': {},
|
||||||
|
}
|
||||||
|
|
||||||
if iso4217 is not None:
|
for r in wikidata_request_result_iterator(SPARQL_WIKIPEDIA_NAMES_REQUEST):
|
||||||
unit = get_property_value(data, 'P558')
|
iso4217 = r['iso4217']['value']
|
||||||
if unit is not None:
|
article_name = r['article_name']['value']
|
||||||
add_currency_name(unit, iso4217)
|
article_lang = r['article_name']['xml:lang']
|
||||||
|
add_currency_name(db, article_name, iso4217)
|
||||||
|
add_currency_label(db, article_name, iso4217, article_lang)
|
||||||
|
|
||||||
labels = data.get('labels', {})
|
for r in wikidata_request_result_iterator(SARQL_REQUEST):
|
||||||
for language in languages:
|
iso4217 = r['iso4217']['value']
|
||||||
name = labels.get(language, {}).get('value', None)
|
if 'label' in r:
|
||||||
if name is not None:
|
label = r['label']['value']
|
||||||
add_currency_name(name, iso4217)
|
label_lang = r['label']['xml:lang']
|
||||||
add_currency_label(name, iso4217, language)
|
add_currency_name(db, label, iso4217)
|
||||||
|
add_currency_label(db, label, iso4217, label_lang)
|
||||||
|
|
||||||
aliases = data.get('aliases', {})
|
if 'alias' in r:
|
||||||
for language in aliases:
|
add_currency_name(db, r['alias']['value'], iso4217)
|
||||||
for i in range(0, len(aliases[language])):
|
|
||||||
alias = aliases[language][i].get('value', None)
|
if 'unicode' in r:
|
||||||
add_currency_name(alias, iso4217)
|
add_currency_name(db, r['unicode']['value'], iso4217, normalize_name=False)
|
||||||
|
|
||||||
|
if 'unit' in r:
|
||||||
|
add_currency_name(db, r['unit']['value'], iso4217, normalize_name=False)
|
||||||
|
|
||||||
|
# reduce memory usage:
|
||||||
|
# replace lists with one item by the item.
|
||||||
|
# see searx.search.processors.online_currency.name_to_iso4217
|
||||||
|
for name in db['names']:
|
||||||
|
if len(db['names'][name]) == 1:
|
||||||
|
db['names'][name] = db['names'][name][0]
|
||||||
|
|
||||||
|
return db
|
||||||
|
|
||||||
|
|
||||||
def fetch_data(wikidata_ids):
|
def get_filename():
|
||||||
url = url_template.format(query=urlencode({'ids': '|'.join(wikidata_ids)}))
|
return join(join(searx_dir, "data"), "currencies.json")
|
||||||
htmlresponse = get(url)
|
|
||||||
jsonresponse = json.loads(htmlresponse.content)
|
|
||||||
entities = jsonresponse.get('entities', {})
|
|
||||||
|
|
||||||
for pname in entities:
|
|
||||||
pvalue = entities.get(pname)
|
|
||||||
parse_currency(pvalue)
|
|
||||||
|
|
||||||
|
|
||||||
def add_q(i):
|
def main():
|
||||||
return "Q" + str(i)
|
#
|
||||||
|
db = fetch_db()
|
||||||
|
# static
|
||||||
|
add_currency_name(db, "euro", 'EUR')
|
||||||
|
add_currency_name(db, "euros", 'EUR')
|
||||||
|
add_currency_name(db, "dollar", 'USD')
|
||||||
|
add_currency_name(db, "dollars", 'USD')
|
||||||
|
add_currency_name(db, "peso", 'MXN')
|
||||||
|
add_currency_name(db, "pesos", 'MXN')
|
||||||
|
|
||||||
|
with open(get_filename(), 'w', encoding='utf8') as f:
|
||||||
|
json.dump(db, f, ensure_ascii=False, indent=4)
|
||||||
|
|
||||||
def fetch_data_batch(wikidata_ids):
|
if __name__ == '__main__':
|
||||||
while len(wikidata_ids) > 0:
|
main()
|
||||||
if len(wikidata_ids) > 50:
|
|
||||||
fetch_data(wikidata_ids[0:49])
|
|
||||||
wikidata_ids = wikidata_ids[50:]
|
|
||||||
else:
|
|
||||||
fetch_data(wikidata_ids)
|
|
||||||
wikidata_ids = []
|
|
||||||
|
|
||||||
|
|
||||||
def wdq_query(query):
|
|
||||||
url = url_wmflabs_template + query
|
|
||||||
htmlresponse = get(url)
|
|
||||||
jsonresponse = json.loads(htmlresponse.content)
|
|
||||||
qlist = list(map(add_q, jsonresponse.get('items', {})))
|
|
||||||
error = jsonresponse.get('status', {}).get('error', None)
|
|
||||||
if error is not None and error != 'OK':
|
|
||||||
print("error for query '" + query + "' :" + error)
|
|
||||||
|
|
||||||
fetch_data_batch(qlist)
|
|
||||||
|
|
||||||
|
|
||||||
def wd_query(query, offset=0):
|
|
||||||
qlist = []
|
|
||||||
|
|
||||||
url = url_wikidata_search_template.format(query=urlencode({'srsearch': query, 'srlimit': 50, 'sroffset': offset}))
|
|
||||||
htmlresponse = get(url)
|
|
||||||
jsonresponse = json.loads(htmlresponse.content)
|
|
||||||
for r in jsonresponse.get('query', {}).get('search', {}):
|
|
||||||
qlist.append(r.get('title', ''))
|
|
||||||
fetch_data_batch(qlist)
|
|
||||||
|
|
||||||
|
|
||||||
# fetch #
|
|
||||||
for q in wmflabs_queries:
|
|
||||||
wdq_query(q)
|
|
||||||
|
|
||||||
# static
|
|
||||||
add_currency_name("euro", 'EUR')
|
|
||||||
add_currency_name("euros", 'EUR')
|
|
||||||
add_currency_name("dollar", 'USD')
|
|
||||||
add_currency_name("dollars", 'USD')
|
|
||||||
add_currency_name("peso", 'MXN')
|
|
||||||
add_currency_name("pesos", 'MXN')
|
|
||||||
|
|
||||||
# write
|
|
||||||
f = open("currencies.json", "wb")
|
|
||||||
json.dump(db, f, indent=4, encoding="utf-8")
|
|
||||||
f.close()
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue