From fa9c9e090b6975987fac187f519dbf66f29eeed4 Mon Sep 17 00:00:00 2001 From: asciimoo Date: Wed, 16 Oct 2013 23:03:26 +0200 Subject: [PATCH] [enh] result ordering and deduplication --- searx/engines/__init__.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 1dc1ea34..d383f360 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -4,6 +4,7 @@ from os import listdir from imp import load_source import grequests from itertools import izip_longest, chain +from operator import itemgetter engine_dir = dirname(realpath(__file__)) @@ -56,4 +57,23 @@ def search(query, request, selected_engines): ) requests.append(req) grequests.map(requests) - return list(filter(None, chain(*izip_longest(*results.values())))) + flat_res = list(filter(None, chain(*izip_longest(*results.values())))) + flat_len = len(flat_res) + results = [] + # deduplication + scoring + for i,res in enumerate(flat_res): + score = flat_len - i + duplicated = False + for new_res in results: + if res['url'] == new_res['url']: + duplicated = new_res + break + if duplicated: + if len(res['content']) > len(duplicated): + duplicated['content'] = res['content'] + duplicated['score'] += score + else: + res['score'] = score + results.append(res) + + return sorted(results, key=itemgetter('score'), reverse=True)