import re from collections import defaultdict from operator import itemgetter from threading import RLock from urllib.parse import urlparse, unquote from searx import logger from searx.engines import engines from searx.metrology.error_recorder import record_error from searx.utils import add_scheme_to_url from searx import settings CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U) WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U) # return the meaningful length of the content for a result def result_content_len(content): if isinstance(content, str): return len(CONTENT_LEN_IGNORED_CHARS_REGEX.sub('', content)) else: return 0 def compare_urls(url_a, url_b): """Lazy compare between two URL. "www.example.com" and "example.com" are equals. "www.example.com/path/" and "www.example.com/path" are equals. "https://www.example.com/" and "http://www.example.com/" are equals. Args: url_a (ParseResult): first URL url_b (ParseResult): second URL Returns: bool: True if url_a and url_b are equals """ # ignore www. in comparison if url_a.netloc.startswith('www.'): host_a = url_a.netloc.replace('www.', '', 1) else: host_a = url_a.netloc if url_b.netloc.startswith('www.'): host_b = url_b.netloc.replace('www.', '', 1) else: host_b = url_b.netloc if host_a != host_b or url_a.query != url_b.query or url_a.fragment != url_b.fragment: return False # remove / from the end of the url if required path_a = url_a.path[:-1]\ if url_a.path.endswith('/')\ else url_a.path path_b = url_b.path[:-1]\ if url_b.path.endswith('/')\ else url_b.path return unquote(path_a) == unquote(path_b) def merge_two_infoboxes(infobox1, infobox2): # get engines weights if hasattr(engines[infobox1['engine']], 'weight'): weight1 = engines[infobox1['engine']].weight else: weight1 = 1 if hasattr(engines[infobox2['engine']], 'weight'): weight2 = engines[infobox2['engine']].weight else: weight2 = 1 if weight2 > weight1: infobox1['engine'] = infobox2['engine'] infobox1['engines'] |= infobox2['engines'] if 'urls' in infobox2: urls1 = infobox1.get('urls', None) if urls1 is None: urls1 = [] for url2 in infobox2.get('urls', []): unique_url = True parsed_url2 = urlparse(url2.get('url', '')) entity_url2 = url2.get('entity') for url1 in urls1: if (entity_url2 is not None and url1.get('entity') == entity_url2)\ or compare_urls(urlparse(url1.get('url', '')), parsed_url2): unique_url = False break if unique_url: urls1.append(url2) infobox1['urls'] = urls1 if 'img_src' in infobox2: img1 = infobox1.get('img_src', None) img2 = infobox2.get('img_src') if img1 is None: infobox1['img_src'] = img2 elif weight2 > weight1: infobox1['img_src'] = img2 if 'attributes' in infobox2: attributes1 = infobox1.get('attributes') if attributes1 is None: infobox1['attributes'] = attributes1 = [] attributeSet = set() for attribute in attributes1: label = attribute.get('label') if label not in attributeSet: attributeSet.add(label) entity = attribute.get('entity') if entity not in attributeSet: attributeSet.add(entity) for attribute in infobox2.get('attributes', []): if attribute.get('label') not in attributeSet\ and attribute.get('entity') not in attributeSet: attributes1.append(attribute) if 'content' in infobox2: content1 = infobox1.get('content', None) content2 = infobox2.get('content', '') if content1 is not None: if result_content_len(content2) > result_content_len(content1): infobox1['content'] = content2 else: infobox1['content'] = content2 def result_score(result, language): weight = 1.0 for result_engine in result['engines']: if hasattr(engines[result_engine], 'weight'): weight *= float(engines[result_engine].weight) if settings['search'].get('prefer_configured_language', False): domain_parts = result['parsed_url'].netloc.split('.') if language in domain_parts: weight *= 1.1 occurrences = len(result['positions']) return sum((occurrences * weight) / position for position in result['positions']) class ResultContainer: """docstring for ResultContainer""" __slots__ = '_merged_results', 'infoboxes', 'suggestions', 'answers', 'corrections', '_number_of_results',\ '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url', 'engine_data',\ '_language' def __init__(self, language): super().__init__() self._merged_results = [] self.infoboxes = [] self.suggestions = set() self.answers = {} self.corrections = set() self._number_of_results = [] self.engine_data = defaultdict(dict) self._ordered = False self.paging = False self.unresponsive_engines = set() self.timings = [] self.redirect_url = None self._language = language.lower().split('-')[0] def extend(self, engine_name, results): standard_result_count = 0 error_msgs = set() for result in list(results): result['engine'] = engine_name if 'suggestion' in result: self.suggestions.add(result['suggestion']) elif 'answer' in result: self.answers[result['answer']] = result elif 'correction' in result: self.corrections.add(result['correction']) elif 'infobox' in result: self._merge_infobox(result) elif 'number_of_results' in result: self._number_of_results.append(result['number_of_results']) elif 'engine_data' in result: self.engine_data[engine_name][result['key']] = result['engine_data'] else: # standard result (url, title, content) if 'url' in result and not isinstance(result['url'], str): logger.debug('result: invalid URL: %s', str(result)) error_msgs.add('invalid URL') elif 'title' in result and not isinstance(result['title'], str): logger.debug('result: invalid title: %s', str(result)) error_msgs.add('invalid title') elif 'content' in result and not isinstance(result['content'], str): logger.debug('result: invalid content: %s', str(result)) error_msgs.add('invalid content') else: self._merge_result(result, standard_result_count + 1) standard_result_count += 1 if len(error_msgs) > 0: for msg in error_msgs: record_error(engine_name, 'some results are invalids: ' + msg) if engine_name in engines: with RLock(): engines[engine_name].stats['search_count'] += 1 engines[engine_name].stats['result_count'] += standard_result_count if not self.paging and standard_result_count > 0 and engine_name in engines\ and engines[engine_name].paging: self.paging = True def _merge_infobox(self, infobox): add_infobox = True infobox_id = infobox.get('id', None) infobox['engines'] = set([infobox['engine']]) if infobox_id is not None: parsed_url_infobox_id = urlparse(infobox_id) for existingIndex in self.infoboxes: if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id): merge_two_infoboxes(existingIndex, infobox) add_infobox = False if add_infobox: self.infoboxes.append(infobox) def _merge_result(self, result, position): if 'url' in result: self.__merge_url_result(result, position) return self.__merge_result_no_url(result, position) def __merge_url_result(self, result, position): result['parsed_url'] = urlparse(result['url']) # if the result has no scheme, use http as default if not result['parsed_url'].scheme or result['parsed_url'].scheme == '': result['parsed_url'] = result['parsed_url']._replace(scheme='http') result['url'] = result['parsed_url'].geturl() if 'thumbnail_src' in result: result['thumbnail_src'] = add_scheme_to_url(result['thumbnail_src']) if 'img_src' in result: result['img_src'] = add_scheme_to_url(result['img_src']) result['engines'] = set([result['engine']]) # strip multiple spaces and carriage returns from content if result.get('content'): result['content'] = WHITESPACE_REGEX.sub(' ', result['content']) duplicated = self.__find_duplicated_http_result(result) if duplicated: self.__merge_duplicated_http_result(duplicated, result, position) return # if there is no duplicate found, append result result['positions'] = [position] with RLock(): self._merged_results.append(result) def __find_duplicated_http_result(self, result): result_template = result.get('template') for merged_result in self._merged_results: if 'parsed_url' not in merged_result: continue if compare_urls(result['parsed_url'], merged_result['parsed_url'])\ and result_template == merged_result.get('template'): if result_template != 'images.html': # not an image, same template, same url : it's a duplicate return merged_result else: # it's an image # it's a duplicate if the parsed_url, template and img_src are different if result.get('img_src', '') == merged_result.get('img_src', ''): return merged_result return None def __merge_duplicated_http_result(self, duplicated, result, position): # using content with more text if result_content_len(result.get('content', '')) >\ result_content_len(duplicated.get('content', '')): duplicated['content'] = result['content'] # merge all result's parameters not found in duplicate for key in result.keys(): if not duplicated.get(key): duplicated[key] = result.get(key) # add the new position duplicated['positions'].append(position) # add engine to list of result-engines duplicated['engines'].add(result['engine']) # using https if possible if duplicated['parsed_url'].scheme != 'https' and result['parsed_url'].scheme == 'https': duplicated['url'] = result['parsed_url'].geturl() duplicated['parsed_url'] = result['parsed_url'] def __merge_result_no_url(self, result, position): result['engines'] = set([result['engine']]) result['positions'] = [position] with RLock(): self._merged_results.append(result) def order_results(self): for result in self._merged_results: score = result_score(result, self._language) result['score'] = score with RLock(): for result_engine in result['engines']: engines[result_engine].stats['score_count'] += score results = sorted(self._merged_results, key=itemgetter('score'), reverse=True) # pass 2 : group results by category and template gresults = [] categoryPositions = {} for res in results: # FIXME : handle more than one category per engine engine = engines[res['engine']] res['category'] = engine.categories[0] if len(engine.categories) > 0 else '' # FIXME : handle more than one category per engine category = res['category']\ + ':' + res.get('template', '')\ + ':' + ('img_src' if 'img_src' in res or 'thumbnail' in res else '') current = None if category not in categoryPositions\ else categoryPositions[category] # group with previous results using the same category # if the group can accept more result and is not too far # from the current position if current is not None and (current['count'] > 0)\ and (len(gresults) - current['index'] < 20): # group with the previous results using # the same category with this one index = current['index'] gresults.insert(index, res) # update every index after the current one # (including the current one) for k in categoryPositions: v = categoryPositions[k]['index'] if v >= index: categoryPositions[k]['index'] = v + 1 # update this category current['count'] -= 1 else: # same category gresults.append(res) # update categoryIndex categoryPositions[category] = {'index': len(gresults), 'count': 8} # update _merged_results self._ordered = True self._merged_results = gresults def get_ordered_results(self): if not self._ordered: self.order_results() return self._merged_results def results_length(self): return len(self._merged_results) def results_number(self): resultnum_sum = sum(self._number_of_results) if not resultnum_sum or not self._number_of_results: return 0 return resultnum_sum / len(self._number_of_results) def add_unresponsive_engine(self, engine_name, error_type, error_message=None): if engines[engine_name].display_error_messages: self.unresponsive_engines.add((engine_name, error_type, error_message)) def add_timing(self, engine_name, engine_time, page_load_time): self.timings.append({ 'engine': engines[engine_name].shortcut, 'total': engine_time, 'load': page_load_time }) def get_timings(self): return self.timings