searx/searx/results.py

import re
from collections import defaultdict
from operator import itemgetter
from threading import RLock
from urllib.parse import urlparse, unquote
from searx import logger
from searx.engines import engines
from searx.metrology.error_recorder import record_error
from searx.utils import add_scheme_to_url
from searx import settings


CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)


# return the meaningful length of the content for a result
def result_content_len(content):
    if isinstance(content, str):
        return len(CONTENT_LEN_IGNORED_CHARS_REGEX.sub('', content))
    else:
        return 0


def compare_urls(url_a, url_b):
    """Lazy compare between two URL.
    "www.example.com" and "example.com" are equals.
    "www.example.com/path/" and "www.example.com/path" are equals.
    "https://www.example.com/" and "http://www.example.com/" are equals.

    Args:
        url_a (ParseResult): first URL
        url_b (ParseResult): second URL

    Returns:
        bool: True if url_a and url_b are equals
    """
    # ignore www. in comparison
    if url_a.netloc.startswith('www.'):
        host_a = url_a.netloc.replace('www.', '', 1)
    else:
        host_a = url_a.netloc
    if url_b.netloc.startswith('www.'):
        host_b = url_b.netloc.replace('www.', '', 1)
    else:
        host_b = url_b.netloc

    if host_a != host_b or url_a.query != url_b.query or url_a.fragment != url_b.fragment:
        return False

    # remove / from the end of the url if required
    path_a = url_a.path[:-1]\
        if url_a.path.endswith('/')\
        else url_a.path
    path_b = url_b.path[:-1]\
        if url_b.path.endswith('/')\
        else url_b.path

    return unquote(path_a) == unquote(path_b)


def merge_two_infoboxes(infobox1, infobox2):
    # get engines weights
    if hasattr(engines[infobox1['engine']], 'weight'):
        weight1 = engines[infobox1['engine']].weight
    else:
        weight1 = 1
    if hasattr(engines[infobox2['engine']], 'weight'):
        weight2 = engines[infobox2['engine']].weight
    else:
        weight2 = 1

    if weight2 > weight1:
        infobox1['engine'] = infobox2['engine']

    infobox1['engines'] |= infobox2['engines']

    if 'urls' in infobox2:
        urls1 = infobox1.get('urls', None)
        if urls1 is None:
            urls1 = []

        for url2 in infobox2.get('urls', []):
            unique_url = True
            parsed_url2 = urlparse(url2.get('url', ''))
            entity_url2 = url2.get('entity')
            for url1 in urls1:
                if (entity_url2 is not None and url1.get('entity') == entity_url2)\
                   or compare_urls(urlparse(url1.get('url', '')), parsed_url2):
                    unique_url = False
                    break
            if unique_url:
                urls1.append(url2)

        infobox1['urls'] = urls1

    if 'img_src' in infobox2:
        img1 = infobox1.get('img_src', None)
        img2 = infobox2.get('img_src')
        if img1 is None:
            infobox1['img_src'] = img2
        elif weight2 > weight1:
            infobox1['img_src'] = img2

    if 'attributes' in infobox2:
        attributes1 = infobox1.get('attributes')
        if attributes1 is None:
            infobox1['attributes'] = attributes1 = []

        attributeSet = set()
        for attribute in attributes1:
            label = attribute.get('label')
            if label not in attributeSet:
                attributeSet.add(label)
            entity = attribute.get('entity')
            if entity not in attributeSet:
                attributeSet.add(entity)

        for attribute in infobox2.get('attributes', []):
            if attribute.get('label') not in attributeSet\
               and attribute.get('entity') not in attributeSet:
                attributes1.append(attribute)

    if 'content' in infobox2:
        content1 = infobox1.get('content', None)
        content2 = infobox2.get('content', '')
        if content1 is not None:
            if result_content_len(content2) > result_content_len(content1):
                infobox1['content'] = content2
        else:
            infobox1['content'] = content2


def result_score(result, language):
    weight = 1.0

    for result_engine in result['engines']:
        if hasattr(engines[result_engine], 'weight'):
            weight *= float(engines[result_engine].weight)

    if settings['search'].get('prefer_configured_language', False):
        domain_parts = result['parsed_url'].netloc.split('.')
        if language in domain_parts:
            weight *= 1.1

    occurrences = len(result['positions'])

    return sum((occurrences * weight) / position for position in result['positions'])


class ResultContainer:
    """docstring for ResultContainer"""

    __slots__ = '_merged_results', 'infoboxes', 'suggestions', 'answers', 'corrections', '_number_of_results',\
                '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url', 'engine_data',\
                '_language'

    def __init__(self, language):
        super().__init__()
        self._merged_results = []
        self.infoboxes = []
        self.suggestions = set()
        self.answers = {}
        self.corrections = set()
        self._number_of_results = []
        self.engine_data = defaultdict(dict)
        self._ordered = False
        self.paging = False
        self.unresponsive_engines = set()
        self.timings = []
        self.redirect_url = None
        self._language = language.lower().split('-')[0]

    def extend(self, engine_name, results):
        standard_result_count = 0
        error_msgs = set()
        for result in list(results):
            result['engine'] = engine_name
            if 'suggestion' in result:
                self.suggestions.add(result['suggestion'])
            elif 'answer' in result:
                self.answers[result['answer']] = result
            elif 'correction' in result:
                self.corrections.add(result['correction'])
            elif 'infobox' in result:
                self._merge_infobox(result)
            elif 'number_of_results' in result:
                self._number_of_results.append(result['number_of_results'])
            elif 'engine_data' in result:
                self.engine_data[engine_name][result['key']] = result['engine_data']
            else:
                # standard result (url, title, content)
                if 'url' in result and not isinstance(result['url'], str):
                    logger.debug('result: invalid URL: %s', str(result))
                    error_msgs.add('invalid URL')
                elif 'title' in result and not isinstance(result['title'], str):
                    logger.debug('result: invalid title: %s', str(result))
                    error_msgs.add('invalid title')
                elif 'content' in result and not isinstance(result['content'], str):
                    logger.debug('result: invalid content: %s', str(result))
                    error_msgs.add('invalid content')
                else:
                    self._merge_result(result, standard_result_count + 1)
                    standard_result_count += 1

        if len(error_msgs) > 0:
            for msg in error_msgs:
                record_error(engine_name, 'some results are invalids: ' + msg)

        if engine_name in engines:
            with RLock():
                engines[engine_name].stats['search_count'] += 1
                engines[engine_name].stats['result_count'] += standard_result_count

        if not self.paging and standard_result_count > 0 and engine_name in engines\
           and engines[engine_name].paging:
            self.paging = True

    def _merge_infobox(self, infobox):
        add_infobox = True
        infobox_id = infobox.get('id', None)
        infobox['engines'] = set([infobox['engine']])
        if infobox_id is not None:
            parsed_url_infobox_id = urlparse(infobox_id)
            for existingIndex in self.infoboxes:
                if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id):
                    merge_two_infoboxes(existingIndex, infobox)
                    add_infobox = False

        if add_infobox:
            self.infoboxes.append(infobox)

    def _merge_result(self, result, position):
        if 'url' in result:
            self.__merge_url_result(result, position)
            return

        self.__merge_result_no_url(result, position)

    def __merge_url_result(self, result, position):
        result['parsed_url'] = urlparse(result['url'])

        # if the result has no scheme, use http as default
        if not result['parsed_url'].scheme or result['parsed_url'].scheme == '':
            result['parsed_url'] = result['parsed_url']._replace(scheme='http')
            result['url'] = result['parsed_url'].geturl()

        if 'thumbnail_src' in result:
            result['thumbnail_src'] = add_scheme_to_url(result['thumbnail_src'])
        if 'img_src' in result:
            result['img_src'] = add_scheme_to_url(result['img_src'])

        result['engines'] = set([result['engine']])

        # strip multiple spaces and carriage returns from content
        if result.get('content'):
            result['content'] = WHITESPACE_REGEX.sub(' ', result['content'])

        duplicated = self.__find_duplicated_http_result(result)
        if duplicated:
            self.__merge_duplicated_http_result(duplicated, result, position)
            return

        # if there is no duplicate found, append result
        result['positions'] = [position]
        with RLock():
            self._merged_results.append(result)

    def __find_duplicated_http_result(self, result):
        result_template = result.get('template')
        for merged_result in self._merged_results:
            if 'parsed_url' not in merged_result:
                continue
            if compare_urls(result['parsed_url'], merged_result['parsed_url'])\
               and result_template == merged_result.get('template'):
                if result_template != 'images.html':
                    # not an image, same template, same url : it's a duplicate
                    return merged_result
                else:
                    # it's an image
                    # it's a duplicate if the parsed_url, template and img_src are different
                    if result.get('img_src', '') == merged_result.get('img_src', ''):
                        return merged_result
        return None

    def __merge_duplicated_http_result(self, duplicated, result, position):
        # using content with more text
        if result_content_len(result.get('content', '')) >\
                result_content_len(duplicated.get('content', '')):
            duplicated['content'] = result['content']

        # merge all result's parameters not found in duplicate
        for key in result.keys():
            if not duplicated.get(key):
                duplicated[key] = result.get(key)

        # add the new position
        duplicated['positions'].append(position)

        # add engine to list of result-engines
        duplicated['engines'].add(result['engine'])

        # using https if possible
        if duplicated['parsed_url'].scheme != 'https' and result['parsed_url'].scheme == 'https':
            duplicated['url'] = result['parsed_url'].geturl()
            duplicated['parsed_url'] = result['parsed_url']

    def __merge_result_no_url(self, result, position):
        result['engines'] = set([result['engine']])
        result['positions'] = [position]
        with RLock():
            self._merged_results.append(result)

    def order_results(self):
        for result in self._merged_results:
            score = result_score(result, self._language)
            result['score'] = score
            with RLock():
                for result_engine in result['engines']:
                    engines[result_engine].stats['score_count'] += score

        results = sorted(self._merged_results, key=itemgetter('score'), reverse=True)

        # pass 2 : group results by category and template
        gresults = []
        categoryPositions = {}

        for res in results:
            # FIXME : handle more than one category per engine
            engine = engines[res['engine']]
            res['category'] = engine.categories[0] if len(engine.categories) > 0 else ''

            # FIXME : handle more than one category per engine
            category = res['category']\
                + ':' + res.get('template', '')\
                + ':' + ('img_src' if 'img_src' in res or 'thumbnail' in res else '')

            current = None if category not in categoryPositions\
                else categoryPositions[category]

            # group with previous results using the same category
            # if the group can accept more result and is not too far
            # from the current position
            if current is not None and (current['count'] > 0)\
                    and (len(gresults) - current['index'] < 20):
                # group with the previous results using
                # the same category with this one
                index = current['index']
                gresults.insert(index, res)

                # update every index after the current one
                # (including the current one)
                for k in categoryPositions:
                    v = categoryPositions[k]['index']
                    if v >= index:
                        categoryPositions[k]['index'] = v + 1

                # update this category
                current['count'] -= 1

            else:
                # same category
                gresults.append(res)

                # update categoryIndex
                categoryPositions[category] = {'index': len(gresults), 'count': 8}

        # update _merged_results
        self._ordered = True
        self._merged_results = gresults

    def get_ordered_results(self):
        if not self._ordered:
            self.order_results()
        return self._merged_results

    def results_length(self):
        return len(self._merged_results)

    def results_number(self):
        resultnum_sum = sum(self._number_of_results)
        if not resultnum_sum or not self._number_of_results:
            return 0
        return resultnum_sum / len(self._number_of_results)

    def add_unresponsive_engine(self, engine_name, error_type, error_message=None):
        if engines[engine_name].display_error_messages:
            self.unresponsive_engines.add((engine_name, error_type, error_message))

    def add_timing(self, engine_name, engine_time, page_load_time):
        self.timings.append({
            'engine': engines[engine_name].shortcut,
            'total': engine_time,
            'load': page_load_time
        })

    def get_timings(self):
        return self.timings