From 44ed4424f60533b73f452252f95f1bc00ec48bc8 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sat, 2 May 2015 21:08:56 +0200 Subject: [PATCH] [fix] yahoo_news engine --- searx/engines/yahoo_news.py | 25 ++-- searx/tests/engines/test_yahoo_news.py | 153 ++++++++++++------------- 2 files changed, 89 insertions(+), 89 deletions(-) diff --git a/searx/engines/yahoo_news.py b/searx/engines/yahoo_news.py index 4a7dd16e..d4cfbeda 100644 --- a/searx/engines/yahoo_news.py +++ b/searx/engines/yahoo_news.py @@ -23,15 +23,15 @@ paging = True language_support = True # search-url -search_url = 'https://news.search.yahoo.com/search?{query}&b={offset}&fl=1&vl=lang_{lang}' # noqa +search_url = 'https://news.search.yahoo.com/search?{query}&b={offset}&{lang}=uh3_news_web_gs_1&pz=10&xargs=0&vl=lang_{lang}' # noqa # specific xpath variables -results_xpath = '//div[@class="res"]' +results_xpath = '//ol[contains(@class,"searchCenterMiddle")]//li' url_xpath = './/h3/a/@href' title_xpath = './/h3/a' -content_xpath = './/div[@class="abstr"]' -publishedDate_xpath = './/span[@class="timestamp"]' -suggestion_xpath = '//div[@id="satat"]//a' +content_xpath = './/div[@class="compText"]' +publishedDate_xpath = './/span[contains(@class,"tri")]' +suggestion_xpath = '//div[contains(@class,"VerALSOTRY")]//a' # do search-request @@ -48,11 +48,18 @@ def request(query, params): lang=language) # TODO required? - params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\ + params['cookies']['sB'] = '"v=1&vm=p&fl=1&vl=lang_{lang}&sh=1&pn=10&rw=new'\ .format(lang=language) return params +def sanitize_url(url): + if ".yahoo.com/" in url: + return re.sub(u"\;\_ylt\=.+$", "", url) + else: + return url + + # get response from search-request def response(resp): results = [] @@ -61,13 +68,17 @@ def response(resp): # parse results for result in dom.xpath(results_xpath): - url = parse_url(extract_url(result.xpath(url_xpath), search_url)) + urls = result.xpath(url_xpath) + if len(urls) != 1: + continue + url = sanitize_url(parse_url(extract_url(urls, search_url))) title = extract_text(result.xpath(title_xpath)[0]) content = extract_text(result.xpath(content_xpath)[0]) # parse publishedDate publishedDate = extract_text(result.xpath(publishedDate_xpath)[0]) + # still useful ? if re.match("^[0-9]+ minute(s|) ago$", publishedDate): publishedDate = datetime.now() - timedelta(minutes=int(re.match(r'\d+', publishedDate).group())) # noqa else: diff --git a/searx/tests/engines/test_yahoo_news.py b/searx/tests/engines/test_yahoo_news.py index 797dc11b..94d819d6 100644 --- a/searx/tests/engines/test_yahoo_news.py +++ b/searx/tests/engines/test_yahoo_news.py @@ -39,19 +39,24 @@ class TestYahooNewsEngine(SearxTestCase): self.assertEqual(yahoo_news.response(response), []) html = """ -
-
-

- - This is - the title... - -

-
- Business via Yahoo! Finance   Feb 03 09:45am -
- This is the content -
+
    +
  1. +
    +

    + + This is + the title... + +

    +
    +
    + Business via Yahoo! + May 01 10:00 AM +
    +
    + This is the content +
    +
""" response = mock.Mock(text=html) @@ -63,48 +68,59 @@ class TestYahooNewsEngine(SearxTestCase): self.assertEqual(results[0]['content'], 'This is the content') html = """ -
-
-

- - This is - the title... - -

-
- Business via Yahoo!   2 hours, 22 minutes ago -
- This is the content -
-
-
-
-

- - This is - the title... - -

-
- Business via Yahoo!   22 minutes ago -
- This is the content -
-
-
-
-

- - This is - the title... - -

-
- Business via Yahoo!   Feb 03 09:45am 1900 -
- This is the content -
-
+
    +
  1. +
    +

    + + This is + the title... + +

    +
    +
    + Business via Yahoo! + 2 hours, 22 minutes ago +
    +
    + This is the content +
    +
    2. +
  2. +
    +

    + + This is + the title... + +

    +
    +
    + Business via Yahoo! + 22 minutes ago +
    +
    + This is the content +
    +
    3. +
  3. +
    +

    + + This is + the title... + +

    +
    +
    + Business via Yahoo! + Feb 03 09:45AM 1900 +
    +
    + This is the content +
    +
    4. +
""" response = mock.Mock(text=html) results = yahoo_news.response(response) @@ -114,30 +130,3 @@ class TestYahooNewsEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'http://this.is.the.url/') self.assertEqual(results[0]['content'], 'This is the content') self.assertEqual(results[2]['publishedDate'].year, datetime.now().year) - - html = """ -
  • -
    -
    -

    - - This should be the title -

    -
    -
    - - this.meta.com - - - - -
    -

    - This should be the content.

    -
    -
    • - """ - response = mock.Mock(text=html) - results = yahoo_news.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0)