From 7afd02c94e7c535af7730b6a058238efcadd750b Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 27 May 2015 13:49:25 +0200 Subject: [PATCH] [fix] flickr engine --- searx/engines/flickr_noapi.py | 17 +- searx/tests/engines/test_flickr_noapi.py | 250 ++++++----------------- 2 files changed, 74 insertions(+), 193 deletions(-) diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py index 2071b8e3..87b912eb 100644 --- a/searx/engines/flickr_noapi.py +++ b/searx/engines/flickr_noapi.py @@ -25,7 +25,7 @@ categories = ['images'] url = 'https://www.flickr.com/' search_url = url + 'search?{query}&page={page}' photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' -regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) +regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's') paging = True @@ -38,6 +38,7 @@ def build_flickr_url(user_id, photo_id): def request(query, params): params['url'] = search_url.format(query=urlencode({'text': query}), page=params['pageno']) + return params @@ -75,10 +76,10 @@ def response(resp): logger.debug('cannot find valid image size: {0}'.format(repr(photo))) continue - if 'id' not in photo['owner']: + if 'ownerNsid' not in photo: continue -# For a bigger thumbnail, keep only the url_z, not the url_n + # For a bigger thumbnail, keep only the url_z, not the url_n if 'n' in photo['sizes']: thumbnail_src = photo['sizes']['n']['url'] elif 'z' in photo['sizes']: @@ -86,20 +87,14 @@ def response(resp): else: thumbnail_src = img_src - url = build_flickr_url(photo['owner']['id'], photo['id']) + url = build_flickr_url(photo['ownerNsid'], photo['id']) title = photo.get('title', '') content = '' +\ - photo['owner']['username'] +\ + photo['username'] +\ '
' - if 'description' in photo: - content = content +\ - '' +\ - photo['description'] +\ - '' - # append result results.append({'url': url, 'title': title, diff --git a/searx/tests/engines/test_flickr_noapi.py b/searx/tests/engines/test_flickr_noapi.py index a1de3a5e..3b337a2d 100644 --- a/searx/tests/engines/test_flickr_noapi.py +++ b/searx/tests/engines/test_flickr_noapi.py @@ -26,19 +26,29 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertRaises(AttributeError, flickr_noapi.response, '') self.assertRaises(AttributeError, flickr_noapi.response, '[]') - response = mock.Mock(text='"search-photos-models","photos":{},"totalItems":') + response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":') self.assertEqual(flickr_noapi.response(response), []) - response = mock.Mock(text='search-photos-models","photos":{"data": []},"totalItems":') + response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":') self.assertEqual(flickr_noapi.response(response), []) + # everthing is ok test json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", "sizes": { "c": { "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg", @@ -117,40 +127,7 @@ class TestFlickrNoapiEngine(SearxTestCase): "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", "key": "z" } - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true, - "id": "59729010@N00" - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "description": "Description", - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, - "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + } } ], "fetchedStart": true, @@ -168,15 +145,24 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertIn('k.jpg', results[0]['img_src']) self.assertIn('n.jpg', results[0]['thumbnail_src']) self.assertIn('Owner', results[0]['content']) - self.assertIn('Description', results[0]['content']) + # no n size, only the z size json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", "sizes": { "z": { "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", @@ -185,40 +171,7 @@ class TestFlickrNoapiEngine(SearxTestCase): "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", "key": "z" } - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true, - "id": "59729010@N00" - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "description": "Description", - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, - "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + } } ], "fetchedStart": true, @@ -235,15 +188,24 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertIn('z.jpg', results[0]['img_src']) self.assertIn('z.jpg', results[0]['thumbnail_src']) self.assertIn('Owner', results[0]['content']) - self.assertIn('Description', results[0]['content']) + # no z or n size json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", "sizes": { "o": { "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", @@ -252,39 +214,7 @@ class TestFlickrNoapiEngine(SearxTestCase): "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", "key": "o" } - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true, - "id": "59729010@N00" - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, - "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + } } ], "fetchedStart": true, @@ -302,48 +232,25 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertIn('o.jpg', results[0]['thumbnail_src']) self.assertIn('Owner', results[0]['content']) + # no image test json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", - "sizes": { - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true, - "id": "59729010@N00" - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "description": "Description", - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", + "sizes": { + } } ], "fetchedStart": true, @@ -356,6 +263,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 0) + # null test json = """ "search-photos-models","photos": { @@ -370,13 +278,22 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 0) + # no ownerNsid test json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", "sizes": { "o": { "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", @@ -385,39 +302,7 @@ class TestFlickrNoapiEngine(SearxTestCase): "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", "key": "o" } - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "description": "Description", - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, - "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + } } ], "fetchedStart": true, @@ -430,6 +315,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 0) + # garbage test json = """ {"toto":[ {"id":200,"name":"Artist Name",