devol
/
wallabag
mirror of https://github.com/wallabag/wallabag.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make PocketImport use HttpClient

This commit is contained in:
Yassine Guedidi 2024-02-08 23:23:11 +01:00
parent ca587fb6b6
commit 62fcfe547a
4 changed files with 110 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
app/config/config.yml
View File

@ -40,6 +40,11 @@ framework:
scoped_clients:
download_images.client:
scope: '.*'
pocket.client:
scope: '.*'
headers:
Content-Type: 'application/json'
X-Accept: 'application/json'
# Twig Configuration
twig:
@ -426,14 +431,6 @@ httplug:
defaults:
timeout: 10
plugins: ['httplug.plugin.logger']
wallabag_core.pocket.client:
factory: 'httplug.factory.auto'
plugins:
- 'httplug.plugin.logger'
- header_defaults:
headers:
'content-type': 'application/json'
'X-Accept': 'application/json'
discovery:
client: false

5
app/config/services.yml
View File

@ -282,12 +282,9 @@ services:
tags:
- { name: kernel.event_listener, event: security.authentication.failure, method: onAuthenticationFailure }
wallabag_core.pocket.client:
alias: 'httplug.client.wallabag_core.pocket.client'
Wallabag\CoreBundle\Import\PocketImport:
calls:
- [ setClient, [ "@wallabag_core.pocket.client" ] ]
- [ setClient, [ '@Symfony\Contracts\HttpClient\HttpClientInterface $pocketClient' ] ]
tags:
- { name: wallabag_core.import, alias: pocket }

129
src/Wallabag/CoreBundle/Import/PocketImport.php
View File

@ -2,22 +2,16 @@
namespace Wallabag\CoreBundle\Import;
use Http\Client\Common\HttpMethodsClient;
use Http\Client\Common\Plugin\ErrorPlugin;
use Http\Client\Common\PluginClient;
use Http\Client\Exception\RequestException;
use Http\Discovery\Psr17FactoryDiscovery;
use Psr\Http\Client\ClientInterface;
use Psr\Http\Message\RequestFactoryInterface;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\StreamFactoryInterface;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Contracts\HttpClient\Exception\ExceptionInterface;
use Symfony\Contracts\HttpClient\HttpClientInterface;
use Wallabag\CoreBundle\Entity\Entry;
class PocketImport extends AbstractImport
{
public const NB_ELEMENTS = 5000;
/**
* @var HttpMethodsClient
* @var HttpClientInterface
*/
private $client;
private $accessToken;
@ -57,17 +51,19 @@ class PocketImport extends AbstractImport
public function getRequestToken($redirectUri)
{
try {
$response = $this->client->post('https://getpocket.com/v3/oauth/request', [], json_encode([
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'redirect_uri' => $redirectUri,
]));
} catch (RequestException $e) {
$response = $this->client->request(Request::METHOD_POST, 'https://getpocket.com/v3/oauth/request', [
'json' => [
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'redirect_uri' => $redirectUri,
],
]);
return $response->toArray()['code'];
} catch (ExceptionInterface $e) {
$this->logger->error(sprintf('PocketImport: Failed to request token: %s', $e->getMessage()), ['exception' => $e]);
return false;
}
return $this->jsonDecode($response)['code'];
}
/**
@ -81,19 +77,21 @@ class PocketImport extends AbstractImport
public function authorize($code)
{
try {
$response = $this->client->post('https://getpocket.com/v3/oauth/authorize', [], json_encode([
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'code' => $code,
]));
} catch (RequestException $e) {
$response = $this->client->request(Request::METHOD_POST, 'https://getpocket.com/v3/oauth/authorize', [
'json' => [
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'code' => $code,
],
]);
$this->accessToken = $response->toArray()['access_token'];
return true;
} catch (ExceptionInterface $e) {
$this->logger->error(sprintf('PocketImport: Failed to authorize client: %s', $e->getMessage()), ['exception' => $e]);
return false;
}
$this->accessToken = $this->jsonDecode($response)['access_token'];
return true;
}
public function import($offset = 0)
@ -101,49 +99,51 @@ class PocketImport extends AbstractImport
static $run = 0;
try {
$response = $this->client->post('https://getpocket.com/v3/get', [], json_encode([
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'access_token' => $this->accessToken,
'detailType' => 'complete',
'state' => 'all',
'sort' => 'newest',
'count' => self::NB_ELEMENTS,
'offset' => $offset,
]));
} catch (RequestException $e) {
$response = $this->client->request(Request::METHOD_POST, 'https://getpocket.com/v3/get', [
'json' => [
'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'access_token' => $this->accessToken,
'detailType' => 'complete',
'state' => 'all',
'sort' => 'newest',
'count' => self::NB_ELEMENTS,
'offset' => $offset,
],
]);
$entries = $response->toArray();
if ($this->producer) {
$this->parseEntriesForProducer($entries['list']);
} else {
$this->parseEntries($entries['list']);
}
// if we retrieve exactly the amount of items requested it means we can get more
// re-call import and offset item by the amount previous received:
// - first call get 5k offset 0
// - second call get 5k offset 5k
// - and so on
if (self::NB_ELEMENTS === \count($entries['list'])) {
++$run;
return $this->import(self::NB_ELEMENTS * $run);
}
return true;
} catch (ExceptionInterface $e) {
$this->logger->error(sprintf('PocketImport: Failed to import: %s', $e->getMessage()), ['exception' => $e]);
return false;
}
$entries = $this->jsonDecode($response);
if ($this->producer) {
$this->parseEntriesForProducer($entries['list']);
} else {
$this->parseEntries($entries['list']);
}
// if we retrieve exactly the amount of items requested it means we can get more
// re-call import and offset item by the amount previous received:
// - first call get 5k offset 0
// - second call get 5k offset 5k
// - and so on
if (self::NB_ELEMENTS === \count($entries['list'])) {
++$run;
return $this->import(self::NB_ELEMENTS * $run);
}
return true;
}
/**
* Set the Http client.
*/
public function setClient(ClientInterface $client, ?RequestFactoryInterface $requestFactory = null, ?StreamFactoryInterface $streamFactory = null)
public function setClient(HttpClientInterface $pocketClient)
{
$this->client = new HttpMethodsClient(new PluginClient($client, [new ErrorPlugin()]), $requestFactory ?: Psr17FactoryDiscovery::findRequestFactory(), $streamFactory ?: Psr17FactoryDiscovery::findStreamFactory());
$this->client = $pocketClient;
}
public function validateEntry(array $importedEntry)
@ -222,15 +222,4 @@ class PocketImport extends AbstractImport
return $importedEntry;
}
protected function jsonDecode(ResponseInterface $response)
{
$data = json_decode((string) $response->getBody(), true);
if (\JSON_ERROR_NONE !== json_last_error()) {
throw new \InvalidArgumentException('Unable to parse JSON data: ' . json_last_error_msg());
}
return $data;
}
}

92
tests/Wallabag/CoreBundle/Import/PocketImportTest.php
View File

@ -4,8 +4,6 @@ namespace Tests\Wallabag\CoreBundle\Import;
use Doctrine\ORM\EntityManager;
use Doctrine\ORM\UnitOfWork;
use GuzzleHttp\Psr7\Response;
use Http\Mock\Client as HttpMockClient;
use M6Web\Component\RedisMock\RedisMockFactory;
use Monolog\Handler\TestHandler;
use Monolog\Logger;
@ -13,6 +11,8 @@ use PHPUnit\Framework\TestCase;
use Predis\Client;
use Simpleue\Queue\RedisQueue;
use Symfony\Component\EventDispatcher\EventDispatcher;
use Symfony\Component\HttpClient\MockHttpClient;
use Symfony\Component\HttpClient\Response\MockResponse;
use Wallabag\CoreBundle\Entity\Config;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\User;
@ -43,11 +43,10 @@ class PocketImportTest extends TestCase
public function testOAuthRequest()
{
$httpMockClient = new HttpMockClient();
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['code' => 'wunderbar_code'])));
$mockHttpClient = new MockHttpClient([new MockResponse(json_encode(['code' => 'wunderbar_code']), ['response_headers' => ['Content-Type: application/json']])]);
$pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$code = $pocketImport->getRequestToken('http://0.0.0.0/redirect');
@ -56,11 +55,10 @@ class PocketImportTest extends TestCase
public function testOAuthRequestBadResponse()
{
$httpMockClient = new HttpMockClient();
$httpMockClient->addResponse(new Response(403));
$mockHttpClient = new MockHttpClient([new MockResponse('', ['http_code' => 403])]);
$pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$code = $pocketImport->getRequestToken('http://0.0.0.0/redirect');
@ -73,11 +71,10 @@ class PocketImportTest extends TestCase
public function testOAuthAuthorize()
{
$httpMockClient = new HttpMockClient();
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token'])));
$mockHttpClient = new MockHttpClient([new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']])]);
$pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$res = $pocketImport->authorize('wunderbar_code');
@ -87,11 +84,10 @@ class PocketImportTest extends TestCase
public function testOAuthAuthorizeBadResponse()
{
$httpMockClient = new HttpMockClient();
$httpMockClient->addResponse(new Response(403));
$mockHttpClient = new MockHttpClient([new MockResponse('', ['http_code' => 403])]);
$pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$res = $pocketImport->authorize('wunderbar_code');
@ -107,9 +103,9 @@ class PocketImportTest extends TestCase
*/
public function testImport()
{
$httpMockClient = new HttpMockClient();
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token'])));
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<'JSON'
$mockHttpClient = new MockHttpClient([
new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
new MockResponse(<<<'JSON'
{
"status": 1,
"list": {
@ -190,7 +186,8 @@ class PocketImportTest extends TestCase
}
}
JSON
));
, ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport('ConsumerKey', 1);
@ -221,7 +218,7 @@ JSON
->method('updateEntry')
->willReturn($entry);
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$pocketImport->authorize('wunderbar_code');
$res = $pocketImport->import();
@ -235,9 +232,9 @@ JSON
*/
public function testImportAndMarkAllAsRead()
{
$httpMockClient = new HttpMockClient();
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token'])));
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<'JSON'
$mockHttpClient = new MockHttpClient([
new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
new MockResponse(<<<'JSON'
{
"status": 1,
"list": {
@ -280,7 +277,8 @@ JSON
}
}
JSON
));
, ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport('ConsumerKey', 2);
@ -312,7 +310,7 @@ JSON
->method('updateEntry')
->willReturn($entry);
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$pocketImport->authorize('wunderbar_code');
$res = $pocketImport->setMarkAsRead(true)->import();
@ -326,8 +324,6 @@ JSON
*/
public function testImportWithRabbit()
{
$httpMockClient = new HttpMockClient();
$body = <<<'JSON'
{
"item_id": "229279689",
@ -351,8 +347,9 @@ JSON
}
JSON;
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token'])));
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<JSON
$mockHttpClient = new MockHttpClient([
new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
new MockResponse(<<<JSON
{
"status": 1,
"list": {
@ -360,7 +357,8 @@ JSON;
}
}
JSON
));
, ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport();
@ -394,7 +392,7 @@ JSON
->method('publish')
->with(json_encode($bodyAsArray));
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$pocketImport->setProducer($producer);
$pocketImport->authorize('wunderbar_code');
@ -409,8 +407,6 @@ JSON
*/
public function testImportWithRedis()
{
$httpMockClient = new HttpMockClient();
$body = <<<'JSON'
{
"item_id": "229279689",
@ -434,8 +430,9 @@ JSON
}
JSON;
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token'])));
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<JSON
$mockHttpClient = new MockHttpClient([
new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
new MockResponse(<<<JSON
{
"status": 1,
"list": {
@ -443,7 +440,8 @@ JSON;
}
}
JSON
));
, ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport();
@ -470,7 +468,7 @@ JSON
$queue = new RedisQueue($redisMock, 'pocket');
$producer = new Producer($queue);
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$pocketImport->setProducer($producer);
$pocketImport->authorize('wunderbar_code');
@ -484,13 +482,13 @@ JSON
public function testImportBadResponse()
{
$httpMockClient = new HttpMockClient();
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token'])));
$httpMockClient->addResponse(new Response(403));
$mockHttpClient = new MockHttpClient([
new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
new MockResponse('', ['http_code' => 403]),
]);
$pocketImport = $this->getPocketImport();
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$pocketImport->authorize('wunderbar_code');
$res = $pocketImport->import();
@ -504,10 +502,9 @@ JSON
public function testImportWithExceptionFromGraby()
{
$httpMockClient = new HttpMockClient();
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode(['access_token' => 'wunderbar_token'])));
$httpMockClient->addResponse(new Response(200, ['Content-Type' => 'application/json'], <<<'JSON'
$mockHttpClient = new MockHttpClient([
new MockResponse(json_encode(['access_token' => 'wunderbar_token']), ['response_headers' => ['Content-Type: application/json']]),
new MockResponse(<<<'JSON'
{
"status": 1,
"list": {
@ -520,7 +517,8 @@ JSON
}
JSON
));
, ['response_headers' => ['Content-Type: application/json']]),
]);
$pocketImport = $this->getPocketImport('ConsumerKey', 1);
@ -544,7 +542,7 @@ JSON
->method('updateEntry')
->will($this->throwException(new \Exception()));
$pocketImport->setClient($httpMockClient);
$pocketImport->setClient($mockHttpClient);
$pocketImport->authorize('wunderbar_code');
$res = $pocketImport->import();