devol
/
wallabag
mirror of https://github.com/wallabag/wallabag.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable Redis async import 

- using javibravo/simpleue
- internal config value are now `import_with_redis` & `import_with_rabbit` which are more clear
- if both option are enable rabbit will be choosen
- services imports related to async are now splitted into 2 files: `redis.yml` & `rabbit.yml`
-
This commit is contained in:
Jeremy Benoist 2016-09-09 21:02:03 +02:00
parent 7f7531171f
commit b3437d58ae
No known key found for this signature in database
GPG Key ID: BCA73962457ACC3C
28 changed files with 846 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/config/config.yml
View File

@ -254,7 +254,7 @@ old_sound_rabbit_mq:
type: topic
queue_options:
name: 'wallabag.import.pocket'
callback: wallabag_import.consumer.pocket
callback: wallabag_import.consumer.ampq.pocket
import_readability:
connection: default
exchange_options:
@ -262,7 +262,7 @@ old_sound_rabbit_mq:
type: topic
queue_options:
name: 'wallabag.import.readability'
callback: wallabag_import.consumer.readability
callback: wallabag_import.consumer.ampq.readability
import_wallabag_v1:
connection: default
exchange_options:
@ -270,7 +270,7 @@ old_sound_rabbit_mq:
type: topic
queue_options:
name: 'wallabag.import.wallabag_v1'
callback: wallabag_import.consumer.wallabag_v1
callback: wallabag_import.consumer.ampq.wallabag_v1
import_wallabag_v2:
connection: default
exchange_options:
@ -278,4 +278,4 @@ old_sound_rabbit_mq:
type: topic
queue_options:
name: 'wallabag.import.wallabag_v2'
callback: wallabag_import.consumer.wallabag_v2
callback: wallabag_import.consumer.ampq.wallabag_v2

4
app/config/parameters.yml.dist
View File

@ -46,3 +46,7 @@ parameters:
rabbitmq_port: 5672
rabbitmq_user: guest
rabbitmq_password: guest
# Redis processing
redis_host: localhost
redis_port: 6379

7
composer.json
View File

@ -82,7 +82,9 @@
"ocramius/proxy-manager": "1.*",
"white-october/pagerfanta-bundle": "^1.0",
"mouf/nodejs-installer": "~1.0",
"php-amqplib/rabbitmq-bundle": "^1.8"
"php-amqplib/rabbitmq-bundle": "^1.8",
"predis/predis": "^1.0",
"javibravo/simpleue": "^1.0"
},
"require-dev": {
"doctrine/doctrine-fixtures-bundle": "~2.2",
@ -90,7 +92,8 @@
"sensio/generator-bundle": "^3.0",
"phpunit/phpunit": "~5.0",
"symfony/phpunit-bridge": "^3.0",
"friendsofphp/php-cs-fixer": "~1.9"
"friendsofphp/php-cs-fixer": "~1.9",
"m6web/redis-mock": "^2.0"
},
"scripts": {
"post-cmd": [

7
src/Wallabag/CoreBundle/Command/InstallCommand.php
View File

@ -322,7 +322,12 @@ class InstallCommand extends ContainerAwareCommand
'section' => 'import',
],
[
'name' => 'rabbitmq',
'name' => 'import_with_redis',
'value' => '0',
'section' => 'import',
],
[
'name' => 'import_with_rabbitmq',
'value' => '0',
'section' => 'import',
],

7
src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php
View File

@ -96,7 +96,12 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
'section' => 'import',
],
[
'name' => 'rabbitmq',
'name' => 'import_with_redis',
'value' => '0',
'section' => 'import',
],
[
'name' => 'import_with_rabbitmq',
'value' => '0',
'section' => 'import',
],

8
src/Wallabag/CoreBundle/Resources/config/services.yml
View File

@ -125,3 +125,11 @@ services:
arguments:
- "@security.token_storage"
- "@router"
wallabag_core.redis.client:
class: Predis\Client
arguments:
-
host: '%redis_host%'
port: '%redis_port%'
schema: tcp

41
src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php Normal file
View File

@ -0,0 +1,41 @@
<?php
namespace Wallabag\ImportBundle\Command;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Config\Definition\Exception\Exception;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Simpleue\Worker\QueueWorker;
class RedisWorkerCommand extends ContainerAwareCommand
{
protected function configure()
{
$this
->setName('wallabag:import:redis-worker')
->setDescription('Launch Redis worker')
->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability')
;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s'));
$output->writeln('Waiting for message ...');
$serviceName = $input->getArgument('serviceName');
if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) {
throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName')));
}
$worker = new QueueWorker(
$this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName),
$this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName)
);
$worker->start();
}
}

6
src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php → src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php
View File

@ -1,6 +1,6 @@
<?php
namespace Wallabag\ImportBundle\Consumer\AMPQ;
namespace Wallabag\ImportBundle\Consumer;
use Doctrine\ORM\EntityManager;
use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
@ -12,7 +12,7 @@ use Wallabag\CoreBundle\Entity\Tag;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
class EntryConsumer implements ConsumerInterface
class AMPQEntryConsumer implements ConsumerInterface
{
private $em;
private $userRepository;
@ -64,5 +64,7 @@ class EntryConsumer implements ConsumerInterface
return;
}
$this->logger->info('Content with url ('.$entry->getUrl().') imported !');
}
}

84
src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php Normal file
View File

@ -0,0 +1,84 @@
<?php
namespace Wallabag\ImportBundle\Consumer;
use Simpleue\Job\Job;
use Doctrine\ORM\EntityManager;
use Wallabag\ImportBundle\Import\AbstractImport;
use Wallabag\UserBundle\Repository\UserRepository;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\Tag;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
class RedisEntryConsumer implements Job
{
private $em;
private $userRepository;
private $import;
private $logger;
public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
{
$this->em = $em;
$this->userRepository = $userRepository;
$this->import = $import;
$this->logger = $logger ?: new NullLogger();
}
/**
* Handle one message by one message.
*
* @param string $job Content of the message (directly from Redis)
*
* @return bool
*/
public function manage($job)
{
$storedEntry = json_decode($job, true);
$user = $this->userRepository->find($storedEntry['userId']);
// no user? Drop message
if (null === $user) {
$this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]);
return false;
}
$this->import->setUser($user);
$entry = $this->import->parseEntry($storedEntry);
if (null === $entry) {
$this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
return false;
}
try {
$this->em->flush();
// clear only affected entities
$this->em->clear(Entry::class);
$this->em->clear(Tag::class);
} catch (\Exception $e) {
$this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]);
return false;
}
$this->logger->info('Content with url ('.$entry->getUrl().') imported !');
return true;
}
/**
* Should tell if the given job will kill the worker.
* We don't want to stop it :).
*/
public function isStopJob($job)
{
return false;
}
}

6
src/Wallabag/ImportBundle/Controller/PocketController.php
View File

@ -20,8 +20,10 @@ class PocketController extends Controller
$pocket = $this->get('wallabag_import.pocket.import');
$pocket->setUser($this->getUser());
if ($this->get('craue_config')->get('rabbitmq')) {
$pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$pocket->setProducer($this->get('wallabag_import.producer.redis.pocket'));
}
return $pocket;

6
src/Wallabag/ImportBundle/Controller/ReadabilityController.php
View File

@ -20,8 +20,10 @@ class ReadabilityController extends Controller
$readability = $this->get('wallabag_import.readability.import');
$readability->setUser($this->getUser());
if ($this->get('craue_config')->get('rabbitmq')) {
$readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$readability->setProducer($this->get('wallabag_import.producer.redis.readability'));
}
if ($form->isValid()) {

6
src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
View File

@ -14,8 +14,10 @@ class WallabagV1Controller extends WallabagController
{
$service = $this->get('wallabag_import.wallabag_v1.import');
if ($this->get('craue_config')->get('rabbitmq')) {
$service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1'));
}
return $service;

6
src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
View File

@ -14,8 +14,10 @@ class WallabagV2Controller extends WallabagController
{
$service = $this->get('wallabag_import.wallabag_v2.import');
if ($this->get('craue_config')->get('rabbitmq')) {
$service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
if ($this->get('craue_config')->get('import_with_rabbitmq')) {
$service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
} elseif ($this->get('craue_config')->get('import_with_redis')) {
$service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2'));
}
return $service;

8
src/Wallabag/ImportBundle/Import/AbstractImport.php
View File

@ -9,7 +9,7 @@ use Wallabag\CoreBundle\Helper\ContentProxy;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\Tag;
use Wallabag\UserBundle\Entity\User;
use OldSound\RabbitMqBundle\RabbitMq\Producer;
use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
abstract class AbstractImport implements ImportInterface
{
@ -35,12 +35,12 @@ abstract class AbstractImport implements ImportInterface
}
/**
* Set RabbitMQ Producer to send each entry to a queue.
* Set RabbitMQ/Redis Producer to send each entry to a queue.
* This method should be called when user has enabled RabbitMQ.
*
* @param Producer $producer
* @param ProducerInterface $producer
*/
public function setRabbitmqProducer(Producer $producer)
public function setProducer(ProducerInterface $producer)
{
$this->producer = $producer;
}

36
src/Wallabag/ImportBundle/Redis/Producer.php Normal file
View File

@ -0,0 +1,36 @@
<?php
namespace Wallabag\ImportBundle\Redis;
use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
use Simpleue\Queue\RedisQueue;
/**
* This is a proxy class for "Simpleue\Queue\RedisQueue".
* It allow us to use the same way to publish a message between RabbitMQ & Redis: publish().
*
* It implements the ProducerInterface of RabbitMQ (yes it's ugly) so we can have the same
* kind of class which implements the same interface.
* So we can inject either a RabbitMQ producer or a Redis producer with the same signature
*/
class Producer implements ProducerInterface
{
private $queue;
public function __construct(RedisQueue $queue)
{
$this->queue = $queue;
}
/**
* Publish a message in the Redis queue.
*
* @param string $msgBody
* @param string $routingKey NOT USED
* @param array $additionalProperties NOT USED
*/
public function publish($msgBody, $routingKey = '', $additionalProperties = array())
{
$this->queue->sendJob($msgBody);
}
}

30
src/Wallabag/ImportBundle/Resources/config/rabbit.yml Normal file
View File

@ -0,0 +1,30 @@
# RabbitMQ stuff
services:
wallabag_import.consumer.ampq.pocket:
class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.pocket.import"
- "@logger"
wallabag_import.consumer.ampq.readability:
class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.readability.import"
- "@logger"
wallabag_import.consumer.ampq.wallabag_v1:
class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v1.import"
- "@logger"
wallabag_import.consumer.ampq.wallabag_v2:
class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v2.import"
- "@logger"

81
src/Wallabag/ImportBundle/Resources/config/redis.yml Normal file
View File

@ -0,0 +1,81 @@
# Redis stuff
services:
# readability
wallabag_import.queue.redis.readability:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.readability"
wallabag_import.producer.redis.readability:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.readability"
wallabag_import.consumer.redis.readability:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.readability.import"
- "@logger"
# pocket
wallabag_import.queue.redis.pocket:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.pocket"
wallabag_import.producer.redis.pocket:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.pocket"
wallabag_import.consumer.redis.pocket:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.pocket.import"
- "@logger"
# wallabag v1
wallabag_import.queue.redis.wallabag_v1:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.wallabag_v1"
wallabag_import.producer.redis.wallabag_v1:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.wallabag_v1"
wallabag_import.consumer.redis.wallabag_v1:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v1.import"
- "@logger"
# wallabag v2
wallabag_import.queue.redis.wallabag_v2:
class: Simpleue\Queue\RedisQueue
arguments:
- "@wallabag_core.redis.client"
- "wallabag.import.wallabag_v2"
wallabag_import.producer.redis.wallabag_v2:
class: Wallabag\ImportBundle\Redis\Producer
arguments:
- "@wallabag_import.queue.redis.wallabag_v2"
wallabag_import.consumer.redis.wallabag_v2:
class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v2.import"
- "@logger"

33
src/Wallabag/ImportBundle/Resources/config/services.yml
View File

@ -1,33 +1,8 @@
services:
wallabag_import.consumer.pocket:
class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.pocket.import"
- "@logger"
wallabag_import.consumer.readability:
class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.readability.import"
- "@logger"
wallabag_import.consumer.wallabag_v1:
class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v1.import"
- "@logger"
wallabag_import.consumer.wallabag_v2:
class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
arguments:
- "@doctrine.orm.entity_manager"
- "@wallabag_user.user_repository"
- "@wallabag_import.wallabag_v2.import"
- "@logger"
imports:
- { resource: rabbit.yml }
- { resource: redis.yml }
services:
wallabag_import.chain:
class: Wallabag\ImportBundle\Import\ImportChain

10
tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php → tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php
View File

@ -2,12 +2,12 @@
namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
use Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer;
use Wallabag\ImportBundle\Consumer\AMPQEntryConsumer;
use PhpAmqpLib\Message\AMQPMessage;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
class EntryConsumerTest extends \PHPUnit_Framework_TestCase
class AMPQEntryConsumerTest extends \PHPUnit_Framework_TestCase
{
public function testMessageOk()
{
@ -112,7 +112,7 @@ JSON;
->with(json_decode($body, true))
->willReturn($entry);
$consumer = new EntryConsumer(
$consumer = new AMPQEntryConsumer(
$em,
$userRepository,
$import
@ -157,7 +157,7 @@ JSON;
->disableOriginalConstructor()
->getMock();
$consumer = new EntryConsumer(
$consumer = new AMPQEntryConsumer(
$em,
$userRepository,
$import
@ -212,7 +212,7 @@ JSON;
->with(json_decode($body, true))
->willReturn(null);
$consumer = new EntryConsumer(
$consumer = new AMPQEntryConsumer(
$em,
$userRepository,
$import

224
tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php Normal file
View File

@ -0,0 +1,224 @@
<?php
namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
use Wallabag\ImportBundle\Consumer\RedisEntryConsumer;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
class RedisEntryConsumerTest extends \PHPUnit_Framework_TestCase
{
public function testMessageOk()
{
$em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$em
->expects($this->once())
->method('flush');
$em
->expects($this->exactly(2))
->method('clear');
$body = <<<'JSON'
{
"item_id": "1402935436",
"resolved_id": "1402935436",
"given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
"given_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
"favorite": "0",
"status": "0",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 0,
"resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
"resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
"excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...",
"is_article": "1",
"is_index": "0",
"has_video": "0",
"has_image": "1",
"word_count": "200",
"tags": {
"ifttt": {
"item_id": "1402935436",
"tag": "ifttt"
},
"mashable": {
"item_id": "1402935436",
"tag": "mashable"
}
},
"authors": {
"2484273": {
"item_id": "1402935436",
"author_id": "2484273",
"name": "Adam Rosenberg",
"url": "http://mashable.com/author/adam-rosenberg/"
}
},
"image": {
"item_id": "1402935436",
"src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
"width": "0",
"height": "0"
},
"images": {
"1": {
"item_id": "1402935436",
"image_id": "1",
"src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
"width": "0",
"height": "0",
"credit": "Image: Steve Eichner/NameFace/Sipa USA",
"caption": ""
}
},
"userId": 1
}
JSON;
$user = new User();
$entry = new Entry($user);
$userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
->disableOriginalConstructor()
->getMock();
$userRepository
->expects($this->once())
->method('find')
// userId from the body json above
->with(1)
->willReturn($user);
$import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
->disableOriginalConstructor()
->getMock();
$import
->expects($this->once())
->method('setUser')
->with($user);
$import
->expects($this->once())
->method('parseEntry')
->with(json_decode($body, true))
->willReturn($entry);
$consumer = new RedisEntryConsumer(
$em,
$userRepository,
$import
);
$res = $consumer->manage($body);
$this->assertTrue($res);
}
public function testMessageWithBadUser()
{
$em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$em
->expects($this->never())
->method('flush');
$em
->expects($this->never())
->method('clear');
$body = '{ "userId": 123 }';
$user = new User();
$entry = new Entry($user);
$userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
->disableOriginalConstructor()
->getMock();
$userRepository
->expects($this->once())
->method('find')
// userId from the body json above
->with(123)
->willReturn(null);
$import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
->disableOriginalConstructor()
->getMock();
$consumer = new RedisEntryConsumer(
$em,
$userRepository,
$import
);
$res = $consumer->manage($body);
$this->assertFalse($res);
}
public function testMessageWithEntryProcessed()
{
$em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
->disableOriginalConstructor()
->getMock();
$em
->expects($this->never())
->method('flush');
$em
->expects($this->never())
->method('clear');
$body = '{ "userId": 123 }';
$user = new User();
$userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
->disableOriginalConstructor()
->getMock();
$userRepository
->expects($this->once())
->method('find')
// userId from the body json above
->with(123)
->willReturn($user);
$import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
->disableOriginalConstructor()
->getMock();
$import
->expects($this->once())
->method('setUser')
->with($user);
$import
->expects($this->once())
->method('parseEntry')
->with(json_decode($body, true))
->willReturn(null);
$consumer = new RedisEntryConsumer(
$em,
$userRepository,
$import
);
$res = $consumer->manage($body);
$this->assertFalse($res);
}
}

19
tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php
View File

@ -22,14 +22,29 @@ class PocketControllerTest extends WallabagCoreTestCase
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('rabbitmq', 1);
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/pocket');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
$client->getContainer()->get('craue_config')->set('rabbitmq', 0);
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
}
public function testImportPocketWithRedisEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_redis', 1);
$crawler = $client->request('GET', '/import/pocket');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
$client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportPocketAuthBadToken()

20
tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php
View File

@ -24,7 +24,7 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('rabbitmq', 1);
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/readability');
@ -32,7 +32,23 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$client->getContainer()->get('craue_config')->set('rabbitmq', 0);
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
}
public function testImportReadabilityWithRedisEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_redis', 1);
$crawler = $client->request('GET', '/import/readability');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportReadabilityWithFile()

20
tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php
View File

@ -24,7 +24,7 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('rabbitmq', 1);
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/wallabag-v1');
@ -32,7 +32,23 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$client->getContainer()->get('craue_config')->set('rabbitmq', 0);
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
}
public function testImportWallabagWithRedisEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_redis', 1);
$crawler = $client->request('GET', '/import/wallabag-v1');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportWallabagWithFile()

20
tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php
View File

@ -24,7 +24,7 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('rabbitmq', 1);
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/wallabag-v2');
@ -32,7 +32,23 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$client->getContainer()->get('craue_config')->set('rabbitmq', 0);
$client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
}
public function testImportWallabagWithRedisEnabled()
{
$this->logInAs('admin');
$client = $this->getClient();
$client->getContainer()->get('craue_config')->set('import_with_redis', 1);
$crawler = $client->request('GET', '/import/wallabag-v2');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
$client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportWallabagWithFile()

86
tests/Wallabag/ImportBundle/Import/PocketImportTest.php
View File

@ -9,8 +9,11 @@ use GuzzleHttp\Client;
use GuzzleHttp\Subscriber\Mock;
use GuzzleHttp\Message\Response;
use GuzzleHttp\Stream\Stream;
use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
use Simpleue\Queue\RedisQueue;
use M6Web\Component\RedisMock\RedisMockFactory;
class PocketImportTest extends \PHPUnit_Framework_TestCase
{
@ -442,7 +445,7 @@ JSON;
->with(json_encode($bodyAsArray));
$pocketImport->setClient($client);
$pocketImport->setRabbitmqProducer($producer);
$pocketImport->setProducer($producer);
$pocketImport->authorize('wunderbar_code');
$res = $pocketImport->setMarkAsRead(true)->import();
@ -451,6 +454,87 @@ JSON;
$this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary());
}
/**
* Will sample results from https://getpocket.com/developer/docs/v3/retrieve.
*/
public function testImportWithRedis()
{
$client = new Client();
$body = <<<'JSON'
{
"item_id": "229279689",
"resolved_id": "229279689",
"given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
"given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
"favorite": "1",
"status": "1",
"time_added": "1473020899",
"time_updated": "1473020899",
"time_read": "0",
"time_favorited": "0",
"sort_id": 0,
"resolved_title": "The Massive Ryder Cup Preview",
"resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
"excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
"is_article": "1",
"has_video": "0",
"has_image": "0",
"word_count": "3197"
}
JSON;
$mock = new Mock([
new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))),
new Response(200, ['Content-Type' => 'application/json'], Stream::factory('
{
"status": 1,
"list": {
"229279690": '.$body.'
}
}
')),
]);
$client->getEmitter()->attach($mock);
$pocketImport = $this->getPocketImport();
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$entry = new Entry($this->user);
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$factory = new RedisMockFactory();
$redisMock = $factory->getAdapter('Predis\Client', true);
$queue = new RedisQueue($redisMock, 'pocket');
$producer = new Producer($queue);
$pocketImport->setClient($client);
$pocketImport->setProducer($producer);
$pocketImport->authorize('wunderbar_code');
$res = $pocketImport->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary());
$this->assertNotEmpty($redisMock->lpop('pocket'));
}
public function testImportBadResponse()
{
$client = new Client();

45
tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php
View File

@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
use Wallabag\ImportBundle\Import\ReadabilityImport;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
use Simpleue\Queue\RedisQueue;
use M6Web\Component\RedisMock\RedisMockFactory;
class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
{
@ -152,7 +155,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
->expects($this->exactly(2))
->method('publish');
$readabilityImport->setRabbitmqProducer($producer);
$readabilityImport->setProducer($producer);
$res = $readabilityImport->setMarkAsRead(true)->import();
@ -160,6 +163,46 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
$this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
}
public function testImportWithRedis()
{
$readabilityImport = $this->getReadabilityImport();
$readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
->disableOriginalConstructor()
->getMock();
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$factory = new RedisMockFactory();
$redisMock = $factory->getAdapter('Predis\Client', true);
$queue = new RedisQueue($redisMock, 'readability');
$producer = new Producer($queue);
$readabilityImport->setProducer($producer);
$res = $readabilityImport->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
$this->assertNotEmpty($redisMock->lpop('readability'));
}
public function testImportBadFile()
{
$readabilityImport = $this->getReadabilityImport();

45
tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php
View File

@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
use Wallabag\ImportBundle\Import\WallabagV1Import;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
use Simpleue\Queue\RedisQueue;
use M6Web\Component\RedisMock\RedisMockFactory;
class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
{
@ -152,7 +155,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
->expects($this->exactly(4))
->method('publish');
$wallabagV1Import->setRabbitmqProducer($producer);
$wallabagV1Import->setProducer($producer);
$res = $wallabagV1Import->setMarkAsRead(true)->import();
@ -160,6 +163,46 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
$this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary());
}
public function testImportWithRedis()
{
$wallabagV1Import = $this->getWallabagV1Import();
$wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
->disableOriginalConstructor()
->getMock();
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$factory = new RedisMockFactory();
$redisMock = $factory->getAdapter('Predis\Client', true);
$queue = new RedisQueue($redisMock, 'wallabag_v1');
$producer = new Producer($queue);
$wallabagV1Import->setProducer($producer);
$res = $wallabagV1Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary());
$this->assertNotEmpty($redisMock->lpop('wallabag_v1'));
}
public function testImportBadFile()
{
$wallabagV1Import = $this->getWallabagV1Import();

41
tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php
View File

@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
use Wallabag\ImportBundle\Import\WallabagV2Import;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
use Simpleue\Queue\RedisQueue;
use M6Web\Component\RedisMock\RedisMockFactory;
class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
{
@ -144,7 +147,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
->expects($this->exactly(24))
->method('publish');
$wallabagV2Import->setRabbitmqProducer($producer);
$wallabagV2Import->setProducer($producer);
$res = $wallabagV2Import->setMarkAsRead(true)->import();
@ -152,6 +155,42 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
$this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary());
}
public function testImportWithRedis()
{
$wallabagV2Import = $this->getWallabagV2Import();
$wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
$entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
->disableOriginalConstructor()
->getMock();
$entryRepo->expects($this->never())
->method('findByUrlAndUserId');
$this->em
->expects($this->never())
->method('getRepository');
$this->contentProxy
->expects($this->never())
->method('updateEntry');
$factory = new RedisMockFactory();
$redisMock = $factory->getAdapter('Predis\Client', true);
$queue = new RedisQueue($redisMock, 'wallabag_v2');
$producer = new Producer($queue);
$wallabagV2Import->setProducer($producer);
$res = $wallabagV2Import->setMarkAsRead(true)->import();
$this->assertTrue($res);
$this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary());
$this->assertNotEmpty($redisMock->lpop('wallabag_v2'));
}
public function testImportBadFile()
{
$wallabagV1Import = $this->getWallabagV2Import();