Switch message queue from MariaDB to beanstalkd (#4418)
This commit is contained in:
parent
e509cde831
commit
11f1f44cea
|
@ -19,6 +19,10 @@ release channel, you can take advantage of these new features and fixes.
|
|||
|
||||
## Code Quality/Technical Changes
|
||||
|
||||
- We have once again switched our message queue implementation, this time from the MariaDB database to a
|
||||
super-lightweight standalone tool called Beanstalkd. We hope this will resolve issues we've encountered with parallel
|
||||
workers causing database lockups and other problems.
|
||||
|
||||
- The main web Docker container will now automatically initialize itself upon startup, performing essential tasks like
|
||||
updating the database, clearing the cache and ensuring the system is set up properly. This means even if you miss a
|
||||
step in installation (or use the Docker images directly) they should still work without issue.
|
||||
|
|
|
@ -62,6 +62,7 @@
|
|||
"spatie/flysystem-dropbox": "^2",
|
||||
"spomky-labs/otphp": "^10.0",
|
||||
"supervisorphp/supervisor": "dev-main",
|
||||
"symfony/beanstalkd-messenger": "^5.3",
|
||||
"symfony/cache": "^5.2",
|
||||
"symfony/console": "^5",
|
||||
"symfony/event-dispatcher": "^5",
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "b8d01546a885d3765652fdf3d8ba59fc",
|
||||
"content-hash": "9238ec827505c7ffc33c58d9ceea2633",
|
||||
"packages": [
|
||||
{
|
||||
"name": "aws/aws-sdk-php",
|
||||
|
@ -4633,6 +4633,61 @@
|
|||
},
|
||||
"time": "2020-12-06T15:14:20+00:00"
|
||||
},
|
||||
{
|
||||
"name": "pda/pheanstalk",
|
||||
"version": "v4.0.3",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/pheanstalk/pheanstalk.git",
|
||||
"reference": "6165573aad525d39b3ac8ae916214cb483a61263"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/pheanstalk/pheanstalk/zipball/6165573aad525d39b3ac8ae916214cb483a61263",
|
||||
"reference": "6165573aad525d39b3ac8ae916214cb483a61263",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-mbstring": "*",
|
||||
"php": ">=7.1.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^7"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Pheanstalk\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Paul Annesley",
|
||||
"email": "paul@annesley.cc",
|
||||
"homepage": "http://paul.annesley.cc/",
|
||||
"role": "Developer"
|
||||
},
|
||||
{
|
||||
"name": "Sam Mousa",
|
||||
"email": "sam@mousa.nl",
|
||||
"role": "Maintainer"
|
||||
}
|
||||
],
|
||||
"description": "PHP client for beanstalkd queue",
|
||||
"homepage": "https://github.com/pheanstalk/pheanstalk",
|
||||
"keywords": [
|
||||
"beanstalkd"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/pheanstalk/pheanstalk/issues",
|
||||
"source": "https://github.com/pheanstalk/pheanstalk/tree/v4.0.3"
|
||||
},
|
||||
"time": "2020-09-22T07:17:48+00:00"
|
||||
},
|
||||
{
|
||||
"name": "php-di/invoker",
|
||||
"version": "2.3.0",
|
||||
|
@ -6310,6 +6365,69 @@
|
|||
],
|
||||
"time": "2021-06-17T12:34:27+00:00"
|
||||
},
|
||||
{
|
||||
"name": "symfony/beanstalkd-messenger",
|
||||
"version": "v5.3.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/symfony/beanstalkd-messenger.git",
|
||||
"reference": "7ca9b2c142ab3a0e13b81910fd5451ebf094bada"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/symfony/beanstalkd-messenger/zipball/7ca9b2c142ab3a0e13b81910fd5451ebf094bada",
|
||||
"reference": "7ca9b2c142ab3a0e13b81910fd5451ebf094bada",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"pda/pheanstalk": "^4.0",
|
||||
"php": ">=7.2.5",
|
||||
"symfony/messenger": "^4.4|^5.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"symfony/property-access": "^4.4|^5.0",
|
||||
"symfony/serializer": "^4.4|^5.0"
|
||||
},
|
||||
"type": "symfony-bridge",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Symfony\\Component\\Messenger\\Bridge\\Beanstalkd\\": ""
|
||||
},
|
||||
"exclude-from-classmap": [
|
||||
"/Tests/"
|
||||
]
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Antonio Pauletich",
|
||||
"email": "antonio.pauletich95@gmail.com"
|
||||
}
|
||||
],
|
||||
"description": "Symfony Beanstalkd Messenger Bridge",
|
||||
"homepage": "https://symfony.com",
|
||||
"support": {
|
||||
"source": "https://github.com/symfony/beanstalkd-messenger/tree/v5.3.0"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://symfony.com/sponsor",
|
||||
"type": "custom"
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/fabpot",
|
||||
"type": "github"
|
||||
},
|
||||
{
|
||||
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
|
||||
"type": "tidelift"
|
||||
}
|
||||
],
|
||||
"time": "2021-05-26T17:33:56+00:00"
|
||||
},
|
||||
{
|
||||
"name": "symfony/cache",
|
||||
"version": "v5.3.3",
|
||||
|
@ -13471,5 +13589,5 @@
|
|||
"ext-xmlwriter": "*"
|
||||
},
|
||||
"platform-dev": [],
|
||||
"plugin-api-version": "2.1.0"
|
||||
"plugin-api-version": "2.0.0"
|
||||
}
|
||||
|
|
|
@ -346,6 +346,22 @@ return [
|
|||
return $builder->getValidator();
|
||||
},
|
||||
|
||||
Pheanstalk\Pheanstalk::class => static function () {
|
||||
return Pheanstalk\Pheanstalk::create('127.0.0.1', 11300);
|
||||
},
|
||||
|
||||
App\MessageQueue\QueueManagerInterface::class => static function (
|
||||
Environment $environment,
|
||||
ContainerInterface $di
|
||||
) {
|
||||
if ($environment->isTesting()) {
|
||||
return new App\MessageQueue\TestQueueManager();
|
||||
}
|
||||
|
||||
$pheanstalk = $di->get(Pheanstalk\Pheanstalk::class);
|
||||
return new App\MessageQueue\QueueManager($pheanstalk);
|
||||
},
|
||||
|
||||
Symfony\Component\Messenger\MessageBus::class => static function (
|
||||
App\MessageQueue\QueueManager $queueManager,
|
||||
App\LockFactory $lockFactory,
|
||||
|
|
|
@ -47,7 +47,6 @@ class InitializeCommand extends CommandAbstract
|
|||
$io->section(__('Reload System Data'));
|
||||
|
||||
$this->runCommand($output, 'cache:clear');
|
||||
$this->runCommand($output, 'queue:clear');
|
||||
|
||||
// Ensure default storage locations exist.
|
||||
$storageLocationRepo->createDefaultStorageLocations();
|
||||
|
|
|
@ -5,22 +5,22 @@ declare(strict_types=1);
|
|||
namespace App\Console\Command\MessageQueue;
|
||||
|
||||
use App\Console\Command\CommandAbstract;
|
||||
use App\Entity\Repository\MessengerMessageRepository;
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\AbstractQueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
|
||||
class ClearCommand extends CommandAbstract
|
||||
{
|
||||
public function __invoke(
|
||||
SymfonyStyle $io,
|
||||
MessengerMessageRepository $messengerMessageRepo,
|
||||
QueueManagerInterface $queueManager,
|
||||
?string $queue = null
|
||||
): int {
|
||||
$allQueues = QueueManager::getAllQueues();
|
||||
$allQueues = AbstractQueueManager::getAllQueues();
|
||||
|
||||
if (!empty($queue)) {
|
||||
if (in_array($queue, $allQueues, true)) {
|
||||
$messengerMessageRepo->clearQueue($queue);
|
||||
$queueManager->clearQueue($queue);
|
||||
|
||||
$io->success(sprintf('Message queue "%s" cleared.', $queue));
|
||||
} else {
|
||||
|
@ -28,7 +28,10 @@ class ClearCommand extends CommandAbstract
|
|||
return 1;
|
||||
}
|
||||
} else {
|
||||
$messengerMessageRepo->clearQueue();
|
||||
foreach ($allQueues as $queueName) {
|
||||
$queueManager->clearQueue($queueName);
|
||||
}
|
||||
|
||||
$io->success('All message queues cleared.');
|
||||
}
|
||||
|
||||
|
|
|
@ -9,7 +9,7 @@ use App\Doctrine\Messenger\ClearEntityManagerSubscriber;
|
|||
use App\Environment;
|
||||
use App\EventDispatcher;
|
||||
use App\MessageQueue\LogWorkerExceptionSubscriber;
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
use App\MessageQueue\ResetArrayCacheMiddleware;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Messenger\EventListener\StopWorkerOnTimeLimitListener;
|
||||
|
@ -22,7 +22,7 @@ class ProcessCommand extends CommandAbstract
|
|||
public function __invoke(
|
||||
MessageBus $messageBus,
|
||||
EventDispatcher $eventDispatcher,
|
||||
QueueManager $queueManager,
|
||||
QueueManagerInterface $queueManager,
|
||||
LoggerInterface $logger,
|
||||
Environment $environment,
|
||||
?int $runtime = 0,
|
||||
|
|
|
@ -10,7 +10,8 @@ use App\Entity;
|
|||
use App\Http\Response;
|
||||
use App\Http\ServerRequest;
|
||||
use App\Message\RunSyncTaskMessage;
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\AbstractQueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
use App\Radio\AutoDJ;
|
||||
use App\Radio\Backend\Liquidsoap;
|
||||
use App\Session\Flash;
|
||||
|
@ -39,9 +40,9 @@ class DebugController extends AbstractLogViewerController
|
|||
Response $response,
|
||||
Entity\Repository\StationRepository $stationRepo,
|
||||
Runner $sync,
|
||||
QueueManager $queueManager
|
||||
QueueManagerInterface $queueManager
|
||||
): ResponseInterface {
|
||||
$queues = QueueManager::getAllQueues();
|
||||
$queues = AbstractQueueManager::getAllQueues();
|
||||
|
||||
$queueTotals = [];
|
||||
foreach ($queues as $queue) {
|
||||
|
|
|
@ -11,7 +11,7 @@ use App\Http\Response;
|
|||
use App\Http\ServerRequest;
|
||||
use App\Media\BatchUtilities;
|
||||
use App\Message;
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
use App\Radio\Backend\Liquidsoap;
|
||||
use App\Utilities\File;
|
||||
use Azura\Files\ExtendedFilesystemInterface;
|
||||
|
@ -28,7 +28,7 @@ class BatchAction
|
|||
protected BatchUtilities $batchUtilities,
|
||||
protected ReloadableEntityManagerInterface $em,
|
||||
protected MessageBus $messageBus,
|
||||
protected QueueManager $queueManager,
|
||||
protected QueueManagerInterface $queueManager,
|
||||
protected Entity\Repository\StationPlaylistMediaRepository $playlistMediaRepo,
|
||||
protected Entity\Repository\StationPlaylistFolderRepository $playlistFolderRepo,
|
||||
) {
|
||||
|
@ -288,7 +288,7 @@ class BatchAction
|
|||
$queuedMediaUpdates = [];
|
||||
$queuedNewFiles = [];
|
||||
|
||||
foreach ($this->queueManager->getMessagesInTransport(QueueManager::QUEUE_MEDIA) as $message) {
|
||||
foreach ($this->queueManager->getMessagesInTransport(QueueManagerInterface::QUEUE_MEDIA) as $message) {
|
||||
if ($message instanceof Message\ReprocessMediaMessage) {
|
||||
$queuedMediaUpdates[$message->media_id] = true;
|
||||
} elseif (
|
||||
|
|
|
@ -1,44 +0,0 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Entity;
|
||||
|
||||
use DateTime;
|
||||
use Doctrine\ORM\Mapping as ORM;
|
||||
|
||||
/**
|
||||
* Internal table used for Symfony Messenger handling.
|
||||
* @internal
|
||||
*/
|
||||
#[
|
||||
ORM\Entity(readOnly: true),
|
||||
ORM\Table(name: 'messenger_messages'),
|
||||
ORM\Index(columns: ['queue_name']),
|
||||
ORM\Index(columns: ['available_at']),
|
||||
ORM\Index(columns: ['delivered_at'])
|
||||
]
|
||||
class MessengerMessage
|
||||
{
|
||||
#[ORM\Column(type: 'bigint')]
|
||||
#[ORM\Id, ORM\GeneratedValue]
|
||||
protected int $id;
|
||||
|
||||
#[ORM\Column(type: 'text')]
|
||||
protected string $body;
|
||||
|
||||
#[ORM\Column(type: 'text')]
|
||||
protected string $headers;
|
||||
|
||||
#[ORM\Column(name: 'queue_name', length: 190)]
|
||||
protected string $queueName;
|
||||
|
||||
#[ORM\Column(name: 'created_at')]
|
||||
protected DateTime $createdAt;
|
||||
|
||||
#[ORM\Column(name: 'available_at')]
|
||||
protected DateTime $availableAt;
|
||||
|
||||
#[ORM\Column(name: 'delivered_at', nullable: true)]
|
||||
protected ?DateTime $deliveredAt = null;
|
||||
}
|
|
@ -0,0 +1,31 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Entity\Migration;
|
||||
|
||||
use Doctrine\DBAL\Schema\Schema;
|
||||
use Doctrine\Migrations\AbstractMigration;
|
||||
|
||||
/**
|
||||
* Auto-generated Migration: Please modify to your needs!
|
||||
*/
|
||||
final class Version20210721011736 extends AbstractMigration
|
||||
{
|
||||
public function getDescription(): string
|
||||
{
|
||||
return 'Retire Message Queue DB table.';
|
||||
}
|
||||
|
||||
public function up(Schema $schema): void
|
||||
{
|
||||
// this up() migration is auto-generated, please modify it to your needs
|
||||
$this->addSql('DROP TABLE messenger_messages');
|
||||
}
|
||||
|
||||
public function down(Schema $schema): void
|
||||
{
|
||||
// this down() migration is auto-generated, please modify it to your needs
|
||||
$this->addSql('CREATE TABLE messenger_messages (id BIGINT AUTO_INCREMENT NOT NULL, body LONGTEXT CHARACTER SET utf8mb4 NOT NULL COLLATE `utf8mb4_general_ci`, headers LONGTEXT CHARACTER SET utf8mb4 NOT NULL COLLATE `utf8mb4_general_ci`, queue_name VARCHAR(190) CHARACTER SET utf8mb4 NOT NULL COLLATE `utf8mb4_general_ci`, created_at DATETIME NOT NULL, available_at DATETIME NOT NULL, delivered_at DATETIME DEFAULT NULL, INDEX IDX_75EA56E0FB7336F0 (queue_name), INDEX IDX_75EA56E0E3BD61CE (available_at), INDEX IDX_75EA56E016BA31DB (delivered_at), PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8 COLLATE `utf8_unicode_ci` ENGINE = InnoDB COMMENT = \'\' ');
|
||||
}
|
||||
}
|
|
@ -1,24 +0,0 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Entity\Repository;
|
||||
|
||||
use App\Doctrine\Repository;
|
||||
use App\Entity;
|
||||
|
||||
class MessengerMessageRepository extends Repository
|
||||
{
|
||||
public function clearQueue(?string $queueName = null): void
|
||||
{
|
||||
$qb = $this->em->createQueryBuilder()
|
||||
->delete(Entity\MessengerMessage::class, 'mm');
|
||||
|
||||
if (!empty($queueName)) {
|
||||
$qb->where('mm.queueName = :queueName')
|
||||
->setParameter('queueName', $queueName);
|
||||
}
|
||||
|
||||
$qb->getQuery()->execute();
|
||||
}
|
||||
}
|
|
@ -4,12 +4,12 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Message;
|
||||
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
|
||||
abstract class AbstractMessage
|
||||
{
|
||||
public function getQueue(): string
|
||||
{
|
||||
return QueueManager::QUEUE_NORMAL_PRIORITY;
|
||||
return QueueManagerInterface::QUEUE_NORMAL_PRIORITY;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Message;
|
||||
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
|
||||
class AddNewMediaMessage extends AbstractUniqueMessage
|
||||
{
|
||||
|
@ -16,6 +16,6 @@ class AddNewMediaMessage extends AbstractUniqueMessage
|
|||
|
||||
public function getQueue(): string
|
||||
{
|
||||
return QueueManager::QUEUE_MEDIA;
|
||||
return QueueManagerInterface::QUEUE_MEDIA;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Message;
|
||||
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
|
||||
class AddNewPodcastMediaMessage extends AbstractUniqueMessage
|
||||
{
|
||||
|
@ -16,6 +16,6 @@ class AddNewPodcastMediaMessage extends AbstractUniqueMessage
|
|||
|
||||
public function getQueue(): string
|
||||
{
|
||||
return QueueManager::QUEUE_PODCAST_MEDIA;
|
||||
return QueueManagerInterface::QUEUE_PODCAST_MEDIA;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Message;
|
||||
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
|
||||
class BackupMessage extends AbstractUniqueMessage
|
||||
{
|
||||
|
@ -33,6 +33,6 @@ class BackupMessage extends AbstractUniqueMessage
|
|||
|
||||
public function getQueue(): string
|
||||
{
|
||||
return QueueManager::QUEUE_LOW_PRIORITY;
|
||||
return QueueManagerInterface::QUEUE_LOW_PRIORITY;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Message;
|
||||
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
|
||||
class ReprocessMediaMessage extends AbstractUniqueMessage
|
||||
{
|
||||
|
@ -21,6 +21,6 @@ class ReprocessMediaMessage extends AbstractUniqueMessage
|
|||
|
||||
public function getQueue(): string
|
||||
{
|
||||
return QueueManager::QUEUE_MEDIA;
|
||||
return QueueManagerInterface::QUEUE_MEDIA;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Message;
|
||||
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
|
||||
class ReprocessPodcastMediaMessage extends AbstractUniqueMessage
|
||||
{
|
||||
|
@ -21,6 +21,6 @@ class ReprocessPodcastMediaMessage extends AbstractUniqueMessage
|
|||
|
||||
public function getQueue(): string
|
||||
{
|
||||
return QueueManager::QUEUE_PODCAST_MEDIA;
|
||||
return QueueManagerInterface::QUEUE_PODCAST_MEDIA;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,7 +5,7 @@ declare(strict_types=1);
|
|||
namespace App\Message;
|
||||
|
||||
use App\Environment;
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
|
||||
class RunSyncTaskMessage extends AbstractUniqueMessage
|
||||
{
|
||||
|
@ -26,6 +26,6 @@ class RunSyncTaskMessage extends AbstractUniqueMessage
|
|||
|
||||
public function getQueue(): string
|
||||
{
|
||||
return QueueManager::QUEUE_HIGH_PRIORITY;
|
||||
return QueueManagerInterface::QUEUE_HIGH_PRIORITY;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Message;
|
||||
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
|
||||
class UpdateNowPlayingMessage extends AbstractMessage
|
||||
{
|
||||
|
@ -12,6 +12,6 @@ class UpdateNowPlayingMessage extends AbstractMessage
|
|||
|
||||
public function getQueue(): string
|
||||
{
|
||||
return QueueManager::QUEUE_HIGH_PRIORITY;
|
||||
return QueueManagerInterface::QUEUE_HIGH_PRIORITY;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,7 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\Message;
|
||||
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
|
||||
class WritePlaylistFileMessage extends AbstractUniqueMessage
|
||||
{
|
||||
|
@ -18,6 +18,6 @@ class WritePlaylistFileMessage extends AbstractUniqueMessage
|
|||
|
||||
public function getQueue(): string
|
||||
{
|
||||
return QueueManager::QUEUE_LOW_PRIORITY;
|
||||
return QueueManagerInterface::QUEUE_LOW_PRIORITY;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,76 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\MessageQueue;
|
||||
|
||||
use App\Message\AbstractMessage;
|
||||
use Generator;
|
||||
use Symfony\Component\Messenger\Envelope;
|
||||
|
||||
abstract class AbstractQueueManager implements QueueManagerInterface
|
||||
{
|
||||
protected string $workerName = 'app';
|
||||
|
||||
public function setWorkerName(string $workerName): void
|
||||
{
|
||||
$this->workerName = $workerName;
|
||||
}
|
||||
|
||||
/**
|
||||
* @inheritDoc
|
||||
*/
|
||||
public function getSenders(Envelope $envelope): iterable
|
||||
{
|
||||
$message = $envelope->getMessage();
|
||||
|
||||
if (!$message instanceof AbstractMessage) {
|
||||
return [
|
||||
$this->getTransport(self::QUEUE_NORMAL_PRIORITY),
|
||||
];
|
||||
}
|
||||
|
||||
$queue = $message->getQueue();
|
||||
return [
|
||||
$this->getTransport($queue),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @inheritDoc
|
||||
*/
|
||||
public function getMessagesInTransport(string $queueName): Generator
|
||||
{
|
||||
foreach ($this->getTransport($queueName)->get() as $envelope) {
|
||||
$message = $envelope->getMessage();
|
||||
if ($message instanceof AbstractMessage) {
|
||||
yield $message;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function getTransports(): array
|
||||
{
|
||||
$allQueues = self::getAllQueues();
|
||||
|
||||
$transports = [];
|
||||
foreach ($allQueues as $queueName) {
|
||||
$transports[$queueName] = $this->getTransport($queueName);
|
||||
}
|
||||
return $transports;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string[]
|
||||
*/
|
||||
public static function getAllQueues(): array
|
||||
{
|
||||
return [
|
||||
self::QUEUE_HIGH_PRIORITY,
|
||||
self::QUEUE_NORMAL_PRIORITY,
|
||||
self::QUEUE_LOW_PRIORITY,
|
||||
self::QUEUE_MEDIA,
|
||||
self::QUEUE_PODCAST_MEDIA,
|
||||
];
|
||||
}
|
||||
}
|
|
@ -4,133 +4,52 @@ declare(strict_types=1);
|
|||
|
||||
namespace App\MessageQueue;
|
||||
|
||||
use App\Message\AbstractMessage;
|
||||
use Doctrine\DBAL\Connection;
|
||||
use Generator;
|
||||
use Symfony\Component\Messenger\Bridge\Doctrine\Transport\Connection as MessengerConnection;
|
||||
use Symfony\Component\Messenger\Bridge\Doctrine\Transport\DoctrineTransport;
|
||||
use Symfony\Component\Messenger\Envelope;
|
||||
use Symfony\Component\Messenger\Transport\Sender\SendersLocatorInterface;
|
||||
use Pheanstalk\Pheanstalk;
|
||||
use Symfony\Component\Messenger\Bridge\Beanstalkd\Transport\BeanstalkdTransport;
|
||||
use Symfony\Component\Messenger\Bridge\Beanstalkd\Transport\Connection as MessengerConnection;
|
||||
use Symfony\Component\Messenger\Exception\TransportException;
|
||||
use Symfony\Component\Messenger\Transport\Serialization\PhpSerializer;
|
||||
|
||||
class QueueManager implements SendersLocatorInterface
|
||||
class QueueManager extends AbstractQueueManager
|
||||
{
|
||||
public const QUEUE_HIGH_PRIORITY = 'high_priority';
|
||||
public const QUEUE_NORMAL_PRIORITY = 'normal_priority';
|
||||
public const QUEUE_LOW_PRIORITY = 'low_priority';
|
||||
public const QUEUE_MEDIA = 'media';
|
||||
public const QUEUE_PODCAST_MEDIA = 'podcast_media';
|
||||
|
||||
protected string $workerName = 'app';
|
||||
|
||||
public function __construct(
|
||||
protected Connection $db
|
||||
protected Pheanstalk $pheanstalk
|
||||
) {
|
||||
}
|
||||
|
||||
public function setWorkerName(string $workerName): void
|
||||
public function clearQueue(string $queueName): void
|
||||
{
|
||||
$this->workerName = $workerName;
|
||||
$pheanstalk = $this->pheanstalk->useTube($queueName);
|
||||
|
||||
while ($job = $pheanstalk->reserve()) {
|
||||
$pheanstalk->delete($job);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @inheritDoc
|
||||
*/
|
||||
public function getSenders(Envelope $envelope): iterable
|
||||
public function getTransport(string $queueName): BeanstalkdTransport
|
||||
{
|
||||
$message = $envelope->getMessage();
|
||||
|
||||
if (!$message instanceof AbstractMessage) {
|
||||
return [
|
||||
$this->getTransport(self::QUEUE_NORMAL_PRIORITY),
|
||||
];
|
||||
}
|
||||
|
||||
$queue = $message->getQueue();
|
||||
return [
|
||||
$this->getTransport($queue),
|
||||
];
|
||||
}
|
||||
|
||||
public function getConnection(string $queueName): MessengerConnection
|
||||
{
|
||||
return new MessengerConnection(
|
||||
[
|
||||
'table_name' => 'messenger_messages',
|
||||
'queue_name' => $queueName,
|
||||
'auto_setup' => false,
|
||||
],
|
||||
$this->db
|
||||
);
|
||||
}
|
||||
|
||||
public function getTransport(string $queueName): DoctrineTransport
|
||||
{
|
||||
return new DoctrineTransport(
|
||||
return new BeanstalkdTransport(
|
||||
$this->getConnection($queueName),
|
||||
new PhpSerializer()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $queueName
|
||||
*
|
||||
* @return Generator<AbstractMessage>
|
||||
*/
|
||||
public function getMessagesInTransport(string $queueName): Generator
|
||||
protected function getConnection(string $queueName): MessengerConnection
|
||||
{
|
||||
foreach ($this->getTransport($queueName)->all() as $envelope) {
|
||||
$message = $envelope->getMessage();
|
||||
if ($message instanceof AbstractMessage) {
|
||||
yield $message;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return DoctrineTransport[]
|
||||
*/
|
||||
public function getTransports(): array
|
||||
{
|
||||
$allQueues = self::getAllQueues();
|
||||
|
||||
$transports = [];
|
||||
foreach ($allQueues as $queueName) {
|
||||
$transports[$queueName] = $this->getTransport($queueName);
|
||||
}
|
||||
return $transports;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return MessengerConnection[]
|
||||
*/
|
||||
public function getConnections(): array
|
||||
{
|
||||
$allQueues = self::getAllQueues();
|
||||
|
||||
$connections = [];
|
||||
foreach ($allQueues as $queueName) {
|
||||
$connections[$queueName] = $this->getConnection($queueName);
|
||||
}
|
||||
return $connections;
|
||||
return new MessengerConnection(
|
||||
[
|
||||
'tube_name' => $queueName,
|
||||
],
|
||||
$this->pheanstalk
|
||||
);
|
||||
}
|
||||
|
||||
public function getQueueCount(string $queueName): int
|
||||
{
|
||||
try {
|
||||
return $this->getConnection($queueName)->getMessageCount();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string[]
|
||||
*/
|
||||
public static function getAllQueues(): array
|
||||
{
|
||||
return [
|
||||
self::QUEUE_HIGH_PRIORITY,
|
||||
self::QUEUE_NORMAL_PRIORITY,
|
||||
self::QUEUE_LOW_PRIORITY,
|
||||
self::QUEUE_MEDIA,
|
||||
self::QUEUE_PODCAST_MEDIA,
|
||||
];
|
||||
} catch (TransportException) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,41 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\MessageQueue;
|
||||
|
||||
use App\Message\AbstractMessage;
|
||||
use Generator;
|
||||
use Symfony\Component\Messenger\Transport\Sender\SendersLocatorInterface;
|
||||
use Symfony\Component\Messenger\Transport\TransportInterface;
|
||||
|
||||
interface QueueManagerInterface extends SendersLocatorInterface
|
||||
{
|
||||
public const QUEUE_HIGH_PRIORITY = 'high_priority';
|
||||
public const QUEUE_NORMAL_PRIORITY = 'normal_priority';
|
||||
public const QUEUE_LOW_PRIORITY = 'low_priority';
|
||||
public const QUEUE_MEDIA = 'media';
|
||||
public const QUEUE_PODCAST_MEDIA = 'podcast_media';
|
||||
|
||||
public function setWorkerName(string $workerName): void;
|
||||
|
||||
public function clearQueue(string $queueName): void;
|
||||
|
||||
public function getTransport(string $queueName): TransportInterface;
|
||||
|
||||
/**
|
||||
* @param string $queueName
|
||||
*
|
||||
* @return Generator<AbstractMessage>
|
||||
*/
|
||||
public function getMessagesInTransport(string $queueName): Generator;
|
||||
|
||||
/**
|
||||
* @return TransportInterface[]
|
||||
*/
|
||||
public function getTransports(): array;
|
||||
|
||||
public function getQueueCount(string $queueName): int;
|
||||
|
||||
public static function getAllQueues(): array;
|
||||
}
|
|
@ -0,0 +1,26 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\MessageQueue;
|
||||
|
||||
use Symfony\Component\Messenger\Transport\InMemoryTransport;
|
||||
use Symfony\Component\Messenger\Transport\TransportInterface;
|
||||
|
||||
class TestQueueManager extends AbstractQueueManager
|
||||
{
|
||||
public function clearQueue(string $queueName): void
|
||||
{
|
||||
return; // Noop
|
||||
}
|
||||
|
||||
public function getTransport(string $queueName): TransportInterface
|
||||
{
|
||||
return new InMemoryTransport();
|
||||
}
|
||||
|
||||
public function getQueueCount(string $queueName): int
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
|
@ -8,7 +8,7 @@ use App\Doctrine\ReloadableEntityManagerInterface;
|
|||
use App\Entity;
|
||||
use App\Media\MimeType;
|
||||
use App\Message;
|
||||
use App\MessageQueue\QueueManager;
|
||||
use App\MessageQueue\QueueManagerInterface;
|
||||
use App\Radio\Quota;
|
||||
use Azura\Files\Attributes\FileAttributes;
|
||||
use Brick\Math\BigInteger;
|
||||
|
@ -26,7 +26,7 @@ class CheckMediaTask extends AbstractTask
|
|||
protected Entity\Repository\StorageLocationRepository $storageLocationRepo,
|
||||
protected Entity\Repository\UnprocessableMediaRepository $unprocessableMediaRepo,
|
||||
protected MessageBus $messageBus,
|
||||
protected QueueManager $queueManager,
|
||||
protected QueueManagerInterface $queueManager,
|
||||
ReloadableEntityManagerInterface $em,
|
||||
LoggerInterface $logger
|
||||
) {
|
||||
|
@ -138,7 +138,7 @@ class CheckMediaTask extends AbstractTask
|
|||
$queuedMediaUpdates = [];
|
||||
$queuedNewFiles = [];
|
||||
|
||||
foreach ($this->queueManager->getMessagesInTransport(QueueManager::QUEUE_MEDIA) as $message) {
|
||||
foreach ($this->queueManager->getMessagesInTransport(QueueManagerInterface::QUEUE_MEDIA) as $message) {
|
||||
if ($message instanceof Message\ReprocessMediaMessage) {
|
||||
$queuedMediaUpdates[$message->media_id] = true;
|
||||
} elseif (
|
||||
|
|
|
@ -34,7 +34,7 @@ else
|
|||
fi
|
||||
|
||||
APP_ENV="${APP_ENV:-production}"
|
||||
UPDATE_REVISION="${UPDATE_REVISION:-64}"
|
||||
UPDATE_REVISION="${UPDATE_REVISION:-65}"
|
||||
|
||||
echo "Updating AzuraCast (Environment: $APP_ENV, Update revision: $UPDATE_REVISION)"
|
||||
|
||||
|
|
|
@ -24,6 +24,7 @@
|
|||
- nginx
|
||||
- php
|
||||
- redis
|
||||
- beanstalkd
|
||||
- mariadb
|
||||
- azuracast-db-install
|
||||
- ufw
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
- name: Install Beanstalkd
|
||||
apt:
|
||||
name: beanstalkd
|
||||
state: latest
|
|
@ -11,3 +11,4 @@
|
|||
- "nginx"
|
||||
- "redis-server"
|
||||
- "redis"
|
||||
- "beanstalkd"
|
||||
|
|
|
@ -21,6 +21,7 @@
|
|||
- { role : mariadb, when : update_revision|int < 63 }
|
||||
- { role : nginx, when : update_revision|int < 60 }
|
||||
- { role : redis, when : update_revision|int < 57 }
|
||||
- { role : beanstalkd, when : update_revision|int < 65 }
|
||||
- { role : php, when : update_revision|int < 62 }
|
||||
- composer
|
||||
- { role : influxdb, when : update_revision|int < 58 }
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
echo 'Spinning up Beanstalkd process...'
|
||||
|
||||
setuser azuracast beanstalkd -p 11300
|
|
@ -0,0 +1,8 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
source /bd_build/buildconfig
|
||||
set -x
|
||||
|
||||
apt-get update
|
||||
|
||||
$minimal_apt_get_install beanstalkd
|
Loading…
Reference in New Issue