rixty
/
AzuraCast
mirror of https://github.com/AzuraCast/AzuraCast.git
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

Switch message queue from MariaDB to beanstalkd (#4418)

This commit is contained in:
Vaalyn 2021-07-21 23:45:22 +02:00 committed by GitHub
parent e509cde831
commit 11f1f44cea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 400 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -19,6 +19,10 @@ release channel, you can take advantage of these new features and fixes.
## Code Quality/Technical Changes
- We have once again switched our message queue implementation, this time from the MariaDB database to a
super-lightweight standalone tool called Beanstalkd. We hope this will resolve issues we've encountered with parallel
workers causing database lockups and other problems.
- The main web Docker container will now automatically initialize itself upon startup, performing essential tasks like
updating the database, clearing the cache and ensuring the system is set up properly. This means even if you miss a
step in installation (or use the Docker images directly) they should still work without issue.

1
composer.json
View File

@ -62,6 +62,7 @@
"spatie/flysystem-dropbox": "^2",
"spomky-labs/otphp": "^10.0",
"supervisorphp/supervisor": "dev-main",
"symfony/beanstalkd-messenger": "^5.3",
"symfony/cache": "^5.2",
"symfony/console": "^5",
"symfony/event-dispatcher": "^5",

122
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "b8d01546a885d3765652fdf3d8ba59fc",
"content-hash": "9238ec827505c7ffc33c58d9ceea2633",
"packages": [
{
"name": "aws/aws-sdk-php",
@ -4633,6 +4633,61 @@
},
"time": "2020-12-06T15:14:20+00:00"
},
{
"name": "pda/pheanstalk",
"version": "v4.0.3",
"source": {
"type": "git",
"url": "https://github.com/pheanstalk/pheanstalk.git",
"reference": "6165573aad525d39b3ac8ae916214cb483a61263"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pheanstalk/pheanstalk/zipball/6165573aad525d39b3ac8ae916214cb483a61263",
"reference": "6165573aad525d39b3ac8ae916214cb483a61263",
"shasum": ""
},
"require": {
"ext-mbstring": "*",
"php": ">=7.1.0"
},
"require-dev": {
"phpunit/phpunit": "^7"
},
"type": "library",
"autoload": {
"psr-4": {
"Pheanstalk\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Paul Annesley",
"email": "paul@annesley.cc",
"homepage": "http://paul.annesley.cc/",
"role": "Developer"
},
{
"name": "Sam Mousa",
"email": "sam@mousa.nl",
"role": "Maintainer"
}
],
"description": "PHP client for beanstalkd queue",
"homepage": "https://github.com/pheanstalk/pheanstalk",
"keywords": [
"beanstalkd"
],
"support": {
"issues": "https://github.com/pheanstalk/pheanstalk/issues",
"source": "https://github.com/pheanstalk/pheanstalk/tree/v4.0.3"
},
"time": "2020-09-22T07:17:48+00:00"
},
{
"name": "php-di/invoker",
"version": "2.3.0",
@ -6310,6 +6365,69 @@
],
"time": "2021-06-17T12:34:27+00:00"
},
{
"name": "symfony/beanstalkd-messenger",
"version": "v5.3.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/beanstalkd-messenger.git",
"reference": "7ca9b2c142ab3a0e13b81910fd5451ebf094bada"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/beanstalkd-messenger/zipball/7ca9b2c142ab3a0e13b81910fd5451ebf094bada",
"reference": "7ca9b2c142ab3a0e13b81910fd5451ebf094bada",
"shasum": ""
},
"require": {
"pda/pheanstalk": "^4.0",
"php": ">=7.2.5",
"symfony/messenger": "^4.4|^5.0"
},
"require-dev": {
"symfony/property-access": "^4.4|^5.0",
"symfony/serializer": "^4.4|^5.0"
},
"type": "symfony-bridge",
"autoload": {
"psr-4": {
"Symfony\\Component\\Messenger\\Bridge\\Beanstalkd\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Antonio Pauletich",
"email": "antonio.pauletich95@gmail.com"
}
],
"description": "Symfony Beanstalkd Messenger Bridge",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/beanstalkd-messenger/tree/v5.3.0"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2021-05-26T17:33:56+00:00"
},
{
"name": "symfony/cache",
"version": "v5.3.3",
@ -13471,5 +13589,5 @@
"ext-xmlwriter": "*"
},
"platform-dev": [],
"plugin-api-version": "2.1.0"
"plugin-api-version": "2.0.0"
}

16
config/services.php
View File

@ -346,6 +346,22 @@ return [
return $builder->getValidator();
},
Pheanstalk\Pheanstalk::class => static function () {
return Pheanstalk\Pheanstalk::create('127.0.0.1', 11300);
},
App\MessageQueue\QueueManagerInterface::class => static function (
Environment $environment,
ContainerInterface $di
) {
if ($environment->isTesting()) {
return new App\MessageQueue\TestQueueManager();
}
$pheanstalk = $di->get(Pheanstalk\Pheanstalk::class);
return new App\MessageQueue\QueueManager($pheanstalk);
},
Symfony\Component\Messenger\MessageBus::class => static function (
App\MessageQueue\QueueManager $queueManager,
App\LockFactory $lockFactory,

1
src/Console/Command/InitializeCommand.php
View File

@ -47,7 +47,6 @@ class InitializeCommand extends CommandAbstract
$io->section(__('Reload System Data'));
$this->runCommand($output, 'cache:clear');
$this->runCommand($output, 'queue:clear');
// Ensure default storage locations exist.
$storageLocationRepo->createDefaultStorageLocations();

15
src/Console/Command/MessageQueue/ClearCommand.php
View File

@ -5,22 +5,22 @@ declare(strict_types=1);
namespace App\Console\Command\MessageQueue;
use App\Console\Command\CommandAbstract;
use App\Entity\Repository\MessengerMessageRepository;
use App\MessageQueue\QueueManager;
use App\MessageQueue\AbstractQueueManager;
use App\MessageQueue\QueueManagerInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
class ClearCommand extends CommandAbstract
{
public function __invoke(
SymfonyStyle $io,
MessengerMessageRepository $messengerMessageRepo,
QueueManagerInterface $queueManager,
?string $queue = null
): int {
$allQueues = QueueManager::getAllQueues();
$allQueues = AbstractQueueManager::getAllQueues();
if (!empty($queue)) {
if (in_array($queue, $allQueues, true)) {
$messengerMessageRepo->clearQueue($queue);
$queueManager->clearQueue($queue);
$io->success(sprintf('Message queue "%s" cleared.', $queue));
} else {
@ -28,7 +28,10 @@ class ClearCommand extends CommandAbstract
return 1;
}
} else {
$messengerMessageRepo->clearQueue();
foreach ($allQueues as $queueName) {
$queueManager->clearQueue($queueName);
}
$io->success('All message queues cleared.');
}

4
src/Console/Command/MessageQueue/ProcessCommand.php
View File

@ -9,7 +9,7 @@ use App\Doctrine\Messenger\ClearEntityManagerSubscriber;
use App\Environment;
use App\EventDispatcher;
use App\MessageQueue\LogWorkerExceptionSubscriber;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
use App\MessageQueue\ResetArrayCacheMiddleware;
use Psr\Log\LoggerInterface;
use Symfony\Component\Messenger\EventListener\StopWorkerOnTimeLimitListener;
@ -22,7 +22,7 @@ class ProcessCommand extends CommandAbstract
public function __invoke(
MessageBus $messageBus,
EventDispatcher $eventDispatcher,
QueueManager $queueManager,
QueueManagerInterface $queueManager,
LoggerInterface $logger,
Environment $environment,
?int $runtime = 0,

7
src/Controller/Admin/DebugController.php
View File

@ -10,7 +10,8 @@ use App\Entity;
use App\Http\Response;
use App\Http\ServerRequest;
use App\Message\RunSyncTaskMessage;
use App\MessageQueue\QueueManager;
use App\MessageQueue\AbstractQueueManager;
use App\MessageQueue\QueueManagerInterface;
use App\Radio\AutoDJ;
use App\Radio\Backend\Liquidsoap;
use App\Session\Flash;
@ -39,9 +40,9 @@ class DebugController extends AbstractLogViewerController
Response $response,
Entity\Repository\StationRepository $stationRepo,
Runner $sync,
QueueManager $queueManager
QueueManagerInterface $queueManager
): ResponseInterface {
$queues = QueueManager::getAllQueues();
$queues = AbstractQueueManager::getAllQueues();
$queueTotals = [];
foreach ($queues as $queue) {

6
src/Controller/Api/Stations/Files/BatchAction.php
View File

@ -11,7 +11,7 @@ use App\Http\Response;
use App\Http\ServerRequest;
use App\Media\BatchUtilities;
use App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
use App\Radio\Backend\Liquidsoap;
use App\Utilities\File;
use Azura\Files\ExtendedFilesystemInterface;
@ -28,7 +28,7 @@ class BatchAction
protected BatchUtilities $batchUtilities,
protected ReloadableEntityManagerInterface $em,
protected MessageBus $messageBus,
protected QueueManager $queueManager,
protected QueueManagerInterface $queueManager,
protected Entity\Repository\StationPlaylistMediaRepository $playlistMediaRepo,
protected Entity\Repository\StationPlaylistFolderRepository $playlistFolderRepo,
) {
@ -288,7 +288,7 @@ class BatchAction
$queuedMediaUpdates = [];
$queuedNewFiles = [];
foreach ($this->queueManager->getMessagesInTransport(QueueManager::QUEUE_MEDIA) as $message) {
foreach ($this->queueManager->getMessagesInTransport(QueueManagerInterface::QUEUE_MEDIA) as $message) {
if ($message instanceof Message\ReprocessMediaMessage) {
$queuedMediaUpdates[$message->media_id] = true;
} elseif (

44
src/Entity/MessengerMessage.php
View File

@ -1,44 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Entity;
use DateTime;
use Doctrine\ORM\Mapping as ORM;
/**
* Internal table used for Symfony Messenger handling.
* @internal
*/
#[
ORM\Entity(readOnly: true),
ORM\Table(name: 'messenger_messages'),
ORM\Index(columns: ['queue_name']),
ORM\Index(columns: ['available_at']),
ORM\Index(columns: ['delivered_at'])
]
class MessengerMessage
{
#[ORM\Column(type: 'bigint')]
#[ORM\Id, ORM\GeneratedValue]
protected int $id;
#[ORM\Column(type: 'text')]
protected string $body;
#[ORM\Column(type: 'text')]
protected string $headers;
#[ORM\Column(name: 'queue_name', length: 190)]
protected string $queueName;
#[ORM\Column(name: 'created_at')]
protected DateTime $createdAt;
#[ORM\Column(name: 'available_at')]
protected DateTime $availableAt;
#[ORM\Column(name: 'delivered_at', nullable: true)]
protected ?DateTime $deliveredAt = null;
}

31
src/Entity/Migration/Version20210721011736.php Normal file
View File

@ -0,0 +1,31 @@
<?php
declare(strict_types=1);
namespace App\Entity\Migration;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\Migrations\AbstractMigration;
/**
* Auto-generated Migration: Please modify to your needs!
*/
final class Version20210721011736 extends AbstractMigration
{
public function getDescription(): string
{
return 'Retire Message Queue DB table.';
}
public function up(Schema $schema): void
{
// this up() migration is auto-generated, please modify it to your needs
$this->addSql('DROP TABLE messenger_messages');
}
public function down(Schema $schema): void
{
// this down() migration is auto-generated, please modify it to your needs
$this->addSql('CREATE TABLE messenger_messages (id BIGINT AUTO_INCREMENT NOT NULL, body LONGTEXT CHARACTER SET utf8mb4 NOT NULL COLLATE `utf8mb4_general_ci`, headers LONGTEXT CHARACTER SET utf8mb4 NOT NULL COLLATE `utf8mb4_general_ci`, queue_name VARCHAR(190) CHARACTER SET utf8mb4 NOT NULL COLLATE `utf8mb4_general_ci`, created_at DATETIME NOT NULL, available_at DATETIME NOT NULL, delivered_at DATETIME DEFAULT NULL, INDEX IDX_75EA56E0FB7336F0 (queue_name), INDEX IDX_75EA56E0E3BD61CE (available_at), INDEX IDX_75EA56E016BA31DB (delivered_at), PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8 COLLATE `utf8_unicode_ci` ENGINE = InnoDB COMMENT = \'\' ');
}
}

24
src/Entity/Repository/MessengerMessageRepository.php
View File

@ -1,24 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Entity\Repository;
use App\Doctrine\Repository;
use App\Entity;
class MessengerMessageRepository extends Repository
{
public function clearQueue(?string $queueName = null): void
{
$qb = $this->em->createQueryBuilder()
->delete(Entity\MessengerMessage::class, 'mm');
if (!empty($queueName)) {
$qb->where('mm.queueName = :queueName')
->setParameter('queueName', $queueName);
}
$qb->getQuery()->execute();
}
}

4
src/Message/AbstractMessage.php
View File

@ -4,12 +4,12 @@ declare(strict_types=1);
namespace App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
abstract class AbstractMessage
{
public function getQueue(): string
{
return QueueManager::QUEUE_NORMAL_PRIORITY;
return QueueManagerInterface::QUEUE_NORMAL_PRIORITY;
}
}

4
src/Message/AddNewMediaMessage.php
View File

@ -4,7 +4,7 @@ declare(strict_types=1);
namespace App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
class AddNewMediaMessage extends AbstractUniqueMessage
{
@ -16,6 +16,6 @@ class AddNewMediaMessage extends AbstractUniqueMessage
public function getQueue(): string
{
return QueueManager::QUEUE_MEDIA;
return QueueManagerInterface::QUEUE_MEDIA;
}
}

4
src/Message/AddNewPodcastMediaMessage.php
View File

@ -4,7 +4,7 @@ declare(strict_types=1);
namespace App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
class AddNewPodcastMediaMessage extends AbstractUniqueMessage
{
@ -16,6 +16,6 @@ class AddNewPodcastMediaMessage extends AbstractUniqueMessage
public function getQueue(): string
{
return QueueManager::QUEUE_PODCAST_MEDIA;
return QueueManagerInterface::QUEUE_PODCAST_MEDIA;
}
}

4
src/Message/BackupMessage.php
View File

@ -4,7 +4,7 @@ declare(strict_types=1);
namespace App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
class BackupMessage extends AbstractUniqueMessage
{
@ -33,6 +33,6 @@ class BackupMessage extends AbstractUniqueMessage
public function getQueue(): string
{
return QueueManager::QUEUE_LOW_PRIORITY;
return QueueManagerInterface::QUEUE_LOW_PRIORITY;
}
}

4
src/Message/ReprocessMediaMessage.php
View File

@ -4,7 +4,7 @@ declare(strict_types=1);
namespace App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
class ReprocessMediaMessage extends AbstractUniqueMessage
{
@ -21,6 +21,6 @@ class ReprocessMediaMessage extends AbstractUniqueMessage
public function getQueue(): string
{
return QueueManager::QUEUE_MEDIA;
return QueueManagerInterface::QUEUE_MEDIA;
}
}

4
src/Message/ReprocessPodcastMediaMessage.php
View File

@ -4,7 +4,7 @@ declare(strict_types=1);
namespace App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
class ReprocessPodcastMediaMessage extends AbstractUniqueMessage
{
@ -21,6 +21,6 @@ class ReprocessPodcastMediaMessage extends AbstractUniqueMessage
public function getQueue(): string
{
return QueueManager::QUEUE_PODCAST_MEDIA;
return QueueManagerInterface::QUEUE_PODCAST_MEDIA;
}
}

4
src/Message/RunSyncTaskMessage.php
View File

@ -5,7 +5,7 @@ declare(strict_types=1);
namespace App\Message;
use App\Environment;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
class RunSyncTaskMessage extends AbstractUniqueMessage
{
@ -26,6 +26,6 @@ class RunSyncTaskMessage extends AbstractUniqueMessage
public function getQueue(): string
{
return QueueManager::QUEUE_HIGH_PRIORITY;
return QueueManagerInterface::QUEUE_HIGH_PRIORITY;
}
}

4
src/Message/UpdateNowPlayingMessage.php
View File

@ -4,7 +4,7 @@ declare(strict_types=1);
namespace App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
class UpdateNowPlayingMessage extends AbstractMessage
{
@ -12,6 +12,6 @@ class UpdateNowPlayingMessage extends AbstractMessage
public function getQueue(): string
{
return QueueManager::QUEUE_HIGH_PRIORITY;
return QueueManagerInterface::QUEUE_HIGH_PRIORITY;
}
}

4
src/Message/WritePlaylistFileMessage.php
View File

@ -4,7 +4,7 @@ declare(strict_types=1);
namespace App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
class WritePlaylistFileMessage extends AbstractUniqueMessage
{
@ -18,6 +18,6 @@ class WritePlaylistFileMessage extends AbstractUniqueMessage
public function getQueue(): string
{
return QueueManager::QUEUE_LOW_PRIORITY;
return QueueManagerInterface::QUEUE_LOW_PRIORITY;
}
}

76
src/MessageQueue/AbstractQueueManager.php Normal file
View File

@ -0,0 +1,76 @@
<?php
declare(strict_types=1);
namespace App\MessageQueue;
use App\Message\AbstractMessage;
use Generator;
use Symfony\Component\Messenger\Envelope;
abstract class AbstractQueueManager implements QueueManagerInterface
{
protected string $workerName = 'app';
public function setWorkerName(string $workerName): void
{
$this->workerName = $workerName;
}
/**
* @inheritDoc
*/
public function getSenders(Envelope $envelope): iterable
{
$message = $envelope->getMessage();
if (!$message instanceof AbstractMessage) {
return [
$this->getTransport(self::QUEUE_NORMAL_PRIORITY),
];
}
$queue = $message->getQueue();
return [
$this->getTransport($queue),
];
}
/**
* @inheritDoc
*/
public function getMessagesInTransport(string $queueName): Generator
{
foreach ($this->getTransport($queueName)->get() as $envelope) {
$message = $envelope->getMessage();
if ($message instanceof AbstractMessage) {
yield $message;
}
}
}
public function getTransports(): array
{
$allQueues = self::getAllQueues();
$transports = [];
foreach ($allQueues as $queueName) {
$transports[$queueName] = $this->getTransport($queueName);
}
return $transports;
}
/**
* @return string[]
*/
public static function getAllQueues(): array
{
return [
self::QUEUE_HIGH_PRIORITY,
self::QUEUE_NORMAL_PRIORITY,
self::QUEUE_LOW_PRIORITY,
self::QUEUE_MEDIA,
self::QUEUE_PODCAST_MEDIA,
];
}
}

129
src/MessageQueue/QueueManager.php
View File

@ -4,133 +4,52 @@ declare(strict_types=1);
namespace App\MessageQueue;
use App\Message\AbstractMessage;
use Doctrine\DBAL\Connection;
use Generator;
use Symfony\Component\Messenger\Bridge\Doctrine\Transport\Connection as MessengerConnection;
use Symfony\Component\Messenger\Bridge\Doctrine\Transport\DoctrineTransport;
use Symfony\Component\Messenger\Envelope;
use Symfony\Component\Messenger\Transport\Sender\SendersLocatorInterface;
use Pheanstalk\Pheanstalk;
use Symfony\Component\Messenger\Bridge\Beanstalkd\Transport\BeanstalkdTransport;
use Symfony\Component\Messenger\Bridge\Beanstalkd\Transport\Connection as MessengerConnection;
use Symfony\Component\Messenger\Exception\TransportException;
use Symfony\Component\Messenger\Transport\Serialization\PhpSerializer;
class QueueManager implements SendersLocatorInterface
class QueueManager extends AbstractQueueManager
{
public const QUEUE_HIGH_PRIORITY = 'high_priority';
public const QUEUE_NORMAL_PRIORITY = 'normal_priority';
public const QUEUE_LOW_PRIORITY = 'low_priority';
public const QUEUE_MEDIA = 'media';
public const QUEUE_PODCAST_MEDIA = 'podcast_media';
protected string $workerName = 'app';
public function __construct(
protected Connection $db
protected Pheanstalk $pheanstalk
) {
}
public function setWorkerName(string $workerName): void
public function clearQueue(string $queueName): void
{
$this->workerName = $workerName;
}
$pheanstalk = $this->pheanstalk->useTube($queueName);
/**
* @inheritDoc
*/
public function getSenders(Envelope $envelope): iterable
{
$message = $envelope->getMessage();
if (!$message instanceof AbstractMessage) {
return [
$this->getTransport(self::QUEUE_NORMAL_PRIORITY),
];
while ($job = $pheanstalk->reserve()) {
$pheanstalk->delete($job);
}
$queue = $message->getQueue();
return [
$this->getTransport($queue),
];
}
public function getConnection(string $queueName): MessengerConnection
public function getTransport(string $queueName): BeanstalkdTransport
{
return new MessengerConnection(
[
'table_name' => 'messenger_messages',
'queue_name' => $queueName,
'auto_setup' => false,
],
$this->db
);
}
public function getTransport(string $queueName): DoctrineTransport
{
return new DoctrineTransport(
return new BeanstalkdTransport(
$this->getConnection($queueName),
new PhpSerializer()
);
}
/**
* @param string $queueName
*
* @return Generator<AbstractMessage>
*/
public function getMessagesInTransport(string $queueName): Generator
protected function getConnection(string $queueName): MessengerConnection
{
foreach ($this->getTransport($queueName)->all() as $envelope) {
$message = $envelope->getMessage();
if ($message instanceof AbstractMessage) {
yield $message;
}
}
}
/**
* @return DoctrineTransport[]
*/
public function getTransports(): array
{
$allQueues = self::getAllQueues();
$transports = [];
foreach ($allQueues as $queueName) {
$transports[$queueName] = $this->getTransport($queueName);
}
return $transports;
}
/**
* @return MessengerConnection[]
*/
public function getConnections(): array
{
$allQueues = self::getAllQueues();
$connections = [];
foreach ($allQueues as $queueName) {
$connections[$queueName] = $this->getConnection($queueName);
}
return $connections;
return new MessengerConnection(
[
'tube_name' => $queueName,
],
$this->pheanstalk
);
}
public function getQueueCount(string $queueName): int
{
return $this->getConnection($queueName)->getMessageCount();
}
/**
* @return string[]
*/
public static function getAllQueues(): array
{
return [
self::QUEUE_HIGH_PRIORITY,
self::QUEUE_NORMAL_PRIORITY,
self::QUEUE_LOW_PRIORITY,
self::QUEUE_MEDIA,
self::QUEUE_PODCAST_MEDIA,
];
try {
return $this->getConnection($queueName)->getMessageCount();
} catch (TransportException) {
return 0;
}
}
}

41
src/MessageQueue/QueueManagerInterface.php Normal file
View File

@ -0,0 +1,41 @@
<?php
declare(strict_types=1);
namespace App\MessageQueue;
use App\Message\AbstractMessage;
use Generator;
use Symfony\Component\Messenger\Transport\Sender\SendersLocatorInterface;
use Symfony\Component\Messenger\Transport\TransportInterface;
interface QueueManagerInterface extends SendersLocatorInterface
{
public const QUEUE_HIGH_PRIORITY = 'high_priority';
public const QUEUE_NORMAL_PRIORITY = 'normal_priority';
public const QUEUE_LOW_PRIORITY = 'low_priority';
public const QUEUE_MEDIA = 'media';
public const QUEUE_PODCAST_MEDIA = 'podcast_media';
public function setWorkerName(string $workerName): void;
public function clearQueue(string $queueName): void;
public function getTransport(string $queueName): TransportInterface;
/**
* @param string $queueName
*
* @return Generator<AbstractMessage>
*/
public function getMessagesInTransport(string $queueName): Generator;
/**
* @return TransportInterface[]
*/
public function getTransports(): array;
public function getQueueCount(string $queueName): int;
public static function getAllQueues(): array;
}

26
src/MessageQueue/TestQueueManager.php Normal file
View File

@ -0,0 +1,26 @@
<?php
declare(strict_types=1);
namespace App\MessageQueue;
use Symfony\Component\Messenger\Transport\InMemoryTransport;
use Symfony\Component\Messenger\Transport\TransportInterface;
class TestQueueManager extends AbstractQueueManager
{
public function clearQueue(string $queueName): void
{
return; // Noop
}
public function getTransport(string $queueName): TransportInterface
{
return new InMemoryTransport();
}
public function getQueueCount(string $queueName): int
{
return 0;
}
}

6
src/Sync/Task/CheckMediaTask.php
View File

@ -8,7 +8,7 @@ use App\Doctrine\ReloadableEntityManagerInterface;
use App\Entity;
use App\Media\MimeType;
use App\Message;
use App\MessageQueue\QueueManager;
use App\MessageQueue\QueueManagerInterface;
use App\Radio\Quota;
use Azura\Files\Attributes\FileAttributes;
use Brick\Math\BigInteger;
@ -26,7 +26,7 @@ class CheckMediaTask extends AbstractTask
protected Entity\Repository\StorageLocationRepository $storageLocationRepo,
protected Entity\Repository\UnprocessableMediaRepository $unprocessableMediaRepo,
protected MessageBus $messageBus,
protected QueueManager $queueManager,
protected QueueManagerInterface $queueManager,
ReloadableEntityManagerInterface $em,
LoggerInterface $logger
) {
@ -138,7 +138,7 @@ class CheckMediaTask extends AbstractTask
$queuedMediaUpdates = [];
$queuedNewFiles = [];
foreach ($this->queueManager->getMessagesInTransport(QueueManager::QUEUE_MEDIA) as $message) {
foreach ($this->queueManager->getMessagesInTransport(QueueManagerInterface::QUEUE_MEDIA) as $message) {
if ($message instanceof Message\ReprocessMediaMessage) {
$queuedMediaUpdates[$message->media_id] = true;
} elseif (

2
update.sh
View File

@ -34,7 +34,7 @@ else
fi
APP_ENV="${APP_ENV:-production}"
UPDATE_REVISION="${UPDATE_REVISION:-64}"
UPDATE_REVISION="${UPDATE_REVISION:-65}"
echo "Updating AzuraCast (Environment: $APP_ENV, Update revision: $UPDATE_REVISION)"

1
util/ansible/deploy.yml
View File

@ -24,6 +24,7 @@
- nginx
- php
- redis
- beanstalkd
- mariadb
- azuracast-db-install
- ufw

5
util/ansible/roles/beanstalkd/tasks/main.yml Normal file
View File

@ -0,0 +1,5 @@
---
- name: Install Beanstalkd
apt:
name: beanstalkd
state: latest

1
util/ansible/roles/services/tasks/main.yml
View File

@ -11,3 +11,4 @@
- "nginx"
- "redis-server"
- "redis"
- "beanstalkd"

1
util/ansible/update.yml
View File

@ -21,6 +21,7 @@
- { role : mariadb, when : update_revision|int < 63 }
- { role : nginx, when : update_revision|int < 60 }
- { role : redis, when : update_revision|int < 57 }
- { role : beanstalkd, when : update_revision|int < 65 }
- { role : php, when : update_revision|int < 62 }
- composer
- { role : influxdb, when : update_revision|int < 58 }

5
util/docker/web/runit/beanstalkd/run Normal file
View File

@ -0,0 +1,5 @@
#!/bin/bash
echo 'Spinning up Beanstalkd process...'
setuser azuracast beanstalkd -p 11300

8
util/docker/web/setup/beanstalkd.sh Normal file
View File

@ -0,0 +1,8 @@
#!/bin/bash
set -e
source /bd_build/buildconfig
set -x
apt-get update
$minimal_apt_get_install beanstalkd