rixty
/
AzuraCast
mirror of https://github.com/AzuraCast/AzuraCast.git
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

Decommission Meilisearch.

This commit is contained in:
Buster Neece 2023-05-21 23:23:35 -05:00
parent 123a7ec822
commit 0a1edbd03a
No known key found for this signature in database
GPG Key ID: F1D2E64A0005E80E
25 changed files with 13 additions and 1067 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -7,6 +7,10 @@ release channel, you can take advantage of these new features and fixes.
## Code Quality/Technical Changes
- We have disabled the Meilisearch search tool, as it consumed a large amount of resources on smaller systems and it was
difficult to ensure the index exactly matched the current state of the filesystem. We will be working to further
optimize search queries to achieve similar improvements without any extra services.
- In sections of our application that depend on IP addresses, we've tightened our allowed IP addresses significantly to
improve security and prevent brute-force flooding. If you're using a reverse proxy or CloudFlare, you should update
your "IP Address Source" under the "System Settings" page.

18
Dockerfile
View File

@ -12,21 +12,6 @@ RUN go install github.com/aptible/supercronic@v0.2.24
RUN go install github.com/centrifugal/centrifugo/v4@v4.1.3
#
# Rust dependencies build step
#
FROM rust:1-bullseye AS rust-dependencies
RUN mkdir -p /tmp/meilisearch
WORKDIR /tmp/meilisearch
RUN curl -fsSL https://github.com/meilisearch/meilisearch/archive/refs/tags/v1.1.1.tar.gz -o meilisearch.tar.gz \
&& tar -xvzf meilisearch.tar.gz --strip-components=1 \
&& cargo build --release \
&& chmod a+x ./target/release/meilisearch \
&& mv ./target/release/meilisearch /usr/local/bin/meilisearch
#
# MariaDB dependencies build step
#
@ -44,9 +29,6 @@ COPY --from=go-dependencies /go/bin/dockerize /usr/local/bin
COPY --from=go-dependencies /go/bin/supercronic /usr/local/bin/supercronic
COPY --from=go-dependencies /go/bin/centrifugo /usr/local/bin/centrifugo
# Add Meilisearch
COPY --from=rust-dependencies /usr/local/bin/meilisearch /usr/local/bin/meilisearch
# Add MariaDB dependencies
COPY --from=mariadb /usr/local/bin/healthcheck.sh /usr/local/bin/db_healthcheck.sh
COPY --from=mariadb /usr/local/bin/docker-entrypoint.sh /usr/local/bin/db_entrypoint.sh

1
config/events.php
View File

@ -144,7 +144,6 @@ return static function (CallableEventDispatcherInterface $dispatcher) {
App\Sync\Task\RunBackupTask::class,
App\Sync\Task\SendTimeOnSocketTask::class,
App\Sync\Task\UpdateGeoLiteTask::class,
App\Sync\Task\UpdateMeilisearchIndex::class,
App\Sync\Task\UpdateStorageLocationSizesTask::class,
]);
}

3
config/messagequeue.php
View File

@ -20,8 +20,5 @@ return [
Message\DispatchWebhookMessage::class => App\Webhook\Dispatcher::class,
Message\TestWebhookMessage::class => App\Webhook\Dispatcher::class,
Message\Meilisearch\AddMediaMessage::class => App\Service\Meilisearch\MessageHandler::class,
Message\Meilisearch\UpdatePlaylistsMessage::class => App\Service\Meilisearch\MessageHandler::class,
Mailer\Messenger\SendEmailMessage::class => Mailer\Messenger\MessageHandler::class,
];

42
src/Controller/Api/Stations/AbstractSearchableListAction.php
View File

@ -4,12 +4,10 @@ declare(strict_types=1);
namespace App\Controller\Api\Stations;
use App\Doctrine\Paginator\HydratingAdapter;
use App\Entity\ApiGenerator\SongApiGenerator;
use App\Entity\StationMedia;
use App\Http\ServerRequest;
use App\Paginator;
use App\Service\Meilisearch;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Cache\CacheItemPoolInterface;
@ -18,7 +16,6 @@ abstract class AbstractSearchableListAction
public function __construct(
protected readonly EntityManagerInterface $em,
protected readonly SongApiGenerator $songApiGenerator,
protected readonly Meilisearch $meilisearch,
protected readonly CacheItemPoolInterface $psr6Cache,
) {
}
@ -43,45 +40,6 @@ abstract class AbstractSearchableListAction
$sortField = (string)($queryParams['sort'] ?? '');
$sortDirection = strtolower($queryParams['sortOrder'] ?? 'asc');
if ($this->meilisearch->isSupported()) {
$index = $this->meilisearch->getIndex($station->getMediaStorageLocation());
$searchParams = [];
if (!empty($sortField)) {
$searchParams['sort'] = [$sortField . ':' . $sortDirection];
}
$searchParams['filter'] = [
'station_' . $station->getIdRequired() . '_playlists IN [' . implode(', ', $playlists) . ']',
];
$paginatorAdapter = $index->getSearchPaginator(
$searchPhrase,
$searchParams,
);
$hydrateCallback = function (iterable $results) {
$ids = array_column([...$results], 'id');
return $this->em->createQuery(
<<<'DQL'
SELECT sm
FROM App\Entity\StationMedia sm
WHERE sm.id IN (:ids)
ORDER BY FIELD(sm.id, :ids)
DQL
)->setParameter('ids', $ids)
->toIterable();
};
$hydrateAdapter = new HydratingAdapter(
$paginatorAdapter,
$hydrateCallback(...)
);
return Paginator::fromAdapter($hydrateAdapter, $request);
}
$qb = $this->em->createQueryBuilder();
$qb->select('sm, spm, sp')
->from(StationMedia::class, 'sm')

22
src/Controller/Api/Stations/Files/BatchAction.php
View File

@ -153,11 +153,7 @@ final class BatchAction
/*
* NOTE: This iteration clears the entity manager.
*/
$mediaToReindex = [];
foreach ($this->batchUtilities->iterateMedia($storageLocation, $result->files) as $media) {
$mediaToReindex[] = $media->getIdRequired();
try {
$mediaPlaylists = $this->playlistMediaRepo->clearPlaylistsFromMedia($media, $station);
foreach ($mediaPlaylists as $playlistId => $playlistRecord) {
@ -196,11 +192,6 @@ final class BatchAction
$this->em->flush();
$this->batchUtilities->queuePlaylistsForUpdate(
$station,
$mediaToReindex
);
$this->writePlaylistChanges($station, $affectedPlaylists);
return $result;
@ -222,18 +213,12 @@ final class BatchAction
$this->batchUtilities->iterateUnprocessableMedia($storageLocation, $result->files),
];
$mediaToReindex = [];
foreach ($toMove as $iterator) {
foreach ($iterator as $record) {
/** @var Entity\Interfaces\PathAwareInterface $record */
$oldPath = $record->getPath();
$newPath = File::renameDirectoryInPath($oldPath, $from, $to);
if ($record instanceof Entity\StationMedia) {
$mediaToReindex[] = $record->getIdRequired();
}
try {
$fs->move($oldPath, $newPath);
$record->setPath($newPath);
@ -256,9 +241,6 @@ final class BatchAction
foreach ($toMove as $iterator) {
foreach ($iterator as $record) {
if ($record instanceof Entity\StationMedia) {
$mediaToReindex[] = $record->getIdRequired();
}
/** @var Entity\Interfaces\PathAwareInterface $record */
try {
@ -273,10 +255,6 @@ final class BatchAction
}
}
if (!empty($mediaToReindex)) {
$this->batchUtilities->queueMediaForIndex($storageLocation, $mediaToReindex);
}
return $result;
}

28
src/Controller/Api/Stations/Files/ListAction.php
View File

@ -12,7 +12,6 @@ use App\Http\RouterInterface;
use App\Http\ServerRequest;
use App\Media\MimeType;
use App\Paginator;
use App\Service\Meilisearch;
use App\Utilities;
use Doctrine\Common\Collections\Criteria;
use Doctrine\ORM\EntityManagerInterface;
@ -29,7 +28,6 @@ final class ListAction
public function __construct(
private readonly EntityManagerInterface $em,
private readonly CacheInterface $cache,
private readonly Meilisearch $meilisearch,
private readonly StationFilesystems $stationFilesystems
) {
}
@ -136,26 +134,16 @@ final class ListAction
} else {
[$searchPhrase, $playlist] = $this->parseSearchQuery($station, $searchPhrase);
if ($this->meilisearch->isSupported()) {
$ids = $this->meilisearch
->getIndex($storageLocation)
->searchMedia($searchPhrase, $playlist);
if (null !== $playlist) {
$mediaQueryBuilder->andWhere(
'sm.id IN (:ids)'
)->setParameter('ids', $ids);
} else {
if (null !== $playlist) {
$mediaQueryBuilder->andWhere(
'sm.id IN (SELECT spm2.media_id FROM App\Entity\StationPlaylistMedia spm2 '
. 'WHERE spm2.playlist = :playlist)'
)->setParameter('playlist', $playlist);
}
'sm.id IN (SELECT spm2.media_id FROM App\Entity\StationPlaylistMedia spm2 '
. 'WHERE spm2.playlist = :playlist)'
)->setParameter('playlist', $playlist);
}
if (!empty($searchPhrase)) {
$mediaQueryBuilder->andWhere('(sm.title LIKE :query OR sm.artist LIKE :query)')
->setParameter('query', '%' . $searchPhrase . '%');
}
if (!empty($searchPhrase)) {
$mediaQueryBuilder->andWhere('(sm.title LIKE :query OR sm.artist LIKE :query)')
->setParameter('query', '%' . $searchPhrase . '%');
}
}

18
src/Controller/Api/Stations/FilesController.php
View File

@ -11,7 +11,6 @@ use App\Flysystem\StationFilesystems;
use App\Http\Response;
use App\Http\ServerRequest;
use App\Media\MediaProcessor;
use App\Message\Meilisearch\AddMediaMessage;
use App\Message\WritePlaylistFileMessage;
use App\OpenApi;
use App\Radio\Adapters;
@ -314,9 +313,6 @@ final class FilesController extends AbstractStationApiCrudController
$this->em->flush();
// Reindex file in search.
$this->reindexMedia($record);
// Handle playlist changes.
$backend = $this->adapters->getBackendAdapter($station);
if ($backend instanceof Liquidsoap) {
@ -393,9 +389,6 @@ final class FilesController extends AbstractStationApiCrudController
throw new InvalidArgumentException(sprintf('Record must be an instance of %s.', $this->entityClass));
}
// Trigger search reindex.
$this->reindexMedia($record);
// Delete the media file off the filesystem.
// Write new PLS playlist configuration.
foreach ($this->mediaRepo->remove($record, true) as $playlist_id => $playlist) {
@ -409,15 +402,4 @@ final class FilesController extends AbstractStationApiCrudController
}
}
}
private function reindexMedia(Entity\StationMedia $media): void
{
$indexMessage = new AddMediaMessage();
$indexMessage->storage_location_id = $media->getStorageLocation()->getIdRequired();
$indexMessage->media_ids = [
$media->getIdRequired(),
];
$indexMessage->include_playlists = true;
$this->messageBus->dispatch($indexMessage);
}
}

51
src/Controller/Api/Stations/PlaylistsController.php
View File

@ -5,21 +5,15 @@ declare(strict_types=1);
namespace App\Controller\Api\Stations;
use App\Controller\Api\Traits\CanSortResults;
use App\Doctrine\ReloadableEntityManagerInterface;
use App\Entity;
use App\Http\Response;
use App\Http\ServerRequest;
use App\Message\Meilisearch\UpdatePlaylistsMessage;
use App\OpenApi;
use App\Radio\AutoDJ\Scheduler;
use Carbon\CarbonInterface;
use InvalidArgumentException;
use OpenApi\Attributes as OA;
use Psr\Http\Message\ResponseInterface;
use Symfony\Component\Messenger\MessageBus;
use Symfony\Component\Serializer\Normalizer\AbstractNormalizer;
use Symfony\Component\Serializer\Serializer;
use Symfony\Component\Validator\Validator\ValidatorInterface;
/** @extends AbstractScheduledEntityController<Entity\StationPlaylist> */
#[
@ -151,17 +145,6 @@ final class PlaylistsController extends AbstractScheduledEntityController
protected string $entityClass = Entity\StationPlaylist::class;
protected string $resourceRouteName = 'api:stations:playlist';
public function __construct(
Entity\Repository\StationScheduleRepository $scheduleRepo,
Scheduler $scheduler,
ReloadableEntityManagerInterface $em,
Serializer $serializer,
ValidatorInterface $validator,
private readonly MessageBus $messageBus
) {
parent::__construct($scheduleRepo, $scheduler, $em, $serializer, $validator);
}
public function listAction(
ServerRequest $request,
Response $response,
@ -342,38 +325,4 @@ final class PlaylistsController extends AbstractScheduledEntityController
)
);
}
public function editAction(
ServerRequest $request,
Response $response,
string $station_id,
string $id
): ResponseInterface {
$result = parent::editAction($request, $response, $station_id, $id);
$this->reindexPlaylists($request->getStation());
return $result;
}
public function deleteAction(
ServerRequest $request,
Response $response,
string $station_id,
string $id
): ResponseInterface {
$result = parent::deleteAction($request, $response, $station_id, $id);
$this->reindexPlaylists($request->getStation());
return $result;
}
private function reindexPlaylists(Entity\Station $station): void
{
$indexMessage = new UpdatePlaylistsMessage();
$indexMessage->station_id = $station->getIdRequired();
$this->messageBus->dispatch($indexMessage);
}
}

4
src/Controller/Api/Stations/Requests/ListAction.php
View File

@ -15,7 +15,6 @@ use App\Http\Response;
use App\Http\ServerRequest;
use App\OpenApi;
use App\Radio\AutoDJ\Scheduler;
use App\Service\Meilisearch;
use Carbon\CarbonImmutable;
use Doctrine\ORM\EntityManagerInterface;
use OpenApi\Attributes as OA;
@ -51,11 +50,10 @@ final class ListAction extends AbstractSearchableListAction
public function __construct(
EntityManagerInterface $em,
SongApiGenerator $songApiGenerator,
Meilisearch $meilisearch,
CacheItemPoolInterface $psr6Cache,
private readonly Scheduler $scheduler
) {
parent::__construct($em, $songApiGenerator, $meilisearch, $psr6Cache);
parent::__construct($em, $songApiGenerator, $psr6Cache);
}
public function __invoke(

17
src/Environment.php
View File

@ -54,9 +54,6 @@ final class Environment
public const ENABLE_WEB_UPDATER = 'ENABLE_WEB_UPDATER';
public const ENABLE_MEILISEARCH = 'ENABLE_MEILISEARCH';
public const MEILISEARCH_MASTER_KEY = 'MEILISEARCH_MASTER_KEY';
// Database and Cache Configuration Variables
public const DB_HOST = 'MYSQL_HOST';
public const DB_PORT = 'MYSQL_PORT';
@ -93,9 +90,6 @@ final class Environment
self::PROFILING_EXTENSION_HTTP_KEY => 'dev',
self::ENABLE_WEB_UPDATER => false,
self::ENABLE_MEILISEARCH => false,
self::MEILISEARCH_MASTER_KEY => '',
];
public function __construct(array $elements = [])
@ -376,17 +370,6 @@ final class Environment
return $this->isDocker() && self::envToBool($this->data[self::ENABLE_WEB_UPDATER] ?? false);
}
public function enableMeilisearch(): bool
{
return $this->isDocker() && !$this->isTesting() &&
self::envToBool($this->data[self::ENABLE_MEILISEARCH] ?? false);
}
public function getMeilisearchMasterKey(): string
{
return $this->data[self::MEILISEARCH_MASTER_KEY] ?? $this->defaults[self::MEILISEARCH_MASTER_KEY];
}
public static function getDefaultsForEnvironment(Environment $existingEnv): self
{
return new self([

44
src/Media/BatchUtilities.php
View File

@ -7,11 +7,8 @@ namespace App\Media;
use App\Doctrine\ReadWriteBatchIteratorAggregate;
use App\Entity;
use App\Flysystem\ExtendedFilesystemInterface;
use App\Message\Meilisearch\AddMediaMessage;
use App\Message\Meilisearch\UpdatePlaylistsMessage;
use App\Utilities\File;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Component\Messenger\MessageBus;
use Throwable;
final class BatchUtilities
@ -21,7 +18,6 @@ final class BatchUtilities
private readonly Entity\Repository\StationMediaRepository $mediaRepo,
private readonly Entity\Repository\UnprocessableMediaRepository $unprocessableMediaRepo,
private readonly Entity\Repository\StorageLocationRepository $storageLocationRepo,
private readonly MessageBus $messageBus,
) {
}
@ -41,14 +37,8 @@ final class BatchUtilities
$this->iteratePlaylistFoldersInDirectory($storageLocation, $from),
];
$mediaToReindex = [];
foreach ($toRename as $iterator) {
foreach ($iterator as $record) {
if ($record instanceof Entity\StationMedia) {
$mediaToReindex[] = $record->getIdRequired();
}
/** @var Entity\Interfaces\PathAwareInterface $record */
$record->setPath(
File::renameDirectoryInPath($record->getPath(), $from, $to)
@ -56,8 +46,6 @@ final class BatchUtilities
$this->em->persist($record);
}
}
$this->queueMediaForIndex($storageLocation, $mediaToReindex);
} else {
$record = $this->mediaRepo->findByPath($from, $storageLocation);
@ -65,8 +53,6 @@ final class BatchUtilities
$record->setPath($to);
$this->em->persist($record);
$this->em->flush();
$this->queueMediaForIndex($storageLocation, [$record->getIdRequired()]);
} else {
$record = $this->unprocessableMediaRepo->findByPath($from, $storageLocation);
@ -99,11 +85,7 @@ final class BatchUtilities
/*
* NOTE: This iteration clears the entity manager.
*/
$mediaToReindex = [];
foreach ($this->iterateMedia($storageLocation, $files) as $media) {
$mediaToReindex[] = $media->getIdRequired();
try {
foreach ($this->mediaRepo->remove($media, false, $fs) as $playlistId => $playlist) {
if (!isset($affectedPlaylists[$playlistId])) {
@ -114,8 +96,6 @@ final class BatchUtilities
}
}
$this->queueMediaForIndex($storageLocation, $mediaToReindex);
/*
* NOTE: This iteration clears the entity manager.
*/
@ -134,30 +114,6 @@ final class BatchUtilities
return $affectedPlaylists;
}
public function queueMediaForIndex(
Entity\StorageLocation $storageLocation,
array $ids,
bool $includePlaylists = false
): void {
$queueMessage = new AddMediaMessage();
$queueMessage->storage_location_id = $storageLocation->getIdRequired();
$queueMessage->media_ids = $ids;
$queueMessage->include_playlists = $includePlaylists;
$this->messageBus->dispatch($queueMessage);
}
public function queuePlaylistsForUpdate(
Entity\Station $station,
?array $ids = null
): void {
$queueMessage = new UpdatePlaylistsMessage();
$queueMessage->station_id = $station->getIdRequired();
$queueMessage->media_ids = $ids;
$this->messageBus->dispatch($queueMessage);
}
/**
* Iterate through the found media records, while occasionally flushing and clearing the entity manager.
*

25
src/Message/Meilisearch/AddMediaMessage.php
View File

@ -1,25 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Message\Meilisearch;
use App\Message\AbstractMessage;
use App\MessageQueue\QueueNames;
final class AddMediaMessage extends AbstractMessage
{
/** @var int The numeric identifier for the StorageLocation entity. */
public int $storage_location_id;
/** @var int[] An array of media IDs to process. */
public array $media_ids;
/** @var bool Whether to include playlist data. */
public bool $include_playlists = false;
public function getQueue(): QueueNames
{
return QueueNames::SearchIndex;
}
}

16
src/Message/Meilisearch/UpdatePlaylistsMessage.php
View File

@ -1,16 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Message\Meilisearch;
use App\Message\AbstractMessage;
final class UpdatePlaylistsMessage extends AbstractMessage
{
/** @var int The numeric identifier for the Station entity. */
public int $station_id;
/** @var int[]|null Only update for specific media IDs. */
public ?array $media_ids = null;
}

70
src/Service/Meilisearch.php
View File

@ -1,70 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Service;
use App\Entity\StorageLocation;
use App\Environment;
use App\Service\Meilisearch\Index;
use DI\FactoryInterface;
use GuzzleHttp\Client as GuzzleClient;
use GuzzleHttp\Psr7\HttpFactory;
use Meilisearch\Client;
final class Meilisearch
{
public const BATCH_SIZE = 100;
public function __construct(
private readonly Environment $environment,
private readonly GuzzleClient $httpClient,
private readonly FactoryInterface $factory
) {
}
public function isSupported(): bool
{
return $this->environment->enableMeilisearch();
}
public function getClient(): Client
{
static $client;
if (!$this->isSupported()) {
throw new \RuntimeException('This feature is not supported on this installation.');
}
if (!isset($client)) {
$psrFactory = new HttpFactory();
$client = new Client(
'http://localhost:6070',
$this->environment->getMeilisearchMasterKey(),
$this->httpClient,
requestFactory: $psrFactory,
streamFactory: $psrFactory
);
}
return $client;
}
public function getIndex(StorageLocation $storageLocation): Index
{
$client = $this->getClient();
return $this->factory->make(
Index::class,
[
'storageLocation' => $storageLocation,
'indexClient' => $client->index(self::getIndexUid($storageLocation)),
]
);
}
public static function getIndexUid(StorageLocation $storageLocation): string
{
return 'media_' . $storageLocation->getIdRequired();
}
}

404
src/Service/Meilisearch/Index.php
View File

@ -1,404 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Service\Meilisearch;
use App\Doctrine\ReloadableEntityManagerInterface;
use App\Entity\Repository\CustomFieldRepository;
use App\Entity\Station;
use App\Entity\StationPlaylist;
use App\Entity\StorageLocation;
use App\Environment;
use App\Service\Meilisearch;
use Doctrine\ORM\AbstractQuery;
use Meilisearch\Contracts\DocumentsQuery;
use Meilisearch\Endpoints\Indexes;
use Meilisearch\Exceptions\ApiException;
use Meilisearch\Search\SearchResult;
final class Index
{
public function __construct(
private readonly ReloadableEntityManagerInterface $em,
private readonly CustomFieldRepository $customFieldRepo,
private readonly Environment $environment,
private readonly StorageLocation $storageLocation,
private readonly Indexes $indexClient,
) {
}
public function configure(): void
{
$filterableAttributes = [];
$mediaFields = [
'id',
'path',
'mtime',
'length',
'title',
'artist',
'album',
'genre',
'isrc',
];
foreach ($this->getStationIds() as $stationId) {
$filterableAttributes[] = 'station_' . $stationId . '_playlists';
$filterableAttributes[] = 'station_' . $stationId . '_is_requestable';
$filterableAttributes[] = 'station_' . $stationId . '_is_on_demand';
}
foreach ($this->customFieldRepo->getFieldIds() as $fieldId => $fieldShortCode) {
$mediaFields[] = 'custom_field_' . $fieldId;
}
$indexSettings = [
'filterableAttributes' => $filterableAttributes,
'sortableAttributes' => $mediaFields,
'displayedAttributes' => $this->environment->isProduction()
? ['id']
: ['*'],
];
// Avoid updating settings unless necessary to avoid triggering a reindex.
try {
$this->indexClient->fetchRawInfo();
} catch (ApiException) {
$response = $this->indexClient->create(
$this->indexClient->getUid() ?? '',
['primaryKey' => 'id']
);
$this->indexClient->waitForTask($response['taskUid']);
$this->indexClient->updatePagination([
'maxTotalHits' => 100000,
]);
}
$currentSettings = $this->indexClient->getSettings();
$settingsToUpdate = [];
foreach ($indexSettings as $settingKey => $setting) {
$currentSetting = $currentSettings[$settingKey] ?? [];
sort($setting);
if ($currentSetting !== $setting) {
$settingsToUpdate[$settingKey] = $setting;
}
}
if (!empty($settingsToUpdate)) {
$response = $this->indexClient->updateSettings($settingsToUpdate);
$this->indexClient->waitForTask($response['taskUid']);
}
}
public function getIdsInIndex(): array
{
$ids = [];
foreach ($this->getAllDocuments(['id', 'mtime']) as $document) {
$ids[$document['id']] = $document['mtime'];
}
return $ids;
}
public function getAllDocuments(array $fields = ['*']): iterable
{
$perPage = Meilisearch::BATCH_SIZE;
$documentsQuery = (new DocumentsQuery())
->setOffset(0)
->setLimit($perPage)
->setFields($fields);
$documents = $this->indexClient->getDocuments($documentsQuery);
yield from $documents->getIterator();
if ($documents->getTotal() <= $perPage) {
return;
}
$totalPages = ceil($documents->getTotal() / $perPage);
for ($page = 1; $page <= $totalPages; $page++) {
$documentsQuery->setOffset($page * $perPage);
$documents = $this->indexClient->getDocuments($documentsQuery);
yield from $documents->getIterator();
}
}
public function deleteIds(array $ids): void
{
$this->indexClient->deleteDocuments($ids);
}
public function refreshMedia(
array $ids,
bool $includePlaylists = false
): void {
if ($includePlaylists) {
$mediaPlaylistsRaw = $this->em->createQuery(
<<<'DQL'
SELECT spm.media_id, spm.playlist_id
FROM App\Entity\StationPlaylistMedia spm
WHERE spm.media_id IN (:mediaIds)
DQL
)->setParameter('mediaIds', $ids)
->getArrayResult();
$mediaPlaylists = [];
$playlistIds = [];
foreach ($mediaPlaylistsRaw as $mediaPlaylistRow) {
$mediaId = $mediaPlaylistRow['media_id'];
$playlistId = $mediaPlaylistRow['playlist_id'];
$playlistIds[$playlistId] = $playlistId;
$mediaPlaylists[$mediaId] ??= [];
$mediaPlaylists[$mediaId][] = $playlistId;
}
$stationIds = $this->getStationIds();
$playlistsRaw = $this->em->createQuery(
<<<'DQL'
SELECT p.id, p.station_id, p.include_in_on_demand, p.include_in_requests
FROM App\Entity\StationPlaylist p
WHERE p.id IN (:playlistIds) AND p.station_id IN (:stationIds)
AND p.is_enabled = 1
DQL
)->setParameter('playlistIds', $playlistIds)
->setParameter('stationIds', $stationIds)
->getArrayResult();
$playlists = [];
foreach ($playlistsRaw as $playlistRow) {
$playlists[$playlistRow['id']] = $playlistRow;
}
}
$customFieldsRaw = $this->em->createQuery(
<<<'DQL'
SELECT smcf.media_id, smcf.field_id, smcf.value
FROM App\Entity\StationMediaCustomField smcf
WHERE smcf.media_id IN (:mediaIds)
DQL
)->setParameter('mediaIds', $ids)
->getArrayResult();
$customFields = [];
foreach ($customFieldsRaw as $customFieldRow) {
$mediaId = $customFieldRow['media_id'];
$customFields[$mediaId] ??= [];
$customFields[$mediaId]['custom_field_' . $customFieldRow['field_id']] = $customFieldRow['value'];
}
$mediaRaw = $this->em->createQuery(
<<<'DQL'
SELECT sm.id,
sm.path,
sm.mtime,
sm.length_text,
sm.title,
sm.artist,
sm.album,
sm.genre,
sm.isrc
FROM App\Entity\StationMedia sm
WHERE sm.storage_location = :storageLocation
AND sm.id IN (:ids)
DQL
)->setParameter('storageLocation', $this->storageLocation)
->setParameter('ids', $ids)
->toIterable([], AbstractQuery::HYDRATE_ARRAY);
$media = [];
foreach ($mediaRaw as $row) {
$mediaId = $row['id'];
$record = [
'id' => $row['id'],
'path' => $row['path'],
'mtime' => $row['mtime'],
'duration' => $row['length_text'],
'title' => $row['title'],
'artist' => $row['artist'],
'album' => $row['album'],
'genre' => $row['genre'],
'isrc' => $row['isrc'],
];
if (isset($customFields[$mediaId])) {
$record = array_merge($record, $customFields[$mediaId]);
}
if ($includePlaylists) {
foreach ($stationIds as $stationId) {
$record['station_' . $stationId . '_playlists'] = [];
}
if (isset($mediaPlaylists[$mediaId])) {
foreach ($mediaPlaylists[$mediaId] as $mediaPlaylistId) {
if (!isset($playlists[$mediaPlaylistId])) {
continue;
}
$playlist = $playlists[$mediaPlaylistId];
$stationId = $playlist['station_id'];
$record['station_' . $stationId . '_playlists'][] = $mediaPlaylistId;
}
}
}
$media[$mediaId] = $record;
}
if ($includePlaylists) {
$this->indexClient->addDocumentsInBatches(
$media,
Meilisearch::BATCH_SIZE
);
} else {
$this->indexClient->updateDocumentsInBatches(
$media,
Meilisearch::BATCH_SIZE
);
}
}
public function refreshPlaylists(
Station $station,
?array $ids = null
): void {
$stationId = $station->getIdRequired();
$playlistsKey = 'station_' . $stationId . '_playlists';
$media = [];
if (null === $ids) {
$allMediaRaw = $this->em->createQuery(
<<<'DQL'
SELECT m.id FROM App\Entity\StationMedia m
WHERE m.storage_location = :storageLocation
DQL
)->setParameter('storageLocation', $this->storageLocation)
->toIterable([], AbstractQuery::HYDRATE_ARRAY);
foreach ($allMediaRaw as $mediaRow) {
$media[$mediaRow['id']] = [
'id' => $mediaRow['id'],
$playlistsKey => [],
];
}
} else {
foreach ($ids as $mediaId) {
$media[$mediaId] = [
'id' => $mediaId,
$playlistsKey => [],
];
}
}
$allPlaylistIds = $this->em->createQuery(
<<<'DQL'
SELECT p.id
FROM App\Entity\StationPlaylist p
WHERE p.station = :station AND p.is_enabled = 1
DQL
)->setParameter('station', $station)
->getSingleColumnResult();
if (null === $ids) {
$mediaInPlaylists = $this->em->createQuery(
<<<'DQL'
SELECT spm.media_id, spm.playlist_id
FROM App\Entity\StationPlaylistMedia spm
WHERE spm.playlist_id IN (:allPlaylistIds)
DQL
)->setParameter('allPlaylistIds', $allPlaylistIds)
->toIterable([], AbstractQuery::HYDRATE_ARRAY);
} else {
$mediaInPlaylists = $this->em->createQuery(
<<<'DQL'
SELECT spm.media_id, spm.playlist_id
FROM App\Entity\StationPlaylistMedia spm
WHERE spm.playlist_id IN (:allPlaylistIds)
AND spm.media_id IN (:mediaIds)
DQL
)->setParameter('allPlaylistIds', $allPlaylistIds)
->setParameter('mediaIds', $ids)
->toIterable([], AbstractQuery::HYDRATE_ARRAY);
}
foreach ($mediaInPlaylists as $spmRow) {
$media[$spmRow['media_id']][$playlistsKey][] = $spmRow['playlist_id'];
}
$this->indexClient->updateDocumentsInBatches(
array_values($media),
Meilisearch::BATCH_SIZE
);
}
/**
* @return PaginatorAdapter<array>
*/
public function getSearchPaginator(
?string $query,
array $searchParams = [],
array $options = [],
): PaginatorAdapter {
return new PaginatorAdapter(
$this->indexClient,
$query,
$searchParams,
$options,
);
}
public function searchMedia(
string $query,
?StationPlaylist $playlist = null
): array {
$searchParams = [
'hitsPerPage' => PHP_INT_MAX,
'page' => 1,
];
if (null !== $playlist) {
$station = $playlist->getStation();
$searchParams['filter'] = [
[
'station_' . $station->getIdRequired() . '_playlists = ' . $playlist->getIdRequired(),
],
];
}
/** @var SearchResult $searchResult */
$searchResult = $this->indexClient->search(
$query,
$searchParams
);
return array_column($searchResult->getHits(), 'id');
}
/** @return int[] */
private function getStationIds(): array
{
return $this->em->createQuery(
<<<'DQL'
SELECT s.id FROM App\Entity\Station s
WHERE s.media_storage_location = :storageLocation
AND s.is_enabled = 1
DQL
)->setParameter('storageLocation', $this->storageLocation)
->getSingleColumnResult();
}
}

65
src/Service/Meilisearch/MessageHandler.php
View File

@ -1,65 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Service\Meilisearch;
use App\Entity\Repository\StationRepository;
use App\Entity\Repository\StorageLocationRepository;
use App\Entity\Station;
use App\Entity\StorageLocation;
use App\Message\AbstractMessage;
use App\Message\Meilisearch\AddMediaMessage;
use App\Message\Meilisearch\UpdatePlaylistsMessage;
use App\Service\Meilisearch;
final class MessageHandler
{
public function __construct(
private readonly Meilisearch $meilisearch,
private readonly StorageLocationRepository $storageLocationRepo,
private readonly StationRepository $stationRepo
) {
}
public function __invoke(AbstractMessage $message): void
{
if (!$this->meilisearch->isSupported()) {
return;
}
match (true) {
$message instanceof AddMediaMessage => $this->addMedia($message),
$message instanceof UpdatePlaylistsMessage => $this->updatePlaylists($message),
default => null,
};
}
private function addMedia(AddMediaMessage $message): void
{
$storageLocation = $this->storageLocationRepo->find($message->storage_location_id);
if (!($storageLocation instanceof StorageLocation)) {
return;
}
$index = $this->meilisearch->getIndex($storageLocation);
$index->refreshMedia(
$message->media_ids,
$message->include_playlists
);
}
private function updatePlaylists(UpdatePlaylistsMessage $message): void
{
$station = $this->stationRepo->find($message->station_id);
if (!($station instanceof Station)) {
return;
}
$storageLocation = $station->getMediaStorageLocation();
$index = $this->meilisearch->getIndex($storageLocation);
$index->refreshPlaylists($station, $message->media_ids);
}
}

57
src/Service/Meilisearch/PaginatorAdapter.php
View File

@ -1,57 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Service\Meilisearch;
use Meilisearch\Endpoints\Indexes;
use Meilisearch\Search\SearchResult;
use Pagerfanta\Adapter\AdapterInterface;
/**
* Adapter which uses Meilisearch to perform a search.
*
* @template T of array
* @implements AdapterInterface<T>
*/
final class PaginatorAdapter implements AdapterInterface
{
public function __construct(
private readonly Indexes $indexClient,
private readonly ?string $query,
private readonly array $searchParams = [],
private readonly array $options = [],
) {
}
public function getNbResults(): int
{
/** @var SearchResult $results */
$results = $this->indexClient->search(
$this->query,
[
...$this->searchParams,
'hitsPerPage' => 0,
],
$this->options
);
return abs($results->getTotalHits() ?? 0);
}
public function getSlice(int $offset, int $length): iterable
{
/** @var SearchResult $results */
$results = $this->indexClient->search(
$this->query,
[
...$this->searchParams,
'offset' => $offset,
'limit' => $length,
],
$this->options
);
yield from $results->getHits();
}
}

1
src/Service/ServiceControl.php
View File

@ -84,7 +84,6 @@ final class ServiceControl
'redis' => __('Cache'),
'sftpgo' => __('SFTP service'),
'centrifugo' => __('Live Now Playing updates'),
'meilisearch' => __('Meilisearch'),
];
if (!$this->centrifugo->isSupported()) {

13
src/Sync/Task/CheckFolderPlaylistsTask.php
View File

@ -8,16 +8,13 @@ use App\Doctrine\ReloadableEntityManagerInterface;
use App\Entity;
use App\Flysystem\ExtendedFilesystemInterface;
use App\Flysystem\StationFilesystems;
use App\Message\Meilisearch\UpdatePlaylistsMessage;
use Doctrine\ORM\Query;
use Psr\Log\LoggerInterface;
use Symfony\Component\Messenger\MessageBus;
final class CheckFolderPlaylistsTask extends AbstractTask
{
public function __construct(
private readonly Entity\Repository\StationPlaylistMediaRepository $spmRepo,
private readonly MessageBus $messageBus,
private readonly StationFilesystems $stationFilesystems,
ReloadableEntityManagerInterface $em,
LoggerInterface $logger,
@ -114,7 +111,6 @@ final class CheckFolderPlaylistsTask extends AbstractTask
->getArrayResult();
$addedRecords = 0;
$mediaToIndex = [];
foreach ($mediaInFolderRaw as $row) {
$mediaId = $row['id'];
@ -125,21 +121,12 @@ final class CheckFolderPlaylistsTask extends AbstractTask
if ($media instanceof Entity\StationMedia) {
$this->spmRepo->addMediaToPlaylist($media, $playlist);
$mediaToIndex[] = $mediaId;
$mediaInPlaylist[$mediaId] = $mediaId;
$addedRecords++;
}
}
}
if (!empty($mediaToIndex)) {
$indexMessage = new UpdatePlaylistsMessage();
$indexMessage->station_id = $station->getIdRequired();
$indexMessage->media_ids = $mediaToIndex;
$this->messageBus->dispatch($indexMessage);
}
$logMessage = (0 === $addedRecords)
? 'No changes detected in folder.'
: sprintf('%d media records added from folder.', $addedRecords);

132
src/Sync/Task/UpdateMeilisearchIndex.php
View File

@ -1,132 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Sync\Task;
use App\Doctrine\ReloadableEntityManagerInterface;
use App\Entity;
use App\Message\Meilisearch\AddMediaMessage;
use App\MessageQueue\QueueManagerInterface;
use App\MessageQueue\QueueNames;
use App\Service\Meilisearch;
use Doctrine\ORM\AbstractQuery;
use Psr\Log\LoggerInterface;
use Symfony\Component\Messenger\MessageBus;
final class UpdateMeilisearchIndex extends AbstractTask
{
public function __construct(
private readonly MessageBus $messageBus,
private readonly QueueManagerInterface $queueManager,
private readonly Meilisearch $meilisearch,
ReloadableEntityManagerInterface $em,
LoggerInterface $logger
) {
parent::__construct($em, $logger);
}
public static function getSchedulePattern(): string
{
return '3-59/5 * * * *';
}
public static function isLongTask(): bool
{
return true;
}
public function run(bool $force = false): void
{
if (!$this->meilisearch->isSupported()) {
$this->logger->debug('Meilisearch is not supported on this instance. Skipping sync task.');
return;
}
$this->queueManager->clearQueue(QueueNames::SearchIndex);
$storageLocations = $this->iterateStorageLocations(Entity\Enums\StorageLocationTypes::StationMedia);
foreach ($storageLocations as $storageLocation) {
$this->logger->info(
sprintf(
'Updating MeiliSearch index for storage location %s...',
$storageLocation
)
);
$this->updateIndex($storageLocation);
}
}
public function updateIndex(Entity\StorageLocation $storageLocation): void
{
$stats = [
'existing' => 0,
'queued' => 0,
'added' => 0,
'updated' => 0,
'deleted' => 0,
];
$index = $this->meilisearch->getIndex($storageLocation);
$index->configure();
$existingIds = $index->getIdsInIndex();
$stats['existing'] = count($existingIds);
$mediaRaw = $this->em->createQuery(
<<<'DQL'
SELECT sm.id, sm.mtime
FROM App\Entity\StationMedia sm
WHERE sm.storage_location = :storageLocation
DQL
)->setParameter('storageLocation', $storageLocation)
->toIterable([], AbstractQuery::HYDRATE_ARRAY);
$newIds = [];
$idsToUpdate = [];
foreach ($mediaRaw as $row) {
$mediaId = $row['id'];
if (isset($existingIds[$mediaId])) {
if ($existingIds[$mediaId] < $row['mtime']) {
$idsToUpdate[] = $mediaId;
$stats['updated']++;
}
unset($existingIds[$mediaId]);
continue;
}
$newIds[] = $mediaId;
$stats['added']++;
}
foreach (array_chunk($idsToUpdate, Meilisearch::BATCH_SIZE) as $batchIds) {
$message = new AddMediaMessage();
$message->storage_location_id = $storageLocation->getIdRequired();
$message->media_ids = $batchIds;
$message->include_playlists = true;
$this->messageBus->dispatch($message);
}
foreach (array_chunk($newIds, Meilisearch::BATCH_SIZE) as $batchIds) {
$message = new AddMediaMessage();
$message->storage_location_id = $storageLocation->getIdRequired();
$message->media_ids = $batchIds;
$message->include_playlists = true;
$this->messageBus->dispatch($message);
}
if (!empty($existingIds)) {
$stats['deleted'] = count($existingIds);
$index->deleteIds($existingIds);
}
$this->logger->debug(sprintf('Meilisearch processed for "%s".', $storageLocation), $stats);
}
}

4
util/docker/web/meilisearch/config.toml
View File

@ -1,4 +0,0 @@
db_path = "/var/azuracast/meilisearch/persist"
http_addr = "0.0.0.0:6070"
env = "production"
no_analytics = true

19
util/docker/web/service.full/meilisearch.conf
View File

@ -1,19 +0,0 @@
[program:meilisearch]
command=meilisearch --config-file-path=/var/azuracast/meilisearch/config.toml
--env %(ENV_APPLICATION_ENV)s
--master-key %(ENV_MEILISEARCH_MASTER_KEY)s
priority=500
numprocs=1
autostart=true
autorestart=true
stopasgroup=true
killasgroup=true
stdout_logfile=/var/azuracast/www_tmp/service_meilisearch.log
stdout_logfile_maxbytes=5MB
stdout_logfile_backups=5
redirect_stderr=true
stdout_events_enabled = true
stderr_events_enabled = true

7
util/docker/web/setup/meilisearch.sh
View File

@ -1,7 +0,0 @@
#!/bin/bash
set -e
set -x
mkdir -p /var/azuracast/meilisearch/persist
cp /bd_build/web/meilisearch/config.toml /var/azuracast/meilisearch/config.toml

15
util/docker/web/startup_scripts/08_disable_meilisearch.sh
View File

@ -1,15 +0,0 @@
#!/bin/bash
bool() {
case "$1" in
Y* | y* | true | TRUE | 1) return 0 ;;
esac
return 1
}
ENABLE_MEILISEARCH=${ENABLE_MEILISEARCH:-true}
if ! bool "$ENABLE_MEILISEARCH"; then
echo "Meilisearch is disabled..."
rm -rf /etc/supervisor/full.conf.d/meilisearch.conf
fi