Switch to using a recursive iterator for media processing.
This commit is contained in:
Buster "Silver Eagle" Neece
parent
8e41e44284
commit
6bee06ee93
|
|
@ -50,7 +50,8 @@
|
|||
"voku/portable-utf8": "^5.4",
|
||||
"wikimedia/composer-merge-plugin": "^1.4",
|
||||
"zendframework/zend-config": "^3.3",
|
||||
"zircote/swagger-php": "^3.0"
|
||||
"zircote/swagger-php": "^3.0",
|
||||
"jhofm/flysystem-iterator": "^2.1"
|
||||
},
|
||||
"require-dev": {
|
||||
"codeception/codeception": "^2.2",
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "1fd0a77c0fa0cd68767a10b46305fbef",
|
||||
"content-hash": "daa64133b5f78f421c131b53091c1edc",
|
||||
"packages": [
|
||||
{
|
||||
"name": "aws/aws-sdk-php",
|
||||
|
|
@ -2245,12 +2245,12 @@
|
|||
"version": "v4.7.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/oscarotero/Gettext.git",
|
||||
"url": "https://github.com/php-gettext/Gettext.git",
|
||||
"reference": "739c935503853759b1607f375960c73a5b03909b"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/oscarotero/Gettext/zipball/739c935503853759b1607f375960c73a5b03909b",
|
||||
"url": "https://api.github.com/repos/php-gettext/Gettext/zipball/739c935503853759b1607f375960c73a5b03909b",
|
||||
"reference": "739c935503853759b1607f375960c73a5b03909b",
|
||||
"shasum": ""
|
||||
},
|
||||
|
|
@ -2885,6 +2885,48 @@
|
|||
],
|
||||
"time": "2018-03-21T22:21:57+00:00"
|
||||
},
|
||||
{
|
||||
"name": "jhofm/flysystem-iterator",
|
||||
"version": "v2.1.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/jhofm/flysystem-iterator.git",
|
||||
"reference": "0192e062ebef4d7f102463b1301aa52513c559ad"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/jhofm/flysystem-iterator/zipball/0192e062ebef4d7f102463b1301aa52513c559ad",
|
||||
"reference": "0192e062ebef4d7f102463b1301aa52513c559ad",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-json": "*",
|
||||
"league/flysystem": "^1.0",
|
||||
"php": "^7.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"league/flysystem-memory": "^1.0",
|
||||
"phpunit/phpunit": "^6.0"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Jhofm\\FlysystemIterator\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Johannes Hofmann",
|
||||
"email": "hofmann.johannes@gmail.com"
|
||||
}
|
||||
],
|
||||
"description": "Iterator plugin for league/flysystem",
|
||||
"time": "2019-04-22T19:59:38+00:00"
|
||||
},
|
||||
{
|
||||
"name": "league/flysystem",
|
||||
"version": "1.0.57",
|
||||
|
|
|
|||
|
|
@ -3,6 +3,10 @@ namespace App\Flysystem;
|
|||
|
||||
use Azura\Exception;
|
||||
use InvalidArgumentException;
|
||||
use Jhofm\FlysystemIterator\FilesystemFilterIterator;
|
||||
use Jhofm\FlysystemIterator\FilesystemIterator;
|
||||
use Jhofm\FlysystemIterator\Options\Options;
|
||||
use Jhofm\FlysystemIterator\RecursiveFilesystemIteratorIterator;
|
||||
use League\Flysystem\Adapter\Local;
|
||||
use League\Flysystem\Cached\CachedAdapter;
|
||||
use League\Flysystem\Cached\Storage\AbstractCache;
|
||||
|
|
@ -105,7 +109,7 @@ class StationFilesystem extends MountManager
|
|||
*/
|
||||
public function getFullPath($uri): string
|
||||
{
|
||||
list($prefix, $path) = $this->getPrefixAndPath($uri);
|
||||
[$prefix, $path] = $this->getPrefixAndPath($uri);
|
||||
|
||||
$fs = $this->getFilesystem($prefix);
|
||||
|
||||
|
|
@ -154,4 +158,29 @@ class StationFilesystem extends MountManager
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an iterator that loops through the entire contents of a given prefix.
|
||||
*
|
||||
* @param string $uri
|
||||
* @param array $iteratorOptions
|
||||
*
|
||||
* @return \Iterator
|
||||
*/
|
||||
public function createIterator(string $uri, array $iteratorOptions = []): \Iterator
|
||||
{
|
||||
[$prefix, $path] = $this->getPrefixAndPath($uri);
|
||||
|
||||
$fs = $this->getFilesystem($prefix);
|
||||
$iterator = new FilesystemIterator($fs, $path, $iteratorOptions);
|
||||
|
||||
$options = Options::fromArray($iteratorOptions);
|
||||
if ($options->{Options::OPTION_IS_RECURSIVE}) {
|
||||
$iterator = new RecursiveFilesystemIteratorIterator($iterator);
|
||||
}
|
||||
if ($options->{Options::OPTION_FILTER} !== null) {
|
||||
$iterator = new FilesystemFilterIterator($iterator, $options->{Options::OPTION_FILTER});
|
||||
}
|
||||
return $iterator;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ use League\Flysystem\Cached\CachedAdapter;
|
|||
use League\Flysystem\Cached\Storage\Psr6Cache;
|
||||
use League\Flysystem\Filesystem as LeagueFilesystem;
|
||||
use Psr\Cache\CacheItemPoolInterface;
|
||||
use Redis;
|
||||
|
||||
/**
|
||||
* A wrapper and manager class for accessing assets on the filesystem.
|
||||
|
|
@ -30,7 +29,7 @@ class Filesystem
|
|||
$this->cachePool = new PrefixedCachePool($cachePool, 'fs|');
|
||||
}
|
||||
|
||||
public function getForStation(Entity\Station $station): StationFilesystem
|
||||
public function getForStation(Entity\Station $station, bool $cached = true): StationFilesystem
|
||||
{
|
||||
$station_id = $station->getId();
|
||||
if (!isset($this->interfaces[$station_id])) {
|
||||
|
|
@ -46,8 +45,12 @@ class Filesystem
|
|||
foreach ($aliases as $alias => $localPath) {
|
||||
$adapter = new Local($localPath);
|
||||
|
||||
$cachedClient = new Psr6Cache($this->cachePool, $this->normalizeCacheKey($localPath), 3600);
|
||||
$filesystems[$alias] = new LeagueFilesystem(new CachedAdapter($adapter, $cachedClient));
|
||||
if ($cached) {
|
||||
$cachedClient = new Psr6Cache($this->cachePool, $this->normalizeCacheKey($localPath), 3600);
|
||||
$adapter = new CachedAdapter($adapter, $cachedClient);
|
||||
}
|
||||
|
||||
$filesystems[$alias] = new LeagueFilesystem($adapter);
|
||||
}
|
||||
|
||||
$this->interfaces[$station_id] = new StationFilesystem($filesystems);
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ use Bernard\Envelope;
|
|||
use Brick\Math\BigInteger;
|
||||
use Doctrine\Common\Persistence\Mapping\MappingException;
|
||||
use Doctrine\ORM\EntityManager;
|
||||
use Jhofm\FlysystemIterator\Filter\FilterFactory;
|
||||
use Symfony\Component\Finder\Finder;
|
||||
|
||||
class Media extends AbstractTask
|
||||
|
|
@ -101,8 +102,7 @@ class Media extends AbstractTask
|
|||
|
||||
public function importMusic(Entity\Station $station)
|
||||
{
|
||||
$fs = $this->filesystem->getForStation($station);
|
||||
$fs->flushAllCaches();
|
||||
$fs = $this->filesystem->getForStation($station, false);
|
||||
|
||||
$stats = [
|
||||
'total_size' => '0',
|
||||
|
|
@ -117,15 +117,11 @@ class Media extends AbstractTask
|
|||
$music_files = [];
|
||||
$total_size = BigInteger::zero();
|
||||
|
||||
foreach ($fs->listContents('media://', true) as $file) {
|
||||
if (!empty($file['size'])) {
|
||||
$total_size = $total_size->plus($file['size']);
|
||||
}
|
||||
|
||||
if ('file' !== $file['type']) {
|
||||
continue;
|
||||
}
|
||||
$fsIterator = $fs->createIterator('media://', [
|
||||
'filter' => FilterFactory::isFile(),
|
||||
]);
|
||||
|
||||
foreach ($fsIterator as $file) {
|
||||
$path_hash = md5($file['path']);
|
||||
$music_files[$path_hash] = $file;
|
||||
}
|
||||
|
|
@ -240,8 +236,6 @@ class Media extends AbstractTask
|
|||
}
|
||||
}
|
||||
|
||||
$fs->flushAllCaches(true);
|
||||
|
||||
Logger::getInstance()->debug(sprintf('Media processed for station "%s".', $station->getName()), $stats);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue