mirror of https://github.com/KDE/kasts.git
Use Entry title to set downloaded file name
Instead of using the MD5 hash of the enclosure download URL, we create a filename which follows `feedname/entry_title.hash.ext`, where feedname is a uniquefied feed title (stored in the DB), a truncated version of the entry title, a shortened hash based on the download URL, and the original file extension extracted from the download URL. BUG: 457848
This commit is contained in:
parent
b2e79dbb51
commit
dc311cac7b
|
@ -7,14 +7,19 @@
|
||||||
|
|
||||||
#include "database.h"
|
#include "database.h"
|
||||||
|
|
||||||
|
#include <QCryptographicHash>
|
||||||
#include <QDateTime>
|
#include <QDateTime>
|
||||||
#include <QDebug>
|
#include <QDebug>
|
||||||
#include <QDir>
|
#include <QDir>
|
||||||
|
#include <QFile>
|
||||||
|
#include <QFileInfo>
|
||||||
#include <QSqlDatabase>
|
#include <QSqlDatabase>
|
||||||
#include <QSqlError>
|
#include <QSqlError>
|
||||||
#include <QStandardPaths>
|
#include <QStandardPaths>
|
||||||
#include <QUrl>
|
#include <QUrl>
|
||||||
|
|
||||||
|
#include "settingsmanager.h"
|
||||||
|
|
||||||
#define TRUE_OR_RETURN(x) \
|
#define TRUE_OR_RETURN(x) \
|
||||||
if (!x) \
|
if (!x) \
|
||||||
return false;
|
return false;
|
||||||
|
@ -64,6 +69,8 @@ bool Database::migrate()
|
||||||
TRUE_OR_RETURN(migrateTo6());
|
TRUE_OR_RETURN(migrateTo6());
|
||||||
if (dbversion < 7)
|
if (dbversion < 7)
|
||||||
TRUE_OR_RETURN(migrateTo7());
|
TRUE_OR_RETURN(migrateTo7());
|
||||||
|
if (dbversion < 8)
|
||||||
|
TRUE_OR_RETURN(migrateTo8());
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -163,6 +170,82 @@ bool Database::migrateTo7()
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool Database::migrateTo8()
|
||||||
|
{
|
||||||
|
qDebug() << "Migrating database to version 8; this can take a while";
|
||||||
|
|
||||||
|
const int maxFilenameLength = 200;
|
||||||
|
QString enclosurePath = QStandardPaths::writableLocation(QStandardPaths::AppDataLocation);
|
||||||
|
if (!SettingsManager::self()->storagePath().isEmpty()) {
|
||||||
|
enclosurePath = SettingsManager::self()->storagePath().toLocalFile();
|
||||||
|
}
|
||||||
|
enclosurePath += QStringLiteral("/enclosures/");
|
||||||
|
|
||||||
|
TRUE_OR_RETURN(transaction());
|
||||||
|
TRUE_OR_RETURN(execute(QStringLiteral("ALTER TABLE Feeds ADD COLUMN dirname TEXT;")));
|
||||||
|
|
||||||
|
QStringList dirNameList;
|
||||||
|
QSqlQuery query(QStringLiteral("SELECT url, name FROM Feeds;"));
|
||||||
|
while (query.next()) {
|
||||||
|
QString url = query.value(QStringLiteral("url")).toString();
|
||||||
|
QString name = query.value(QStringLiteral("name")).toString();
|
||||||
|
|
||||||
|
// Generate directory name for enclosures based on feed name
|
||||||
|
QString dirBaseName = name.left(maxFilenameLength);
|
||||||
|
QString dirName = dirBaseName;
|
||||||
|
|
||||||
|
// Check for duplicate names
|
||||||
|
int numDups = 1; // Minimum to append is " (1)" if file already exists
|
||||||
|
while (dirNameList.contains(dirName)) {
|
||||||
|
dirName = QStringLiteral("%1 (%2)").arg(dirBaseName, QString::number(numDups));
|
||||||
|
numDups++;
|
||||||
|
}
|
||||||
|
|
||||||
|
dirNameList << dirName;
|
||||||
|
|
||||||
|
QSqlQuery writeQuery;
|
||||||
|
writeQuery.prepare(QStringLiteral("UPDATE Feeds SET dirname=:dirname WHERE url=:url;"));
|
||||||
|
writeQuery.bindValue(QStringLiteral(":dirname"), dirName);
|
||||||
|
writeQuery.bindValue(QStringLiteral(":url"), url);
|
||||||
|
TRUE_OR_RETURN(execute(writeQuery));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rename enclosures to new filename convention
|
||||||
|
query.prepare(
|
||||||
|
QStringLiteral("SELECT entry.title, enclosure.id, enclosure.url, feed.dirname FROM Enclosures enclosure JOIN Entries entry ON enclosure.id = entry.id "
|
||||||
|
"JOIN Feeds feed ON enclosure.feed = feed.url;"));
|
||||||
|
TRUE_OR_RETURN(execute(query));
|
||||||
|
while (query.next()) {
|
||||||
|
QString queryTitle = query.value(QStringLiteral("title")).toString();
|
||||||
|
QString queryId = query.value(QStringLiteral("id")).toString();
|
||||||
|
QString queryUrl = query.value(QStringLiteral("url")).toString();
|
||||||
|
QString feedDirName = query.value(QStringLiteral("dirname")).toString();
|
||||||
|
|
||||||
|
// Rename any existing files with the new filename generated above
|
||||||
|
QString legacyPath = enclosurePath + QString::fromStdString(QCryptographicHash::hash(queryUrl.toUtf8(), QCryptographicHash::Md5).toHex().toStdString());
|
||||||
|
|
||||||
|
if (QFileInfo::exists(legacyPath) && QFileInfo(legacyPath).isFile()) {
|
||||||
|
// Generate filename based on episode name and url hash with feedname as subdirectory
|
||||||
|
QString enclosureFilenameBase = queryTitle.left(maxFilenameLength) + QStringLiteral(".")
|
||||||
|
+ QString::fromStdString(QCryptographicHash::hash(queryUrl.toUtf8(), QCryptographicHash::Md5).toHex().toStdString()).left(6);
|
||||||
|
QString enclosureFilenameExt = QFileInfo(QUrl::fromUserInput(queryUrl).fileName()).suffix();
|
||||||
|
|
||||||
|
QString enclosureFilename =
|
||||||
|
!enclosureFilenameExt.isEmpty() ? enclosureFilenameBase + QStringLiteral(".") + enclosureFilenameExt : enclosureFilenameBase;
|
||||||
|
|
||||||
|
QString newDirPath = enclosurePath + feedDirName + QStringLiteral("/");
|
||||||
|
QString newFilePath = newDirPath + enclosureFilename;
|
||||||
|
|
||||||
|
QFileInfo().absoluteDir().mkpath(newDirPath);
|
||||||
|
QFile::rename(legacyPath, newFilePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 8;")));
|
||||||
|
TRUE_OR_RETURN(commit());
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool Database::execute(const QString &query, const QString &connectionName)
|
bool Database::execute(const QString &query, const QString &connectionName)
|
||||||
{
|
{
|
||||||
QSqlQuery q(connectionName);
|
QSqlQuery q(connectionName);
|
||||||
|
|
|
@ -43,6 +43,7 @@ private:
|
||||||
bool migrateTo5();
|
bool migrateTo5();
|
||||||
bool migrateTo6();
|
bool migrateTo6();
|
||||||
bool migrateTo7();
|
bool migrateTo7();
|
||||||
|
bool migrateTo8();
|
||||||
void cleanup();
|
void cleanup();
|
||||||
void setWalMode();
|
void setWalMode();
|
||||||
|
|
||||||
|
|
|
@ -27,11 +27,16 @@
|
||||||
|
|
||||||
DataManager::DataManager()
|
DataManager::DataManager()
|
||||||
{
|
{
|
||||||
connect(
|
connect(&Fetcher::instance(),
|
||||||
&Fetcher::instance(),
|
|
||||||
&Fetcher::feedDetailsUpdated,
|
&Fetcher::feedDetailsUpdated,
|
||||||
this,
|
this,
|
||||||
[this](const QString &url, const QString &name, const QString &image, const QString &link, const QString &description, const QDateTime &lastUpdated) {
|
[this](const QString &url,
|
||||||
|
const QString &name,
|
||||||
|
const QString &image,
|
||||||
|
const QString &link,
|
||||||
|
const QString &description,
|
||||||
|
const QDateTime &lastUpdated,
|
||||||
|
const QString &dirname) {
|
||||||
qCDebug(kastsDataManager) << "Start updating feed details for" << url;
|
qCDebug(kastsDataManager) << "Start updating feed details for" << url;
|
||||||
Feed *feed = getFeed(url);
|
Feed *feed = getFeed(url);
|
||||||
if (feed != nullptr) {
|
if (feed != nullptr) {
|
||||||
|
@ -40,6 +45,7 @@ DataManager::DataManager()
|
||||||
feed->setLink(link);
|
feed->setLink(link);
|
||||||
feed->setDescription(description);
|
feed->setDescription(description);
|
||||||
feed->setLastUpdated(lastUpdated);
|
feed->setLastUpdated(lastUpdated);
|
||||||
|
feed->setDirname(dirname);
|
||||||
qCDebug(kastsDataManager) << "Retrieving authors";
|
qCDebug(kastsDataManager) << "Retrieving authors";
|
||||||
feed->updateAuthors();
|
feed->updateAuthors();
|
||||||
// For feeds that have just been added, this is probably the point
|
// For feeds that have just been added, this is probably the point
|
||||||
|
@ -289,6 +295,11 @@ void DataManager::removeFeeds(const QList<Feed *> &feeds)
|
||||||
m_entrymap.remove(feedurl); // remove all the entry mappings belonging to the feed
|
m_entrymap.remove(feedurl); // remove all the entry mappings belonging to the feed
|
||||||
|
|
||||||
qCDebug(kastsDataManager) << "Remove feed image" << feed->image() << "for feed" << feedurl;
|
qCDebug(kastsDataManager) << "Remove feed image" << feed->image() << "for feed" << feedurl;
|
||||||
|
qCDebug(kastsDataManager) << "Remove feed enclosure download directory" << feed->dirname() << "for feed" << feedurl;
|
||||||
|
QDir enclosureDir = QDir(StorageManager::instance().enclosureDirPath() + feed->dirname());
|
||||||
|
if (!feed->dirname().isEmpty() && enclosureDir.exists()) {
|
||||||
|
enclosureDir.removeRecursively();
|
||||||
|
}
|
||||||
if (!feed->image().isEmpty())
|
if (!feed->image().isEmpty())
|
||||||
StorageManager::instance().removeImage(feed->image());
|
StorageManager::instance().removeImage(feed->image());
|
||||||
m_feeds.remove(m_feedmap[index]); // remove from m_feeds
|
m_feeds.remove(m_feedmap[index]); // remove from m_feeds
|
||||||
|
@ -362,7 +373,8 @@ void DataManager::addFeeds(const QStringList &urls, const bool fetch)
|
||||||
// TODO: Add more checks like checking if URLs exist; however this will mean async...
|
// TODO: Add more checks like checking if URLs exist; however this will mean async...
|
||||||
QStringList newUrls;
|
QStringList newUrls;
|
||||||
for (const QString &url : urls) {
|
for (const QString &url : urls) {
|
||||||
if (!url.trimmed().isEmpty()) {
|
if (!url.trimmed().isEmpty() && !feedExists(url)) {
|
||||||
|
qCDebug(kastsDataManager) << "Feed already exists or URL is empty" << url.trimmed();
|
||||||
newUrls << url.trimmed();
|
newUrls << url.trimmed();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -375,18 +387,13 @@ void DataManager::addFeeds(const QStringList &urls, const bool fetch)
|
||||||
// authors and enclosures) will be updated by calling Fetcher::fetch() which
|
// authors and enclosures) will be updated by calling Fetcher::fetch() which
|
||||||
// will trigger a full update of the feed and all related items.
|
// will trigger a full update of the feed and all related items.
|
||||||
for (const QString &url : newUrls) {
|
for (const QString &url : newUrls) {
|
||||||
qCDebug(kastsDataManager) << "Adding feed";
|
qCDebug(kastsDataManager) << "Adding new feed:" << url;
|
||||||
if (feedExists(url)) {
|
|
||||||
qCDebug(kastsDataManager) << "Feed already exists";
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
qCDebug(kastsDataManager) << "Feed does not yet exist";
|
|
||||||
|
|
||||||
QUrl urlFromInput = QUrl::fromUserInput(url);
|
QUrl urlFromInput = QUrl::fromUserInput(url);
|
||||||
QSqlQuery query;
|
QSqlQuery query;
|
||||||
query.prepare(
|
query.prepare(
|
||||||
QStringLiteral("INSERT INTO Feeds VALUES (:name, :url, :image, :link, :description, :deleteAfterCount, :deleteAfterType, :subscribed, "
|
QStringLiteral("INSERT INTO Feeds VALUES (:name, :url, :image, :link, :description, :deleteAfterCount, :deleteAfterType, :subscribed, "
|
||||||
":lastUpdated, :new, :notify);"));
|
":lastUpdated, :new, :notify, :dirname);"));
|
||||||
query.bindValue(QStringLiteral(":name"), urlFromInput.toString());
|
query.bindValue(QStringLiteral(":name"), urlFromInput.toString());
|
||||||
query.bindValue(QStringLiteral(":url"), urlFromInput.toString());
|
query.bindValue(QStringLiteral(":url"), urlFromInput.toString());
|
||||||
query.bindValue(QStringLiteral(":image"), QLatin1String(""));
|
query.bindValue(QStringLiteral(":image"), QLatin1String(""));
|
||||||
|
@ -398,6 +405,7 @@ void DataManager::addFeeds(const QStringList &urls, const bool fetch)
|
||||||
query.bindValue(QStringLiteral(":lastUpdated"), 0);
|
query.bindValue(QStringLiteral(":lastUpdated"), 0);
|
||||||
query.bindValue(QStringLiteral(":new"), true);
|
query.bindValue(QStringLiteral(":new"), true);
|
||||||
query.bindValue(QStringLiteral(":notify"), false);
|
query.bindValue(QStringLiteral(":notify"), false);
|
||||||
|
query.bindValue(QStringLiteral(":dirname"), QLatin1String(""));
|
||||||
Database::instance().execute(query);
|
Database::instance().execute(query);
|
||||||
|
|
||||||
m_feeds[urlFromInput.toString()] = nullptr;
|
m_feeds[urlFromInput.toString()] = nullptr;
|
||||||
|
|
|
@ -290,7 +290,7 @@ QString Enclosure::url() const
|
||||||
|
|
||||||
QString Enclosure::path() const
|
QString Enclosure::path() const
|
||||||
{
|
{
|
||||||
return StorageManager::instance().enclosurePath(m_url);
|
return StorageManager::instance().enclosurePath(m_entry->title(), m_url, m_entry->feed()->dirname());
|
||||||
}
|
}
|
||||||
|
|
||||||
Enclosure::Status Enclosure::status() const
|
Enclosure::Status Enclosure::status() const
|
||||||
|
@ -301,7 +301,7 @@ Enclosure::Status Enclosure::status() const
|
||||||
QString Enclosure::cachedEmbeddedImage() const
|
QString Enclosure::cachedEmbeddedImage() const
|
||||||
{
|
{
|
||||||
// if image is already cached, then return the path
|
// if image is already cached, then return the path
|
||||||
QString cachedpath = StorageManager::instance().imagePath(path());
|
QString cachedpath = StorageManager::instance().imagePath(m_url);
|
||||||
if (QFileInfo::exists(cachedpath)) {
|
if (QFileInfo::exists(cachedpath)) {
|
||||||
if (QFileInfo(cachedpath).size() != 0) {
|
if (QFileInfo(cachedpath).size() != 0) {
|
||||||
return QUrl::fromLocalFile(cachedpath).toString();
|
return QUrl::fromLocalFile(cachedpath).toString();
|
||||||
|
|
14
src/feed.cpp
14
src/feed.cpp
|
@ -37,6 +37,7 @@ Feed::Feed(const QString &feedurl)
|
||||||
m_deleteAfterCount = query.value(QStringLiteral("deleteAfterCount")).toInt();
|
m_deleteAfterCount = query.value(QStringLiteral("deleteAfterCount")).toInt();
|
||||||
m_deleteAfterType = query.value(QStringLiteral("deleteAfterType")).toInt();
|
m_deleteAfterType = query.value(QStringLiteral("deleteAfterType")).toInt();
|
||||||
m_notify = query.value(QStringLiteral("notify")).toBool();
|
m_notify = query.value(QStringLiteral("notify")).toBool();
|
||||||
|
m_dirname = query.value(QStringLiteral("dirname")).toString();
|
||||||
|
|
||||||
m_errorId = 0;
|
m_errorId = 0;
|
||||||
m_errorString = QLatin1String("");
|
m_errorString = QLatin1String("");
|
||||||
|
@ -200,6 +201,11 @@ bool Feed::notify() const
|
||||||
return m_notify;
|
return m_notify;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
QString Feed::dirname() const
|
||||||
|
{
|
||||||
|
return m_dirname;
|
||||||
|
}
|
||||||
|
|
||||||
int Feed::entryCount() const
|
int Feed::entryCount() const
|
||||||
{
|
{
|
||||||
return DataManager::instance().entryCount(this);
|
return DataManager::instance().entryCount(this);
|
||||||
|
@ -306,6 +312,14 @@ void Feed::setNotify(bool notify)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Feed::setDirname(const QString &dirname)
|
||||||
|
{
|
||||||
|
if (dirname != m_dirname) {
|
||||||
|
m_dirname = dirname;
|
||||||
|
Q_EMIT dirnameChanged(m_dirname);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void Feed::setUnreadEntryCount(const int count)
|
void Feed::setUnreadEntryCount(const int count)
|
||||||
{
|
{
|
||||||
if (count != m_unreadEntryCount) {
|
if (count != m_unreadEntryCount) {
|
||||||
|
|
|
@ -32,6 +32,7 @@ class Feed : public QObject
|
||||||
Q_PROPERTY(QDateTime subscribed READ subscribed CONSTANT)
|
Q_PROPERTY(QDateTime subscribed READ subscribed CONSTANT)
|
||||||
Q_PROPERTY(QDateTime lastUpdated READ lastUpdated WRITE setLastUpdated NOTIFY lastUpdatedChanged)
|
Q_PROPERTY(QDateTime lastUpdated READ lastUpdated WRITE setLastUpdated NOTIFY lastUpdatedChanged)
|
||||||
Q_PROPERTY(bool notify READ notify WRITE setNotify NOTIFY notifyChanged)
|
Q_PROPERTY(bool notify READ notify WRITE setNotify NOTIFY notifyChanged)
|
||||||
|
Q_PROPERTY(QString dirname READ dirname WRITE setDirname NOTIFY dirnameChanged)
|
||||||
Q_PROPERTY(int entryCount READ entryCount NOTIFY entryCountChanged)
|
Q_PROPERTY(int entryCount READ entryCount NOTIFY entryCountChanged)
|
||||||
Q_PROPERTY(int unreadEntryCount READ unreadEntryCount WRITE setUnreadEntryCount NOTIFY unreadEntryCountChanged)
|
Q_PROPERTY(int unreadEntryCount READ unreadEntryCount WRITE setUnreadEntryCount NOTIFY unreadEntryCountChanged)
|
||||||
Q_PROPERTY(int newEntryCount READ newEntryCount NOTIFY newEntryCountChanged)
|
Q_PROPERTY(int newEntryCount READ newEntryCount NOTIFY newEntryCountChanged)
|
||||||
|
@ -59,6 +60,7 @@ public:
|
||||||
QDateTime subscribed() const;
|
QDateTime subscribed() const;
|
||||||
QDateTime lastUpdated() const;
|
QDateTime lastUpdated() const;
|
||||||
bool notify() const;
|
bool notify() const;
|
||||||
|
QString dirname() const;
|
||||||
int entryCount() const;
|
int entryCount() const;
|
||||||
int unreadEntryCount() const;
|
int unreadEntryCount() const;
|
||||||
int newEntryCount() const;
|
int newEntryCount() const;
|
||||||
|
@ -78,6 +80,7 @@ public:
|
||||||
void setDeleteAfterType(int type);
|
void setDeleteAfterType(int type);
|
||||||
void setLastUpdated(const QDateTime &lastUpdated);
|
void setLastUpdated(const QDateTime &lastUpdated);
|
||||||
void setNotify(bool notify);
|
void setNotify(bool notify);
|
||||||
|
void setDirname(const QString &dirname);
|
||||||
void setUnreadEntryCount(const int count);
|
void setUnreadEntryCount(const int count);
|
||||||
void setRefreshing(bool refreshing);
|
void setRefreshing(bool refreshing);
|
||||||
void setErrorId(int errorId);
|
void setErrorId(int errorId);
|
||||||
|
@ -96,6 +99,7 @@ Q_SIGNALS:
|
||||||
void deleteAfterTypeChanged(int type);
|
void deleteAfterTypeChanged(int type);
|
||||||
void lastUpdatedChanged(const QDateTime &lastUpdated);
|
void lastUpdatedChanged(const QDateTime &lastUpdated);
|
||||||
void notifyChanged(bool notify);
|
void notifyChanged(bool notify);
|
||||||
|
void dirnameChanged(const QString &dirname);
|
||||||
void entryCountChanged();
|
void entryCountChanged();
|
||||||
void unreadEntryCountChanged();
|
void unreadEntryCountChanged();
|
||||||
void newEntryCountChanged();
|
void newEntryCountChanged();
|
||||||
|
@ -119,6 +123,7 @@ private:
|
||||||
QDateTime m_subscribed;
|
QDateTime m_subscribed;
|
||||||
QDateTime m_lastUpdated;
|
QDateTime m_lastUpdated;
|
||||||
bool m_notify;
|
bool m_notify;
|
||||||
|
QString m_dirname;
|
||||||
int m_errorId;
|
int m_errorId;
|
||||||
QString m_errorString;
|
QString m_errorString;
|
||||||
int m_unreadEntryCount = -1;
|
int m_unreadEntryCount = -1;
|
||||||
|
|
|
@ -51,7 +51,8 @@ Q_SIGNALS:
|
||||||
const QString &image,
|
const QString &image,
|
||||||
const QString &link,
|
const QString &link,
|
||||||
const QString &description,
|
const QString &description,
|
||||||
const QDateTime &lastUpdated);
|
const QDateTime &lastUpdated,
|
||||||
|
const QString &dirname);
|
||||||
void feedUpdateStatusChanged(const QString &url, bool status);
|
void feedUpdateStatusChanged(const QString &url, bool status);
|
||||||
void cancelFetching();
|
void cancelFetching();
|
||||||
|
|
||||||
|
|
|
@ -65,7 +65,13 @@ void StorageManager::setStoragePath(QUrl url)
|
||||||
qCDebug(kastsStorageManager) << "New storage path will be:" << newPath;
|
qCDebug(kastsStorageManager) << "New storage path will be:" << newPath;
|
||||||
|
|
||||||
if (oldPath != newPath) {
|
if (oldPath != newPath) {
|
||||||
QStringList list = {QStringLiteral("enclosures"), QStringLiteral("images")};
|
// make list of dirs to be moved (images/ and enclosures/*/)
|
||||||
|
QStringList list = {QStringLiteral("images")};
|
||||||
|
for (const QString &subdir : QDir(oldPath + QStringLiteral("/enclosures/")).entryList(QDir::Dirs | QDir::NoDotAndDotDot)) {
|
||||||
|
list << QStringLiteral("enclosures/") + subdir;
|
||||||
|
}
|
||||||
|
list << QStringLiteral("enclosures");
|
||||||
|
|
||||||
StorageMoveJob *moveJob = new StorageMoveJob(oldPath, newPath, list);
|
StorageMoveJob *moveJob = new StorageMoveJob(oldPath, newPath, list);
|
||||||
connect(moveJob, &KJob::processedAmountChanged, this, [this, moveJob]() {
|
connect(moveJob, &KJob::processedAmountChanged, this, [this, moveJob]() {
|
||||||
m_storageMoveProgress = moveJob->processedAmount(KJob::Files);
|
m_storageMoveProgress = moveJob->processedAmount(KJob::Files);
|
||||||
|
@ -112,24 +118,44 @@ QString StorageManager::imagePath(const QString &url) const
|
||||||
}
|
}
|
||||||
|
|
||||||
QString StorageManager::enclosureDirPath() const
|
QString StorageManager::enclosureDirPath() const
|
||||||
|
{
|
||||||
|
return enclosureDirPath(QStringLiteral(""));
|
||||||
|
}
|
||||||
|
|
||||||
|
QString StorageManager::enclosureDirPath(const QString &feedname) const
|
||||||
{
|
{
|
||||||
QString path = storagePath() + QStringLiteral("/enclosures/");
|
QString path = storagePath() + QStringLiteral("/enclosures/");
|
||||||
|
|
||||||
|
if (!feedname.isEmpty()) {
|
||||||
|
path += feedname + QStringLiteral("/");
|
||||||
|
}
|
||||||
|
|
||||||
// Create path if it doesn't exist yet
|
// Create path if it doesn't exist yet
|
||||||
QFileInfo().absoluteDir().mkpath(path);
|
QFileInfo().absoluteDir().mkpath(path);
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
||||||
QString StorageManager::enclosurePath(const QString &url) const
|
QString StorageManager::enclosurePath(const QString &name, const QString &url, const QString &feedname) const
|
||||||
{
|
{
|
||||||
return enclosureDirPath() + QString::fromStdString(QCryptographicHash::hash(url.toUtf8(), QCryptographicHash::Md5).toHex().toStdString());
|
// Generate filename based on episode name and url hash with feedname as subdirectory
|
||||||
|
QString enclosureFilenameBase = name.left(maxFilenameLength) + QStringLiteral(".")
|
||||||
|
+ QString::fromStdString(QCryptographicHash::hash(url.toUtf8(), QCryptographicHash::Md5).toHex().toStdString()).left(6);
|
||||||
|
QString enclosureFilenameExt = QFileInfo(QUrl::fromUserInput(url).fileName()).suffix();
|
||||||
|
|
||||||
|
QString enclosureFilename = !enclosureFilenameExt.isEmpty() ? enclosureFilenameBase + QStringLiteral(".") + enclosureFilenameExt : enclosureFilenameBase;
|
||||||
|
|
||||||
|
return enclosureDirPath(feedname) + enclosureFilename;
|
||||||
}
|
}
|
||||||
|
|
||||||
qint64 StorageManager::dirSize(const QString &path) const
|
qint64 StorageManager::dirSize(const QString &path) const
|
||||||
{
|
{
|
||||||
qint64 size = 0;
|
qint64 size = 0;
|
||||||
QFileInfoList files = QDir(path).entryInfoList(QDir::Files);
|
QFileInfoList files = QDir(path).entryInfoList(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot);
|
||||||
|
|
||||||
for (QFileInfo info : files) {
|
for (QFileInfo info : files) {
|
||||||
|
if (info.isDir()) {
|
||||||
|
size += dirSize(info.filePath());
|
||||||
|
}
|
||||||
size += info.size();
|
size += info.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -145,11 +171,11 @@ void StorageManager::removeImage(const QString &url)
|
||||||
|
|
||||||
void StorageManager::clearImageCache()
|
void StorageManager::clearImageCache()
|
||||||
{
|
{
|
||||||
qDebug() << imageDirPath();
|
qCDebug(kastsStorageManager) << imageDirPath();
|
||||||
QStringList images = QDir(imageDirPath()).entryList(QDir::Files);
|
QStringList images = QDir(imageDirPath()).entryList(QDir::Files);
|
||||||
qDebug() << images;
|
qCDebug(kastsStorageManager) << images;
|
||||||
for (QString image : images) {
|
for (QString image : images) {
|
||||||
qDebug() << image;
|
qCDebug(kastsStorageManager) << image;
|
||||||
QFile(QDir(imageDirPath()).absoluteFilePath(image)).remove();
|
QFile(QDir(imageDirPath()).absoluteFilePath(image)).remove();
|
||||||
}
|
}
|
||||||
Q_EMIT imageDirSizeChanged();
|
Q_EMIT imageDirSizeChanged();
|
||||||
|
|
|
@ -34,6 +34,8 @@ public:
|
||||||
return _instance;
|
return _instance;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static const int maxFilenameLength = 200;
|
||||||
|
|
||||||
QString storagePath() const;
|
QString storagePath() const;
|
||||||
Q_INVOKABLE void setStoragePath(QUrl url);
|
Q_INVOKABLE void setStoragePath(QUrl url);
|
||||||
|
|
||||||
|
@ -41,7 +43,8 @@ public:
|
||||||
QString imagePath(const QString &url) const;
|
QString imagePath(const QString &url) const;
|
||||||
|
|
||||||
QString enclosureDirPath() const;
|
QString enclosureDirPath() const;
|
||||||
QString enclosurePath(const QString &url) const;
|
QString enclosureDirPath(const QString &feedname) const;
|
||||||
|
QString enclosurePath(const QString &name, const QString &url, const QString &feedname) const;
|
||||||
|
|
||||||
qint64 enclosureDirSize() const;
|
qint64 enclosureDirSize() const;
|
||||||
qint64 imageDirSize() const;
|
qint64 imageDirSize() const;
|
||||||
|
|
|
@ -101,6 +101,13 @@ void StorageMoveJob::moveFiles()
|
||||||
QFile(QDir(m_from).absoluteFilePath(file)).remove();
|
QFile(QDir(m_from).absoluteFilePath(file)).remove();
|
||||||
qCDebug(kastsStorageMoveJob) << "Removing file" << QDir(m_from).absoluteFilePath(file);
|
qCDebug(kastsStorageMoveJob) << "Removing file" << QDir(m_from).absoluteFilePath(file);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// delete the directories as well
|
||||||
|
for (const QString &item : m_list) {
|
||||||
|
if (!item.isEmpty() && QFileInfo(m_from + QStringLiteral("/") + item).isDir()) {
|
||||||
|
QDir(m_from).rmdir(item);
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
setError(2);
|
setError(2);
|
||||||
setErrorText(i18n("An error occurred while copying data"));
|
setErrorText(i18n("An error occurred while copying data"));
|
||||||
|
|
|
@ -23,6 +23,7 @@
|
||||||
#include "fetcherlogging.h"
|
#include "fetcherlogging.h"
|
||||||
#include "kasts-version.h"
|
#include "kasts-version.h"
|
||||||
#include "settingsmanager.h"
|
#include "settingsmanager.h"
|
||||||
|
#include "storagemanager.h"
|
||||||
|
|
||||||
using namespace ThreadWeaver;
|
using namespace ThreadWeaver;
|
||||||
|
|
||||||
|
@ -69,27 +70,32 @@ void UpdateFeedJob::processFeed(Syndication::FeedPtr feed)
|
||||||
if (feed.isNull())
|
if (feed.isNull())
|
||||||
return;
|
return;
|
||||||
|
|
||||||
// First check if this is a newly added feed
|
// First check if this is a newly added feed and get current name and dirname
|
||||||
m_isNewFeed = false;
|
m_isNewFeed = false;
|
||||||
|
QString oldName, oldDirname;
|
||||||
QSqlQuery query(QSqlDatabase::database(m_url));
|
QSqlQuery query(QSqlDatabase::database(m_url));
|
||||||
query.prepare(QStringLiteral("SELECT new FROM Feeds WHERE url=:url;"));
|
query.prepare(QStringLiteral("SELECT new, name, dirname FROM Feeds WHERE url=:url;"));
|
||||||
query.bindValue(QStringLiteral(":url"), m_url);
|
query.bindValue(QStringLiteral(":url"), m_url);
|
||||||
Database::execute(query);
|
Database::execute(query);
|
||||||
if (query.next()) {
|
if (query.next()) {
|
||||||
m_isNewFeed = query.value(QStringLiteral("new")).toBool();
|
m_isNewFeed = query.value(QStringLiteral("new")).toBool();
|
||||||
|
oldName = query.value(QStringLiteral("name")).toString();
|
||||||
|
oldDirname = query.value(QStringLiteral("dirname")).toString();
|
||||||
} else {
|
} else {
|
||||||
qCDebug(kastsFetcher) << "Feed not found in database" << m_url;
|
qCDebug(kastsFetcher) << "Feed not found in database" << m_url;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (m_isNewFeed)
|
if (m_isNewFeed) {
|
||||||
qCDebug(kastsFetcher) << "New feed" << feed->title();
|
qCDebug(kastsFetcher) << "New feed" << feed->title();
|
||||||
|
}
|
||||||
|
|
||||||
m_markUnreadOnNewFeed = !(SettingsManager::self()->markUnreadOnNewFeed() == 2);
|
m_markUnreadOnNewFeed = !(SettingsManager::self()->markUnreadOnNewFeed() == 2);
|
||||||
|
|
||||||
// Retrieve "other" fields; this will include the "itunes" tags
|
// Retrieve "other" fields; this will include the "itunes" tags
|
||||||
QMultiMap<QString, QDomElement> otherItems = feed->additionalProperties();
|
QMultiMap<QString, QDomElement> otherItems = feed->additionalProperties();
|
||||||
|
|
||||||
query.prepare(QStringLiteral("UPDATE Feeds SET name=:name, image=:image, link=:link, description=:description, lastUpdated=:lastUpdated WHERE url=:url;"));
|
query.prepare(QStringLiteral(
|
||||||
|
"UPDATE Feeds SET name=:name, image=:image, link=:link, description=:description, lastUpdated=:lastUpdated, dirname=:dirname WHERE url=:url;"));
|
||||||
query.bindValue(QStringLiteral(":name"), feed->title());
|
query.bindValue(QStringLiteral(":name"), feed->title());
|
||||||
query.bindValue(QStringLiteral(":url"), m_url);
|
query.bindValue(QStringLiteral(":url"), m_url);
|
||||||
query.bindValue(QStringLiteral(":link"), feed->link());
|
query.bindValue(QStringLiteral(":link"), feed->link());
|
||||||
|
@ -110,6 +116,23 @@ void UpdateFeedJob::processFeed(Syndication::FeedPtr feed)
|
||||||
image = QUrl(m_url).adjusted(QUrl::RemovePath).toString() + image;
|
image = QUrl(m_url).adjusted(QUrl::RemovePath).toString() + image;
|
||||||
query.bindValue(QStringLiteral(":image"), image);
|
query.bindValue(QStringLiteral(":image"), image);
|
||||||
|
|
||||||
|
// if the title has changed, we need to rename the corresponding enclosure
|
||||||
|
// download directory name and move the files
|
||||||
|
m_dirname = oldDirname;
|
||||||
|
if (oldName != feed->title() || oldDirname.isEmpty() || m_isNewFeed) {
|
||||||
|
QString generatedDirname = generateFeedDirname(feed->title());
|
||||||
|
if (generatedDirname != oldDirname) {
|
||||||
|
m_dirname = generatedDirname;
|
||||||
|
QString enclosurePath = StorageManager::instance().enclosureDirPath();
|
||||||
|
if (QDir(enclosurePath + oldDirname).exists()) {
|
||||||
|
QDir().rename(enclosurePath + oldDirname, enclosurePath + m_dirname);
|
||||||
|
} else {
|
||||||
|
QDir().mkpath(enclosurePath + m_dirname);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
query.bindValue(QStringLiteral(":dirname"), m_dirname);
|
||||||
|
|
||||||
// Do the actual database UPDATE of this feed
|
// Do the actual database UPDATE of this feed
|
||||||
Database::execute(query);
|
Database::execute(query);
|
||||||
|
|
||||||
|
@ -210,7 +233,7 @@ void UpdateFeedJob::processFeed(Syndication::FeedPtr feed)
|
||||||
qCDebug(kastsFetcher) << "Updated feed details:" << feed->title();
|
qCDebug(kastsFetcher) << "Updated feed details:" << feed->title();
|
||||||
|
|
||||||
// TODO: Only emit signal if the details have really changed
|
// TODO: Only emit signal if the details have really changed
|
||||||
Q_EMIT feedDetailsUpdated(m_url, feed->title(), image, feed->link(), feed->description(), current);
|
Q_EMIT feedDetailsUpdated(m_url, feed->title(), image, feed->link(), feed->description(), current, m_dirname);
|
||||||
|
|
||||||
if (m_abort)
|
if (m_abort)
|
||||||
return;
|
return;
|
||||||
|
@ -359,7 +382,7 @@ bool UpdateFeedJob::processEntry(Syndication::ItemPtr entry)
|
||||||
// the first one is probably the podcast author's preferred version
|
// the first one is probably the podcast author's preferred version
|
||||||
// TODO: handle more than one enclosure?
|
// TODO: handle more than one enclosure?
|
||||||
if (entry->enclosures().count() > 0) {
|
if (entry->enclosures().count() > 0) {
|
||||||
isUpdateDependencies = isUpdateDependencies | processEnclosure(entry->enclosures()[0], entry);
|
isUpdateDependencies = isUpdateDependencies | processEnclosure(entry->enclosures()[0], entryDetails, currentEntry);
|
||||||
}
|
}
|
||||||
|
|
||||||
return isNewEntry | isUpdateEntry | isUpdateDependencies; // this is a new or updated entry, or an enclosure, chapter or author has been changed/added
|
return isNewEntry | isUpdateEntry | isUpdateDependencies; // this is a new or updated entry, or an enclosure, chapter or author has been changed/added
|
||||||
|
@ -402,7 +425,7 @@ bool UpdateFeedJob::processAuthor(const QString &entryId, const QString &authorN
|
||||||
return isNewAuthor | isUpdateAuthor;
|
return isNewAuthor | isUpdateAuthor;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool UpdateFeedJob::processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry)
|
bool UpdateFeedJob::processEnclosure(Syndication::EnclosurePtr enclosure, const EntryDetails &newEntry, const EntryDetails &oldEntry)
|
||||||
{
|
{
|
||||||
bool isNewEnclosure = true;
|
bool isNewEnclosure = true;
|
||||||
bool isUpdateEnclosure = false;
|
bool isUpdateEnclosure = false;
|
||||||
|
@ -410,7 +433,7 @@ bool UpdateFeedJob::processEnclosure(Syndication::EnclosurePtr enclosure, Syndic
|
||||||
|
|
||||||
// check against existing enclosures already in database
|
// check against existing enclosures already in database
|
||||||
for (const EnclosureDetails &enclosureDetails : (m_enclosures + m_newEnclosures)) {
|
for (const EnclosureDetails &enclosureDetails : (m_enclosures + m_newEnclosures)) {
|
||||||
if (enclosureDetails.id == entry->id()) {
|
if (enclosureDetails.id == newEntry.id) {
|
||||||
isNewEnclosure = false;
|
isNewEnclosure = false;
|
||||||
currentEnclosure = enclosureDetails;
|
currentEnclosure = enclosureDetails;
|
||||||
}
|
}
|
||||||
|
@ -418,7 +441,7 @@ bool UpdateFeedJob::processEnclosure(Syndication::EnclosurePtr enclosure, Syndic
|
||||||
|
|
||||||
EnclosureDetails enclosureDetails;
|
EnclosureDetails enclosureDetails;
|
||||||
enclosureDetails.feed = m_url;
|
enclosureDetails.feed = m_url;
|
||||||
enclosureDetails.id = entry->id();
|
enclosureDetails.id = newEntry.id;
|
||||||
enclosureDetails.duration = enclosure->duration();
|
enclosureDetails.duration = enclosure->duration();
|
||||||
enclosureDetails.size = enclosure->length();
|
enclosureDetails.size = enclosure->length();
|
||||||
enclosureDetails.title = enclosure->title();
|
enclosureDetails.title = enclosure->title();
|
||||||
|
@ -430,14 +453,32 @@ bool UpdateFeedJob::processEnclosure(Syndication::EnclosurePtr enclosure, Syndic
|
||||||
if (!isNewEnclosure) {
|
if (!isNewEnclosure) {
|
||||||
if ((currentEnclosure.url != enclosureDetails.url) || (currentEnclosure.title != enclosureDetails.title)
|
if ((currentEnclosure.url != enclosureDetails.url) || (currentEnclosure.title != enclosureDetails.title)
|
||||||
|| (currentEnclosure.type != enclosureDetails.type)) {
|
|| (currentEnclosure.type != enclosureDetails.type)) {
|
||||||
qCDebug(kastsFetcher) << "enclosure details have been updated for:" << entry->id();
|
qCDebug(kastsFetcher) << "enclosure details have been updated for:" << newEntry.id;
|
||||||
isUpdateEnclosure = true;
|
isUpdateEnclosure = true;
|
||||||
m_updateEnclosures += enclosureDetails;
|
m_updateEnclosures += enclosureDetails;
|
||||||
} else {
|
} else {
|
||||||
qCDebug(kastsFetcher) << "enclosure details are unchanged:" << entry->id();
|
qCDebug(kastsFetcher) << "enclosure details are unchanged:" << newEntry.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if entry title or enclosure URL has changed
|
||||||
|
if (newEntry.title != oldEntry.title) {
|
||||||
|
QString oldFilename = StorageManager::instance().enclosurePath(oldEntry.title, currentEnclosure.url, m_dirname);
|
||||||
|
QString newFilename = StorageManager::instance().enclosurePath(newEntry.title, enclosureDetails.url, m_dirname);
|
||||||
|
|
||||||
|
if (oldFilename != newFilename) {
|
||||||
|
if (currentEnclosure.url == enclosureDetails.url) {
|
||||||
|
// If entry title has changed but URL is still the same, the existing enclosure needs to be renamed
|
||||||
|
QFile::rename(oldFilename, newFilename);
|
||||||
|
} else {
|
||||||
|
// If enclosure URL has changed, the old enclosure needs to be deleted
|
||||||
|
if (QFile(oldFilename).exists()) {
|
||||||
|
QFile(oldFilename).remove();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
qCDebug(kastsFetcher) << "this is a new enclosure:" << entry->id();
|
qCDebug(kastsFetcher) << "this is a new enclosure:" << newEntry.id;
|
||||||
m_newEnclosures += enclosureDetails;
|
m_newEnclosures += enclosureDetails;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -653,6 +694,29 @@ void UpdateFeedJob::writeToDatabase()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
QString UpdateFeedJob::generateFeedDirname(const QString &name) const
|
||||||
|
{
|
||||||
|
// Generate directory name for enclosures based on feed name
|
||||||
|
// NOTE: Any changes here require a database migration!
|
||||||
|
QString dirBaseName = name.left(StorageManager::maxFilenameLength);
|
||||||
|
QString dirName = dirBaseName;
|
||||||
|
|
||||||
|
QStringList dirNameList;
|
||||||
|
QSqlQuery query(QSqlDatabase::database(m_url));
|
||||||
|
query.prepare(QStringLiteral("SELECT name FROM Feeds;"));
|
||||||
|
while (query.next()) {
|
||||||
|
dirNameList << query.value(QStringLiteral("name")).toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for duplicate names in database and on filesystem
|
||||||
|
int numDups = 1; // Minimum to append is " (1)" if file already exists
|
||||||
|
while (dirNameList.contains(dirName) || QDir(StorageManager::instance().enclosureDirPath() + dirName).exists()) {
|
||||||
|
dirName = QStringLiteral("%1 (%2)").arg(dirBaseName, QString::number(numDups));
|
||||||
|
numDups++;
|
||||||
|
}
|
||||||
|
return dirName;
|
||||||
|
}
|
||||||
|
|
||||||
void UpdateFeedJob::abort()
|
void UpdateFeedJob::abort()
|
||||||
{
|
{
|
||||||
m_abort = true;
|
m_abort = true;
|
||||||
|
|
|
@ -76,7 +76,8 @@ Q_SIGNALS:
|
||||||
const QString &image,
|
const QString &image,
|
||||||
const QString &link,
|
const QString &link,
|
||||||
const QString &description,
|
const QString &description,
|
||||||
const QDateTime &lastUpdated);
|
const QDateTime &lastUpdated,
|
||||||
|
const QString &dirname);
|
||||||
void feedUpdated(const QString &url);
|
void feedUpdated(const QString &url);
|
||||||
void entryAdded(const QString &feedurl, const QString &id);
|
void entryAdded(const QString &feedurl, const QString &id);
|
||||||
void entryUpdated(const QString &feedurl, const QString &id);
|
void entryUpdated(const QString &feedurl, const QString &id);
|
||||||
|
@ -88,13 +89,15 @@ private:
|
||||||
void processFeed(Syndication::FeedPtr feed);
|
void processFeed(Syndication::FeedPtr feed);
|
||||||
bool processEntry(Syndication::ItemPtr entry);
|
bool processEntry(Syndication::ItemPtr entry);
|
||||||
bool processAuthor(const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail);
|
bool processAuthor(const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail);
|
||||||
bool processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry);
|
bool processEnclosure(Syndication::EnclosurePtr enclosure, const EntryDetails &newEntry, const EntryDetails &oldEntry);
|
||||||
bool processChapter(const QString &entryId, const int &start, const QString &chapterTitle, const QString &link, const QString &image);
|
bool processChapter(const QString &entryId, const int &start, const QString &chapterTitle, const QString &link, const QString &image);
|
||||||
void writeToDatabase();
|
void writeToDatabase();
|
||||||
|
|
||||||
|
QString generateFeedDirname(const QString &name) const;
|
||||||
bool m_abort = false;
|
bool m_abort = false;
|
||||||
|
|
||||||
QString m_url;
|
QString m_url;
|
||||||
|
QString m_dirname;
|
||||||
QByteArray m_data;
|
QByteArray m_data;
|
||||||
|
|
||||||
bool m_isNewFeed;
|
bool m_isNewFeed;
|
||||||
|
|
Loading…
Reference in New Issue