mirror of https://github.com/KDE/kasts.git
Wrap feed update routine in KJob and make it more efficient
The feed update routine which is now spread over several methods in Fetcher, is now put into a self-contained KJob. This will allow to re-use this job later on in e.g. gpodder sync, where it's required to update feeds before syncing episode statuses. This also makes the feed update abortable. Lastly, but most importantly, the feed update procedure has been optimized to minimize database transactions, resulting in a dramatic speed-up. This is especially true for importing new feeds, which will now be at least 5x faster on slow hardware.
This commit is contained in:
Bart De Vries
parent
e4ee0f19fd
commit
c061a01c59
|
|
@ -16,6 +16,8 @@ set(SRCS_base
|
|||
enclosuredownloadjob.cpp
|
||||
storagemanager.cpp
|
||||
storagemovejob.cpp
|
||||
updatefeedjob.cpp
|
||||
fetchfeedsjob.cpp
|
||||
models/chaptermodel.cpp
|
||||
models/feedsmodel.cpp
|
||||
models/feedsproxymodel.cpp
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ bool Database::migrate()
|
|||
bool Database::migrateTo1()
|
||||
{
|
||||
qDebug() << "Migrating database to version 1";
|
||||
TRUE_OR_RETURN(transaction());
|
||||
TRUE_OR_RETURN(
|
||||
execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Feeds (name TEXT, url TEXT, image TEXT, link TEXT, description TEXT, deleteAfterCount INTEGER, "
|
||||
"deleteAfterType INTEGER, subscribed INTEGER, lastUpdated INTEGER, new BOOL, notify BOOL);")));
|
||||
|
|
@ -68,24 +69,25 @@ bool Database::migrateTo1()
|
|||
TRUE_OR_RETURN(execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Queue (listnr INTEGER, feed TEXT, id TEXT, playing BOOL);")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Errors (url TEXT, id TEXT, code INTEGER, message TEXT, date INTEGER);")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 1;")));
|
||||
TRUE_OR_RETURN(commit());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Database::migrateTo2()
|
||||
{
|
||||
qDebug() << "Migrating database to version 2";
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("BEGIN TRANSACTION;")));
|
||||
TRUE_OR_RETURN(transaction());
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("DROP TABLE Errors;")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Errors (type INTEGER, url TEXT, id TEXT, code INTEGER, message TEXT, date INTEGER);")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 2;")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("COMMIT;")));
|
||||
TRUE_OR_RETURN(commit());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Database::migrateTo3()
|
||||
{
|
||||
qDebug() << "Migrating database to version 3";
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("BEGIN TRANSACTION;")));
|
||||
TRUE_OR_RETURN(transaction());
|
||||
TRUE_OR_RETURN(
|
||||
execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Enclosurestemp (feed TEXT, id TEXT, duration INTEGER, size INTEGER, title TEXT, type TEXT, url "
|
||||
"TEXT, playposition INTEGER, downloaded INTEGER);")));
|
||||
|
|
@ -96,31 +98,32 @@ bool Database::migrateTo3()
|
|||
TRUE_OR_RETURN(execute(QStringLiteral("DROP TABLE Enclosures;")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("ALTER TABLE Enclosurestemp RENAME TO Enclosures;")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 3;")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("COMMIT;")));
|
||||
TRUE_OR_RETURN(commit());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Database::migrateTo4()
|
||||
{
|
||||
qDebug() << "Migrating database to version 4";
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("BEGIN TRANSACTION;")));
|
||||
TRUE_OR_RETURN(transaction());
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("DROP TABLE Errors;")));
|
||||
TRUE_OR_RETURN(
|
||||
execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Errors (type INTEGER, url TEXT, id TEXT, code INTEGER, message TEXT, date INTEGER, title TEXT);")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 4;")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("COMMIT;")));
|
||||
TRUE_OR_RETURN(commit());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Database::migrateTo5()
|
||||
{
|
||||
qDebug() << "Migrating database to version 5";
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("BEGIN TRANSACTION;")));
|
||||
TRUE_OR_RETURN(transaction());
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Chapters (feed TEXT, id TEXT, start INTEGER, title TEXT, link TEXT, image TEXT);")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 5;")));
|
||||
TRUE_OR_RETURN(execute(QStringLiteral("COMMIT;")));
|
||||
TRUE_OR_RETURN(commit());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Database::execute(const QString &query)
|
||||
{
|
||||
QSqlQuery q;
|
||||
|
|
@ -139,6 +142,16 @@ bool Database::execute(QSqlQuery &query)
|
|||
return true;
|
||||
}
|
||||
|
||||
bool Database::transaction()
|
||||
{
|
||||
return QSqlDatabase::database().transaction();
|
||||
}
|
||||
|
||||
bool Database::commit()
|
||||
{
|
||||
return QSqlDatabase::database().commit();
|
||||
}
|
||||
|
||||
int Database::version()
|
||||
{
|
||||
QSqlQuery query;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
#include <QObject>
|
||||
#include <QSqlQuery>
|
||||
#include <QString>
|
||||
|
||||
class Database : public QObject
|
||||
{
|
||||
|
|
@ -23,6 +24,9 @@ public:
|
|||
bool execute(QSqlQuery &query);
|
||||
bool execute(const QString &query);
|
||||
|
||||
bool transaction();
|
||||
bool commit();
|
||||
|
||||
private:
|
||||
Database();
|
||||
int version();
|
||||
|
|
|
|||
|
|
@ -49,11 +49,11 @@ DataManager::DataManager()
|
|||
}
|
||||
});
|
||||
connect(&Fetcher::instance(), &Fetcher::entryAdded, this, [this](const QString &feedurl, const QString &id) {
|
||||
Q_UNUSED(feedurl)
|
||||
// Only add the new entry to m_entries
|
||||
// we will repopulate m_entrymap once all new entries have been added,
|
||||
// such that m_entrymap will show all new entries in the correct order
|
||||
m_entries[id] = nullptr;
|
||||
Q_EMIT entryAdded(feedurl, id);
|
||||
});
|
||||
connect(&Fetcher::instance(), &Fetcher::feedUpdated, this, [this](const QString &feedurl) {
|
||||
// Update m_entrymap for feedurl, such that the new and old entries show
|
||||
|
|
@ -563,7 +563,7 @@ void DataManager::bulkMarkReadByIndex(bool state, QModelIndexList list)
|
|||
|
||||
void DataManager::bulkMarkRead(bool state, QStringList list)
|
||||
{
|
||||
Database::instance().execute(QStringLiteral("BEGIN TRANSACTION;"));
|
||||
Database::instance().transaction();
|
||||
|
||||
if (state) { // Mark as read
|
||||
// This needs special attention as the DB operations are very intensive.
|
||||
|
|
@ -577,7 +577,7 @@ void DataManager::bulkMarkRead(bool state, QStringList list)
|
|||
getEntry(id)->setReadInternal(state);
|
||||
}
|
||||
}
|
||||
Database::instance().execute(QStringLiteral("COMMIT;"));
|
||||
Database::instance().commit();
|
||||
|
||||
Q_EMIT bulkReadStatusActionFinished();
|
||||
}
|
||||
|
|
@ -589,11 +589,11 @@ void DataManager::bulkMarkNewByIndex(bool state, QModelIndexList list)
|
|||
|
||||
void DataManager::bulkMarkNew(bool state, QStringList list)
|
||||
{
|
||||
Database::instance().execute(QStringLiteral("BEGIN TRANSACTION;"));
|
||||
Database::instance().transaction();
|
||||
for (QString id : list) {
|
||||
getEntry(id)->setNewInternal(state);
|
||||
}
|
||||
Database::instance().execute(QStringLiteral("COMMIT;"));
|
||||
Database::instance().commit();
|
||||
|
||||
Q_EMIT bulkNewStatusActionFinished();
|
||||
}
|
||||
|
|
@ -605,7 +605,7 @@ void DataManager::bulkQueueStatusByIndex(bool state, QModelIndexList list)
|
|||
|
||||
void DataManager::bulkQueueStatus(bool state, QStringList list)
|
||||
{
|
||||
Database::instance().execute(QStringLiteral("BEGIN TRANSACTION;"));
|
||||
Database::instance().transaction();
|
||||
if (state) { // i.e. add to queue
|
||||
for (QString id : list) {
|
||||
getEntry(id)->setQueueStatusInternal(state);
|
||||
|
|
@ -619,7 +619,7 @@ void DataManager::bulkQueueStatus(bool state, QStringList list)
|
|||
}
|
||||
updateQueueListnrs();
|
||||
}
|
||||
Database::instance().execute(QStringLiteral("COMMIT;"));
|
||||
Database::instance().commit();
|
||||
|
||||
Q_EMIT bulkReadStatusActionFinished();
|
||||
Q_EMIT bulkNewStatusActionFinished();
|
||||
|
|
@ -647,13 +647,13 @@ void DataManager::bulkDeleteEnclosuresByIndex(QModelIndexList list)
|
|||
|
||||
void DataManager::bulkDeleteEnclosures(QStringList list)
|
||||
{
|
||||
Database::instance().execute(QStringLiteral("BEGIN TRANSACTION;"));
|
||||
Database::instance().transaction();
|
||||
for (QString id : list) {
|
||||
if (getEntry(id)->hasEnclosure()) {
|
||||
getEntry(id)->enclosure()->deleteFile();
|
||||
}
|
||||
}
|
||||
Database::instance().execute(QStringLiteral("COMMIT;"));
|
||||
Database::instance().commit();
|
||||
}
|
||||
|
||||
QStringList DataManager::getIdsFromModelIndexList(const QModelIndexList &list) const
|
||||
|
|
|
|||
|
|
@ -77,8 +77,6 @@ public:
|
|||
Q_SIGNALS:
|
||||
void feedAdded(const QString &url);
|
||||
void feedRemoved(const int &index);
|
||||
void entryAdded(const QString &feedurl, const QString &id);
|
||||
// void entryRemoved(const Feed*, const int &index); // TODO: implement this signal, is this needed?
|
||||
void feedEntriesUpdated(const QString &url);
|
||||
void queueEntryAdded(const int &index, const QString &id);
|
||||
void queueEntryRemoved(const int &index, const QString &id);
|
||||
|
|
|
|||
15
src/feed.cpp
15
src/feed.cpp
|
|
@ -44,11 +44,9 @@ Feed::Feed(const QString &feedurl)
|
|||
|
||||
updateAuthors();
|
||||
|
||||
connect(&Fetcher::instance(), &Fetcher::startedFetchingFeed, this, [this](const QString &url) {
|
||||
connect(&Fetcher::instance(), &Fetcher::feedUpdateStatusChanged, this, [this](const QString &url, bool status) {
|
||||
if (url == m_url) {
|
||||
m_errorId = 0;
|
||||
m_errorString = QString();
|
||||
setRefreshing(true);
|
||||
setRefreshing(status);
|
||||
}
|
||||
});
|
||||
connect(&DataManager::instance(), &DataManager::feedEntriesUpdated, this, [this](const QString &url) {
|
||||
|
|
@ -71,11 +69,6 @@ Feed::Feed(const QString &feedurl)
|
|||
setRefreshing(false);
|
||||
}
|
||||
});
|
||||
connect(&Fetcher::instance(), &Fetcher::feedUpdateFinished, this, [this](const QString &url) {
|
||||
if (url == m_url) {
|
||||
setRefreshing(false);
|
||||
}
|
||||
});
|
||||
connect(&Fetcher::instance(), &Fetcher::downloadFinished, this, [this](QString url) {
|
||||
if (url == m_image) {
|
||||
Q_EMIT imageChanged(url);
|
||||
|
|
@ -300,6 +293,10 @@ void Feed::setRefreshing(bool refreshing)
|
|||
{
|
||||
if (refreshing != m_refreshing) {
|
||||
m_refreshing = refreshing;
|
||||
if (!m_refreshing) {
|
||||
m_errorId = 0;
|
||||
m_errorString = QString();
|
||||
}
|
||||
Q_EMIT refreshingChanged(m_refreshing);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
369
src/fetcher.cpp
369
src/fetcher.cpp
|
|
@ -24,12 +24,16 @@
|
|||
|
||||
#include "database.h"
|
||||
#include "enclosure.h"
|
||||
#include "fetchfeedsjob.h"
|
||||
#include "kasts-version.h"
|
||||
#include "models/errorlogmodel.h"
|
||||
#include "settingsmanager.h"
|
||||
#include "storagemanager.h"
|
||||
|
||||
Fetcher::Fetcher()
|
||||
{
|
||||
connect(this, &Fetcher::error, &ErrorLogModel::instance(), &ErrorLogModel::monitorErrorMessages);
|
||||
|
||||
m_updateProgress = -1;
|
||||
m_updateTotal = -1;
|
||||
m_updating = false;
|
||||
|
|
@ -53,24 +57,6 @@ void Fetcher::fetch(const QString &url)
|
|||
fetch(urls);
|
||||
}
|
||||
|
||||
void Fetcher::fetch(const QStringList &urls)
|
||||
{
|
||||
if (m_updating)
|
||||
return; // update is already running, do nothing
|
||||
|
||||
m_updating = true;
|
||||
m_updateProgress = 0;
|
||||
m_updateTotal = urls.count();
|
||||
connect(this, &Fetcher::updateProgressChanged, this, &Fetcher::updateMonitor);
|
||||
Q_EMIT updatingChanged(m_updating);
|
||||
Q_EMIT updateProgressChanged(m_updateProgress);
|
||||
Q_EMIT updateTotalChanged(m_updateTotal);
|
||||
|
||||
for (int i = 0; i < urls.count(); i++) {
|
||||
retrieveFeed(urls[i]);
|
||||
}
|
||||
}
|
||||
|
||||
void Fetcher::fetchAll()
|
||||
{
|
||||
QStringList urls;
|
||||
|
|
@ -79,7 +65,6 @@ void Fetcher::fetchAll()
|
|||
Database::instance().execute(query);
|
||||
while (query.next()) {
|
||||
urls += query.value(0).toString();
|
||||
;
|
||||
}
|
||||
|
||||
if (urls.count() > 0) {
|
||||
|
|
@ -87,318 +72,41 @@ void Fetcher::fetchAll()
|
|||
}
|
||||
}
|
||||
|
||||
void Fetcher::retrieveFeed(const QString &url)
|
||||
void Fetcher::fetch(const QStringList &urls)
|
||||
{
|
||||
if (isMeteredConnection() && !SettingsManager::self()->allowMeteredFeedUpdates()) {
|
||||
Q_EMIT error(Error::Type::MeteredNotAllowed, url, QString(), 0, i18n("Podcast updates not allowed due to user setting"), QString());
|
||||
m_updateProgress++;
|
||||
if (m_updating)
|
||||
return; // update is already running, do nothing
|
||||
|
||||
m_updating = true;
|
||||
m_updateProgress = 0;
|
||||
m_updateTotal = urls.count();
|
||||
Q_EMIT updatingChanged(m_updating);
|
||||
Q_EMIT updateProgressChanged(m_updateProgress);
|
||||
Q_EMIT updateTotalChanged(m_updateTotal);
|
||||
|
||||
qCDebug(kastsFetcher) << "Create fetchFeedsJob";
|
||||
FetchFeedsJob *fetchFeedsJob = new FetchFeedsJob(urls, this);
|
||||
connect(this, &Fetcher::cancelFetching, fetchFeedsJob, &FetchFeedsJob::abort);
|
||||
connect(fetchFeedsJob, &FetchFeedsJob::processedAmountChanged, this, [this](KJob *job, KJob::Unit unit, qulonglong amount) {
|
||||
qCDebug(kastsFetcher) << "FetchFeedsJob::processedAmountChanged:" << amount;
|
||||
Q_UNUSED(job);
|
||||
Q_ASSERT(unit == KJob::Unit::Items);
|
||||
m_updateProgress = amount;
|
||||
Q_EMIT updateProgressChanged(m_updateProgress);
|
||||
return;
|
||||
}
|
||||
|
||||
qCDebug(kastsFetcher) << "Starting to fetch" << url;
|
||||
|
||||
Q_EMIT startedFetchingFeed(url);
|
||||
|
||||
QNetworkRequest request((QUrl(url)));
|
||||
request.setTransferTimeout();
|
||||
QNetworkReply *reply = get(request);
|
||||
connect(reply, &QNetworkReply::finished, this, [this, url, reply]() {
|
||||
if (reply->error()) {
|
||||
qWarning() << "Error fetching feed";
|
||||
qWarning() << reply->errorString();
|
||||
Q_EMIT error(Error::Type::FeedUpdate, url, QString(), reply->error(), reply->errorString(), QString());
|
||||
} else {
|
||||
QByteArray data = reply->readAll();
|
||||
Syndication::DocumentSource *document = new Syndication::DocumentSource(data, url);
|
||||
Syndication::FeedPtr feed = Syndication::parserCollection()->parse(*document, QStringLiteral("Atom"));
|
||||
processFeed(feed, url);
|
||||
}
|
||||
m_updateProgress++;
|
||||
Q_EMIT updateProgressChanged(m_updateProgress);
|
||||
delete reply;
|
||||
});
|
||||
}
|
||||
|
||||
void Fetcher::updateMonitor(int progress)
|
||||
{
|
||||
qCDebug(kastsFetcher) << "Update monitor" << progress << "/" << m_updateTotal;
|
||||
// this method will watch for the end of the update process
|
||||
if (progress > -1 && m_updateTotal > -1 && progress == m_updateTotal) {
|
||||
m_updating = false;
|
||||
m_updateProgress = -1;
|
||||
m_updateTotal = -1;
|
||||
disconnect(this, &Fetcher::updateProgressChanged, this, &Fetcher::updateMonitor);
|
||||
Q_EMIT updatingChanged(m_updating);
|
||||
// Q_EMIT updateProgressChanged(m_updateProgress);
|
||||
// Q_EMIT updateTotalChanged(m_updateTotal);
|
||||
}
|
||||
}
|
||||
|
||||
void Fetcher::processFeed(Syndication::FeedPtr feed, const QString &url)
|
||||
{
|
||||
if (feed.isNull())
|
||||
return;
|
||||
|
||||
// First check if this is a newly added feed
|
||||
bool isNewFeed = false;
|
||||
QSqlQuery query;
|
||||
query.prepare(QStringLiteral("SELECT new FROM Feeds WHERE url=:url;"));
|
||||
query.bindValue(QStringLiteral(":url"), url);
|
||||
Database::instance().execute(query);
|
||||
if (query.next()) {
|
||||
isNewFeed = query.value(QStringLiteral("new")).toBool();
|
||||
} else {
|
||||
qCDebug(kastsFetcher) << "Feed not found in database" << url;
|
||||
return;
|
||||
}
|
||||
if (isNewFeed)
|
||||
qCDebug(kastsFetcher) << "New feed" << feed->title() << ":" << isNewFeed;
|
||||
|
||||
// Retrieve "other" fields; this will include the "itunes" tags
|
||||
QMultiMap<QString, QDomElement> otherItems = feed->additionalProperties();
|
||||
|
||||
query.prepare(QStringLiteral("UPDATE Feeds SET name=:name, image=:image, link=:link, description=:description, lastUpdated=:lastUpdated WHERE url=:url;"));
|
||||
query.bindValue(QStringLiteral(":name"), feed->title());
|
||||
query.bindValue(QStringLiteral(":url"), url);
|
||||
query.bindValue(QStringLiteral(":link"), feed->link());
|
||||
query.bindValue(QStringLiteral(":description"), feed->description());
|
||||
|
||||
QDateTime current = QDateTime::currentDateTime();
|
||||
query.bindValue(QStringLiteral(":lastUpdated"), current.toSecsSinceEpoch());
|
||||
|
||||
// Process authors
|
||||
QString authorname, authoremail;
|
||||
if (feed->authors().count() > 0) {
|
||||
for (auto &author : feed->authors()) {
|
||||
processAuthor(url, QLatin1String(""), author->name(), QLatin1String(""), QLatin1String(""));
|
||||
connect(fetchFeedsJob, &FetchFeedsJob::result, this, [this, fetchFeedsJob]() {
|
||||
qCDebug(kastsFetcher) << "result slot of FetchFeedsJob";
|
||||
if (fetchFeedsJob->error()) {
|
||||
Q_EMIT error(Error::Type::FeedUpdate, QString(), QString(), fetchFeedsJob->error(), fetchFeedsJob->errorString(), QString());
|
||||
}
|
||||
} else {
|
||||
// Try to find itunes fields if plain author doesn't exist
|
||||
QString authorname, authoremail;
|
||||
// First try the "itunes:owner" tag, if that doesn't succeed, then try the "itunes:author" tag
|
||||
if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).hasChildNodes()) {
|
||||
QDomNodeList nodelist = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).childNodes();
|
||||
for (int i = 0; i < nodelist.length(); i++) {
|
||||
if (nodelist.item(i).nodeName() == QStringLiteral("itunes:name")) {
|
||||
authorname = nodelist.item(i).toElement().text();
|
||||
} else if (nodelist.item(i).nodeName() == QStringLiteral("itunes:email")) {
|
||||
authoremail = nodelist.item(i).toElement().text();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
authorname = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text();
|
||||
qCDebug(kastsFetcher) << "authorname" << authorname;
|
||||
if (m_updating) {
|
||||
m_updating = false;
|
||||
Q_EMIT updatingChanged(m_updating);
|
||||
}
|
||||
if (!authorname.isEmpty()) {
|
||||
processAuthor(url, QLatin1String(""), authorname, QLatin1String(""), authoremail);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
QString image = feed->image()->url();
|
||||
// If there is no regular image tag, then try the itunes tags
|
||||
if (image.isEmpty()) {
|
||||
if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) {
|
||||
image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href"));
|
||||
}
|
||||
}
|
||||
if (image.startsWith(QStringLiteral("/")))
|
||||
image = QUrl(url).adjusted(QUrl::RemovePath).toString() + image;
|
||||
query.bindValue(QStringLiteral(":image"), image);
|
||||
Database::instance().execute(query);
|
||||
|
||||
qCDebug(kastsFetcher) << "Updated feed details:" << feed->title();
|
||||
|
||||
Q_EMIT feedDetailsUpdated(url, feed->title(), image, feed->link(), feed->description(), current);
|
||||
|
||||
bool updatedEntries = false;
|
||||
for (const auto &entry : feed->items()) {
|
||||
QCoreApplication::processEvents(); // keep the main thread semi-responsive
|
||||
bool isNewEntry = processEntry(entry, url, isNewFeed);
|
||||
updatedEntries = updatedEntries || isNewEntry;
|
||||
}
|
||||
|
||||
// Now mark the appropriate number of recent entries "new" and "read" only for new feeds
|
||||
if (isNewFeed) {
|
||||
query.prepare(QStringLiteral("SELECT * FROM Entries WHERE feed=:feed ORDER BY updated DESC LIMIT :recentNew;"));
|
||||
query.bindValue(QStringLiteral(":feed"), url);
|
||||
query.bindValue(QStringLiteral(":recentNew"), 0); // hardcode to marking no episode as new on a new feed
|
||||
Database::instance().execute(query);
|
||||
QSqlQuery updateQuery;
|
||||
while (query.next()) {
|
||||
qCDebug(kastsFetcher) << "Marked as new:" << query.value(QStringLiteral("id")).toString();
|
||||
updateQuery.prepare(QStringLiteral("UPDATE Entries SET read=:read, new=:new WHERE id=:id AND feed=:feed;"));
|
||||
updateQuery.bindValue(QStringLiteral(":read"), false);
|
||||
updateQuery.bindValue(QStringLiteral(":new"), true);
|
||||
updateQuery.bindValue(QStringLiteral(":feed"), url);
|
||||
updateQuery.bindValue(QStringLiteral(":id"), query.value(QStringLiteral("id")).toString());
|
||||
Database::instance().execute(updateQuery);
|
||||
}
|
||||
// Finally, reset the new flag to false now that the new feed has been fully processed
|
||||
// If we would reset the flag sooner, then too many episodes will get flagged as new if
|
||||
// the initial import gets interrupted somehow.
|
||||
query.prepare(QStringLiteral("UPDATE Feeds SET new=:new WHERE url=:url;"));
|
||||
query.bindValue(QStringLiteral(":url"), url);
|
||||
query.bindValue(QStringLiteral(":new"), false);
|
||||
Database::instance().execute(query);
|
||||
}
|
||||
|
||||
if (updatedEntries || isNewFeed)
|
||||
Q_EMIT feedUpdated(url);
|
||||
Q_EMIT feedUpdateFinished(url);
|
||||
}
|
||||
|
||||
bool Fetcher::processEntry(Syndication::ItemPtr entry, const QString &url, bool isNewFeed)
|
||||
{
|
||||
qCDebug(kastsFetcher) << "Processing" << entry->title();
|
||||
|
||||
// Retrieve "other" fields; this will include the "itunes" tags
|
||||
QMultiMap<QString, QDomElement> otherItems = entry->additionalProperties();
|
||||
|
||||
QSqlQuery query;
|
||||
query.prepare(QStringLiteral("SELECT COUNT (id) FROM Entries WHERE id=:id;"));
|
||||
query.bindValue(QStringLiteral(":id"), entry->id());
|
||||
Database::instance().execute(query);
|
||||
query.next();
|
||||
|
||||
if (query.value(0).toInt() != 0)
|
||||
return false; // entry already exists
|
||||
|
||||
query.prepare(QStringLiteral("INSERT INTO Entries VALUES (:feed, :id, :title, :content, :created, :updated, :link, :read, :new, :hasEnclosure, :image);"));
|
||||
query.bindValue(QStringLiteral(":feed"), url);
|
||||
query.bindValue(QStringLiteral(":id"), entry->id());
|
||||
query.bindValue(QStringLiteral(":title"), QTextDocumentFragment::fromHtml(entry->title()).toPlainText());
|
||||
query.bindValue(QStringLiteral(":created"), static_cast<int>(entry->datePublished()));
|
||||
query.bindValue(QStringLiteral(":updated"), static_cast<int>(entry->dateUpdated()));
|
||||
query.bindValue(QStringLiteral(":link"), entry->link());
|
||||
query.bindValue(QStringLiteral(":hasEnclosure"), entry->enclosures().length() == 0 ? 0 : 1);
|
||||
query.bindValue(QStringLiteral(":read"), isNewFeed); // if new feed, then mark all as read
|
||||
query.bindValue(QStringLiteral(":new"), !isNewFeed); // if new feed, then mark none as new
|
||||
|
||||
if (!entry->content().isEmpty())
|
||||
query.bindValue(QStringLiteral(":content"), entry->content());
|
||||
else
|
||||
query.bindValue(QStringLiteral(":content"), entry->description());
|
||||
|
||||
// Look for image in itunes tags
|
||||
QString image;
|
||||
if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) {
|
||||
image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href"));
|
||||
}
|
||||
if (image.startsWith(QStringLiteral("/")))
|
||||
image = QUrl(url).adjusted(QUrl::RemovePath).toString() + image;
|
||||
query.bindValue(QStringLiteral(":image"), image);
|
||||
qCDebug(kastsFetcher) << "Entry image found" << image;
|
||||
|
||||
Database::instance().execute(query);
|
||||
|
||||
if (entry->authors().count() > 0) {
|
||||
for (const auto &author : entry->authors()) {
|
||||
processAuthor(url, entry->id(), author->name(), author->uri(), author->email());
|
||||
}
|
||||
} else {
|
||||
// As fallback, check if there is itunes "author" information
|
||||
QString authorName = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text();
|
||||
if (!authorName.isEmpty())
|
||||
processAuthor(url, entry->id(), authorName, QLatin1String(""), QLatin1String(""));
|
||||
}
|
||||
|
||||
/* Process chapters */
|
||||
if (otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).hasChildNodes()) {
|
||||
QDomNodeList nodelist = otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).childNodes();
|
||||
for (int i = 0; i < nodelist.length(); i++) {
|
||||
if (nodelist.item(i).nodeName() == QStringLiteral("psc:chapter")) {
|
||||
QDomElement element = nodelist.at(i).toElement();
|
||||
QString title = element.attribute(QStringLiteral("title"));
|
||||
QString start = element.attribute(QStringLiteral("start"));
|
||||
QTime startString = QTime::fromString(start, QStringLiteral("hh:mm:ss.zzz"));
|
||||
int startInt = startString.hour() * 60 * 60 + startString.minute() * 60 + startString.second();
|
||||
QString images = element.attribute(QStringLiteral("image"));
|
||||
processChapter(url, entry->id(), startInt, title, entry->link(), images);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// only process first enclosure if there are multiple (e.g. mp3 and ogg);
|
||||
// the first one is probably the podcast author's preferred version
|
||||
// TODO: handle more than one enclosure?
|
||||
if (entry->enclosures().count() > 0) {
|
||||
processEnclosure(entry->enclosures()[0], entry, url);
|
||||
}
|
||||
|
||||
Q_EMIT entryAdded(url, entry->id());
|
||||
return true; // this is a new entry
|
||||
}
|
||||
|
||||
void Fetcher::processAuthor(const QString &url, const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail)
|
||||
{
|
||||
QSqlQuery query;
|
||||
query.prepare(QStringLiteral("SELECT COUNT (id) FROM Authors WHERE feed=:feed AND id=:id AND name=:name;"));
|
||||
query.bindValue(QStringLiteral(":feed"), url);
|
||||
query.bindValue(QStringLiteral(":id"), entryId);
|
||||
query.bindValue(QStringLiteral(":name"), authorName);
|
||||
Database::instance().execute(query);
|
||||
query.next();
|
||||
|
||||
if (query.value(0).toInt() != 0)
|
||||
query.prepare(QStringLiteral("UPDATE Authors SET feed=:feed, id=:id, name=:name, uri=:uri, email=:email WHERE feed=:feed AND id=:id;"));
|
||||
else
|
||||
query.prepare(QStringLiteral("INSERT INTO Authors VALUES(:feed, :id, :name, :uri, :email);"));
|
||||
|
||||
query.bindValue(QStringLiteral(":feed"), url);
|
||||
query.bindValue(QStringLiteral(":id"), entryId);
|
||||
query.bindValue(QStringLiteral(":name"), authorName);
|
||||
query.bindValue(QStringLiteral(":uri"), authorUri);
|
||||
query.bindValue(QStringLiteral(":email"), authorEmail);
|
||||
Database::instance().execute(query);
|
||||
}
|
||||
|
||||
void Fetcher::processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry, const QString &feedUrl)
|
||||
{
|
||||
QSqlQuery query;
|
||||
query.prepare(QStringLiteral("SELECT COUNT (id) FROM Enclosures WHERE feed=:feed AND id=:id;"));
|
||||
query.bindValue(QStringLiteral(":feed"), feedUrl);
|
||||
query.bindValue(QStringLiteral(":id"), entry->id());
|
||||
Database::instance().execute(query);
|
||||
query.next();
|
||||
|
||||
if (query.value(0).toInt() != 0)
|
||||
query.prepare(QStringLiteral(
|
||||
"UPDATE Enclosures SET feed=:feed, id=:id, duration=:duration, size=:size, title=:title, type=:type, url=:url WHERE feed=:feed AND id=:id;"));
|
||||
// NOTE: In case more than one enclosure is present per episode, only
|
||||
// the last one will end up in the database
|
||||
else
|
||||
query.prepare(QStringLiteral("INSERT INTO Enclosures VALUES (:feed, :id, :duration, :size, :title, :type, :url, :playposition, :downloaded);"));
|
||||
|
||||
query.bindValue(QStringLiteral(":feed"), feedUrl);
|
||||
query.bindValue(QStringLiteral(":id"), entry->id());
|
||||
query.bindValue(QStringLiteral(":duration"), enclosure->duration());
|
||||
query.bindValue(QStringLiteral(":size"), enclosure->length());
|
||||
query.bindValue(QStringLiteral(":title"), enclosure->title());
|
||||
query.bindValue(QStringLiteral(":type"), enclosure->type());
|
||||
query.bindValue(QStringLiteral(":url"), enclosure->url());
|
||||
query.bindValue(QStringLiteral(":playposition"), 0);
|
||||
query.bindValue(QStringLiteral(":downloaded"), Enclosure::statusToDb(Enclosure::Downloadable));
|
||||
Database::instance().execute(query);
|
||||
}
|
||||
|
||||
void Fetcher::processChapter(const QString &url,
|
||||
const QString &entryId,
|
||||
const int &start,
|
||||
const QString &chapterTitle,
|
||||
const QString &link,
|
||||
const QString &image)
|
||||
{
|
||||
QSqlQuery query;
|
||||
|
||||
query.prepare(QStringLiteral("INSERT INTO Chapters VALUES(:feed, :id, :start, :title, :link, :image);"));
|
||||
query.bindValue(QStringLiteral(":feed"), url);
|
||||
query.bindValue(QStringLiteral(":id"), entryId);
|
||||
query.bindValue(QStringLiteral(":start"), start);
|
||||
query.bindValue(QStringLiteral(":title"), chapterTitle);
|
||||
query.bindValue(QStringLiteral(":link"), link);
|
||||
query.bindValue(QStringLiteral(":image"), image);
|
||||
Database::instance().execute(query);
|
||||
fetchFeedsJob->start();
|
||||
qCDebug(kastsFetcher) << "end of Fetcher::fetch";
|
||||
}
|
||||
|
||||
QString Fetcher::image(const QString &url) const
|
||||
|
|
@ -449,17 +157,6 @@ QNetworkReply *Fetcher::download(const QString &url, const QString &filePath) co
|
|||
file->open(QIODevice::WriteOnly);
|
||||
}
|
||||
|
||||
/*
|
||||
QNetworkReply *headerReply = head(request);
|
||||
connect(headerReply, &QNetworkReply::finished, this, [=]() {
|
||||
if (headerReply->isOpen()) {
|
||||
int fileSize = headerReply->header(QNetworkRequest::ContentLengthHeader).toInt();
|
||||
qCDebug(kastsFetcher) << "Reported download size" << fileSize;
|
||||
}
|
||||
headerReply->deleteLater();
|
||||
});
|
||||
*/
|
||||
|
||||
QNetworkReply *reply = get(request);
|
||||
|
||||
connect(reply, &QNetworkReply::readyRead, this, [=]() {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
#include <QUrl>
|
||||
#include <Syndication/Syndication>
|
||||
|
||||
#include "enclosure.h"
|
||||
#include "error.h"
|
||||
|
||||
#if !defined Q_OS_ANDROID && !defined Q_OS_WIN
|
||||
|
|
@ -49,7 +50,7 @@ public:
|
|||
Q_INVOKABLE bool isMeteredConnection() const;
|
||||
|
||||
Q_SIGNALS:
|
||||
void startedFetchingFeed(const QString &url);
|
||||
void entryAdded(const QString &feedurl, const QString &id);
|
||||
void feedUpdated(const QString &url);
|
||||
void feedDetailsUpdated(const QString &url,
|
||||
const QString &name,
|
||||
|
|
@ -57,28 +58,19 @@ Q_SIGNALS:
|
|||
const QString &link,
|
||||
const QString &description,
|
||||
const QDateTime &lastUpdated);
|
||||
void feedUpdateFinished(const QString &url);
|
||||
void error(Error::Type type, const QString &url, const QString &id, const int errorId, const QString &errorString, const QString &title);
|
||||
void entryAdded(const QString &feedurl, const QString &id);
|
||||
void downloadFinished(QString url) const;
|
||||
void feedUpdateStatusChanged(const QString &url, bool status);
|
||||
void cancelFetching();
|
||||
|
||||
void updateProgressChanged(int progress);
|
||||
void updateTotalChanged(int nrOfFeeds);
|
||||
void updatingChanged(bool state);
|
||||
|
||||
private Q_SLOTS:
|
||||
void updateMonitor(int progress);
|
||||
void error(Error::Type type, const QString &url, const QString &id, const int errorId, const QString &errorString, const QString &title);
|
||||
void downloadFinished(QString url) const;
|
||||
|
||||
private:
|
||||
Fetcher();
|
||||
|
||||
void retrieveFeed(const QString &url);
|
||||
void processFeed(Syndication::FeedPtr feed, const QString &url);
|
||||
bool processEntry(Syndication::ItemPtr entry, const QString &url, bool isNewFeed); // returns true if this is a new entry; false if it already existed
|
||||
void processAuthor(const QString &url, const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail);
|
||||
void processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry, const QString &feedUrl);
|
||||
void processChapter(const QString &url, const QString &entryId, const int &start, const QString &chapterTitle, const QString &link, const QString &image);
|
||||
|
||||
QNetworkReply *head(QNetworkRequest &request) const;
|
||||
void setHeader(QNetworkRequest &request) const;
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,71 @@
|
|||
/**
|
||||
* SPDX-FileCopyrightText: 2021 Bart De Vries <bart@mogwai.be>
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
|
||||
*/
|
||||
|
||||
#include "fetchfeedsjob.h"
|
||||
|
||||
#include <QTimer>
|
||||
|
||||
#include <KLocalizedString>
|
||||
|
||||
#include "error.h"
|
||||
#include "fetcher.h"
|
||||
#include "fetcherlogging.h"
|
||||
#include "models/errorlogmodel.h"
|
||||
#include "settingsmanager.h"
|
||||
#include "updatefeedjob.h"
|
||||
|
||||
FetchFeedsJob::FetchFeedsJob(const QStringList &urls, QObject *parent)
|
||||
: KJob(parent)
|
||||
, m_urls(urls)
|
||||
{
|
||||
for (int i = 0; i < m_urls.count(); i++) {
|
||||
m_feedjobs += nullptr;
|
||||
}
|
||||
connect(this, &FetchFeedsJob::processedAmountChanged, this, &FetchFeedsJob::monitorProgress);
|
||||
connect(this, &FetchFeedsJob::logError, &ErrorLogModel::instance(), &ErrorLogModel::monitorErrorMessages);
|
||||
}
|
||||
|
||||
void FetchFeedsJob::start()
|
||||
{
|
||||
QTimer::singleShot(0, this, &FetchFeedsJob::fetch);
|
||||
}
|
||||
|
||||
void FetchFeedsJob::fetch()
|
||||
{
|
||||
if (Fetcher::instance().isMeteredConnection() && !SettingsManager::self()->allowMeteredFeedUpdates()) {
|
||||
setError(0);
|
||||
setErrorText(i18n("Podcast updates not allowed due to user setting"));
|
||||
emitResult();
|
||||
return;
|
||||
}
|
||||
|
||||
setTotalAmount(KJob::Unit::Items, m_urls.count());
|
||||
setProcessedAmount(KJob::Unit::Items, 0);
|
||||
|
||||
for (int i = 0; i < m_urls.count(); i++) {
|
||||
QString url = m_urls[i];
|
||||
|
||||
UpdateFeedJob *updateFeedJob = new UpdateFeedJob(url, this);
|
||||
m_feedjobs[i] = updateFeedJob;
|
||||
connect(this, &FetchFeedsJob::abort, updateFeedJob, &UpdateFeedJob::abort);
|
||||
connect(updateFeedJob, &UpdateFeedJob::result, this, [this, url, updateFeedJob]() {
|
||||
if (updateFeedJob->error()) {
|
||||
Q_EMIT logError(Error::Type::FeedUpdate, url, QString(), updateFeedJob->error(), updateFeedJob->errorString(), QString());
|
||||
}
|
||||
setProcessedAmount(KJob::Unit::Items, processedAmount(KJob::Unit::Items) + 1);
|
||||
});
|
||||
updateFeedJob->start();
|
||||
qCDebug(kastsFetcher) << "Just started updateFeedJob" << i + 1;
|
||||
}
|
||||
qCDebug(kastsFetcher) << "End of FetchFeedsJob::fetch";
|
||||
}
|
||||
|
||||
void FetchFeedsJob::monitorProgress()
|
||||
{
|
||||
if (processedAmount(KJob::Unit::Items) == totalAmount(KJob::Unit::Items)) {
|
||||
emitResult();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/**
|
||||
* SPDX-FileCopyrightText: 2021 Bart De Vries <bart@mogwai.be>
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <KJob>
|
||||
#include <QVector>
|
||||
|
||||
#include "error.h"
|
||||
#include "updatefeedjob.h"
|
||||
|
||||
class FetchFeedsJob : public KJob
|
||||
{
|
||||
Q_OBJECT
|
||||
|
||||
public:
|
||||
explicit FetchFeedsJob(const QStringList &urls, QObject *parent = nullptr);
|
||||
|
||||
void start() override;
|
||||
|
||||
Q_SIGNALS:
|
||||
void abort();
|
||||
void logError(Error::Type type, const QString &url, const QString &id, const int errorId, const QString &errorString, const QString &title);
|
||||
|
||||
private:
|
||||
QStringList m_urls;
|
||||
|
||||
void fetch();
|
||||
void monitorProgress();
|
||||
|
||||
bool m_abort = false;
|
||||
QVector<UpdateFeedJob *> m_feedjobs;
|
||||
};
|
||||
|
|
@ -132,8 +132,8 @@ int main(int argc, char *argv[])
|
|||
qmlRegisterUncreatableType<FeedsModel>("org.kde.kasts", 1, 0, "FeedsModel", QStringLiteral("Only for enums"));
|
||||
|
||||
qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "AboutType", &AboutType::instance());
|
||||
qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "Fetcher", &Fetcher::instance());
|
||||
qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "Database", &Database::instance());
|
||||
qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "Fetcher", &Fetcher::instance());
|
||||
qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "DataManager", &DataManager::instance());
|
||||
qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "SettingsManager", SettingsManager::self());
|
||||
qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "DownloadModel", &DownloadModel::instance());
|
||||
|
|
|
|||
|
|
@ -6,19 +6,15 @@
|
|||
|
||||
#include "models/errorlogmodel.h"
|
||||
|
||||
#include <QDebug>
|
||||
#include <QSqlQuery>
|
||||
|
||||
#include "database.h"
|
||||
#include "datamanager.h"
|
||||
#include "fetcher.h"
|
||||
#include "storagemanager.h"
|
||||
|
||||
ErrorLogModel::ErrorLogModel()
|
||||
: QAbstractListModel(nullptr)
|
||||
{
|
||||
connect(&Fetcher::instance(), &Fetcher::error, this, &ErrorLogModel::monitorErrorMessages);
|
||||
connect(&StorageManager::instance(), &StorageManager::error, this, &ErrorLogModel::monitorErrorMessages);
|
||||
|
||||
QSqlQuery query;
|
||||
query.prepare(QStringLiteral("SELECT * FROM Errors ORDER BY date DESC;"));
|
||||
Database::instance().execute(query);
|
||||
|
|
|
|||
|
|
@ -234,6 +234,12 @@ Kirigami.ApplicationWindow {
|
|||
Fetcher.updateTotal,
|
||||
Fetcher.updateProgress)
|
||||
|
||||
showAbortButton: true
|
||||
|
||||
function abortAction() {
|
||||
Fetcher.cancelFetching();
|
||||
}
|
||||
|
||||
Connections {
|
||||
target: Fetcher
|
||||
function onUpdatingChanged() {
|
||||
|
|
|
|||
|
|
@ -17,11 +17,13 @@
|
|||
#include <QStandardPaths>
|
||||
|
||||
#include "enclosure.h"
|
||||
#include "models/errorlogmodel.h"
|
||||
#include "settingsmanager.h"
|
||||
#include "storagemovejob.h"
|
||||
|
||||
StorageManager::StorageManager()
|
||||
{
|
||||
connect(this, &StorageManager::error, &ErrorLogModel::instance(), &ErrorLogModel::monitorErrorMessages);
|
||||
}
|
||||
|
||||
QString StorageManager::storagePath() const
|
||||
|
|
|
|||
|
|
@ -0,0 +1,423 @@
|
|||
/**
|
||||
* SPDX-FileCopyrightText: 2021 Bart De Vries <bart@mogwai.be>
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
|
||||
*/
|
||||
|
||||
#include "updatefeedjob.h"
|
||||
|
||||
#include <QDomElement>
|
||||
#include <QMultiMap>
|
||||
#include <QNetworkReply>
|
||||
#include <QSqlQuery>
|
||||
#include <QTextDocumentFragment>
|
||||
#include <QTimer>
|
||||
|
||||
#include <KLocalizedString>
|
||||
|
||||
#include "database.h"
|
||||
#include "enclosure.h"
|
||||
#include "fetcher.h"
|
||||
#include "fetcherlogging.h"
|
||||
#include "settingsmanager.h"
|
||||
|
||||
UpdateFeedJob::UpdateFeedJob(const QString &url, QObject *parent)
|
||||
: KJob(parent)
|
||||
, m_url(url)
|
||||
{
|
||||
// connect to signals in Fetcher such that GUI can pick up the changes
|
||||
connect(this, &UpdateFeedJob::feedDetailsUpdated, &Fetcher::instance(), &Fetcher::feedDetailsUpdated);
|
||||
connect(this, &UpdateFeedJob::feedUpdated, &Fetcher::instance(), &Fetcher::feedUpdated);
|
||||
connect(this, &UpdateFeedJob::entryAdded, &Fetcher::instance(), &Fetcher::entryAdded);
|
||||
connect(this, &UpdateFeedJob::feedUpdateStatusChanged, &Fetcher::instance(), &Fetcher::feedUpdateStatusChanged);
|
||||
}
|
||||
|
||||
void UpdateFeedJob::start()
|
||||
{
|
||||
QTimer::singleShot(0, this, &UpdateFeedJob::retrieveFeed);
|
||||
}
|
||||
|
||||
void UpdateFeedJob::retrieveFeed()
|
||||
{
|
||||
if (m_abort)
|
||||
return;
|
||||
|
||||
qCDebug(kastsFetcher) << "Starting to fetch" << m_url;
|
||||
Q_EMIT feedUpdateStatusChanged(m_url, true);
|
||||
|
||||
QNetworkRequest request((QUrl(m_url)));
|
||||
request.setTransferTimeout();
|
||||
m_reply = Fetcher::instance().get(request);
|
||||
connect(this, &UpdateFeedJob::aborting, m_reply, &QNetworkReply::abort);
|
||||
connect(m_reply, &QNetworkReply::finished, this, [this]() {
|
||||
qCDebug(kastsFetcher) << "got networkreply for" << m_reply;
|
||||
if (m_reply->error()) {
|
||||
qWarning() << "Error fetching feed";
|
||||
qWarning() << m_reply->errorString();
|
||||
setError(m_reply->error());
|
||||
setErrorText(m_reply->errorString());
|
||||
} else {
|
||||
QByteArray data = m_reply->readAll();
|
||||
Syndication::DocumentSource *document = new Syndication::DocumentSource(data, m_url);
|
||||
Syndication::FeedPtr feed = Syndication::parserCollection()->parse(*document, QStringLiteral("Atom"));
|
||||
processFeed(feed);
|
||||
}
|
||||
Q_EMIT feedUpdateStatusChanged(m_url, false);
|
||||
m_reply->deleteLater();
|
||||
emitResult();
|
||||
});
|
||||
qCDebug(kastsFetcher) << "End of retrieveFeed for" << m_url;
|
||||
}
|
||||
|
||||
void UpdateFeedJob::processFeed(Syndication::FeedPtr feed)
|
||||
{
|
||||
qCDebug(kastsFetcher) << "start process feed" << feed;
|
||||
|
||||
if (feed.isNull())
|
||||
return;
|
||||
|
||||
// First check if this is a newly added feed
|
||||
m_isNewFeed = false;
|
||||
QSqlQuery query;
|
||||
query.prepare(QStringLiteral("SELECT new FROM Feeds WHERE url=:url;"));
|
||||
query.bindValue(QStringLiteral(":url"), m_url);
|
||||
Database::instance().execute(query);
|
||||
if (query.next()) {
|
||||
m_isNewFeed = query.value(QStringLiteral("new")).toBool();
|
||||
} else {
|
||||
qCDebug(kastsFetcher) << "Feed not found in database" << m_url;
|
||||
return;
|
||||
}
|
||||
if (m_isNewFeed)
|
||||
qCDebug(kastsFetcher) << "New feed" << feed->title();
|
||||
|
||||
// Retrieve "other" fields; this will include the "itunes" tags
|
||||
QMultiMap<QString, QDomElement> otherItems = feed->additionalProperties();
|
||||
|
||||
query.prepare(QStringLiteral("UPDATE Feeds SET name=:name, image=:image, link=:link, description=:description, lastUpdated=:lastUpdated WHERE url=:url;"));
|
||||
query.bindValue(QStringLiteral(":name"), feed->title());
|
||||
query.bindValue(QStringLiteral(":url"), m_url);
|
||||
query.bindValue(QStringLiteral(":link"), feed->link());
|
||||
query.bindValue(QStringLiteral(":description"), feed->description());
|
||||
|
||||
QDateTime current = QDateTime::currentDateTime();
|
||||
query.bindValue(QStringLiteral(":lastUpdated"), current.toSecsSinceEpoch());
|
||||
|
||||
QString image = feed->image()->url();
|
||||
// If there is no regular image tag, then try the itunes tags
|
||||
if (image.isEmpty()) {
|
||||
if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) {
|
||||
image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href"));
|
||||
}
|
||||
}
|
||||
if (image.startsWith(QStringLiteral("/")))
|
||||
image = QUrl(m_url).adjusted(QUrl::RemovePath).toString() + image;
|
||||
query.bindValue(QStringLiteral(":image"), image);
|
||||
|
||||
// Do the actual database UPDATE of this feed
|
||||
Database::instance().execute(query);
|
||||
|
||||
// Now that we have the feed details, we make vectors of the data that's
|
||||
// already in the database relating to this feed
|
||||
// NOTE: We will do the feed authors after this step, because otherwise
|
||||
// we can't check for duplicates and we'll keep adding more of the same!
|
||||
query.prepare(QStringLiteral("SELECT id FROM Entries WHERE feed=:feed;"));
|
||||
query.bindValue(QStringLiteral(":feed"), m_url);
|
||||
Database::instance().execute(query);
|
||||
while (query.next()) {
|
||||
m_existingEntryIds += query.value(QStringLiteral("id")).toString();
|
||||
}
|
||||
|
||||
query.prepare(QStringLiteral("SELECT id, url FROM Enclosures WHERE feed=:feed;"));
|
||||
query.bindValue(QStringLiteral(":feed"), m_url);
|
||||
Database::instance().execute(query);
|
||||
while (query.next()) {
|
||||
m_existingEnclosures += qMakePair(query.value(QStringLiteral("id")).toString(), query.value(QStringLiteral("url")).toString());
|
||||
}
|
||||
|
||||
query.prepare(QStringLiteral("SELECT id, name FROM Authors WHERE feed=:feed;"));
|
||||
query.bindValue(QStringLiteral(":feed"), m_url);
|
||||
Database::instance().execute(query);
|
||||
while (query.next()) {
|
||||
m_existingAuthors += qMakePair(query.value(QStringLiteral("id")).toString(), query.value(QStringLiteral("name")).toString());
|
||||
}
|
||||
|
||||
query.prepare(QStringLiteral("SELECT id, start FROM Chapters WHERE feed=:feed;"));
|
||||
query.bindValue(QStringLiteral(":feed"), m_url);
|
||||
Database::instance().execute(query);
|
||||
while (query.next()) {
|
||||
m_existingChapters += qMakePair(query.value(QStringLiteral("id")).toString(), query.value(QStringLiteral("start")).toInt());
|
||||
}
|
||||
|
||||
// Process feed authors
|
||||
QString authorname, authoremail;
|
||||
if (feed->authors().count() > 0) {
|
||||
for (auto &author : feed->authors()) {
|
||||
processAuthor(QLatin1String(""), author->name(), QLatin1String(""), QLatin1String(""));
|
||||
}
|
||||
} else {
|
||||
// Try to find itunes fields if plain author doesn't exist
|
||||
QString authorname, authoremail;
|
||||
// First try the "itunes:owner" tag, if that doesn't succeed, then try the "itunes:author" tag
|
||||
if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).hasChildNodes()) {
|
||||
QDomNodeList nodelist = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).childNodes();
|
||||
for (int i = 0; i < nodelist.length(); i++) {
|
||||
if (nodelist.item(i).nodeName() == QStringLiteral("itunes:name")) {
|
||||
authorname = nodelist.item(i).toElement().text();
|
||||
} else if (nodelist.item(i).nodeName() == QStringLiteral("itunes:email")) {
|
||||
authoremail = nodelist.item(i).toElement().text();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
authorname = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text();
|
||||
qCDebug(kastsFetcher) << "authorname" << authorname;
|
||||
}
|
||||
if (!authorname.isEmpty()) {
|
||||
processAuthor(QLatin1String(""), authorname, QLatin1String(""), authoremail);
|
||||
}
|
||||
}
|
||||
|
||||
qCDebug(kastsFetcher) << "Updated feed details:" << feed->title();
|
||||
|
||||
// TODO: Only emit signal if the details have really changed
|
||||
Q_EMIT feedDetailsUpdated(m_url, feed->title(), image, feed->link(), feed->description(), current);
|
||||
|
||||
if (m_abort)
|
||||
return;
|
||||
|
||||
// Now deal with the entries, enclosures, entry authors and chapter marks
|
||||
bool updatedEntries = false;
|
||||
for (const auto &entry : feed->items()) {
|
||||
if (m_abort)
|
||||
return;
|
||||
QCoreApplication::processEvents(); // keep the main thread semi-responsive
|
||||
bool isNewEntry = processEntry(entry);
|
||||
updatedEntries = updatedEntries || isNewEntry;
|
||||
}
|
||||
|
||||
writeToDatabase();
|
||||
|
||||
if (m_isNewFeed) {
|
||||
// Finally, reset the new flag to false now that the new feed has been
|
||||
// fully processed. If we would reset the flag sooner, then too many
|
||||
// episodes will get flagged as new if the initial import gets
|
||||
// interrupted somehow.
|
||||
query.prepare(QStringLiteral("UPDATE Feeds SET new=:new WHERE url=:url;"));
|
||||
query.bindValue(QStringLiteral(":url"), m_url);
|
||||
query.bindValue(QStringLiteral(":new"), false);
|
||||
Database::instance().execute(query);
|
||||
}
|
||||
|
||||
if (updatedEntries || m_isNewFeed)
|
||||
Q_EMIT feedUpdated(m_url);
|
||||
qCDebug(kastsFetcher) << "done processing feed" << feed;
|
||||
}
|
||||
|
||||
bool UpdateFeedJob::processEntry(Syndication::ItemPtr entry)
|
||||
{
|
||||
qCDebug(kastsFetcher) << "Processing" << entry->title();
|
||||
|
||||
// Retrieve "other" fields; this will include the "itunes" tags
|
||||
QMultiMap<QString, QDomElement> otherItems = entry->additionalProperties();
|
||||
|
||||
// check against existing entries in database
|
||||
if (m_existingEntryIds.contains(entry->id()))
|
||||
return false;
|
||||
|
||||
// also check against the list of new entries
|
||||
for (EntryDetails entryDetails : m_entries) {
|
||||
if (entryDetails.id == entry->id())
|
||||
return false; // entry already exists
|
||||
}
|
||||
|
||||
EntryDetails entryDetails;
|
||||
entryDetails.feed = m_url;
|
||||
entryDetails.id = entry->id();
|
||||
entryDetails.title = QTextDocumentFragment::fromHtml(entry->title()).toPlainText();
|
||||
entryDetails.created = static_cast<int>(entry->datePublished());
|
||||
entryDetails.updated = static_cast<int>(entry->dateUpdated());
|
||||
entryDetails.link = entry->link();
|
||||
entryDetails.hasEnclosure = (entry->enclosures().length() > 0);
|
||||
entryDetails.read = m_isNewFeed; // if new feed, then mark all as read
|
||||
entryDetails.isNew = !m_isNewFeed; // if new feed, then mark none as new
|
||||
|
||||
if (!entry->content().isEmpty())
|
||||
entryDetails.content = entry->content();
|
||||
else
|
||||
entryDetails.content = entry->description();
|
||||
|
||||
// Look for image in itunes tags
|
||||
if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) {
|
||||
entryDetails.image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href"));
|
||||
}
|
||||
if (entryDetails.image.startsWith(QStringLiteral("/")))
|
||||
entryDetails.image = QUrl(m_url).adjusted(QUrl::RemovePath).toString() + entryDetails.image;
|
||||
qCDebug(kastsFetcher) << "Entry image found" << entryDetails.image;
|
||||
|
||||
m_entries += entryDetails;
|
||||
|
||||
// Process authors
|
||||
if (entry->authors().count() > 0) {
|
||||
for (const auto &author : entry->authors()) {
|
||||
processAuthor(entry->id(), author->name(), author->uri(), author->email());
|
||||
}
|
||||
} else {
|
||||
// As fallback, check if there is itunes "author" information
|
||||
QString authorName = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text();
|
||||
if (!authorName.isEmpty())
|
||||
processAuthor(entry->id(), authorName, QLatin1String(""), QLatin1String(""));
|
||||
}
|
||||
|
||||
// Process chapters
|
||||
if (otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).hasChildNodes()) {
|
||||
QDomNodeList nodelist = otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).childNodes();
|
||||
for (int i = 0; i < nodelist.length(); i++) {
|
||||
if (nodelist.item(i).nodeName() == QStringLiteral("psc:chapter")) {
|
||||
QDomElement element = nodelist.at(i).toElement();
|
||||
QString title = element.attribute(QStringLiteral("title"));
|
||||
QString start = element.attribute(QStringLiteral("start"));
|
||||
QTime startString = QTime::fromString(start, QStringLiteral("hh:mm:ss.zzz"));
|
||||
int startInt = startString.hour() * 60 * 60 + startString.minute() * 60 + startString.second();
|
||||
QString images = element.attribute(QStringLiteral("image"));
|
||||
processChapter(entry->id(), startInt, title, entry->link(), images);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process enclosures
|
||||
// only process first enclosure if there are multiple (e.g. mp3 and ogg);
|
||||
// the first one is probably the podcast author's preferred version
|
||||
// TODO: handle more than one enclosure?
|
||||
if (entry->enclosures().count() > 0) {
|
||||
processEnclosure(entry->enclosures()[0], entry);
|
||||
}
|
||||
|
||||
return true; // this is a new entry
|
||||
}
|
||||
|
||||
void UpdateFeedJob::processAuthor(const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail)
|
||||
{
|
||||
// check against existing authors already in database
|
||||
if (m_existingAuthors.contains(qMakePair(entryId, authorName)))
|
||||
return;
|
||||
|
||||
AuthorDetails authorDetails;
|
||||
authorDetails.feed = m_url;
|
||||
authorDetails.id = entryId;
|
||||
authorDetails.name = authorName;
|
||||
authorDetails.uri = authorUri;
|
||||
authorDetails.email = authorEmail;
|
||||
m_authors += authorDetails;
|
||||
}
|
||||
|
||||
void UpdateFeedJob::processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry)
|
||||
{
|
||||
// check against existing enclosures already in database
|
||||
if (m_existingEnclosures.contains(qMakePair(entry->id(), enclosure->url())))
|
||||
return;
|
||||
|
||||
EnclosureDetails enclosureDetails;
|
||||
enclosureDetails.feed = m_url;
|
||||
enclosureDetails.id = entry->id();
|
||||
enclosureDetails.duration = enclosure->duration();
|
||||
enclosureDetails.size = enclosure->length();
|
||||
enclosureDetails.title = enclosure->title();
|
||||
enclosureDetails.type = enclosure->type();
|
||||
enclosureDetails.url = enclosure->url();
|
||||
enclosureDetails.playPosition = 0;
|
||||
enclosureDetails.downloaded = Enclosure::Downloadable;
|
||||
|
||||
m_enclosures += enclosureDetails;
|
||||
}
|
||||
|
||||
void UpdateFeedJob::processChapter(const QString &entryId, const int &start, const QString &chapterTitle, const QString &link, const QString &image)
|
||||
{
|
||||
// check against existing enclosures already in database
|
||||
if (m_existingChapters.contains(qMakePair(entryId, start)))
|
||||
return;
|
||||
|
||||
ChapterDetails chapterDetails;
|
||||
chapterDetails.feed = m_url;
|
||||
chapterDetails.id = entryId;
|
||||
chapterDetails.start = start;
|
||||
chapterDetails.title = chapterTitle;
|
||||
chapterDetails.link = link;
|
||||
chapterDetails.image = image;
|
||||
|
||||
m_chapters += chapterDetails;
|
||||
}
|
||||
|
||||
void UpdateFeedJob::writeToDatabase()
|
||||
{
|
||||
QSqlQuery writeQuery;
|
||||
|
||||
Database::instance().transaction();
|
||||
|
||||
// Entries
|
||||
writeQuery.prepare(
|
||||
QStringLiteral("INSERT INTO Entries VALUES (:feed, :id, :title, :content, :created, :updated, :link, :read, :new, :hasEnclosure, :image);"));
|
||||
for (EntryDetails entryDetails : m_entries) {
|
||||
writeQuery.bindValue(QStringLiteral(":feed"), entryDetails.feed);
|
||||
writeQuery.bindValue(QStringLiteral(":id"), entryDetails.id);
|
||||
writeQuery.bindValue(QStringLiteral(":title"), entryDetails.title);
|
||||
writeQuery.bindValue(QStringLiteral(":content"), entryDetails.content);
|
||||
writeQuery.bindValue(QStringLiteral(":created"), entryDetails.created);
|
||||
writeQuery.bindValue(QStringLiteral(":updated"), entryDetails.updated);
|
||||
writeQuery.bindValue(QStringLiteral(":link"), entryDetails.link);
|
||||
writeQuery.bindValue(QStringLiteral(":hasEnclosure"), entryDetails.hasEnclosure);
|
||||
writeQuery.bindValue(QStringLiteral(":read"), entryDetails.read);
|
||||
writeQuery.bindValue(QStringLiteral(":new"), entryDetails.isNew);
|
||||
writeQuery.bindValue(QStringLiteral(":image"), entryDetails.image);
|
||||
Database::instance().execute(writeQuery);
|
||||
}
|
||||
|
||||
// Authors
|
||||
writeQuery.prepare(QStringLiteral("INSERT INTO Authors VALUES(:feed, :id, :name, :uri, :email);"));
|
||||
for (AuthorDetails authorDetails : m_authors) {
|
||||
writeQuery.bindValue(QStringLiteral(":feed"), authorDetails.feed);
|
||||
writeQuery.bindValue(QStringLiteral(":id"), authorDetails.id);
|
||||
writeQuery.bindValue(QStringLiteral(":name"), authorDetails.name);
|
||||
writeQuery.bindValue(QStringLiteral(":uri"), authorDetails.uri);
|
||||
writeQuery.bindValue(QStringLiteral(":email"), authorDetails.email);
|
||||
Database::instance().execute(writeQuery);
|
||||
}
|
||||
|
||||
// Enclosures
|
||||
writeQuery.prepare(QStringLiteral("INSERT INTO Enclosures VALUES (:feed, :id, :duration, :size, :title, :type, :url, :playposition, :downloaded);"));
|
||||
for (EnclosureDetails enclosureDetails : m_enclosures) {
|
||||
writeQuery.bindValue(QStringLiteral(":feed"), enclosureDetails.feed);
|
||||
writeQuery.bindValue(QStringLiteral(":id"), enclosureDetails.id);
|
||||
writeQuery.bindValue(QStringLiteral(":duration"), enclosureDetails.duration);
|
||||
writeQuery.bindValue(QStringLiteral(":size"), enclosureDetails.size);
|
||||
writeQuery.bindValue(QStringLiteral(":title"), enclosureDetails.title);
|
||||
writeQuery.bindValue(QStringLiteral(":type"), enclosureDetails.type);
|
||||
writeQuery.bindValue(QStringLiteral(":url"), enclosureDetails.url);
|
||||
writeQuery.bindValue(QStringLiteral(":playposition"), enclosureDetails.playPosition);
|
||||
writeQuery.bindValue(QStringLiteral(":downloaded"), Enclosure::statusToDb(enclosureDetails.downloaded));
|
||||
Database::instance().execute(writeQuery);
|
||||
}
|
||||
|
||||
// Chapters
|
||||
writeQuery.prepare(QStringLiteral("INSERT INTO Chapters VALUES(:feed, :id, :start, :title, :link, :image);"));
|
||||
for (ChapterDetails chapterDetails : m_chapters) {
|
||||
writeQuery.bindValue(QStringLiteral(":feed"), chapterDetails.feed);
|
||||
writeQuery.bindValue(QStringLiteral(":id"), chapterDetails.id);
|
||||
writeQuery.bindValue(QStringLiteral(":start"), chapterDetails.start);
|
||||
writeQuery.bindValue(QStringLiteral(":title"), chapterDetails.title);
|
||||
writeQuery.bindValue(QStringLiteral(":link"), chapterDetails.link);
|
||||
writeQuery.bindValue(QStringLiteral(":image"), chapterDetails.image);
|
||||
Database::instance().execute(writeQuery);
|
||||
}
|
||||
|
||||
if (Database::instance().commit()) {
|
||||
for (EntryDetails entryDetails : m_entries) {
|
||||
Q_EMIT entryAdded(m_url, entryDetails.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void UpdateFeedJob::abort()
|
||||
{
|
||||
m_abort = true;
|
||||
Q_EMIT aborting();
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/**
|
||||
* SPDX-FileCopyrightText: 2021 Bart De Vries <bart@mogwai.be>
|
||||
*
|
||||
* SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <KJob>
|
||||
#include <QNetworkReply>
|
||||
#include <QString>
|
||||
|
||||
#include <Syndication/Syndication>
|
||||
|
||||
#include "enclosure.h"
|
||||
|
||||
class UpdateFeedJob : public KJob
|
||||
{
|
||||
Q_OBJECT
|
||||
|
||||
public:
|
||||
explicit UpdateFeedJob(const QString &url, QObject *parent = nullptr);
|
||||
|
||||
void start() override;
|
||||
void abort();
|
||||
|
||||
struct EntryDetails {
|
||||
QString feed;
|
||||
QString id;
|
||||
QString title;
|
||||
QString content;
|
||||
int created;
|
||||
int updated;
|
||||
QString link;
|
||||
bool read;
|
||||
bool isNew;
|
||||
bool hasEnclosure;
|
||||
QString image;
|
||||
};
|
||||
|
||||
struct AuthorDetails {
|
||||
QString feed;
|
||||
QString id;
|
||||
QString name;
|
||||
QString uri;
|
||||
QString email;
|
||||
};
|
||||
|
||||
struct EnclosureDetails {
|
||||
QString feed;
|
||||
QString id;
|
||||
int duration;
|
||||
int size;
|
||||
QString title;
|
||||
QString type;
|
||||
QString url;
|
||||
int playPosition;
|
||||
Enclosure::Status downloaded;
|
||||
};
|
||||
|
||||
struct ChapterDetails {
|
||||
QString feed;
|
||||
QString id;
|
||||
int start;
|
||||
QString title;
|
||||
QString link;
|
||||
QString image;
|
||||
};
|
||||
|
||||
Q_SIGNALS:
|
||||
void feedDetailsUpdated(const QString &url,
|
||||
const QString &name,
|
||||
const QString &image,
|
||||
const QString &link,
|
||||
const QString &description,
|
||||
const QDateTime &lastUpdated);
|
||||
void feedUpdated(const QString &url);
|
||||
void entryAdded(const QString &feedurl, const QString &id);
|
||||
void feedUpdateStatusChanged(const QString &url, bool status);
|
||||
void aborting();
|
||||
|
||||
private:
|
||||
void retrieveFeed();
|
||||
void processFeed(Syndication::FeedPtr feed);
|
||||
bool processEntry(Syndication::ItemPtr entry);
|
||||
void processAuthor(const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail);
|
||||
void processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry);
|
||||
void processChapter(const QString &entryId, const int &start, const QString &chapterTitle, const QString &link, const QString &image);
|
||||
void writeToDatabase();
|
||||
|
||||
bool m_abort = false;
|
||||
|
||||
QString m_url;
|
||||
QNetworkReply *m_reply = nullptr;
|
||||
|
||||
bool m_isNewFeed;
|
||||
QVector<EntryDetails> m_entries;
|
||||
QVector<AuthorDetails> m_authors;
|
||||
QVector<EnclosureDetails> m_enclosures;
|
||||
QVector<ChapterDetails> m_chapters;
|
||||
|
||||
QStringList m_existingEntryIds;
|
||||
QVector<QPair<QString, QString>> m_existingEnclosures;
|
||||
QVector<QPair<QString, QString>> m_existingAuthors;
|
||||
QVector<QPair<QString, int>> m_existingChapters;
|
||||
};
|
||||
Loading…
Reference in New Issue