From c061a01c597e635db4ea33430386eead32bf1363 Mon Sep 17 00:00:00 2001 From: Bart De Vries Date: Thu, 23 Sep 2021 19:23:39 +0200 Subject: [PATCH] Wrap feed update routine in KJob and make it more efficient The feed update routine which is now spread over several methods in Fetcher, is now put into a self-contained KJob. This will allow to re-use this job later on in e.g. gpodder sync, where it's required to update feeds before syncing episode statuses. This also makes the feed update abortable. Lastly, but most importantly, the feed update procedure has been optimized to minimize database transactions, resulting in a dramatic speed-up. This is especially true for importing new feeds, which will now be at least 5x faster on slow hardware. --- src/CMakeLists.txt | 2 + src/database.cpp | 29 ++- src/database.h | 4 + src/datamanager.cpp | 18 +- src/datamanager.h | 2 - src/feed.cpp | 15 +- src/fetcher.cpp | 369 +++--------------------------- src/fetcher.h | 20 +- src/fetchfeedsjob.cpp | 71 ++++++ src/fetchfeedsjob.h | 36 +++ src/main.cpp | 2 +- src/models/errorlogmodel.cpp | 6 +- src/qml/main.qml | 6 + src/storagemanager.cpp | 2 + src/updatefeedjob.cpp | 423 +++++++++++++++++++++++++++++++++++ src/updatefeedjob.h | 106 +++++++++ 16 files changed, 727 insertions(+), 384 deletions(-) create mode 100644 src/fetchfeedsjob.cpp create mode 100644 src/fetchfeedsjob.h create mode 100644 src/updatefeedjob.cpp create mode 100644 src/updatefeedjob.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 16528068..78d144f0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,6 +16,8 @@ set(SRCS_base enclosuredownloadjob.cpp storagemanager.cpp storagemovejob.cpp + updatefeedjob.cpp + fetchfeedsjob.cpp models/chaptermodel.cpp models/feedsmodel.cpp models/feedsproxymodel.cpp diff --git a/src/database.cpp b/src/database.cpp index 73f0468f..cf000119 100644 --- a/src/database.cpp +++ b/src/database.cpp @@ -55,6 +55,7 @@ bool Database::migrate() bool Database::migrateTo1() { qDebug() << "Migrating database to version 1"; + TRUE_OR_RETURN(transaction()); TRUE_OR_RETURN( execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Feeds (name TEXT, url TEXT, image TEXT, link TEXT, description TEXT, deleteAfterCount INTEGER, " "deleteAfterType INTEGER, subscribed INTEGER, lastUpdated INTEGER, new BOOL, notify BOOL);"))); @@ -68,24 +69,25 @@ bool Database::migrateTo1() TRUE_OR_RETURN(execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Queue (listnr INTEGER, feed TEXT, id TEXT, playing BOOL);"))); TRUE_OR_RETURN(execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Errors (url TEXT, id TEXT, code INTEGER, message TEXT, date INTEGER);"))); TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 1;"))); + TRUE_OR_RETURN(commit()); return true; } bool Database::migrateTo2() { qDebug() << "Migrating database to version 2"; - TRUE_OR_RETURN(execute(QStringLiteral("BEGIN TRANSACTION;"))); + TRUE_OR_RETURN(transaction()); TRUE_OR_RETURN(execute(QStringLiteral("DROP TABLE Errors;"))); TRUE_OR_RETURN(execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Errors (type INTEGER, url TEXT, id TEXT, code INTEGER, message TEXT, date INTEGER);"))); TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 2;"))); - TRUE_OR_RETURN(execute(QStringLiteral("COMMIT;"))); + TRUE_OR_RETURN(commit()); return true; } bool Database::migrateTo3() { qDebug() << "Migrating database to version 3"; - TRUE_OR_RETURN(execute(QStringLiteral("BEGIN TRANSACTION;"))); + TRUE_OR_RETURN(transaction()); TRUE_OR_RETURN( execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Enclosurestemp (feed TEXT, id TEXT, duration INTEGER, size INTEGER, title TEXT, type TEXT, url " "TEXT, playposition INTEGER, downloaded INTEGER);"))); @@ -96,31 +98,32 @@ bool Database::migrateTo3() TRUE_OR_RETURN(execute(QStringLiteral("DROP TABLE Enclosures;"))); TRUE_OR_RETURN(execute(QStringLiteral("ALTER TABLE Enclosurestemp RENAME TO Enclosures;"))); TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 3;"))); - TRUE_OR_RETURN(execute(QStringLiteral("COMMIT;"))); + TRUE_OR_RETURN(commit()); return true; } bool Database::migrateTo4() { qDebug() << "Migrating database to version 4"; - TRUE_OR_RETURN(execute(QStringLiteral("BEGIN TRANSACTION;"))); + TRUE_OR_RETURN(transaction()); TRUE_OR_RETURN(execute(QStringLiteral("DROP TABLE Errors;"))); TRUE_OR_RETURN( execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Errors (type INTEGER, url TEXT, id TEXT, code INTEGER, message TEXT, date INTEGER, title TEXT);"))); TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 4;"))); - TRUE_OR_RETURN(execute(QStringLiteral("COMMIT;"))); + TRUE_OR_RETURN(commit()); return true; } bool Database::migrateTo5() { qDebug() << "Migrating database to version 5"; - TRUE_OR_RETURN(execute(QStringLiteral("BEGIN TRANSACTION;"))); + TRUE_OR_RETURN(transaction()); TRUE_OR_RETURN(execute(QStringLiteral("CREATE TABLE IF NOT EXISTS Chapters (feed TEXT, id TEXT, start INTEGER, title TEXT, link TEXT, image TEXT);"))); TRUE_OR_RETURN(execute(QStringLiteral("PRAGMA user_version = 5;"))); - TRUE_OR_RETURN(execute(QStringLiteral("COMMIT;"))); + TRUE_OR_RETURN(commit()); return true; } + bool Database::execute(const QString &query) { QSqlQuery q; @@ -139,6 +142,16 @@ bool Database::execute(QSqlQuery &query) return true; } +bool Database::transaction() +{ + return QSqlDatabase::database().transaction(); +} + +bool Database::commit() +{ + return QSqlDatabase::database().commit(); +} + int Database::version() { QSqlQuery query; diff --git a/src/database.h b/src/database.h index b022acfb..ab6d661c 100644 --- a/src/database.h +++ b/src/database.h @@ -9,6 +9,7 @@ #include #include +#include class Database : public QObject { @@ -23,6 +24,9 @@ public: bool execute(QSqlQuery &query); bool execute(const QString &query); + bool transaction(); + bool commit(); + private: Database(); int version(); diff --git a/src/datamanager.cpp b/src/datamanager.cpp index 0ec5ac71..eeddc280 100644 --- a/src/datamanager.cpp +++ b/src/datamanager.cpp @@ -49,11 +49,11 @@ DataManager::DataManager() } }); connect(&Fetcher::instance(), &Fetcher::entryAdded, this, [this](const QString &feedurl, const QString &id) { + Q_UNUSED(feedurl) // Only add the new entry to m_entries // we will repopulate m_entrymap once all new entries have been added, // such that m_entrymap will show all new entries in the correct order m_entries[id] = nullptr; - Q_EMIT entryAdded(feedurl, id); }); connect(&Fetcher::instance(), &Fetcher::feedUpdated, this, [this](const QString &feedurl) { // Update m_entrymap for feedurl, such that the new and old entries show @@ -563,7 +563,7 @@ void DataManager::bulkMarkReadByIndex(bool state, QModelIndexList list) void DataManager::bulkMarkRead(bool state, QStringList list) { - Database::instance().execute(QStringLiteral("BEGIN TRANSACTION;")); + Database::instance().transaction(); if (state) { // Mark as read // This needs special attention as the DB operations are very intensive. @@ -577,7 +577,7 @@ void DataManager::bulkMarkRead(bool state, QStringList list) getEntry(id)->setReadInternal(state); } } - Database::instance().execute(QStringLiteral("COMMIT;")); + Database::instance().commit(); Q_EMIT bulkReadStatusActionFinished(); } @@ -589,11 +589,11 @@ void DataManager::bulkMarkNewByIndex(bool state, QModelIndexList list) void DataManager::bulkMarkNew(bool state, QStringList list) { - Database::instance().execute(QStringLiteral("BEGIN TRANSACTION;")); + Database::instance().transaction(); for (QString id : list) { getEntry(id)->setNewInternal(state); } - Database::instance().execute(QStringLiteral("COMMIT;")); + Database::instance().commit(); Q_EMIT bulkNewStatusActionFinished(); } @@ -605,7 +605,7 @@ void DataManager::bulkQueueStatusByIndex(bool state, QModelIndexList list) void DataManager::bulkQueueStatus(bool state, QStringList list) { - Database::instance().execute(QStringLiteral("BEGIN TRANSACTION;")); + Database::instance().transaction(); if (state) { // i.e. add to queue for (QString id : list) { getEntry(id)->setQueueStatusInternal(state); @@ -619,7 +619,7 @@ void DataManager::bulkQueueStatus(bool state, QStringList list) } updateQueueListnrs(); } - Database::instance().execute(QStringLiteral("COMMIT;")); + Database::instance().commit(); Q_EMIT bulkReadStatusActionFinished(); Q_EMIT bulkNewStatusActionFinished(); @@ -647,13 +647,13 @@ void DataManager::bulkDeleteEnclosuresByIndex(QModelIndexList list) void DataManager::bulkDeleteEnclosures(QStringList list) { - Database::instance().execute(QStringLiteral("BEGIN TRANSACTION;")); + Database::instance().transaction(); for (QString id : list) { if (getEntry(id)->hasEnclosure()) { getEntry(id)->enclosure()->deleteFile(); } } - Database::instance().execute(QStringLiteral("COMMIT;")); + Database::instance().commit(); } QStringList DataManager::getIdsFromModelIndexList(const QModelIndexList &list) const diff --git a/src/datamanager.h b/src/datamanager.h index 87d262c9..a492a45f 100644 --- a/src/datamanager.h +++ b/src/datamanager.h @@ -77,8 +77,6 @@ public: Q_SIGNALS: void feedAdded(const QString &url); void feedRemoved(const int &index); - void entryAdded(const QString &feedurl, const QString &id); - // void entryRemoved(const Feed*, const int &index); // TODO: implement this signal, is this needed? void feedEntriesUpdated(const QString &url); void queueEntryAdded(const int &index, const QString &id); void queueEntryRemoved(const int &index, const QString &id); diff --git a/src/feed.cpp b/src/feed.cpp index 2bf6e9d1..124ea8f9 100644 --- a/src/feed.cpp +++ b/src/feed.cpp @@ -44,11 +44,9 @@ Feed::Feed(const QString &feedurl) updateAuthors(); - connect(&Fetcher::instance(), &Fetcher::startedFetchingFeed, this, [this](const QString &url) { + connect(&Fetcher::instance(), &Fetcher::feedUpdateStatusChanged, this, [this](const QString &url, bool status) { if (url == m_url) { - m_errorId = 0; - m_errorString = QString(); - setRefreshing(true); + setRefreshing(status); } }); connect(&DataManager::instance(), &DataManager::feedEntriesUpdated, this, [this](const QString &url) { @@ -71,11 +69,6 @@ Feed::Feed(const QString &feedurl) setRefreshing(false); } }); - connect(&Fetcher::instance(), &Fetcher::feedUpdateFinished, this, [this](const QString &url) { - if (url == m_url) { - setRefreshing(false); - } - }); connect(&Fetcher::instance(), &Fetcher::downloadFinished, this, [this](QString url) { if (url == m_image) { Q_EMIT imageChanged(url); @@ -300,6 +293,10 @@ void Feed::setRefreshing(bool refreshing) { if (refreshing != m_refreshing) { m_refreshing = refreshing; + if (!m_refreshing) { + m_errorId = 0; + m_errorString = QString(); + } Q_EMIT refreshingChanged(m_refreshing); } } diff --git a/src/fetcher.cpp b/src/fetcher.cpp index 4172828d..32904f03 100644 --- a/src/fetcher.cpp +++ b/src/fetcher.cpp @@ -24,12 +24,16 @@ #include "database.h" #include "enclosure.h" +#include "fetchfeedsjob.h" #include "kasts-version.h" +#include "models/errorlogmodel.h" #include "settingsmanager.h" #include "storagemanager.h" Fetcher::Fetcher() { + connect(this, &Fetcher::error, &ErrorLogModel::instance(), &ErrorLogModel::monitorErrorMessages); + m_updateProgress = -1; m_updateTotal = -1; m_updating = false; @@ -53,24 +57,6 @@ void Fetcher::fetch(const QString &url) fetch(urls); } -void Fetcher::fetch(const QStringList &urls) -{ - if (m_updating) - return; // update is already running, do nothing - - m_updating = true; - m_updateProgress = 0; - m_updateTotal = urls.count(); - connect(this, &Fetcher::updateProgressChanged, this, &Fetcher::updateMonitor); - Q_EMIT updatingChanged(m_updating); - Q_EMIT updateProgressChanged(m_updateProgress); - Q_EMIT updateTotalChanged(m_updateTotal); - - for (int i = 0; i < urls.count(); i++) { - retrieveFeed(urls[i]); - } -} - void Fetcher::fetchAll() { QStringList urls; @@ -79,7 +65,6 @@ void Fetcher::fetchAll() Database::instance().execute(query); while (query.next()) { urls += query.value(0).toString(); - ; } if (urls.count() > 0) { @@ -87,318 +72,41 @@ void Fetcher::fetchAll() } } -void Fetcher::retrieveFeed(const QString &url) +void Fetcher::fetch(const QStringList &urls) { - if (isMeteredConnection() && !SettingsManager::self()->allowMeteredFeedUpdates()) { - Q_EMIT error(Error::Type::MeteredNotAllowed, url, QString(), 0, i18n("Podcast updates not allowed due to user setting"), QString()); - m_updateProgress++; + if (m_updating) + return; // update is already running, do nothing + + m_updating = true; + m_updateProgress = 0; + m_updateTotal = urls.count(); + Q_EMIT updatingChanged(m_updating); + Q_EMIT updateProgressChanged(m_updateProgress); + Q_EMIT updateTotalChanged(m_updateTotal); + + qCDebug(kastsFetcher) << "Create fetchFeedsJob"; + FetchFeedsJob *fetchFeedsJob = new FetchFeedsJob(urls, this); + connect(this, &Fetcher::cancelFetching, fetchFeedsJob, &FetchFeedsJob::abort); + connect(fetchFeedsJob, &FetchFeedsJob::processedAmountChanged, this, [this](KJob *job, KJob::Unit unit, qulonglong amount) { + qCDebug(kastsFetcher) << "FetchFeedsJob::processedAmountChanged:" << amount; + Q_UNUSED(job); + Q_ASSERT(unit == KJob::Unit::Items); + m_updateProgress = amount; Q_EMIT updateProgressChanged(m_updateProgress); - return; - } - - qCDebug(kastsFetcher) << "Starting to fetch" << url; - - Q_EMIT startedFetchingFeed(url); - - QNetworkRequest request((QUrl(url))); - request.setTransferTimeout(); - QNetworkReply *reply = get(request); - connect(reply, &QNetworkReply::finished, this, [this, url, reply]() { - if (reply->error()) { - qWarning() << "Error fetching feed"; - qWarning() << reply->errorString(); - Q_EMIT error(Error::Type::FeedUpdate, url, QString(), reply->error(), reply->errorString(), QString()); - } else { - QByteArray data = reply->readAll(); - Syndication::DocumentSource *document = new Syndication::DocumentSource(data, url); - Syndication::FeedPtr feed = Syndication::parserCollection()->parse(*document, QStringLiteral("Atom")); - processFeed(feed, url); - } - m_updateProgress++; - Q_EMIT updateProgressChanged(m_updateProgress); - delete reply; }); -} - -void Fetcher::updateMonitor(int progress) -{ - qCDebug(kastsFetcher) << "Update monitor" << progress << "/" << m_updateTotal; - // this method will watch for the end of the update process - if (progress > -1 && m_updateTotal > -1 && progress == m_updateTotal) { - m_updating = false; - m_updateProgress = -1; - m_updateTotal = -1; - disconnect(this, &Fetcher::updateProgressChanged, this, &Fetcher::updateMonitor); - Q_EMIT updatingChanged(m_updating); - // Q_EMIT updateProgressChanged(m_updateProgress); - // Q_EMIT updateTotalChanged(m_updateTotal); - } -} - -void Fetcher::processFeed(Syndication::FeedPtr feed, const QString &url) -{ - if (feed.isNull()) - return; - - // First check if this is a newly added feed - bool isNewFeed = false; - QSqlQuery query; - query.prepare(QStringLiteral("SELECT new FROM Feeds WHERE url=:url;")); - query.bindValue(QStringLiteral(":url"), url); - Database::instance().execute(query); - if (query.next()) { - isNewFeed = query.value(QStringLiteral("new")).toBool(); - } else { - qCDebug(kastsFetcher) << "Feed not found in database" << url; - return; - } - if (isNewFeed) - qCDebug(kastsFetcher) << "New feed" << feed->title() << ":" << isNewFeed; - - // Retrieve "other" fields; this will include the "itunes" tags - QMultiMap otherItems = feed->additionalProperties(); - - query.prepare(QStringLiteral("UPDATE Feeds SET name=:name, image=:image, link=:link, description=:description, lastUpdated=:lastUpdated WHERE url=:url;")); - query.bindValue(QStringLiteral(":name"), feed->title()); - query.bindValue(QStringLiteral(":url"), url); - query.bindValue(QStringLiteral(":link"), feed->link()); - query.bindValue(QStringLiteral(":description"), feed->description()); - - QDateTime current = QDateTime::currentDateTime(); - query.bindValue(QStringLiteral(":lastUpdated"), current.toSecsSinceEpoch()); - - // Process authors - QString authorname, authoremail; - if (feed->authors().count() > 0) { - for (auto &author : feed->authors()) { - processAuthor(url, QLatin1String(""), author->name(), QLatin1String(""), QLatin1String("")); + connect(fetchFeedsJob, &FetchFeedsJob::result, this, [this, fetchFeedsJob]() { + qCDebug(kastsFetcher) << "result slot of FetchFeedsJob"; + if (fetchFeedsJob->error()) { + Q_EMIT error(Error::Type::FeedUpdate, QString(), QString(), fetchFeedsJob->error(), fetchFeedsJob->errorString(), QString()); } - } else { - // Try to find itunes fields if plain author doesn't exist - QString authorname, authoremail; - // First try the "itunes:owner" tag, if that doesn't succeed, then try the "itunes:author" tag - if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).hasChildNodes()) { - QDomNodeList nodelist = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).childNodes(); - for (int i = 0; i < nodelist.length(); i++) { - if (nodelist.item(i).nodeName() == QStringLiteral("itunes:name")) { - authorname = nodelist.item(i).toElement().text(); - } else if (nodelist.item(i).nodeName() == QStringLiteral("itunes:email")) { - authoremail = nodelist.item(i).toElement().text(); - } - } - } else { - authorname = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text(); - qCDebug(kastsFetcher) << "authorname" << authorname; + if (m_updating) { + m_updating = false; + Q_EMIT updatingChanged(m_updating); } - if (!authorname.isEmpty()) { - processAuthor(url, QLatin1String(""), authorname, QLatin1String(""), authoremail); - } - } + }); - QString image = feed->image()->url(); - // If there is no regular image tag, then try the itunes tags - if (image.isEmpty()) { - if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) { - image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href")); - } - } - if (image.startsWith(QStringLiteral("/"))) - image = QUrl(url).adjusted(QUrl::RemovePath).toString() + image; - query.bindValue(QStringLiteral(":image"), image); - Database::instance().execute(query); - - qCDebug(kastsFetcher) << "Updated feed details:" << feed->title(); - - Q_EMIT feedDetailsUpdated(url, feed->title(), image, feed->link(), feed->description(), current); - - bool updatedEntries = false; - for (const auto &entry : feed->items()) { - QCoreApplication::processEvents(); // keep the main thread semi-responsive - bool isNewEntry = processEntry(entry, url, isNewFeed); - updatedEntries = updatedEntries || isNewEntry; - } - - // Now mark the appropriate number of recent entries "new" and "read" only for new feeds - if (isNewFeed) { - query.prepare(QStringLiteral("SELECT * FROM Entries WHERE feed=:feed ORDER BY updated DESC LIMIT :recentNew;")); - query.bindValue(QStringLiteral(":feed"), url); - query.bindValue(QStringLiteral(":recentNew"), 0); // hardcode to marking no episode as new on a new feed - Database::instance().execute(query); - QSqlQuery updateQuery; - while (query.next()) { - qCDebug(kastsFetcher) << "Marked as new:" << query.value(QStringLiteral("id")).toString(); - updateQuery.prepare(QStringLiteral("UPDATE Entries SET read=:read, new=:new WHERE id=:id AND feed=:feed;")); - updateQuery.bindValue(QStringLiteral(":read"), false); - updateQuery.bindValue(QStringLiteral(":new"), true); - updateQuery.bindValue(QStringLiteral(":feed"), url); - updateQuery.bindValue(QStringLiteral(":id"), query.value(QStringLiteral("id")).toString()); - Database::instance().execute(updateQuery); - } - // Finally, reset the new flag to false now that the new feed has been fully processed - // If we would reset the flag sooner, then too many episodes will get flagged as new if - // the initial import gets interrupted somehow. - query.prepare(QStringLiteral("UPDATE Feeds SET new=:new WHERE url=:url;")); - query.bindValue(QStringLiteral(":url"), url); - query.bindValue(QStringLiteral(":new"), false); - Database::instance().execute(query); - } - - if (updatedEntries || isNewFeed) - Q_EMIT feedUpdated(url); - Q_EMIT feedUpdateFinished(url); -} - -bool Fetcher::processEntry(Syndication::ItemPtr entry, const QString &url, bool isNewFeed) -{ - qCDebug(kastsFetcher) << "Processing" << entry->title(); - - // Retrieve "other" fields; this will include the "itunes" tags - QMultiMap otherItems = entry->additionalProperties(); - - QSqlQuery query; - query.prepare(QStringLiteral("SELECT COUNT (id) FROM Entries WHERE id=:id;")); - query.bindValue(QStringLiteral(":id"), entry->id()); - Database::instance().execute(query); - query.next(); - - if (query.value(0).toInt() != 0) - return false; // entry already exists - - query.prepare(QStringLiteral("INSERT INTO Entries VALUES (:feed, :id, :title, :content, :created, :updated, :link, :read, :new, :hasEnclosure, :image);")); - query.bindValue(QStringLiteral(":feed"), url); - query.bindValue(QStringLiteral(":id"), entry->id()); - query.bindValue(QStringLiteral(":title"), QTextDocumentFragment::fromHtml(entry->title()).toPlainText()); - query.bindValue(QStringLiteral(":created"), static_cast(entry->datePublished())); - query.bindValue(QStringLiteral(":updated"), static_cast(entry->dateUpdated())); - query.bindValue(QStringLiteral(":link"), entry->link()); - query.bindValue(QStringLiteral(":hasEnclosure"), entry->enclosures().length() == 0 ? 0 : 1); - query.bindValue(QStringLiteral(":read"), isNewFeed); // if new feed, then mark all as read - query.bindValue(QStringLiteral(":new"), !isNewFeed); // if new feed, then mark none as new - - if (!entry->content().isEmpty()) - query.bindValue(QStringLiteral(":content"), entry->content()); - else - query.bindValue(QStringLiteral(":content"), entry->description()); - - // Look for image in itunes tags - QString image; - if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) { - image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href")); - } - if (image.startsWith(QStringLiteral("/"))) - image = QUrl(url).adjusted(QUrl::RemovePath).toString() + image; - query.bindValue(QStringLiteral(":image"), image); - qCDebug(kastsFetcher) << "Entry image found" << image; - - Database::instance().execute(query); - - if (entry->authors().count() > 0) { - for (const auto &author : entry->authors()) { - processAuthor(url, entry->id(), author->name(), author->uri(), author->email()); - } - } else { - // As fallback, check if there is itunes "author" information - QString authorName = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text(); - if (!authorName.isEmpty()) - processAuthor(url, entry->id(), authorName, QLatin1String(""), QLatin1String("")); - } - - /* Process chapters */ - if (otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).hasChildNodes()) { - QDomNodeList nodelist = otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).childNodes(); - for (int i = 0; i < nodelist.length(); i++) { - if (nodelist.item(i).nodeName() == QStringLiteral("psc:chapter")) { - QDomElement element = nodelist.at(i).toElement(); - QString title = element.attribute(QStringLiteral("title")); - QString start = element.attribute(QStringLiteral("start")); - QTime startString = QTime::fromString(start, QStringLiteral("hh:mm:ss.zzz")); - int startInt = startString.hour() * 60 * 60 + startString.minute() * 60 + startString.second(); - QString images = element.attribute(QStringLiteral("image")); - processChapter(url, entry->id(), startInt, title, entry->link(), images); - } - } - } - - // only process first enclosure if there are multiple (e.g. mp3 and ogg); - // the first one is probably the podcast author's preferred version - // TODO: handle more than one enclosure? - if (entry->enclosures().count() > 0) { - processEnclosure(entry->enclosures()[0], entry, url); - } - - Q_EMIT entryAdded(url, entry->id()); - return true; // this is a new entry -} - -void Fetcher::processAuthor(const QString &url, const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail) -{ - QSqlQuery query; - query.prepare(QStringLiteral("SELECT COUNT (id) FROM Authors WHERE feed=:feed AND id=:id AND name=:name;")); - query.bindValue(QStringLiteral(":feed"), url); - query.bindValue(QStringLiteral(":id"), entryId); - query.bindValue(QStringLiteral(":name"), authorName); - Database::instance().execute(query); - query.next(); - - if (query.value(0).toInt() != 0) - query.prepare(QStringLiteral("UPDATE Authors SET feed=:feed, id=:id, name=:name, uri=:uri, email=:email WHERE feed=:feed AND id=:id;")); - else - query.prepare(QStringLiteral("INSERT INTO Authors VALUES(:feed, :id, :name, :uri, :email);")); - - query.bindValue(QStringLiteral(":feed"), url); - query.bindValue(QStringLiteral(":id"), entryId); - query.bindValue(QStringLiteral(":name"), authorName); - query.bindValue(QStringLiteral(":uri"), authorUri); - query.bindValue(QStringLiteral(":email"), authorEmail); - Database::instance().execute(query); -} - -void Fetcher::processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry, const QString &feedUrl) -{ - QSqlQuery query; - query.prepare(QStringLiteral("SELECT COUNT (id) FROM Enclosures WHERE feed=:feed AND id=:id;")); - query.bindValue(QStringLiteral(":feed"), feedUrl); - query.bindValue(QStringLiteral(":id"), entry->id()); - Database::instance().execute(query); - query.next(); - - if (query.value(0).toInt() != 0) - query.prepare(QStringLiteral( - "UPDATE Enclosures SET feed=:feed, id=:id, duration=:duration, size=:size, title=:title, type=:type, url=:url WHERE feed=:feed AND id=:id;")); - // NOTE: In case more than one enclosure is present per episode, only - // the last one will end up in the database - else - query.prepare(QStringLiteral("INSERT INTO Enclosures VALUES (:feed, :id, :duration, :size, :title, :type, :url, :playposition, :downloaded);")); - - query.bindValue(QStringLiteral(":feed"), feedUrl); - query.bindValue(QStringLiteral(":id"), entry->id()); - query.bindValue(QStringLiteral(":duration"), enclosure->duration()); - query.bindValue(QStringLiteral(":size"), enclosure->length()); - query.bindValue(QStringLiteral(":title"), enclosure->title()); - query.bindValue(QStringLiteral(":type"), enclosure->type()); - query.bindValue(QStringLiteral(":url"), enclosure->url()); - query.bindValue(QStringLiteral(":playposition"), 0); - query.bindValue(QStringLiteral(":downloaded"), Enclosure::statusToDb(Enclosure::Downloadable)); - Database::instance().execute(query); -} - -void Fetcher::processChapter(const QString &url, - const QString &entryId, - const int &start, - const QString &chapterTitle, - const QString &link, - const QString &image) -{ - QSqlQuery query; - - query.prepare(QStringLiteral("INSERT INTO Chapters VALUES(:feed, :id, :start, :title, :link, :image);")); - query.bindValue(QStringLiteral(":feed"), url); - query.bindValue(QStringLiteral(":id"), entryId); - query.bindValue(QStringLiteral(":start"), start); - query.bindValue(QStringLiteral(":title"), chapterTitle); - query.bindValue(QStringLiteral(":link"), link); - query.bindValue(QStringLiteral(":image"), image); - Database::instance().execute(query); + fetchFeedsJob->start(); + qCDebug(kastsFetcher) << "end of Fetcher::fetch"; } QString Fetcher::image(const QString &url) const @@ -449,17 +157,6 @@ QNetworkReply *Fetcher::download(const QString &url, const QString &filePath) co file->open(QIODevice::WriteOnly); } - /* - QNetworkReply *headerReply = head(request); - connect(headerReply, &QNetworkReply::finished, this, [=]() { - if (headerReply->isOpen()) { - int fileSize = headerReply->header(QNetworkRequest::ContentLengthHeader).toInt(); - qCDebug(kastsFetcher) << "Reported download size" << fileSize; - } - headerReply->deleteLater(); - }); - */ - QNetworkReply *reply = get(request); connect(reply, &QNetworkReply::readyRead, this, [=]() { diff --git a/src/fetcher.h b/src/fetcher.h index 34fe22a6..6aa54d09 100644 --- a/src/fetcher.h +++ b/src/fetcher.h @@ -14,6 +14,7 @@ #include #include +#include "enclosure.h" #include "error.h" #if !defined Q_OS_ANDROID && !defined Q_OS_WIN @@ -49,7 +50,7 @@ public: Q_INVOKABLE bool isMeteredConnection() const; Q_SIGNALS: - void startedFetchingFeed(const QString &url); + void entryAdded(const QString &feedurl, const QString &id); void feedUpdated(const QString &url); void feedDetailsUpdated(const QString &url, const QString &name, @@ -57,28 +58,19 @@ Q_SIGNALS: const QString &link, const QString &description, const QDateTime &lastUpdated); - void feedUpdateFinished(const QString &url); - void error(Error::Type type, const QString &url, const QString &id, const int errorId, const QString &errorString, const QString &title); - void entryAdded(const QString &feedurl, const QString &id); - void downloadFinished(QString url) const; + void feedUpdateStatusChanged(const QString &url, bool status); + void cancelFetching(); void updateProgressChanged(int progress); void updateTotalChanged(int nrOfFeeds); void updatingChanged(bool state); -private Q_SLOTS: - void updateMonitor(int progress); + void error(Error::Type type, const QString &url, const QString &id, const int errorId, const QString &errorString, const QString &title); + void downloadFinished(QString url) const; private: Fetcher(); - void retrieveFeed(const QString &url); - void processFeed(Syndication::FeedPtr feed, const QString &url); - bool processEntry(Syndication::ItemPtr entry, const QString &url, bool isNewFeed); // returns true if this is a new entry; false if it already existed - void processAuthor(const QString &url, const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail); - void processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry, const QString &feedUrl); - void processChapter(const QString &url, const QString &entryId, const int &start, const QString &chapterTitle, const QString &link, const QString &image); - QNetworkReply *head(QNetworkRequest &request) const; void setHeader(QNetworkRequest &request) const; diff --git a/src/fetchfeedsjob.cpp b/src/fetchfeedsjob.cpp new file mode 100644 index 00000000..5d070101 --- /dev/null +++ b/src/fetchfeedsjob.cpp @@ -0,0 +1,71 @@ +/** + * SPDX-FileCopyrightText: 2021 Bart De Vries + * + * SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL + */ + +#include "fetchfeedsjob.h" + +#include + +#include + +#include "error.h" +#include "fetcher.h" +#include "fetcherlogging.h" +#include "models/errorlogmodel.h" +#include "settingsmanager.h" +#include "updatefeedjob.h" + +FetchFeedsJob::FetchFeedsJob(const QStringList &urls, QObject *parent) + : KJob(parent) + , m_urls(urls) +{ + for (int i = 0; i < m_urls.count(); i++) { + m_feedjobs += nullptr; + } + connect(this, &FetchFeedsJob::processedAmountChanged, this, &FetchFeedsJob::monitorProgress); + connect(this, &FetchFeedsJob::logError, &ErrorLogModel::instance(), &ErrorLogModel::monitorErrorMessages); +} + +void FetchFeedsJob::start() +{ + QTimer::singleShot(0, this, &FetchFeedsJob::fetch); +} + +void FetchFeedsJob::fetch() +{ + if (Fetcher::instance().isMeteredConnection() && !SettingsManager::self()->allowMeteredFeedUpdates()) { + setError(0); + setErrorText(i18n("Podcast updates not allowed due to user setting")); + emitResult(); + return; + } + + setTotalAmount(KJob::Unit::Items, m_urls.count()); + setProcessedAmount(KJob::Unit::Items, 0); + + for (int i = 0; i < m_urls.count(); i++) { + QString url = m_urls[i]; + + UpdateFeedJob *updateFeedJob = new UpdateFeedJob(url, this); + m_feedjobs[i] = updateFeedJob; + connect(this, &FetchFeedsJob::abort, updateFeedJob, &UpdateFeedJob::abort); + connect(updateFeedJob, &UpdateFeedJob::result, this, [this, url, updateFeedJob]() { + if (updateFeedJob->error()) { + Q_EMIT logError(Error::Type::FeedUpdate, url, QString(), updateFeedJob->error(), updateFeedJob->errorString(), QString()); + } + setProcessedAmount(KJob::Unit::Items, processedAmount(KJob::Unit::Items) + 1); + }); + updateFeedJob->start(); + qCDebug(kastsFetcher) << "Just started updateFeedJob" << i + 1; + } + qCDebug(kastsFetcher) << "End of FetchFeedsJob::fetch"; +} + +void FetchFeedsJob::monitorProgress() +{ + if (processedAmount(KJob::Unit::Items) == totalAmount(KJob::Unit::Items)) { + emitResult(); + } +} diff --git a/src/fetchfeedsjob.h b/src/fetchfeedsjob.h new file mode 100644 index 00000000..bdcac5ee --- /dev/null +++ b/src/fetchfeedsjob.h @@ -0,0 +1,36 @@ +/** + * SPDX-FileCopyrightText: 2021 Bart De Vries + * + * SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL + */ + +#pragma once + +#include +#include + +#include "error.h" +#include "updatefeedjob.h" + +class FetchFeedsJob : public KJob +{ + Q_OBJECT + +public: + explicit FetchFeedsJob(const QStringList &urls, QObject *parent = nullptr); + + void start() override; + +Q_SIGNALS: + void abort(); + void logError(Error::Type type, const QString &url, const QString &id, const int errorId, const QString &errorString, const QString &title); + +private: + QStringList m_urls; + + void fetch(); + void monitorProgress(); + + bool m_abort = false; + QVector m_feedjobs; +}; diff --git a/src/main.cpp b/src/main.cpp index 6c178f41..73c9da3c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -132,8 +132,8 @@ int main(int argc, char *argv[]) qmlRegisterUncreatableType("org.kde.kasts", 1, 0, "FeedsModel", QStringLiteral("Only for enums")); qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "AboutType", &AboutType::instance()); - qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "Fetcher", &Fetcher::instance()); qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "Database", &Database::instance()); + qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "Fetcher", &Fetcher::instance()); qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "DataManager", &DataManager::instance()); qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "SettingsManager", SettingsManager::self()); qmlRegisterSingletonInstance("org.kde.kasts", 1, 0, "DownloadModel", &DownloadModel::instance()); diff --git a/src/models/errorlogmodel.cpp b/src/models/errorlogmodel.cpp index 71f494c5..2cffcd99 100644 --- a/src/models/errorlogmodel.cpp +++ b/src/models/errorlogmodel.cpp @@ -6,19 +6,15 @@ #include "models/errorlogmodel.h" +#include #include #include "database.h" #include "datamanager.h" -#include "fetcher.h" -#include "storagemanager.h" ErrorLogModel::ErrorLogModel() : QAbstractListModel(nullptr) { - connect(&Fetcher::instance(), &Fetcher::error, this, &ErrorLogModel::monitorErrorMessages); - connect(&StorageManager::instance(), &StorageManager::error, this, &ErrorLogModel::monitorErrorMessages); - QSqlQuery query; query.prepare(QStringLiteral("SELECT * FROM Errors ORDER BY date DESC;")); Database::instance().execute(query); diff --git a/src/qml/main.qml b/src/qml/main.qml index 4684e888..7f1e61f7 100644 --- a/src/qml/main.qml +++ b/src/qml/main.qml @@ -234,6 +234,12 @@ Kirigami.ApplicationWindow { Fetcher.updateTotal, Fetcher.updateProgress) + showAbortButton: true + + function abortAction() { + Fetcher.cancelFetching(); + } + Connections { target: Fetcher function onUpdatingChanged() { diff --git a/src/storagemanager.cpp b/src/storagemanager.cpp index a4adc980..f9826e9a 100644 --- a/src/storagemanager.cpp +++ b/src/storagemanager.cpp @@ -17,11 +17,13 @@ #include #include "enclosure.h" +#include "models/errorlogmodel.h" #include "settingsmanager.h" #include "storagemovejob.h" StorageManager::StorageManager() { + connect(this, &StorageManager::error, &ErrorLogModel::instance(), &ErrorLogModel::monitorErrorMessages); } QString StorageManager::storagePath() const diff --git a/src/updatefeedjob.cpp b/src/updatefeedjob.cpp new file mode 100644 index 00000000..cf65f080 --- /dev/null +++ b/src/updatefeedjob.cpp @@ -0,0 +1,423 @@ +/** + * SPDX-FileCopyrightText: 2021 Bart De Vries + * + * SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL + */ + +#include "updatefeedjob.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "database.h" +#include "enclosure.h" +#include "fetcher.h" +#include "fetcherlogging.h" +#include "settingsmanager.h" + +UpdateFeedJob::UpdateFeedJob(const QString &url, QObject *parent) + : KJob(parent) + , m_url(url) +{ + // connect to signals in Fetcher such that GUI can pick up the changes + connect(this, &UpdateFeedJob::feedDetailsUpdated, &Fetcher::instance(), &Fetcher::feedDetailsUpdated); + connect(this, &UpdateFeedJob::feedUpdated, &Fetcher::instance(), &Fetcher::feedUpdated); + connect(this, &UpdateFeedJob::entryAdded, &Fetcher::instance(), &Fetcher::entryAdded); + connect(this, &UpdateFeedJob::feedUpdateStatusChanged, &Fetcher::instance(), &Fetcher::feedUpdateStatusChanged); +} + +void UpdateFeedJob::start() +{ + QTimer::singleShot(0, this, &UpdateFeedJob::retrieveFeed); +} + +void UpdateFeedJob::retrieveFeed() +{ + if (m_abort) + return; + + qCDebug(kastsFetcher) << "Starting to fetch" << m_url; + Q_EMIT feedUpdateStatusChanged(m_url, true); + + QNetworkRequest request((QUrl(m_url))); + request.setTransferTimeout(); + m_reply = Fetcher::instance().get(request); + connect(this, &UpdateFeedJob::aborting, m_reply, &QNetworkReply::abort); + connect(m_reply, &QNetworkReply::finished, this, [this]() { + qCDebug(kastsFetcher) << "got networkreply for" << m_reply; + if (m_reply->error()) { + qWarning() << "Error fetching feed"; + qWarning() << m_reply->errorString(); + setError(m_reply->error()); + setErrorText(m_reply->errorString()); + } else { + QByteArray data = m_reply->readAll(); + Syndication::DocumentSource *document = new Syndication::DocumentSource(data, m_url); + Syndication::FeedPtr feed = Syndication::parserCollection()->parse(*document, QStringLiteral("Atom")); + processFeed(feed); + } + Q_EMIT feedUpdateStatusChanged(m_url, false); + m_reply->deleteLater(); + emitResult(); + }); + qCDebug(kastsFetcher) << "End of retrieveFeed for" << m_url; +} + +void UpdateFeedJob::processFeed(Syndication::FeedPtr feed) +{ + qCDebug(kastsFetcher) << "start process feed" << feed; + + if (feed.isNull()) + return; + + // First check if this is a newly added feed + m_isNewFeed = false; + QSqlQuery query; + query.prepare(QStringLiteral("SELECT new FROM Feeds WHERE url=:url;")); + query.bindValue(QStringLiteral(":url"), m_url); + Database::instance().execute(query); + if (query.next()) { + m_isNewFeed = query.value(QStringLiteral("new")).toBool(); + } else { + qCDebug(kastsFetcher) << "Feed not found in database" << m_url; + return; + } + if (m_isNewFeed) + qCDebug(kastsFetcher) << "New feed" << feed->title(); + + // Retrieve "other" fields; this will include the "itunes" tags + QMultiMap otherItems = feed->additionalProperties(); + + query.prepare(QStringLiteral("UPDATE Feeds SET name=:name, image=:image, link=:link, description=:description, lastUpdated=:lastUpdated WHERE url=:url;")); + query.bindValue(QStringLiteral(":name"), feed->title()); + query.bindValue(QStringLiteral(":url"), m_url); + query.bindValue(QStringLiteral(":link"), feed->link()); + query.bindValue(QStringLiteral(":description"), feed->description()); + + QDateTime current = QDateTime::currentDateTime(); + query.bindValue(QStringLiteral(":lastUpdated"), current.toSecsSinceEpoch()); + + QString image = feed->image()->url(); + // If there is no regular image tag, then try the itunes tags + if (image.isEmpty()) { + if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) { + image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href")); + } + } + if (image.startsWith(QStringLiteral("/"))) + image = QUrl(m_url).adjusted(QUrl::RemovePath).toString() + image; + query.bindValue(QStringLiteral(":image"), image); + + // Do the actual database UPDATE of this feed + Database::instance().execute(query); + + // Now that we have the feed details, we make vectors of the data that's + // already in the database relating to this feed + // NOTE: We will do the feed authors after this step, because otherwise + // we can't check for duplicates and we'll keep adding more of the same! + query.prepare(QStringLiteral("SELECT id FROM Entries WHERE feed=:feed;")); + query.bindValue(QStringLiteral(":feed"), m_url); + Database::instance().execute(query); + while (query.next()) { + m_existingEntryIds += query.value(QStringLiteral("id")).toString(); + } + + query.prepare(QStringLiteral("SELECT id, url FROM Enclosures WHERE feed=:feed;")); + query.bindValue(QStringLiteral(":feed"), m_url); + Database::instance().execute(query); + while (query.next()) { + m_existingEnclosures += qMakePair(query.value(QStringLiteral("id")).toString(), query.value(QStringLiteral("url")).toString()); + } + + query.prepare(QStringLiteral("SELECT id, name FROM Authors WHERE feed=:feed;")); + query.bindValue(QStringLiteral(":feed"), m_url); + Database::instance().execute(query); + while (query.next()) { + m_existingAuthors += qMakePair(query.value(QStringLiteral("id")).toString(), query.value(QStringLiteral("name")).toString()); + } + + query.prepare(QStringLiteral("SELECT id, start FROM Chapters WHERE feed=:feed;")); + query.bindValue(QStringLiteral(":feed"), m_url); + Database::instance().execute(query); + while (query.next()) { + m_existingChapters += qMakePair(query.value(QStringLiteral("id")).toString(), query.value(QStringLiteral("start")).toInt()); + } + + // Process feed authors + QString authorname, authoremail; + if (feed->authors().count() > 0) { + for (auto &author : feed->authors()) { + processAuthor(QLatin1String(""), author->name(), QLatin1String(""), QLatin1String("")); + } + } else { + // Try to find itunes fields if plain author doesn't exist + QString authorname, authoremail; + // First try the "itunes:owner" tag, if that doesn't succeed, then try the "itunes:author" tag + if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).hasChildNodes()) { + QDomNodeList nodelist = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).childNodes(); + for (int i = 0; i < nodelist.length(); i++) { + if (nodelist.item(i).nodeName() == QStringLiteral("itunes:name")) { + authorname = nodelist.item(i).toElement().text(); + } else if (nodelist.item(i).nodeName() == QStringLiteral("itunes:email")) { + authoremail = nodelist.item(i).toElement().text(); + } + } + } else { + authorname = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text(); + qCDebug(kastsFetcher) << "authorname" << authorname; + } + if (!authorname.isEmpty()) { + processAuthor(QLatin1String(""), authorname, QLatin1String(""), authoremail); + } + } + + qCDebug(kastsFetcher) << "Updated feed details:" << feed->title(); + + // TODO: Only emit signal if the details have really changed + Q_EMIT feedDetailsUpdated(m_url, feed->title(), image, feed->link(), feed->description(), current); + + if (m_abort) + return; + + // Now deal with the entries, enclosures, entry authors and chapter marks + bool updatedEntries = false; + for (const auto &entry : feed->items()) { + if (m_abort) + return; + QCoreApplication::processEvents(); // keep the main thread semi-responsive + bool isNewEntry = processEntry(entry); + updatedEntries = updatedEntries || isNewEntry; + } + + writeToDatabase(); + + if (m_isNewFeed) { + // Finally, reset the new flag to false now that the new feed has been + // fully processed. If we would reset the flag sooner, then too many + // episodes will get flagged as new if the initial import gets + // interrupted somehow. + query.prepare(QStringLiteral("UPDATE Feeds SET new=:new WHERE url=:url;")); + query.bindValue(QStringLiteral(":url"), m_url); + query.bindValue(QStringLiteral(":new"), false); + Database::instance().execute(query); + } + + if (updatedEntries || m_isNewFeed) + Q_EMIT feedUpdated(m_url); + qCDebug(kastsFetcher) << "done processing feed" << feed; +} + +bool UpdateFeedJob::processEntry(Syndication::ItemPtr entry) +{ + qCDebug(kastsFetcher) << "Processing" << entry->title(); + + // Retrieve "other" fields; this will include the "itunes" tags + QMultiMap otherItems = entry->additionalProperties(); + + // check against existing entries in database + if (m_existingEntryIds.contains(entry->id())) + return false; + + // also check against the list of new entries + for (EntryDetails entryDetails : m_entries) { + if (entryDetails.id == entry->id()) + return false; // entry already exists + } + + EntryDetails entryDetails; + entryDetails.feed = m_url; + entryDetails.id = entry->id(); + entryDetails.title = QTextDocumentFragment::fromHtml(entry->title()).toPlainText(); + entryDetails.created = static_cast(entry->datePublished()); + entryDetails.updated = static_cast(entry->dateUpdated()); + entryDetails.link = entry->link(); + entryDetails.hasEnclosure = (entry->enclosures().length() > 0); + entryDetails.read = m_isNewFeed; // if new feed, then mark all as read + entryDetails.isNew = !m_isNewFeed; // if new feed, then mark none as new + + if (!entry->content().isEmpty()) + entryDetails.content = entry->content(); + else + entryDetails.content = entry->description(); + + // Look for image in itunes tags + if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) { + entryDetails.image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href")); + } + if (entryDetails.image.startsWith(QStringLiteral("/"))) + entryDetails.image = QUrl(m_url).adjusted(QUrl::RemovePath).toString() + entryDetails.image; + qCDebug(kastsFetcher) << "Entry image found" << entryDetails.image; + + m_entries += entryDetails; + + // Process authors + if (entry->authors().count() > 0) { + for (const auto &author : entry->authors()) { + processAuthor(entry->id(), author->name(), author->uri(), author->email()); + } + } else { + // As fallback, check if there is itunes "author" information + QString authorName = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text(); + if (!authorName.isEmpty()) + processAuthor(entry->id(), authorName, QLatin1String(""), QLatin1String("")); + } + + // Process chapters + if (otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).hasChildNodes()) { + QDomNodeList nodelist = otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).childNodes(); + for (int i = 0; i < nodelist.length(); i++) { + if (nodelist.item(i).nodeName() == QStringLiteral("psc:chapter")) { + QDomElement element = nodelist.at(i).toElement(); + QString title = element.attribute(QStringLiteral("title")); + QString start = element.attribute(QStringLiteral("start")); + QTime startString = QTime::fromString(start, QStringLiteral("hh:mm:ss.zzz")); + int startInt = startString.hour() * 60 * 60 + startString.minute() * 60 + startString.second(); + QString images = element.attribute(QStringLiteral("image")); + processChapter(entry->id(), startInt, title, entry->link(), images); + } + } + } + + // Process enclosures + // only process first enclosure if there are multiple (e.g. mp3 and ogg); + // the first one is probably the podcast author's preferred version + // TODO: handle more than one enclosure? + if (entry->enclosures().count() > 0) { + processEnclosure(entry->enclosures()[0], entry); + } + + return true; // this is a new entry +} + +void UpdateFeedJob::processAuthor(const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail) +{ + // check against existing authors already in database + if (m_existingAuthors.contains(qMakePair(entryId, authorName))) + return; + + AuthorDetails authorDetails; + authorDetails.feed = m_url; + authorDetails.id = entryId; + authorDetails.name = authorName; + authorDetails.uri = authorUri; + authorDetails.email = authorEmail; + m_authors += authorDetails; +} + +void UpdateFeedJob::processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry) +{ + // check against existing enclosures already in database + if (m_existingEnclosures.contains(qMakePair(entry->id(), enclosure->url()))) + return; + + EnclosureDetails enclosureDetails; + enclosureDetails.feed = m_url; + enclosureDetails.id = entry->id(); + enclosureDetails.duration = enclosure->duration(); + enclosureDetails.size = enclosure->length(); + enclosureDetails.title = enclosure->title(); + enclosureDetails.type = enclosure->type(); + enclosureDetails.url = enclosure->url(); + enclosureDetails.playPosition = 0; + enclosureDetails.downloaded = Enclosure::Downloadable; + + m_enclosures += enclosureDetails; +} + +void UpdateFeedJob::processChapter(const QString &entryId, const int &start, const QString &chapterTitle, const QString &link, const QString &image) +{ + // check against existing enclosures already in database + if (m_existingChapters.contains(qMakePair(entryId, start))) + return; + + ChapterDetails chapterDetails; + chapterDetails.feed = m_url; + chapterDetails.id = entryId; + chapterDetails.start = start; + chapterDetails.title = chapterTitle; + chapterDetails.link = link; + chapterDetails.image = image; + + m_chapters += chapterDetails; +} + +void UpdateFeedJob::writeToDatabase() +{ + QSqlQuery writeQuery; + + Database::instance().transaction(); + + // Entries + writeQuery.prepare( + QStringLiteral("INSERT INTO Entries VALUES (:feed, :id, :title, :content, :created, :updated, :link, :read, :new, :hasEnclosure, :image);")); + for (EntryDetails entryDetails : m_entries) { + writeQuery.bindValue(QStringLiteral(":feed"), entryDetails.feed); + writeQuery.bindValue(QStringLiteral(":id"), entryDetails.id); + writeQuery.bindValue(QStringLiteral(":title"), entryDetails.title); + writeQuery.bindValue(QStringLiteral(":content"), entryDetails.content); + writeQuery.bindValue(QStringLiteral(":created"), entryDetails.created); + writeQuery.bindValue(QStringLiteral(":updated"), entryDetails.updated); + writeQuery.bindValue(QStringLiteral(":link"), entryDetails.link); + writeQuery.bindValue(QStringLiteral(":hasEnclosure"), entryDetails.hasEnclosure); + writeQuery.bindValue(QStringLiteral(":read"), entryDetails.read); + writeQuery.bindValue(QStringLiteral(":new"), entryDetails.isNew); + writeQuery.bindValue(QStringLiteral(":image"), entryDetails.image); + Database::instance().execute(writeQuery); + } + + // Authors + writeQuery.prepare(QStringLiteral("INSERT INTO Authors VALUES(:feed, :id, :name, :uri, :email);")); + for (AuthorDetails authorDetails : m_authors) { + writeQuery.bindValue(QStringLiteral(":feed"), authorDetails.feed); + writeQuery.bindValue(QStringLiteral(":id"), authorDetails.id); + writeQuery.bindValue(QStringLiteral(":name"), authorDetails.name); + writeQuery.bindValue(QStringLiteral(":uri"), authorDetails.uri); + writeQuery.bindValue(QStringLiteral(":email"), authorDetails.email); + Database::instance().execute(writeQuery); + } + + // Enclosures + writeQuery.prepare(QStringLiteral("INSERT INTO Enclosures VALUES (:feed, :id, :duration, :size, :title, :type, :url, :playposition, :downloaded);")); + for (EnclosureDetails enclosureDetails : m_enclosures) { + writeQuery.bindValue(QStringLiteral(":feed"), enclosureDetails.feed); + writeQuery.bindValue(QStringLiteral(":id"), enclosureDetails.id); + writeQuery.bindValue(QStringLiteral(":duration"), enclosureDetails.duration); + writeQuery.bindValue(QStringLiteral(":size"), enclosureDetails.size); + writeQuery.bindValue(QStringLiteral(":title"), enclosureDetails.title); + writeQuery.bindValue(QStringLiteral(":type"), enclosureDetails.type); + writeQuery.bindValue(QStringLiteral(":url"), enclosureDetails.url); + writeQuery.bindValue(QStringLiteral(":playposition"), enclosureDetails.playPosition); + writeQuery.bindValue(QStringLiteral(":downloaded"), Enclosure::statusToDb(enclosureDetails.downloaded)); + Database::instance().execute(writeQuery); + } + + // Chapters + writeQuery.prepare(QStringLiteral("INSERT INTO Chapters VALUES(:feed, :id, :start, :title, :link, :image);")); + for (ChapterDetails chapterDetails : m_chapters) { + writeQuery.bindValue(QStringLiteral(":feed"), chapterDetails.feed); + writeQuery.bindValue(QStringLiteral(":id"), chapterDetails.id); + writeQuery.bindValue(QStringLiteral(":start"), chapterDetails.start); + writeQuery.bindValue(QStringLiteral(":title"), chapterDetails.title); + writeQuery.bindValue(QStringLiteral(":link"), chapterDetails.link); + writeQuery.bindValue(QStringLiteral(":image"), chapterDetails.image); + Database::instance().execute(writeQuery); + } + + if (Database::instance().commit()) { + for (EntryDetails entryDetails : m_entries) { + Q_EMIT entryAdded(m_url, entryDetails.id); + } + } +} + +void UpdateFeedJob::abort() +{ + m_abort = true; + Q_EMIT aborting(); +} diff --git a/src/updatefeedjob.h b/src/updatefeedjob.h new file mode 100644 index 00000000..63838471 --- /dev/null +++ b/src/updatefeedjob.h @@ -0,0 +1,106 @@ +/** + * SPDX-FileCopyrightText: 2021 Bart De Vries + * + * SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL + */ + +#pragma once + +#include +#include +#include + +#include + +#include "enclosure.h" + +class UpdateFeedJob : public KJob +{ + Q_OBJECT + +public: + explicit UpdateFeedJob(const QString &url, QObject *parent = nullptr); + + void start() override; + void abort(); + + struct EntryDetails { + QString feed; + QString id; + QString title; + QString content; + int created; + int updated; + QString link; + bool read; + bool isNew; + bool hasEnclosure; + QString image; + }; + + struct AuthorDetails { + QString feed; + QString id; + QString name; + QString uri; + QString email; + }; + + struct EnclosureDetails { + QString feed; + QString id; + int duration; + int size; + QString title; + QString type; + QString url; + int playPosition; + Enclosure::Status downloaded; + }; + + struct ChapterDetails { + QString feed; + QString id; + int start; + QString title; + QString link; + QString image; + }; + +Q_SIGNALS: + void feedDetailsUpdated(const QString &url, + const QString &name, + const QString &image, + const QString &link, + const QString &description, + const QDateTime &lastUpdated); + void feedUpdated(const QString &url); + void entryAdded(const QString &feedurl, const QString &id); + void feedUpdateStatusChanged(const QString &url, bool status); + void aborting(); + +private: + void retrieveFeed(); + void processFeed(Syndication::FeedPtr feed); + bool processEntry(Syndication::ItemPtr entry); + void processAuthor(const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail); + void processEnclosure(Syndication::EnclosurePtr enclosure, Syndication::ItemPtr entry); + void processChapter(const QString &entryId, const int &start, const QString &chapterTitle, const QString &link, const QString &image); + void writeToDatabase(); + + bool m_abort = false; + + QString m_url; + QNetworkReply *m_reply = nullptr; + + bool m_isNewFeed; + QVector m_entries; + QVector m_authors; + QVector m_enclosures; + QVector m_chapters; + + QStringList m_existingEntryIds; + QVector> m_existingEnclosures; + QVector> m_existingAuthors; + QVector> m_existingChapters; +};