From fd251a1e78de880d52d149b0e1ee1a06ca43a42a Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Wed, 14 Jul 2021 14:46:45 +0200 Subject: [PATCH] provide local IDs --- src/librssguard/core/feeddownloader.cpp | 37 +++++++++-- src/librssguard/database/databasequeries.cpp | 66 +++++++++++++++++++ src/librssguard/database/databasequeries.h | 2 + .../services/abstract/serviceroot.cpp | 4 ++ .../services/abstract/serviceroot.h | 15 ++++- .../services/feedly/feedlyserviceroot.cpp | 7 +- .../services/feedly/feedlyserviceroot.h | 4 +- .../services/gmail/gmailserviceroot.cpp | 7 +- .../services/gmail/gmailserviceroot.h | 4 +- .../services/greader/greaderserviceroot.cpp | 11 +++- .../services/greader/greaderserviceroot.h | 5 +- .../inoreader/inoreaderserviceroot.cpp | 7 +- .../services/inoreader/inoreaderserviceroot.h | 4 +- .../services/owncloud/owncloudserviceroot.cpp | 7 +- .../services/owncloud/owncloudserviceroot.h | 4 +- .../services/standard/standardserviceroot.cpp | 7 +- .../services/standard/standardserviceroot.h | 4 +- .../services/tt-rss/ttrssserviceroot.cpp | 7 +- .../services/tt-rss/ttrssserviceroot.h | 4 +- 19 files changed, 185 insertions(+), 21 deletions(-) diff --git a/src/librssguard/core/feeddownloader.cpp b/src/librssguard/core/feeddownloader.cpp index 9ab44cee7..2f47a0ff4 100644 --- a/src/librssguard/core/feeddownloader.cpp +++ b/src/librssguard/core/feeddownloader.cpp @@ -5,6 +5,7 @@ #include "3rd-party/boolinq/boolinq.h" #include "core/feedsmodel.h" #include "core/messagefilter.h" +#include "database/databasequeries.h" #include "definitions/definitions.h" #include "exceptions/feedfetchexception.h" #include "exceptions/filteringexception.h" @@ -122,7 +123,36 @@ void FeedDownloader::updateOneFeed(Feed* feed) { QElapsedTimer tmr; tmr.start(); try { - QList msgs = feed->getParentServiceRoot()->obtainNewMessages({ feed }); + bool is_main_thread = QThread::currentThread() == qApp->thread(); + QSqlDatabase database = is_main_thread ? + qApp->database()->driver()->connection(metaObject()->className()) : + qApp->database()->driver()->connection(QSL("feed_upd")); + QHash stated_messages; + QHash tagged_messages; + + if (feed->getParentServiceRoot()->wantsBaggedIdsOfExistingMessages()) { + // This account has activated intelligent downloading of messages. + // Prepare bags. + stated_messages.insert(ServiceRoot::BagOfMessages::Read, + DatabaseQueries::bagOfMessages(database, + ServiceRoot::BagOfMessages::Read, + { feed })); + stated_messages.insert(ServiceRoot::BagOfMessages::Unread, + DatabaseQueries::bagOfMessages(database, + ServiceRoot::BagOfMessages::Unread, + { feed })); + stated_messages.insert(ServiceRoot::BagOfMessages::Starred, + DatabaseQueries::bagOfMessages(database, + ServiceRoot::BagOfMessages::Starred, + { feed })); + + tagged_messages = DatabaseQueries::bagsOfMessages(database, + feed->getParentServiceRoot()->labelsNode()->labels()); + } + + QList msgs = feed->getParentServiceRoot()->obtainNewMessages({ feed }, + stated_messages, + tagged_messages); qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded " << msgs.size() << " messages for feed ID '" << feed->customId() << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '" @@ -137,11 +167,6 @@ void FeedDownloader::updateOneFeed(Feed* feed) { if (!feed->messageFilters().isEmpty()) { tmr.restart(); - bool is_main_thread = QThread::currentThread() == qApp->thread(); - QSqlDatabase database = is_main_thread ? - qApp->database()->driver()->connection(metaObject()->className()) : - qApp->database()->driver()->connection(QSL("feed_upd")); - // Perform per-message filtering. QJSEngine filter_engine; diff --git a/src/librssguard/database/databasequeries.cpp b/src/librssguard/database/databasequeries.cpp index 57e5e3c74..804b3cae8 100755 --- a/src/librssguard/database/databasequeries.cpp +++ b/src/librssguard/database/databasequeries.cpp @@ -958,6 +958,72 @@ QList DatabaseQueries::getUndeletedMessagesForAccount(const QSqlDatabas return messages; } +QStringList DatabaseQueries::bagOfMessages(const QSqlDatabase& db, ServiceRoot::BagOfMessages bag, const QList& feeds) { + QStringList ids; + QSqlQuery q(db); + QString query; + + q.setForwardOnly(true); + + switch (bag) { + case ServiceRoot::BagOfMessages::Unread: + query = QSL("is_read = 0"); + break; + + case ServiceRoot::BagOfMessages::Starred: + query = QSL("is_important = 1"); + break; + + case ServiceRoot::BagOfMessages::Read: + default: + query = QSL("is_read = 1"); + break; + } + + q.prepare(QSL("SELECT custom_id " + "FROM Messages " + "WHERE %1 AND account_id = :account_id;").arg(query)); + + for (Feed* feed: feeds) { + q.bindValue(QSL(":account_id"), feed->getParentServiceRoot()->accountId()); + q.exec(); + + while (q.next()) { + ids.append(q.value(0).toString()); + } + } + + return ids; +} + +QHash DatabaseQueries::bagsOfMessages(const QSqlDatabase& db, const QList& labels) { + QHash ids; + QSqlQuery q(db); + QString query; + + q.setForwardOnly(true); + + q.prepare(QSL("SELECT message " + "FROM LabelsInMessages " + "WHERE label = :label AND account_id = :account_id;").arg(query)); + + for (const Label* lbl :labels) { + q.bindValue(QSL(":label"), lbl->customId()); + q.bindValue(QSL(":account_id"), lbl->getParentServiceRoot()->accountId()); + q.exec(); + + QStringList ids_one_label; + + while (q.next()) { + ids_one_label.append(q.value(0).toString()); + } + + ids.insert(lbl->customId(), ids_one_label); + } + + return ids; +} + QPair DatabaseQueries::updateMessages(QSqlDatabase db, const QList& messages, const QString& feed_custom_id, diff --git a/src/librssguard/database/databasequeries.h b/src/librssguard/database/databasequeries.h index 6da941e6e..0a76d0e22 100644 --- a/src/librssguard/database/databasequeries.h +++ b/src/librssguard/database/databasequeries.h @@ -93,6 +93,8 @@ class DatabaseQueries { static QList getUndeletedMessagesForAccount(const QSqlDatabase& db, int account_id, bool* ok = nullptr); // Custom ID accumulators. + static QStringList bagOfMessages(const QSqlDatabase& db, ServiceRoot::BagOfMessages bag, const QList& feeds); + static QHash bagsOfMessages(const QSqlDatabase& db, const QList& labels); static QStringList customIdsOfMessagesFromLabel(const QSqlDatabase& db, Label* label, bool* ok = nullptr); static QStringList customIdsOfImportantMessages(const QSqlDatabase& db, int account_id, bool* ok = nullptr); static QStringList customIdsOfUnreadMessages(const QSqlDatabase& db, int account_id, bool* ok = nullptr); diff --git a/src/librssguard/services/abstract/serviceroot.cpp b/src/librssguard/services/abstract/serviceroot.cpp index 58f7ca895..940641892 100644 --- a/src/librssguard/services/abstract/serviceroot.cpp +++ b/src/librssguard/services/abstract/serviceroot.cpp @@ -311,6 +311,10 @@ void ServiceRoot::setCustomDatabaseData(const QVariantHash& data) { Q_UNUSED(data) } +bool ServiceRoot::wantsBaggedIdsOfExistingMessages() const { + return false; +} + void ServiceRoot::itemChanged(const QList& items) { emit dataChanged(items); } diff --git a/src/librssguard/services/abstract/serviceroot.h b/src/librssguard/services/abstract/serviceroot.h index 088add514..c10158101 100644 --- a/src/librssguard/services/abstract/serviceroot.h +++ b/src/librssguard/services/abstract/serviceroot.h @@ -36,6 +36,12 @@ class ServiceRoot : public RootItem { Deleting = 4 }; + enum class BagOfMessages { + Read, + Unread, + Starred + }; + public: explicit ServiceRoot(RootItem* parent = nullptr); virtual ~ServiceRoot(); @@ -58,6 +64,7 @@ class ServiceRoot : public RootItem { virtual void saveAccountDataToDatabase(); virtual QVariantHash customDatabaseData() const; virtual void setCustomDatabaseData(const QVariantHash& data); + virtual bool wantsBaggedIdsOfExistingMessages() const; // Returns list of specific actions for "Add new item" main window menu. // So typical list of returned actions could look like: @@ -94,7 +101,9 @@ class ServiceRoot : public RootItem { // Obtains list of messages. // Throws exception subclassed from ApplicationException, preferably FeedFetchException // if any problems arise. - virtual QList obtainNewMessages(const QList& feeds) = 0; + virtual QList obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages) = 0; // This method should prepare messages for given "item" (download them maybe?) // into predefined "Messages" table @@ -266,6 +275,10 @@ class ServiceRoot : public RootItem { QNetworkProxy m_networkProxy; }; +inline uint qHash(ServiceRoot::BagOfMessages key, uint seed) { + return ::qHash(static_cast(key), seed); +} + ServiceRoot::LabelOperation operator|(ServiceRoot::LabelOperation lhs, ServiceRoot::LabelOperation rhs); ServiceRoot::LabelOperation operator&(ServiceRoot::LabelOperation lhs, ServiceRoot::LabelOperation rhs); diff --git a/src/librssguard/services/feedly/feedlyserviceroot.cpp b/src/librssguard/services/feedly/feedlyserviceroot.cpp index f701beede..bff9130c6 100755 --- a/src/librssguard/services/feedly/feedlyserviceroot.cpp +++ b/src/librssguard/services/feedly/feedlyserviceroot.cpp @@ -72,7 +72,12 @@ void FeedlyServiceRoot::setCustomDatabaseData(const QVariantHash& data) { m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool()); } -QList FeedlyServiceRoot::obtainNewMessages(const QList& feeds) { +QList FeedlyServiceRoot::obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages) { + Q_UNUSED(stated_messages) + Q_UNUSED(tagged_messages) + QList messages; for (Feed* feed : feeds) { diff --git a/src/librssguard/services/feedly/feedlyserviceroot.h b/src/librssguard/services/feedly/feedlyserviceroot.h index c2a4f0c62..79edf203e 100755 --- a/src/librssguard/services/feedly/feedlyserviceroot.h +++ b/src/librssguard/services/feedly/feedlyserviceroot.h @@ -23,7 +23,9 @@ class FeedlyServiceRoot : public ServiceRoot, public CacheForServiceRoot { virtual LabelOperation supportedLabelOperations() const; virtual QVariantHash customDatabaseData() const; virtual void setCustomDatabaseData(const QVariantHash& data); - virtual QList obtainNewMessages(const QList& feeds); + virtual QList obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages); FeedlyNetwork* network() const; diff --git a/src/librssguard/services/gmail/gmailserviceroot.cpp b/src/librssguard/services/gmail/gmailserviceroot.cpp index a47fb7374..0c8180471 100644 --- a/src/librssguard/services/gmail/gmailserviceroot.cpp +++ b/src/librssguard/services/gmail/gmailserviceroot.cpp @@ -74,7 +74,12 @@ void GmailServiceRoot::setCustomDatabaseData(const QVariantHash& data) { m_network->oauth()->setRedirectUrl(data["redirect_uri"].toString()); } -QList GmailServiceRoot::obtainNewMessages(const QList& feeds) { +QList GmailServiceRoot::obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages) { + Q_UNUSED(stated_messages) + Q_UNUSED(tagged_messages) + QList messages; for (Feed* feed : feeds) { diff --git a/src/librssguard/services/gmail/gmailserviceroot.h b/src/librssguard/services/gmail/gmailserviceroot.h index 688b02232..eb8c42c11 100644 --- a/src/librssguard/services/gmail/gmailserviceroot.h +++ b/src/librssguard/services/gmail/gmailserviceroot.h @@ -31,7 +31,9 @@ class GmailServiceRoot : public ServiceRoot, public CacheForServiceRoot { virtual void saveAllCachedData(bool ignore_errors); virtual QVariantHash customDatabaseData() const; virtual void setCustomDatabaseData(const QVariantHash& data); - virtual QList obtainNewMessages(const QList& feeds); + virtual QList obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages); protected: virtual RootItem* obtainNewTreeForSyncIn() const; diff --git a/src/librssguard/services/greader/greaderserviceroot.cpp b/src/librssguard/services/greader/greaderserviceroot.cpp index b8b91d72b..5a9c0b8f9 100755 --- a/src/librssguard/services/greader/greaderserviceroot.cpp +++ b/src/librssguard/services/greader/greaderserviceroot.cpp @@ -57,7 +57,12 @@ void GreaderServiceRoot::setCustomDatabaseData(const QVariantHash& data) { m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool()); } -QList GreaderServiceRoot::obtainNewMessages(const QList& feeds) { +QList GreaderServiceRoot::obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages) { + Q_UNUSED(stated_messages) + Q_UNUSED(tagged_messages) + QList messages; for (Feed* feed : feeds) { @@ -73,6 +78,10 @@ QList GreaderServiceRoot::obtainNewMessages(const QList& feeds) return messages; } +bool GreaderServiceRoot::wantsBaggedIdsOfExistingMessages() const { + return true; +} + void GreaderServiceRoot::start(bool freshly_activated) { if (!freshly_activated) { DatabaseQueries::loadFromDatabase(this); diff --git a/src/librssguard/services/greader/greaderserviceroot.h b/src/librssguard/services/greader/greaderserviceroot.h index 6306a1432..79b590519 100755 --- a/src/librssguard/services/greader/greaderserviceroot.h +++ b/src/librssguard/services/greader/greaderserviceroot.h @@ -31,7 +31,10 @@ class GreaderServiceRoot : public ServiceRoot, public CacheForServiceRoot { virtual LabelOperation supportedLabelOperations() const; virtual QVariantHash customDatabaseData() const; virtual void setCustomDatabaseData(const QVariantHash& data); - virtual QList obtainNewMessages(const QList& feeds); + virtual QList obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages); + virtual bool wantsBaggedIdsOfExistingMessages() const; GreaderNetwork* network() const; diff --git a/src/librssguard/services/inoreader/inoreaderserviceroot.cpp b/src/librssguard/services/inoreader/inoreaderserviceroot.cpp index 49b43a1e6..dad13f2d6 100644 --- a/src/librssguard/services/inoreader/inoreaderserviceroot.cpp +++ b/src/librssguard/services/inoreader/inoreaderserviceroot.cpp @@ -56,7 +56,12 @@ void InoreaderServiceRoot::setCustomDatabaseData(const QVariantHash& data) { m_network->oauth()->setRedirectUrl(data["redirect_uri"].toString()); } -QList InoreaderServiceRoot::obtainNewMessages(const QList& feeds) { +QList InoreaderServiceRoot::obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages) { + Q_UNUSED(stated_messages) + Q_UNUSED(tagged_messages) + QList messages; for (Feed* feed : feeds) { diff --git a/src/librssguard/services/inoreader/inoreaderserviceroot.h b/src/librssguard/services/inoreader/inoreaderserviceroot.h index 01106fdce..c8954a6f8 100644 --- a/src/librssguard/services/inoreader/inoreaderserviceroot.h +++ b/src/librssguard/services/inoreader/inoreaderserviceroot.h @@ -29,7 +29,9 @@ class InoreaderServiceRoot : public ServiceRoot, public CacheForServiceRoot { virtual void saveAllCachedData(bool ignore_errors); virtual QVariantHash customDatabaseData() const; virtual void setCustomDatabaseData(const QVariantHash& data); - virtual QList obtainNewMessages(const QList& feeds); + virtual QList obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages); protected: virtual RootItem* obtainNewTreeForSyncIn() const; diff --git a/src/librssguard/services/owncloud/owncloudserviceroot.cpp b/src/librssguard/services/owncloud/owncloudserviceroot.cpp index be9b1fe7c..61f3ec3d0 100644 --- a/src/librssguard/services/owncloud/owncloudserviceroot.cpp +++ b/src/librssguard/services/owncloud/owncloudserviceroot.cpp @@ -150,7 +150,12 @@ void OwnCloudServiceRoot::setCustomDatabaseData(const QVariantHash& data) { m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool()); } -QList OwnCloudServiceRoot::obtainNewMessages(const QList& feeds) { +QList OwnCloudServiceRoot::obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages) { + Q_UNUSED(stated_messages) + Q_UNUSED(tagged_messages) + QList msgs; for (Feed* feed : feeds) { diff --git a/src/librssguard/services/owncloud/owncloudserviceroot.h b/src/librssguard/services/owncloud/owncloudserviceroot.h index 3ec6423db..1e5cb9b5d 100644 --- a/src/librssguard/services/owncloud/owncloudserviceroot.h +++ b/src/librssguard/services/owncloud/owncloudserviceroot.h @@ -28,7 +28,9 @@ class OwnCloudServiceRoot : public ServiceRoot, public CacheForServiceRoot { virtual void saveAllCachedData(bool ignore_errors); virtual QVariantHash customDatabaseData() const; virtual void setCustomDatabaseData(const QVariantHash& data); - virtual QList obtainNewMessages(const QList& feeds); + virtual QList obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages); OwnCloudNetworkFactory* network() const; diff --git a/src/librssguard/services/standard/standardserviceroot.cpp b/src/librssguard/services/standard/standardserviceroot.cpp index 6f50c58a1..ecf920d7e 100644 --- a/src/librssguard/services/standard/standardserviceroot.cpp +++ b/src/librssguard/services/standard/standardserviceroot.cpp @@ -144,7 +144,12 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const { return Qt::ItemFlag::ItemIsDropEnabled; } -QList StandardServiceRoot::obtainNewMessages(const QList& feeds) { +QList StandardServiceRoot::obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages) { + Q_UNUSED(stated_messages) + Q_UNUSED(tagged_messages) + QList msgs; for (Feed* f : feeds) { diff --git a/src/librssguard/services/standard/standardserviceroot.h b/src/librssguard/services/standard/standardserviceroot.h index 65c5e7851..4bc453102 100644 --- a/src/librssguard/services/standard/standardserviceroot.h +++ b/src/librssguard/services/standard/standardserviceroot.h @@ -32,7 +32,9 @@ class StandardServiceRoot : public ServiceRoot { virtual bool supportsFeedAdding() const; virtual bool supportsCategoryAdding() const; virtual Qt::ItemFlags additionalFlags() const; - virtual QList obtainNewMessages(const QList& feeds); + virtual QList obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages); QList serviceMenu(); QList getContextMenuForFeed(StandardFeed* feed); diff --git a/src/librssguard/services/tt-rss/ttrssserviceroot.cpp b/src/librssguard/services/tt-rss/ttrssserviceroot.cpp index 5d1967a65..1dc4cabe1 100644 --- a/src/librssguard/services/tt-rss/ttrssserviceroot.cpp +++ b/src/librssguard/services/tt-rss/ttrssserviceroot.cpp @@ -214,7 +214,12 @@ void TtRssServiceRoot::setCustomDatabaseData(const QVariantHash& data) { m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool()); } -QList TtRssServiceRoot::obtainNewMessages(const QList& feeds) { +QList TtRssServiceRoot::obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages) { + Q_UNUSED(stated_messages) + Q_UNUSED(tagged_messages) + QList messages; for (Feed* feed : feeds) { diff --git a/src/librssguard/services/tt-rss/ttrssserviceroot.h b/src/librssguard/services/tt-rss/ttrssserviceroot.h index e3c21a5bf..021ce21a9 100644 --- a/src/librssguard/services/tt-rss/ttrssserviceroot.h +++ b/src/librssguard/services/tt-rss/ttrssserviceroot.h @@ -33,7 +33,9 @@ class TtRssServiceRoot : public ServiceRoot, public CacheForServiceRoot { virtual void saveAllCachedData(bool ignore_errors); virtual QVariantHash customDatabaseData() const; virtual void setCustomDatabaseData(const QVariantHash& data); - virtual QList obtainNewMessages(const QList& feeds); + virtual QList obtainNewMessages(const QList& feeds, + const QHash& stated_messages, + const QHash& tagged_messages); // Access to network. TtRssNetworkFactory* network() const;