provide local IDs

src/librssguard/core/feeddownloader.cpp

@@ -5,6 +5,7 @@
 #include "3rd-party/boolinq/boolinq.h"
 #include "core/feedsmodel.h"
 #include "core/messagefilter.h"
+#include "database/databasequeries.h"
 #include "definitions/definitions.h"
 #include "exceptions/feedfetchexception.h"
 #include "exceptions/filteringexception.h"
@@ -122,7 +123,36 @@ void FeedDownloader::updateOneFeed(Feed* feed) {
   QElapsedTimer tmr; tmr.start();
 
   try {
-    QList<Message> msgs = feed->getParentServiceRoot()->obtainNewMessages({ feed });
+    bool is_main_thread = QThread::currentThread() == qApp->thread();
+    QSqlDatabase database = is_main_thread ?
+                            qApp->database()->driver()->connection(metaObject()->className()) :
+                            qApp->database()->driver()->connection(QSL("feed_upd"));
+    QHash<ServiceRoot::BagOfMessages, QStringList> stated_messages;
+    QHash<QString, QStringList> tagged_messages;
+
+    if (feed->getParentServiceRoot()->wantsBaggedIdsOfExistingMessages()) {
+      // This account has activated intelligent downloading of messages.
+      // Prepare bags.
+      stated_messages.insert(ServiceRoot::BagOfMessages::Read,
+                             DatabaseQueries::bagOfMessages(database,
+                                                            ServiceRoot::BagOfMessages::Read,
+                                                            { feed }));
+      stated_messages.insert(ServiceRoot::BagOfMessages::Unread,
+                             DatabaseQueries::bagOfMessages(database,
+                                                            ServiceRoot::BagOfMessages::Unread,
+                                                            { feed }));
+      stated_messages.insert(ServiceRoot::BagOfMessages::Starred,
+                             DatabaseQueries::bagOfMessages(database,
+                                                            ServiceRoot::BagOfMessages::Starred,
+                                                            { feed }));
+
+      tagged_messages = DatabaseQueries::bagsOfMessages(database,
+                                                        feed->getParentServiceRoot()->labelsNode()->labels());
+    }
+
+    QList<Message> msgs = feed->getParentServiceRoot()->obtainNewMessages({ feed },
+                                                                          stated_messages,
+                                                                          tagged_messages);
 
     qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded " << msgs.size() << " messages for feed ID '"
              << feed->customId() << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '"
@@ -137,11 +167,6 @@ void FeedDownloader::updateOneFeed(Feed* feed) {
     if (!feed->messageFilters().isEmpty()) {
       tmr.restart();
 
-      bool is_main_thread = QThread::currentThread() == qApp->thread();
-      QSqlDatabase database = is_main_thread ?
-                              qApp->database()->driver()->connection(metaObject()->className()) :
-                              qApp->database()->driver()->connection(QSL("feed_upd"));
-
       // Perform per-message filtering.
       QJSEngine filter_engine;
 

src/librssguard/database/databasequeries.cpp

@@ -958,6 +958,72 @@ QList<Message> DatabaseQueries::getUndeletedMessagesForAccount(const QSqlDatabas
   return messages;
 }
 
+QStringList DatabaseQueries::bagOfMessages(const QSqlDatabase& db, ServiceRoot::BagOfMessages bag, const QList<Feed*>& feeds) {
+  QStringList ids;
+  QSqlQuery q(db);
+  QString query;
+
+  q.setForwardOnly(true);
+
+  switch (bag) {
+    case ServiceRoot::BagOfMessages::Unread:
+      query = QSL("is_read = 0");
+      break;
+
+    case ServiceRoot::BagOfMessages::Starred:
+      query = QSL("is_important = 1");
+      break;
+
+    case ServiceRoot::BagOfMessages::Read:
+    default:
+      query = QSL("is_read = 1");
+      break;
+  }
+
+  q.prepare(QSL("SELECT custom_id "
+                "FROM Messages "
+                "WHERE %1 AND account_id = :account_id;").arg(query));
+
+  for (Feed* feed: feeds) {
+    q.bindValue(QSL(":account_id"), feed->getParentServiceRoot()->accountId());
+    q.exec();
+
+    while (q.next()) {
+      ids.append(q.value(0).toString());
+    }
+  }
+
+  return ids;
+}
+
+QHash<QString, QStringList> DatabaseQueries::bagsOfMessages(const QSqlDatabase& db, const QList<Label*>& labels) {
+  QHash<QString, QStringList> ids;
+  QSqlQuery q(db);
+  QString query;
+
+  q.setForwardOnly(true);
+
+  q.prepare(QSL("SELECT message "
+                "FROM LabelsInMessages "
+                "WHERE label = :label AND account_id = :account_id;").arg(query));
+
+  for (const Label* lbl :labels) {
+    q.bindValue(QSL(":label"), lbl->customId());
+    q.bindValue(QSL(":account_id"), lbl->getParentServiceRoot()->accountId());
+    q.exec();
+
+    QStringList ids_one_label;
+
+    while (q.next()) {
+      ids_one_label.append(q.value(0).toString());
+    }
+
+    ids.insert(lbl->customId(), ids_one_label);
+  }
+
+  return ids;
+}
+
 QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
                                                 const QList<Message>& messages,
                                                 const QString& feed_custom_id,

src/librssguard/database/databasequeries.h

@@ -93,6 +93,8 @@ class DatabaseQueries {
     static QList<Message> getUndeletedMessagesForAccount(const QSqlDatabase& db, int account_id, bool* ok = nullptr);
 
     // Custom ID accumulators.
+    static QStringList bagOfMessages(const QSqlDatabase& db, ServiceRoot::BagOfMessages bag, const QList<Feed*>& feeds);
+    static QHash<QString, QStringList> bagsOfMessages(const QSqlDatabase& db, const QList<Label*>& labels);
     static QStringList customIdsOfMessagesFromLabel(const QSqlDatabase& db, Label* label, bool* ok = nullptr);
     static QStringList customIdsOfImportantMessages(const QSqlDatabase& db, int account_id, bool* ok = nullptr);
     static QStringList customIdsOfUnreadMessages(const QSqlDatabase& db, int account_id, bool* ok = nullptr);

src/librssguard/services/abstract/serviceroot.cpp

@@ -311,6 +311,10 @@ void ServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
   Q_UNUSED(data)
 }
 
+bool ServiceRoot::wantsBaggedIdsOfExistingMessages() const {
+  return false;
+}
+
 void ServiceRoot::itemChanged(const QList<RootItem*>& items) {
   emit dataChanged(items);
 }

src/librssguard/services/abstract/serviceroot.h

@@ -36,6 +36,12 @@ class ServiceRoot : public RootItem {
       Deleting = 4
     };
 
+    enum class BagOfMessages {
+      Read,
+      Unread,
+      Starred
+    };
+
   public:
     explicit ServiceRoot(RootItem* parent = nullptr);
     virtual ~ServiceRoot();
@@ -58,6 +64,7 @@ class ServiceRoot : public RootItem {
     virtual void saveAccountDataToDatabase();
     virtual QVariantHash customDatabaseData() const;
     virtual void setCustomDatabaseData(const QVariantHash& data);
+    virtual bool wantsBaggedIdsOfExistingMessages() const;
 
     // Returns list of specific actions for "Add new item" main window menu.
     // So typical list of returned actions could look like:
@@ -94,7 +101,9 @@ class ServiceRoot : public RootItem {
     // Obtains list of messages.
     // Throws exception subclassed from ApplicationException, preferably FeedFetchException
     // if any problems arise.
-    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds) = 0;
+    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
+                                             const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
+                                             const QHash<QString, QStringList>& tagged_messages) = 0;
 
     // This method should prepare messages for given "item" (download them maybe?)
     // into predefined "Messages" table
@@ -266,6 +275,10 @@ class ServiceRoot : public RootItem {
     QNetworkProxy m_networkProxy;
 };
 
+inline uint qHash(ServiceRoot::BagOfMessages key, uint seed) {
+  return ::qHash(static_cast<uint>(key), seed);
+}
+
 ServiceRoot::LabelOperation operator|(ServiceRoot::LabelOperation lhs, ServiceRoot::LabelOperation rhs);
 ServiceRoot::LabelOperation operator&(ServiceRoot::LabelOperation lhs, ServiceRoot::LabelOperation rhs);
 

src/librssguard/services/feedly/feedlyserviceroot.cpp

@@ -72,7 +72,12 @@ void FeedlyServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
   m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool());
 }
 
-QList<Message> FeedlyServiceRoot::obtainNewMessages(const QList<Feed*>& feeds) {
+QList<Message> FeedlyServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
+                                                    const QHash<BagOfMessages, QStringList>& stated_messages,
+                                                    const QHash<QString, QStringList>& tagged_messages) {
+  Q_UNUSED(stated_messages)
+  Q_UNUSED(tagged_messages)
+
   QList<Message> messages;
 
   for (Feed* feed : feeds) {

src/librssguard/services/feedly/feedlyserviceroot.h

@@ -23,7 +23,9 @@ class FeedlyServiceRoot : public ServiceRoot, public CacheForServiceRoot {
     virtual LabelOperation supportedLabelOperations() const;
     virtual QVariantHash customDatabaseData() const;
     virtual void setCustomDatabaseData(const QVariantHash& data);
-    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds);
+    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
+                                             const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
+                                             const QHash<QString, QStringList>& tagged_messages);
 
     FeedlyNetwork* network() const;
 

src/librssguard/services/gmail/gmailserviceroot.cpp

@@ -74,7 +74,12 @@ void GmailServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
   m_network->oauth()->setRedirectUrl(data["redirect_uri"].toString());
 }
 
-QList<Message> GmailServiceRoot::obtainNewMessages(const QList<Feed*>& feeds) {
+QList<Message> GmailServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
+                                                   const QHash<BagOfMessages, QStringList>& stated_messages,
+                                                   const QHash<QString, QStringList>& tagged_messages) {
+  Q_UNUSED(stated_messages)
+  Q_UNUSED(tagged_messages)
+
   QList<Message> messages;
 
   for (Feed* feed : feeds) {

src/librssguard/services/gmail/gmailserviceroot.h

@@ -31,7 +31,9 @@ class GmailServiceRoot : public ServiceRoot, public CacheForServiceRoot {
     virtual void saveAllCachedData(bool ignore_errors);
     virtual QVariantHash customDatabaseData() const;
     virtual void setCustomDatabaseData(const QVariantHash& data);
-    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds);
+    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
+                                             const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
+                                             const QHash<QString, QStringList>& tagged_messages);
 
   protected:
     virtual RootItem* obtainNewTreeForSyncIn() const;

src/librssguard/services/greader/greaderserviceroot.cpp

@@ -57,7 +57,12 @@ void GreaderServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
   m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool());
 }
 
-QList<Message> GreaderServiceRoot::obtainNewMessages(const QList<Feed*>& feeds) {
+QList<Message> GreaderServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
+                                                     const QHash<BagOfMessages, QStringList>& stated_messages,
+                                                     const QHash<QString, QStringList>& tagged_messages) {
+  Q_UNUSED(stated_messages)
+  Q_UNUSED(tagged_messages)
+
   QList<Message> messages;
 
   for (Feed* feed : feeds) {
@@ -73,6 +78,10 @@ QList<Message> GreaderServiceRoot::obtainNewMessages(const QList<Feed*>& feeds)
   return messages;
 }
 
+bool GreaderServiceRoot::wantsBaggedIdsOfExistingMessages() const {
+  return true;
+}
+
 void GreaderServiceRoot::start(bool freshly_activated) {
   if (!freshly_activated) {
     DatabaseQueries::loadFromDatabase<Category, Feed>(this);

src/librssguard/services/greader/greaderserviceroot.h

@@ -31,7 +31,10 @@ class GreaderServiceRoot : public ServiceRoot, public CacheForServiceRoot {
     virtual LabelOperation supportedLabelOperations() const;
     virtual QVariantHash customDatabaseData() const;
     virtual void setCustomDatabaseData(const QVariantHash& data);
-    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds);
+    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
+                                             const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
+                                             const QHash<QString, QStringList>& tagged_messages);
+    virtual bool wantsBaggedIdsOfExistingMessages() const;
 
     GreaderNetwork* network() const;
 

src/librssguard/services/inoreader/inoreaderserviceroot.cpp

@@ -56,7 +56,12 @@ void InoreaderServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
   m_network->oauth()->setRedirectUrl(data["redirect_uri"].toString());
 }
 
-QList<Message> InoreaderServiceRoot::obtainNewMessages(const QList<Feed*>& feeds) {
+QList<Message> InoreaderServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
+                                                       const QHash<BagOfMessages, QStringList>& stated_messages,
+                                                       const QHash<QString, QStringList>& tagged_messages) {
+  Q_UNUSED(stated_messages)
+  Q_UNUSED(tagged_messages)
+
   QList<Message> messages;
 
   for (Feed* feed : feeds) {

src/librssguard/services/inoreader/inoreaderserviceroot.h

@@ -29,7 +29,9 @@ class InoreaderServiceRoot : public ServiceRoot, public CacheForServiceRoot {
     virtual void saveAllCachedData(bool ignore_errors);
     virtual QVariantHash customDatabaseData() const;
     virtual void setCustomDatabaseData(const QVariantHash& data);
-    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds);
+    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
+                                             const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
+                                             const QHash<QString, QStringList>& tagged_messages);
 
   protected:
     virtual RootItem* obtainNewTreeForSyncIn() const;

src/librssguard/services/owncloud/owncloudserviceroot.cpp

@@ -150,7 +150,12 @@ void OwnCloudServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
   m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool());
 }
 
-QList<Message> OwnCloudServiceRoot::obtainNewMessages(const QList<Feed*>& feeds) {
+QList<Message> OwnCloudServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
+                                                      const QHash<BagOfMessages, QStringList>& stated_messages,
+                                                      const QHash<QString, QStringList>& tagged_messages) {
+  Q_UNUSED(stated_messages)
+  Q_UNUSED(tagged_messages)
+
   QList<Message> msgs;
 
   for (Feed* feed : feeds) {

src/librssguard/services/owncloud/owncloudserviceroot.h

@@ -28,7 +28,9 @@ class OwnCloudServiceRoot : public ServiceRoot, public CacheForServiceRoot {
     virtual void saveAllCachedData(bool ignore_errors);
     virtual QVariantHash customDatabaseData() const;
     virtual void setCustomDatabaseData(const QVariantHash& data);
-    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds);
+    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
+                                             const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
+                                             const QHash<QString, QStringList>& tagged_messages);
 
     OwnCloudNetworkFactory* network() const;
 

src/librssguard/services/standard/standardserviceroot.cpp

@@ -144,7 +144,12 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const {
   return Qt::ItemFlag::ItemIsDropEnabled;
 }
 
-QList<Message> StandardServiceRoot::obtainNewMessages(const QList<Feed*>& feeds) {
+QList<Message> StandardServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
+                                                      const QHash<BagOfMessages, QStringList>& stated_messages,
+                                                      const QHash<QString, QStringList>& tagged_messages) {
+  Q_UNUSED(stated_messages)
+  Q_UNUSED(tagged_messages)
+
   QList<Message> msgs;
 
   for (Feed* f : feeds) {

src/librssguard/services/standard/standardserviceroot.h

@@ -32,7 +32,9 @@ class StandardServiceRoot : public ServiceRoot {
     virtual bool supportsFeedAdding() const;
     virtual bool supportsCategoryAdding() const;
     virtual Qt::ItemFlags additionalFlags() const;
-    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds);
+    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
+                                             const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
+                                             const QHash<QString, QStringList>& tagged_messages);
 
     QList<QAction*> serviceMenu();
     QList<QAction*> getContextMenuForFeed(StandardFeed* feed);

src/librssguard/services/tt-rss/ttrssserviceroot.cpp

@@ -214,7 +214,12 @@ void TtRssServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
   m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool());
 }
 
-QList<Message> TtRssServiceRoot::obtainNewMessages(const QList<Feed*>& feeds) {
+QList<Message> TtRssServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
+                                                   const QHash<BagOfMessages, QStringList>& stated_messages,
+                                                   const QHash<QString, QStringList>& tagged_messages) {
+  Q_UNUSED(stated_messages)
+  Q_UNUSED(tagged_messages)
+
   QList<Message> messages;
 
   for (Feed* feed : feeds) {

src/librssguard/services/tt-rss/ttrssserviceroot.h

@@ -33,7 +33,9 @@ class TtRssServiceRoot : public ServiceRoot, public CacheForServiceRoot {
     virtual void saveAllCachedData(bool ignore_errors);
     virtual QVariantHash customDatabaseData() const;
     virtual void setCustomDatabaseData(const QVariantHash& data);
-    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds);
+    virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
+                                             const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
+                                             const QHash<QString, QStringList>& tagged_messages);
 
     // Access to network.
     TtRssNetworkFactory* network() const;