very experimental version of better syncing algorithm for greader plugin

resources/desktop/com.github.rssguard.appdata.xml

@@ -30,7 +30,7 @@
   <url type="donation">https://martinrotter.github.io/donate/</url>
   <content_rating type="oars-1.1" />
   <releases>
-    <release version="3.9.2" date="2021-07-27"/>
+    <release version="3.9.2" date="2021-07-28"/>
   </releases>
   <content_rating type="oars-1.0">
     <content_attribute id="violence-cartoon">none</content_attribute>

src/librssguard/services/greader/definitions.h

@@ -14,7 +14,6 @@
 // Means "read" message. If both "reading-list" and "read" are specified, message is READ. If this state
 // is not present, message is UNREAD.
 #define GREADER_API_STATE_READ            "state/com.google/read"
-
 #define GREADER_API_STATE_IMPORTANT       "state/com.google/starred"
 
 #define GREADER_API_FULL_STATE_READING_LIST    "user/-/state/com.google/reading-list"
@@ -27,8 +26,8 @@
 #define GREADER_API_SUBSCRIPTION_LIST   "reader/api/0/subscription/list?output=json"
 #define GREADER_API_STREAM_CONTENTS     "reader/api/0/stream/contents/%1?output=json&n=%2"
 #define GREADER_API_EDIT_TAG            "reader/api/0/edit-tag"
-#define GREADER_API_ITEM_IDS            "reader/api/0/stream/items/ids?output=json&s=%1"
+#define GREADER_API_ITEM_IDS            "reader/api/0/stream/items/ids?output=json&n=200000&s=%1"
-#define GREADER_API_ITEM_CONTENTS       "reader/api/0/stream/items/contents?output=json&n=%1"
+#define GREADER_API_ITEM_CONTENTS       "reader/api/0/stream/items/contents?output=json&n=200000"
 #define GREADER_API_TOKEN               "reader/api/0/token"
 #define GREADER_API_USER_INFO           "reader/api/0/user-info?output=json"
 

src/librssguard/services/greader/greadernetwork.cpp

@@ -128,13 +128,35 @@ QList<Message> GreaderNetwork::getMessagesIntelligently(ServiceRoot* root,
                                                         const QNetworkProxy& proxy) {
   // 1. Get unread IDs for a feed.
   // 2. Get read IDs for a feed.
+  // 3. Get starred IDs. Unfortunately, we cannot get only starred IDs for a feed, but all
+  // instead.
   // 3. Download messages/contents for missing or changed IDs.
   // 4. Download all starred messages and append those belonging to this feed.
 
   auto remote_all_ids_list = itemIds(stream_id, false, proxy);
   auto remote_unread_ids_list = itemIds(stream_id, true, proxy);
+  auto remote_starred_ids_list = itemIds(GREADER_API_FULL_STATE_IMPORTANT, false, proxy);
+
+  // Convert item IDs to long form.
+  for (int i = 0; i < remote_all_ids_list.size(); i++) {
+    remote_all_ids_list.replace(i,
+                                convertShortStreamIdToLongStreamId(remote_all_ids_list.at(i)));
+  }
+
+  for (int i = 0; i < remote_unread_ids_list.size(); i++) {
+    remote_unread_ids_list.replace(i,
+                                   convertShortStreamIdToLongStreamId(remote_unread_ids_list.at(i)));
+  }
+
+  for (int i = 0; i < remote_starred_ids_list.size(); i++) {
+    remote_starred_ids_list.replace(i,
+                                    convertShortStreamIdToLongStreamId(remote_starred_ids_list.at(i)));
+  }
+
   QSet<QString> remote_all_ids(remote_all_ids_list.begin(), remote_all_ids_list.end());
 
+  //remote_all_ids += QSet<QString>(remote_starred_ids_list.begin(), remote_starred_ids_list.end());
+
   // 1.
   auto local_unread_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Unread);
   QSet<QString> remote_unread_ids(remote_unread_ids_list.begin(), remote_unread_ids_list.end());
@@ -146,13 +168,37 @@ QList<Message> GreaderNetwork::getMessagesIntelligently(ServiceRoot* root,
   QSet<QString> remote_read_ids = remote_all_ids - remote_unread_ids;
   QSet<QString> local_read_ids(local_read_ids_list.begin(),
                                local_read_ids_list.end());
+
+  // 3.
+  auto local_starred_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Starred);
+  QSet<QString> remote_starred_ids(remote_starred_ids_list.begin(), remote_starred_ids_list.end());
+  QSet<QString> local_starred_ids(local_starred_ids_list.begin(), local_starred_ids_list.end());
+  QSet<QString> local_unstarred_ids((local_read_ids + local_unread_ids) - local_starred_ids);
+
+  // 4.
   auto not_downloaded = remote_all_ids - local_read_ids - local_unread_ids;
   auto moved_unread = local_unread_ids.intersect(remote_read_ids);
   auto moved_read = local_read_ids.intersect(remote_unread_ids);
-  auto to_download = not_downloaded + moved_read + moved_unread;
+  auto moved_unstarred = local_unstarred_ids.intersect(remote_starred_ids);
-  QList<QString> to_download_list(to_download.begin(), to_download.end());
+  auto moved_starred = local_starred_ids - remote_starred_ids;
+  auto to_download = not_downloaded + moved_read + moved_unread + moved_starred + moved_unstarred + remote_starred_ids;
 
-  return itemContents(root, to_download_list, error, proxy);
+  if (to_download.isEmpty()) {
+    return {};
+  }
+
+  QList<QString> to_download_list(to_download.begin(), to_download.end());
+  auto msgs = itemContents(root, to_download_list, error, proxy);
+
+  // Filter out (starred) messages from other feeds.
+  // TODO: Cache them instead?
+  for (int i = 0; i < msgs.size(); i++) {
+    if (msgs.at(i).m_feedId != stream_id) {
+      msgs.removeAt(i--);
+    }
+  }
+
+  return msgs;
 }
 
 QNetworkReply::NetworkError GreaderNetwork::markMessagesRead(RootItem::ReadStatus status,
@@ -229,10 +275,11 @@ QList<Message> GreaderNetwork::itemContents(ServiceRoot* root, const QList<QStri
   }
 
   QList<Message> msgs;
-  int target_msgs_size = 999;
+
+  //int target_msgs_size = 999;
 
   do {
-    QString full_url = generateFullUrl(Operations::ItemContents).arg(QString::number(target_msgs_size));
+    QString full_url = generateFullUrl(Operations::ItemContents);
     auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
 
     if (!continuation.isEmpty()) {
@@ -267,7 +314,7 @@ QList<Message> GreaderNetwork::itemContents(ServiceRoot* root, const QList<QStri
       return {};
     }
     else {
-      msgs.append(decodeStreamContents(root, output_stream, "", continuation));
+      msgs.append(decodeStreamContents(root, output_stream, QString(), continuation));
     }
   }
   while (!continuation.isEmpty());
@@ -688,6 +735,18 @@ bool GreaderNetwork::ensureLogin(const QNetworkProxy& proxy, QNetworkReply::Netw
   return true;
 }
 
+QString GreaderNetwork::convertShortStreamIdToLongStreamId(const QString& stream_id) const {
+  if (m_service == GreaderServiceRoot::Service::TheOldReader) {
+    return QSL("tag:google.com,2005:reader/item/%1").arg(stream_id);
+  }
+  else {
+    return QSL("tag:google.com,2005:reader/item/%1").arg(stream_id.toULongLong(),
+                                                         16,
+                                                         16,
+                                                         QL1C('0'));
+  }
+}
+
 QString GreaderNetwork::simplifyStreamId(const QString& stream_id) const {
   return QString(stream_id).replace(QRegularExpression("\\/\\d+\\/"), QSL("/-/"));
 }

src/librssguard/services/greader/greadernetwork.h

@@ -91,6 +91,7 @@ class GreaderNetwork : public QObject {
     // Make sure we are logged in and if we are not, return error.
     bool ensureLogin(const QNetworkProxy& proxy, QNetworkReply::NetworkError* output = nullptr);
 
+    QString convertShortStreamIdToLongStreamId(const QString& stream_id) const;
     QString simplifyStreamId(const QString& stream_id) const;
     QStringList decodeItemIds(const QString& stream_json_data, QString& continuation);
     QList<Message> decodeStreamContents(ServiceRoot* root, const QString& stream_json_data, const QString& stream_id, QString& continuation);