From 465f5d254f083a42b1daac7818d8c4205ab822f9 Mon Sep 17 00:00:00 2001
From: Martin Rotter <rotter@praktik.cz>
Date: Wed, 14 Jul 2021 10:50:20 +0200
Subject: [PATCH] download all messages with greader, REALLY ALL if user wants
 to

---
 .../services/greader/greadernetwork.cpp       | 85 +++++++++++--------
 .../services/greader/greadernetwork.h         |  2 +-
 2 files changed, 51 insertions(+), 36 deletions(-)

diff --git a/src/librssguard/services/greader/greadernetwork.cpp b/src/librssguard/services/greader/greadernetwork.cpp
index 3ffc72a06..2eeffeac3 100755
--- a/src/librssguard/services/greader/greadernetwork.cpp
+++ b/src/librssguard/services/greader/greadernetwork.cpp
@@ -134,48 +134,60 @@ QNetworkReply::NetworkError GreaderNetwork::markMessagesStarred(RootItem::Import
 
 QList<Message> GreaderNetwork::streamContents(ServiceRoot* root, const QString& stream_id,
                                               Feed::Status& error, const QNetworkProxy& proxy) {
-  QString full_url = generateFullUrl(Operations::StreamContents).arg(m_service == GreaderServiceRoot::Service::TheOldReader
-                                                                     ? stream_id
-                                                                     : QUrl::toPercentEncoding(stream_id),
-                                                                     QString::number(batchSize() <= 0
-                                                                                     ? 2000000
-                                                                                     : batchSize()));
-  auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
+  QString continuation;
 
   if (!ensureLogin(proxy)) {
     error = Feed::Status::AuthError;
     return {};
   }
 
-  if (downloadOnlyUnreadMessages()) {
-    full_url += QSL("&xt=%1").arg(GREADER_API_FULL_STATE_READ);
-  }
+  QList<Message> msgs;
+  int target_msgs_size = batchSize() <= 0 ? 2000000: batchSize();
 
-  QByteArray output_stream;
-  auto result_stream = NetworkFactory::performNetworkOperation(full_url,
-                                                               timeout,
-                                                               {},
-                                                               output_stream,
-                                                               QNetworkAccessManager::Operation::GetOperation,
-                                                               { authHeader() },
-                                                               false,
-                                                               {},
-                                                               {},
-                                                               proxy);
+  do {
+    QString full_url = generateFullUrl(Operations::StreamContents).arg(m_service == GreaderServiceRoot::Service::TheOldReader
+                                                                     ? stream_id
+                                                                     : QUrl::toPercentEncoding(stream_id),
+                                                                       QString::number(target_msgs_size));
+    auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
 
-  if (result_stream.first != QNetworkReply::NetworkError::NoError) {
-    qCriticalNN << LOGSEC_GREADER
-                << "Cannot download messages for "
-                << QUOTE_NO_SPACE(stream_id)
-                << ", network error:"
-                << QUOTE_W_SPACE_DOT(result_stream.first);
-    error = Feed::Status::NetworkError;
-    return {};
-  }
-  else {
-    error = Feed::Status::Normal;
-    return decodeStreamContents(root, output_stream, stream_id);
+    if (downloadOnlyUnreadMessages()) {
+      full_url += QSL("&xt=%1").arg(GREADER_API_FULL_STATE_READ);
+    }
+
+    if (!continuation.isEmpty()) {
+      full_url += QSL("&c=%1").arg(continuation);
+    }
+
+    QByteArray output_stream;
+    auto result_stream = NetworkFactory::performNetworkOperation(full_url,
+                                                                 timeout,
+                                                                 {},
+                                                                 output_stream,
+                                                                 QNetworkAccessManager::Operation::GetOperation,
+                                                                 { authHeader() },
+                                                                 false,
+                                                                 {},
+                                                                 {},
+                                                                 proxy);
+
+    if (result_stream.first != QNetworkReply::NetworkError::NoError) {
+      qCriticalNN << LOGSEC_GREADER
+                  << "Cannot download messages for "
+                  << QUOTE_NO_SPACE(stream_id)
+                  << ", network error:"
+                  << QUOTE_W_SPACE_DOT(result_stream.first);
+      error = Feed::Status::NetworkError;
+      return {};
+    }
+    else {
+      msgs.append(decodeStreamContents(root, output_stream, stream_id, continuation));
+    }
   }
+  while (!continuation.isEmpty() && msgs.size() < target_msgs_size);
+
+  error = Feed::Status::Normal;
+  return msgs;
 }
 
 RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNetworkProxy& proxy) {
@@ -539,11 +551,14 @@ QString GreaderNetwork::simplifyStreamId(const QString& stream_id) const {
 
 QList<Message> GreaderNetwork::decodeStreamContents(ServiceRoot* root,
                                                     const QString& stream_json_data,
-                                                    const QString& stream_id) {
+                                                    const QString& stream_id,
+                                                    QString& continuation) {
   QList<Message> messages;
-  QJsonArray json = QJsonDocument::fromJson(stream_json_data.toUtf8()).object()["items"].toArray();
+  QJsonDocument json_doc = QJsonDocument::fromJson(stream_json_data.toUtf8());
+  QJsonArray json = json_doc.object()["items"].toArray();
   auto active_labels = root->labelsNode() != nullptr ? root->labelsNode()->labels() : QList<Label*>();
 
+  continuation = json_doc.object()["continuation"].toString();
   messages.reserve(json.count());
 
   for (const QJsonValue& obj : json) {
diff --git a/src/librssguard/services/greader/greadernetwork.h b/src/librssguard/services/greader/greadernetwork.h
index a50681821..5ba364245 100755
--- a/src/librssguard/services/greader/greadernetwork.h
+++ b/src/librssguard/services/greader/greadernetwork.h
@@ -78,7 +78,7 @@ class GreaderNetwork : public QObject {
     bool ensureLogin(const QNetworkProxy& proxy, QNetworkReply::NetworkError* output = nullptr);
 
     QString simplifyStreamId(const QString& stream_id) const;
-    QList<Message> decodeStreamContents(ServiceRoot* root, const QString& stream_json_data, const QString& stream_id);
+    QList<Message> decodeStreamContents(ServiceRoot* root, const QString& stream_json_data, const QString& stream_id, QString& continuation);
     RootItem* decodeTagsSubscriptions(const QString& categories, const QString& feeds, bool obtain_icons, const QNetworkProxy& proxy);
     QString sanitizedBaseUrl() const;
     QString generateFullUrl(Operations operation) const;