Add exponential back-off to Amazon requests.

3rdparty/vreen/CMakeLists.txt

@@ -14,6 +14,7 @@ if(NOT VREEN_FOUND)
   if(NOT CMAKE_INSTALL_PREFIX STREQUAL "/usr")
     set(VREEN_IMPORTS_DIR bin)
   endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++0x")
   set(VREEN_WITH_QMLAPI OFF CACHE INTERNAL "")
   set(VREEN_WITH_OAUTH ON CACHE INTERNAL "")
   set(VREEN_INSTALL_HEADERS OFF CACHE INTERNAL "")

ext/libclementine-common/core/closure.cpp

@@ -65,3 +65,11 @@ void DoInAMinuteOrSo(QObject* receiver, const char* slot) {
   int msec = (60 + (qrand() % 60)) * kMsecPerSec;
   DoAfter(receiver, slot, msec);
 }
+
+void DoAfter(std::function<void()> callback, int msec) {
+  QTimer* timer = new QTimer;
+  timer->setSingleShot(true);
+  NewClosure(timer, SIGNAL(timeout()), callback);
+  QObject::connect(timer, SIGNAL(timeout()), timer, SLOT(deleteLater()));
+  timer->start(msec);
+}

ext/libclementine-common/core/closure.h

@@ -188,6 +188,7 @@ _detail::ClosureBase* NewClosure(QObject* sender, const char* signal,
 }
 
 void DoAfter(QObject* receiver, const char* slot, int msec);
+void DoAfter(std::function<void()> callback, int msec);
 void DoInAMinuteOrSo(QObject* receiver, const char* slot);
 
 #endif  // CLOSURE_H

src/internet/amazon/amazonclouddrive.cpp

@@ -17,6 +17,9 @@
 
 #include "internet/amazon/amazonclouddrive.h"
 
+#include <cmath>
+
+#include <QtGlobal>
 #include <QIcon>
 
 #include <qjson/parser.h>
@@ -27,12 +30,15 @@
 #include "core/logging.h"
 #include "core/network.h"
 #include "core/player.h"
+#include "core/timeconstants.h"
 #include "core/waitforsignal.h"
 #include "internet/core/oauthenticator.h"
 #include "internet/amazon/amazonurlhandler.h"
 #include "library/librarybackend.h"
 #include "ui/settingsdialog.h"
 
+using std::placeholders::_1;
+
 const char* AmazonCloudDrive::kServiceName = "Amazon Cloud Drive";
 const char* AmazonCloudDrive::kSettingsGroup = "AmazonCloudDrive";
 
@@ -91,7 +97,7 @@ void AmazonCloudDrive::Connect() {
   }
 
   NewClosure(oauth, SIGNAL(Finished()), this,
-            SLOT(ConnectFinished(OAuthenticator*)), oauth);
+             SLOT(ConnectFinished(OAuthenticator*)), oauth);
 }
 
 void AmazonCloudDrive::EnsureConnected() {
@@ -126,18 +132,20 @@ void AmazonCloudDrive::ConnectFinished(OAuthenticator* oauth) {
 void AmazonCloudDrive::FetchEndpoint() {
   QUrl url(kEndpointEndpoint);
   QNetworkRequest request(url);
-  AddAuthorizationHeader(&request);
-  QNetworkReply* reply = network_->get(request);
-  NewClosure(reply, SIGNAL(finished()), this,
-             SLOT(FetchEndpointFinished(QNetworkReply*)), reply);
+  Get(request, std::bind(&AmazonCloudDrive::FetchEndpointFinished, this, _1));
 }
 
 void AmazonCloudDrive::FetchEndpointFinished(QNetworkReply* reply) {
   reply->deleteLater();
+
   QJson::Parser parser;
   QVariantMap response = parser.parse(reply).toMap();
   content_url_ = response["contentUrl"].toString();
   metadata_url_ = response["metadataUrl"].toString();
+  if (content_url_.isEmpty() || metadata_url_.isEmpty()) {
+    qLog(Debug) << "Couldn't fetch Amazon endpoint";
+    return;
+  }
   QSettings s;
   s.beginGroup(kSettingsGroup);
   QString checkpoint = s.value("checkpoint", "").toString();
@@ -160,10 +168,52 @@ void AmazonCloudDrive::RequestChanges(const QString& checkpoint) {
   QByteArray json = serializer.serialize(data);
 
   QNetworkRequest request(url);
+  Post(request, json,
+       std::bind(&AmazonCloudDrive::RequestChangesFinished, this, _1));
+}
+
+void AmazonCloudDrive::Get(QNetworkRequest request,
+                           std::function<void(QNetworkReply*)> done,
+                           int retries) {
   AddAuthorizationHeader(&request);
-  QNetworkReply* reply = network_->post(request, json);
-  NewClosure(reply, SIGNAL(finished()), this,
-             SLOT(RequestChangesFinished(QNetworkReply*)), reply);
+  MonitorReply(network_->get(request), done, QByteArray(), retries);
+}
+
+void AmazonCloudDrive::Post(QNetworkRequest request, const QByteArray& data,
+                            std::function<void(QNetworkReply*)> done,
+                            int retries) {
+  AddAuthorizationHeader(&request);
+  MonitorReply(network_->post(request, data), done, data, retries);
+}
+
+void AmazonCloudDrive::MonitorReply(QNetworkReply* reply,
+                                    std::function<void(QNetworkReply*)> done,
+                                    const QByteArray& post_data, int retries) {
+  NewClosure(reply, SIGNAL(finished()), [=]() {
+    if (reply->error() == QNetworkReply::NoError) {
+      done(reply);
+    } else {
+      int code =
+          reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
+      if (code >= 500) {  // Retry with exponential backoff.
+        int max_delay_s = std::pow(std::min(retries + 1, 8), 2);
+        int delay_s = qrand() % max_delay_s;
+        qLog(Debug) << max_delay_s << delay_s;
+        qLog(Debug) << "Request failed with code:" << code << "- retrying after"
+                    << delay_s << "seconds";
+        DoAfter([=]() {
+          if (post_data.isEmpty()) {
+            Get(reply->request(), done, retries + 1);
+          } else {
+            Post(reply->request(), post_data, done, retries + 1);
+          }
+        }, delay_s * kMsecPerSec);
+      } else {
+        // Request failed permanently.
+        done(reply);
+      }
+    }
+  });
 }
 
 void AmazonCloudDrive::RequestChangesFinished(QNetworkReply* reply) {
@@ -223,7 +273,8 @@ void AmazonCloudDrive::RequestChangesFinished(QNetworkReply* reply) {
       song.set_title(node["name"].toString());
       song.set_filesize(content_properties["size"].toInt());
 
-      MaybeAddFileToDatabase(song, mime_type, content_url, QString("Bearer %1").arg(access_token_));
+      MaybeAddFileToDatabase(song, mime_type, content_url,
+                             QString("Bearer %1").arg(access_token_));
     }
   }
 

src/internet/amazon/amazonclouddrive.h

@@ -43,7 +43,7 @@ class AmazonCloudDrive : public CloudFileService {
 
   void ForgetCredentials();
 
- signals:
+signals:
   void Connected();
 
  public slots:
@@ -54,6 +54,14 @@ class AmazonCloudDrive : public CloudFileService {
   void RequestChanges(const QString& checkpoint);
   void AddAuthorizationHeader(QNetworkRequest* request);
   void EnsureConnected();
+  void Get(QNetworkRequest, std::function<void(QNetworkReply*)>,
+           int retries = 0);
+  void Post(QNetworkRequest, const QByteArray& data,
+            std::function<void(QNetworkReply*)>, int retries = 0);
+  void MonitorReply(QNetworkReply* reply,
+                    std::function<void(QNetworkReply*)> done,
+                    const QByteArray& post_data = QByteArray(),
+                    int retries = 0);
 
  private slots:
   void ConnectFinished(OAuthenticator*);