From 8fae3bfa9c1ac3a5e4398e8266d1c14ac458eab1 Mon Sep 17 00:00:00 2001 From: David Sansome Date: Tue, 22 Jan 2013 11:52:15 +0100 Subject: [PATCH] Upload minidumps to Cloud Storage instead of blobstore. --- dist/dump_all_symbols.py | 47 +++++------ src/core/crashsender.cpp | 131 +++++++++++++++++------------ src/core/crashsender.h | 20 ++++- src/core/utilities.cpp | 19 +++++ src/core/utilities.h | 11 +++ src/covers/amazoncoverprovider.cpp | 41 ++++----- 6 files changed, 156 insertions(+), 113 deletions(-) diff --git a/dist/dump_all_symbols.py b/dist/dump_all_symbols.py index 67e7e1187..c17d9c0c5 100644 --- a/dist/dump_all_symbols.py +++ b/dist/dump_all_symbols.py @@ -11,6 +11,7 @@ import shutil import subprocess import sys import tempfile +import traceback LDD_RE = re.compile(r'^\t([^ ]+) => ([^ ]+) ', re.MULTILINE) OTOOL_RE = re.compile(r'^\t([^ ]+) \(', re.MULTILINE) @@ -19,8 +20,7 @@ DEFAULT_CRASHREPORTING_HOSTNAME = "crashes.clementine-player.org" DEFAULT_SYMBOLS_DIRECTORY = "symbols" DEFAULT_DUMP_SYMS_BINARY = "3rdparty/google-breakpad/dump_syms" -UPLOAD_URL = "http://%s/upload/symbols" -SYMBOLCHECK_URL = "http://%s/symbolcheck" +SYMBOLUPLOAD_URL = "http://%s/api/upload/symbols" class BaseDumperImpl(object): @@ -160,35 +160,26 @@ class Uploader(object): return stringio def UploadMissing(self): + name_hashes = {"%s/%s" % x: x for x in self.dumper.binary_namehashes} + # Which symbols aren't on the server yet? - url = SYMBOLCHECK_URL % self.crashreporting_hostname - missing_symbols = requests.post(url, data=json.dumps({ - "binary_info": [ - { - "name": x[0], - "hash": x[1], - } for x in self.dumper.binary_namehashes - ], - })).json() + url = SYMBOLUPLOAD_URL % self.crashreporting_hostname + response = requests.post(url, data={ + "symbol": name_hashes.keys(), + }).json() - for info in missing_symbols["missing_symbols"]: - symbol_filename = self.dumper.SymbolFilename((info["name"], info["hash"])) - logging.info("Uploading '%s'", symbol_filename) + for path, url in response["urls"].items(): + try: + symbol_filename = self.dumper.SymbolFilename(name_hashes[path]) + logging.info("Uploading '%s'", symbol_filename) - # Get the upload URL - url = UPLOAD_URL % self.crashreporting_hostname - redirect_response = requests.get(url, allow_redirects=False) - blobstore_url = redirect_response.headers["Location"] - - # Upload the symbols - requests.post(blobstore_url, - files={"data": (symbol_filename + ".gz", - self.Compress(symbol_filename))}, - data={ - "name": info["name"], - "hash": info["hash"], - } - ) + upload_response = requests.put(url, + data=self.Compress(symbol_filename).read(), + headers={'content-type': 'application/gzip'}) + upload_response.raise_for_status() + except Exception: + logging.warning("Failed to upload file '%s': %s" % ( + symbol_filename, traceback.format_exc())) def main(): diff --git a/src/core/crashsender.cpp b/src/core/crashsender.cpp index df2126545..2d4507295 100644 --- a/src/core/crashsender.cpp +++ b/src/core/crashsender.cpp @@ -37,7 +37,7 @@ const char* CrashSender::kUploadURL = - "http://" CRASHREPORTING_HOSTNAME "/upload/crash"; + "http://" CRASHREPORTING_HOSTNAME "/api/upload/minidump"; CrashSender::CrashSender(const QString& minidump_filename, const QString& log_filename) @@ -75,8 +75,12 @@ bool CrashSender::Start() { progress_ = new QProgressDialog("Uploading crash report", "Cancel", 0, 0); progress_->show(); - // We'll get a redirect first, so don't start POSTing data yet. - QNetworkReply* reply = network_->get(QNetworkRequest(QUrl(kUploadURL))); + Utilities::ArgList args; + ClientInfo(&args); + QByteArray data = Utilities::UrlEncode(args).toAscii(); + + // POST the metadata. + QNetworkReply* reply = network_->post(QNetworkRequest(QUrl(kUploadURL)), data); connect(reply, SIGNAL(finished()), SLOT(RedirectFinished())); return true; @@ -91,57 +95,54 @@ void CrashSender::RedirectFinished() { reply->deleteLater(); - QUrl url = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl(); - if (!url.isValid()) { - printf("Response didn't have a redirection target - HTTP %d\n", - reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + int code = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (code > 400) { + printf("Metadata upload caused HTTP %d: %s\n", code, reply->readAll().constData()); progress_->close(); return; } - printf("Uploading crash report to %s\n", url.toEncoded().constData()); - QNetworkRequest req(url); + // "Parse" the response. + QUrl minidump_upload_url = QUrl::fromEncoded(reply->readLine()); + QUrl log_upload_url = QUrl::fromEncoded(reply->readLine()); - QHttpMultiPart* multi_part = new QHttpMultiPart(QHttpMultiPart::FormDataType); - - // Create the HTTP part for the crash report file - QHttpPart minidump_part; - minidump_part.setHeader(QNetworkRequest::ContentDispositionHeader, - "form-data; name=\"data\"; filename=\"data.dmp\""); - minidump_part.setBodyDevice(minidump_); - multi_part->append(minidump_part); - - // Create the HTTP part for the log file. - if (log_->isOpen()) { - QHttpPart log_part; - log_part.setHeader(QNetworkRequest::ContentDispositionHeader, - "form-data; name=\"log\"; filename=\"log.txt\""); - log_part.setBodyDevice(log_); - multi_part->append(log_part); + // Start the uploads. + if (!minidump_upload_url.isEmpty()) { + StartUpload(minidump_upload_url, "application/octet-stream", minidump_); + printf("Uploading crash report to %s\n", + minidump_upload_url.toEncoded().constData()); } - // Get some information about the thing that crashed and add that to the - // request as well. - QList info(ClientInfo()); - foreach (const ClientInfoPair& pair, info) { - QHttpPart part; - part.setHeader(QNetworkRequest::ContentDispositionHeader, - QString("form-data; name=\"" + pair.first + "\"")); - part.setBody(pair.second.toUtf8()); - multi_part->append(part); + if (!log_upload_url.isEmpty()) { + StartUpload(log_upload_url, "text/plain", log_); + printf("Uploading log to %s\n", log_upload_url.toEncoded().constData()); } - // Start uploading the crash report - reply = network_->post(req, multi_part); + CheckUploadsFinished(); +} - connect(reply, SIGNAL(uploadProgress(qint64,qint64)), SLOT(UploadProgress(qint64,qint64))); +void CrashSender::StartUpload(const QUrl& url, const QString& content_type, + QIODevice* file) { + QNetworkRequest request(url); + request.setHeader(QNetworkRequest::ContentTypeHeader, content_type); + + QNetworkReply* reply = network_->put(request, file); + connect(reply, SIGNAL(uploadProgress(qint64,qint64)), + SLOT(UploadProgress(qint64,qint64))); connect(reply, SIGNAL(finished()), SLOT(UploadFinished())); + + UploadState state; + state.total_size_ = file->size(); + state.progress_ = 0; + state.done_ = false; + upload_state_[reply] = state; } void CrashSender::UploadProgress(qint64 bytes, qint64 total) { - printf("Uploaded %lld of %lld bytes\n", bytes, total); - progress_->setMaximum(total); - progress_->setValue(bytes); + UploadState* state = &upload_state_[qobject_cast(sender())]; + state->progress_ = bytes; + state->total_size_ = total; + CheckUploadsFinished(); } void CrashSender::UploadFinished() { @@ -151,35 +152,55 @@ void CrashSender::UploadFinished() { reply->deleteLater(); int code = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (code > 400) { + if (code != 200) { printf("Upload caused HTTP %d: %s\n", code, reply->readAll().constData()); } - progress_->close(); + UploadState* state = &upload_state_[reply]; + state->done_ = true; + + CheckUploadsFinished(); } -QList CrashSender::ClientInfo() const { - QList ret; +void CrashSender::CheckUploadsFinished() { + qint64 total_sum = 0; + qint64 progress_sum = 0; + bool downloads_pending = false; - ret.append(ClientInfoPair("version", CLEMENTINE_VERSION_DISPLAY)); - ret.append(ClientInfoPair("qt_version", qVersion())); + foreach (const UploadState& state, upload_state_) { + total_sum += state.total_size_; + progress_sum += state.progress_; + if (!state.done_) { + downloads_pending = true; + } + } + + if (!downloads_pending) { + progress_->close(); + } else { + progress_->setMaximum(int(total_sum / 1024)); + progress_->setValue(int(progress_sum / 1024)); + } +} + +void CrashSender::ClientInfo(Utilities::ArgList* args) const { + args->append(Utilities::Arg("version", CLEMENTINE_VERSION_DISPLAY)); + args->append(Utilities::Arg("qt_version", qVersion())); // Get the OS version #if defined(Q_OS_MAC) - ret.append(ClientInfoPair("os", "mac")); - ret.append(ClientInfoPair("os_version", mac::GetOSXVersion())); + args->append(Utilities::Arg("os", "mac")); + args->append(Utilities::Arg("os_version", mac::GetOSXVersion())); #elif defined(Q_OS_WIN) - ret.append(ClientInfoPair("os", "win")); - ret.append(ClientInfoPair("os_version", QString::number(QSysInfo::WindowsVersion))); + args->append(Utilities::Arg("os", "win")); + args->append(Utilities::Arg("os_version", QString::number(QSysInfo::WindowsVersion))); #else - ret.append(ClientInfoPair("os", "linux")); + args->append(Utilities::Arg("os", "linux")); QFile lsb_release("/etc/lsb-release"); if (lsb_release.open(QIODevice::ReadOnly)) { - ret.append(ClientInfoPair("os_version", - QString::fromUtf8(lsb_release.readAll()).simplified())); + args->append(Utilities::Arg("os_version", + QString::fromUtf8(lsb_release.readAll()).simplified())); } #endif - - return ret; } diff --git a/src/core/crashsender.h b/src/core/crashsender.h index 5d4f14aba..26d966b00 100644 --- a/src/core/crashsender.h +++ b/src/core/crashsender.h @@ -22,8 +22,11 @@ #include #include +#include "core/utilities.h" + class QFile; class QNetworkAccessManager; +class QNetworkReply; class QProgressDialog; @@ -39,15 +42,16 @@ public: // should exit), or true if he does (caller should start the Qt event loop). bool Start(); -private: - typedef QPair ClientInfoPair; - private slots: void RedirectFinished(); void UploadProgress(qint64 bytes, qint64 total); void UploadFinished(); - QList ClientInfo() const; + void ClientInfo(Utilities::ArgList* args) const; + +private: + void StartUpload(const QUrl& url, const QString& content_type, QIODevice* file); + void CheckUploadsFinished(); private: static const char* kUploadURL; @@ -59,6 +63,14 @@ private: QFile* minidump_; QFile* log_; QProgressDialog* progress_; + + struct UploadState { + qint64 total_size_; + qint64 progress_; + bool done_; + }; + + QMap upload_state_; }; #endif // CRASHSENDER_H diff --git a/src/core/utilities.cpp b/src/core/utilities.cpp index 490fa4420..b3117090a 100644 --- a/src/core/utilities.cpp +++ b/src/core/utilities.cpp @@ -567,6 +567,25 @@ bool IsLaptop() { #endif } +QString UrlEncode(const ArgList& args, EncodedArgList* encoded_args) { + EncodedArgList temp_encoded_args; + QStringList query_items; + + if (!encoded_args) { + encoded_args = &temp_encoded_args; + } + + // Encode the arguments + foreach (const Arg& arg, args) { + EncodedArg encoded_arg(QUrl::toPercentEncoding(arg.first), + QUrl::toPercentEncoding(arg.second)); + *encoded_args << encoded_arg; + query_items << QString(encoded_arg.first + "=" + encoded_arg.second); + } + + return query_items.join("&"); +} + } // namespace Utilities diff --git a/src/core/utilities.h b/src/core/utilities.h index 2f1744607..9a40a2e46 100644 --- a/src/core/utilities.h +++ b/src/core/utilities.h @@ -95,6 +95,17 @@ namespace Utilities { // Replaces some HTML entities with their normal characters. QString DecodeHtmlEntities(const QString& text); + // URL encodes the key/value pairs and returns a string with them joined by + // & characters. Identical to Python's urllib.urlencode. + // If encoded_args is not NULL, appends the encoded arguments to that list as + // well so you can use them for generating signatures for web services. + typedef QPair Arg; + typedef QList ArgList; + typedef QPair EncodedArg; + typedef QList EncodedArgList; + QString UrlEncode(const ArgList& args, + EncodedArgList* encoded_args = NULL); + // Shortcut for getting a Qt-aware enum value as a string. // Pass in the QMetaObject of the class that owns the enum, the string name of // the enum and a valid value from that enum. diff --git a/src/covers/amazoncoverprovider.cpp b/src/covers/amazoncoverprovider.cpp index e935c8e28..66257ee84 100644 --- a/src/covers/amazoncoverprovider.cpp +++ b/src/covers/amazoncoverprovider.cpp @@ -38,44 +38,33 @@ AmazonCoverProvider::AmazonCoverProvider(QObject* parent) } bool AmazonCoverProvider::StartSearch(const QString& artist, const QString& album, int id) { - typedef QPair Arg; - typedef QList ArgList; - - typedef QPair EncodedArg; - typedef QList EncodedArgList; - // Must be sorted by parameter name - ArgList args = ArgList() - << Arg("AWSAccessKeyId", kAccessKey) - << Arg("AssociateTag", kAssociateTag) - << Arg("Keywords", artist + " " + album) - << Arg("Operation", "ItemSearch") - << Arg("ResponseGroup", "Images") - << Arg("SearchIndex", "All") - << Arg("Service", "AWSECommerceService") - << Arg("Timestamp", QDateTime::currentDateTime().toString("yyyy-MM-ddThh:mm:ss.zzzZ")) - << Arg("Version", "2009-11-01"); + Utilities::ArgList args = Utilities::ArgList() + << Utilities::Arg("AWSAccessKeyId", kAccessKey) + << Utilities::Arg("AssociateTag", kAssociateTag) + << Utilities::Arg("Keywords", artist + " " + album) + << Utilities::Arg("Operation", "ItemSearch") + << Utilities::Arg("ResponseGroup", "Images") + << Utilities::Arg("SearchIndex", "All") + << Utilities::Arg("Service", "AWSECommerceService") + << Utilities::Arg("Timestamp", QDateTime::currentDateTime().toString("yyyy-MM-ddThh:mm:ss.zzzZ")) + << Utilities::Arg("Version", "2009-11-01"); - EncodedArgList encoded_args; - QStringList query_items; + Utilities::EncodedArgList encoded_args; // Encode the arguments - foreach (const Arg& arg, args) { - EncodedArg encoded_arg(QUrl::toPercentEncoding(arg.first), - QUrl::toPercentEncoding(arg.second)); - encoded_args << encoded_arg; - query_items << QString(encoded_arg.first + "=" + encoded_arg.second); - } + QString query_items = Utilities::UrlEncode(args, &encoded_args); // Sign the request QUrl url(kUrl); const QByteArray data_to_sign = QString("GET\n%1\n%2\n%3").arg( - url.host(), url.path(), query_items.join("&")).toAscii(); + url.host(), url.path(), query_items).toAscii(); const QByteArray signature(Utilities::HmacSha256(kSecretAccessKey, data_to_sign)); // Add the signature to the request - encoded_args << EncodedArg("Signature", QUrl::toPercentEncoding(signature.toBase64())); + encoded_args << Utilities::EncodedArg + ("Signature", QUrl::toPercentEncoding(signature.toBase64())); url.setEncodedQueryItems(encoded_args); QNetworkReply* reply = network_->get(QNetworkRequest(url));