Funkwhale/Clementine-audio-player-Mac-Linux-Windows
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Upload minidumps to Cloud Storage instead of blobstore.

Browse Source
This commit is contained in:
David Sansome 2013-01-22 11:52:15 +01:00
parent c64358406e
commit 8fae3bfa9c
6 changed files with 156 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
dist/dump_all_symbols.py vendored
View File

@ -11,6 +11,7 @@ import shutil
import subprocess
import sys
import tempfile
import traceback
LDD_RE = re.compile(r'^\t([^ ]+) => ([^ ]+) ', re.MULTILINE)
OTOOL_RE = re.compile(r'^\t([^ ]+) \(', re.MULTILINE)
@ -19,8 +20,7 @@ DEFAULT_CRASHREPORTING_HOSTNAME = "crashes.clementine-player.org"
DEFAULT_SYMBOLS_DIRECTORY = "symbols"
DEFAULT_DUMP_SYMS_BINARY = "3rdparty/google-breakpad/dump_syms"
UPLOAD_URL = "http://%s/upload/symbols"
SYMBOLCHECK_URL = "http://%s/symbolcheck"
SYMBOLUPLOAD_URL = "http://%s/api/upload/symbols"
class BaseDumperImpl(object):
@ -160,35 +160,26 @@ class Uploader(object):
return stringio
def UploadMissing(self):
name_hashes = {"%s/%s" % x: x for x in self.dumper.binary_namehashes}
# Which symbols aren't on the server yet?
url = SYMBOLCHECK_URL % self.crashreporting_hostname
missing_symbols = requests.post(url, data=json.dumps({
"binary_info": [
{
"name": x[0],
"hash": x[1],
} for x in self.dumper.binary_namehashes
],
})).json()
url = SYMBOLUPLOAD_URL % self.crashreporting_hostname
response = requests.post(url, data={
"symbol": name_hashes.keys(),
}).json()
for info in missing_symbols["missing_symbols"]:
symbol_filename = self.dumper.SymbolFilename((info["name"], info["hash"]))
logging.info("Uploading '%s'", symbol_filename)
for path, url in response["urls"].items():
try:
symbol_filename = self.dumper.SymbolFilename(name_hashes[path])
logging.info("Uploading '%s'", symbol_filename)
# Get the upload URL
url = UPLOAD_URL % self.crashreporting_hostname
redirect_response = requests.get(url, allow_redirects=False)
blobstore_url = redirect_response.headers["Location"]
# Upload the symbols
requests.post(blobstore_url,
files={"data": (symbol_filename + ".gz",
self.Compress(symbol_filename))},
data={
"name": info["name"],
"hash": info["hash"],
}
)
upload_response = requests.put(url,
data=self.Compress(symbol_filename).read(),
headers={'content-type': 'application/gzip'})
upload_response.raise_for_status()
except Exception:
logging.warning("Failed to upload file '%s': %s" % (
symbol_filename, traceback.format_exc()))
def main():

131
src/core/crashsender.cpp
View File

@ -37,7 +37,7 @@
const char* CrashSender::kUploadURL =
"http://" CRASHREPORTING_HOSTNAME "/upload/crash";
"http://" CRASHREPORTING_HOSTNAME "/api/upload/minidump";
CrashSender::CrashSender(const QString& minidump_filename,
const QString& log_filename)
@ -75,8 +75,12 @@ bool CrashSender::Start() {
progress_ = new QProgressDialog("Uploading crash report", "Cancel", 0, 0);
progress_->show();
// We'll get a redirect first, so don't start POSTing data yet.
QNetworkReply* reply = network_->get(QNetworkRequest(QUrl(kUploadURL)));
Utilities::ArgList args;
ClientInfo(&args);
QByteArray data = Utilities::UrlEncode(args).toAscii();
// POST the metadata.
QNetworkReply* reply = network_->post(QNetworkRequest(QUrl(kUploadURL)), data);
connect(reply, SIGNAL(finished()), SLOT(RedirectFinished()));
return true;
@ -91,57 +95,54 @@ void CrashSender::RedirectFinished() {
reply->deleteLater();
QUrl url = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
if (!url.isValid()) {
printf("Response didn't have a redirection target - HTTP %d\n",
reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt());
int code = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (code > 400) {
printf("Metadata upload caused HTTP %d: %s\n", code, reply->readAll().constData());
progress_->close();
return;
}
printf("Uploading crash report to %s\n", url.toEncoded().constData());
QNetworkRequest req(url);
// "Parse" the response.
QUrl minidump_upload_url = QUrl::fromEncoded(reply->readLine());
QUrl log_upload_url = QUrl::fromEncoded(reply->readLine());
QHttpMultiPart* multi_part = new QHttpMultiPart(QHttpMultiPart::FormDataType);
// Create the HTTP part for the crash report file
QHttpPart minidump_part;
minidump_part.setHeader(QNetworkRequest::ContentDispositionHeader,
"form-data; name=\"data\"; filename=\"data.dmp\"");
minidump_part.setBodyDevice(minidump_);
multi_part->append(minidump_part);
// Create the HTTP part for the log file.
if (log_->isOpen()) {
QHttpPart log_part;
log_part.setHeader(QNetworkRequest::ContentDispositionHeader,
"form-data; name=\"log\"; filename=\"log.txt\"");
log_part.setBodyDevice(log_);
multi_part->append(log_part);
// Start the uploads.
if (!minidump_upload_url.isEmpty()) {
StartUpload(minidump_upload_url, "application/octet-stream", minidump_);
printf("Uploading crash report to %s\n",
minidump_upload_url.toEncoded().constData());
}
// Get some information about the thing that crashed and add that to the
// request as well.
QList<ClientInfoPair> info(ClientInfo());
foreach (const ClientInfoPair& pair, info) {
QHttpPart part;
part.setHeader(QNetworkRequest::ContentDispositionHeader,
QString("form-data; name=\"" + pair.first + "\""));
part.setBody(pair.second.toUtf8());
multi_part->append(part);
if (!log_upload_url.isEmpty()) {
StartUpload(log_upload_url, "text/plain", log_);
printf("Uploading log to %s\n", log_upload_url.toEncoded().constData());
}
// Start uploading the crash report
reply = network_->post(req, multi_part);
CheckUploadsFinished();
}
connect(reply, SIGNAL(uploadProgress(qint64,qint64)), SLOT(UploadProgress(qint64,qint64)));
void CrashSender::StartUpload(const QUrl& url, const QString& content_type,
QIODevice* file) {
QNetworkRequest request(url);
request.setHeader(QNetworkRequest::ContentTypeHeader, content_type);
QNetworkReply* reply = network_->put(request, file);
connect(reply, SIGNAL(uploadProgress(qint64,qint64)),
SLOT(UploadProgress(qint64,qint64)));
connect(reply, SIGNAL(finished()), SLOT(UploadFinished()));
UploadState state;
state.total_size_ = file->size();
state.progress_ = 0;
state.done_ = false;
upload_state_[reply] = state;
}
void CrashSender::UploadProgress(qint64 bytes, qint64 total) {
printf("Uploaded %lld of %lld bytes\n", bytes, total);
progress_->setMaximum(total);
progress_->setValue(bytes);
UploadState* state = &upload_state_[qobject_cast<QNetworkReply*>(sender())];
state->progress_ = bytes;
state->total_size_ = total;
CheckUploadsFinished();
}
void CrashSender::UploadFinished() {
@ -151,35 +152,55 @@ void CrashSender::UploadFinished() {
reply->deleteLater();
int code = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (code > 400) {
if (code != 200) {
printf("Upload caused HTTP %d: %s\n", code, reply->readAll().constData());
}
progress_->close();
UploadState* state = &upload_state_[reply];
state->done_ = true;
CheckUploadsFinished();
}
QList<CrashSender::ClientInfoPair> CrashSender::ClientInfo() const {
QList<ClientInfoPair> ret;
void CrashSender::CheckUploadsFinished() {
qint64 total_sum = 0;
qint64 progress_sum = 0;
bool downloads_pending = false;
ret.append(ClientInfoPair("version", CLEMENTINE_VERSION_DISPLAY));
ret.append(ClientInfoPair("qt_version", qVersion()));
foreach (const UploadState& state, upload_state_) {
total_sum += state.total_size_;
progress_sum += state.progress_;
if (!state.done_) {
downloads_pending = true;
}
}
if (!downloads_pending) {
progress_->close();
} else {
progress_->setMaximum(int(total_sum / 1024));
progress_->setValue(int(progress_sum / 1024));
}
}
void CrashSender::ClientInfo(Utilities::ArgList* args) const {
args->append(Utilities::Arg("version", CLEMENTINE_VERSION_DISPLAY));
args->append(Utilities::Arg("qt_version", qVersion()));
// Get the OS version
#if defined(Q_OS_MAC)
ret.append(ClientInfoPair("os", "mac"));
ret.append(ClientInfoPair("os_version", mac::GetOSXVersion()));
args->append(Utilities::Arg("os", "mac"));
args->append(Utilities::Arg("os_version", mac::GetOSXVersion()));
#elif defined(Q_OS_WIN)
ret.append(ClientInfoPair("os", "win"));
ret.append(ClientInfoPair("os_version", QString::number(QSysInfo::WindowsVersion)));
args->append(Utilities::Arg("os", "win"));
args->append(Utilities::Arg("os_version", QString::number(QSysInfo::WindowsVersion)));
#else
ret.append(ClientInfoPair("os", "linux"));
args->append(Utilities::Arg("os", "linux"));
QFile lsb_release("/etc/lsb-release");
if (lsb_release.open(QIODevice::ReadOnly)) {
ret.append(ClientInfoPair("os_version",
QString::fromUtf8(lsb_release.readAll()).simplified()));
args->append(Utilities::Arg("os_version",
QString::fromUtf8(lsb_release.readAll()).simplified()));
}
#endif
return ret;
}

20
src/core/crashsender.h
View File

@ -22,8 +22,11 @@
#include <QObject>
#include <QPair>
#include "core/utilities.h"
class QFile;
class QNetworkAccessManager;
class QNetworkReply;
class QProgressDialog;
@ -39,15 +42,16 @@ public:
// should exit), or true if he does (caller should start the Qt event loop).
bool Start();
private:
typedef QPair<QString, QString> ClientInfoPair;
private slots:
void RedirectFinished();
void UploadProgress(qint64 bytes, qint64 total);
void UploadFinished();
QList<ClientInfoPair> ClientInfo() const;
void ClientInfo(Utilities::ArgList* args) const;
private:
void StartUpload(const QUrl& url, const QString& content_type, QIODevice* file);
void CheckUploadsFinished();
private:
static const char* kUploadURL;
@ -59,6 +63,14 @@ private:
QFile* minidump_;
QFile* log_;
QProgressDialog* progress_;
struct UploadState {
qint64 total_size_;
qint64 progress_;
bool done_;
};
QMap<QNetworkReply*, UploadState> upload_state_;
};
#endif // CRASHSENDER_H

19
src/core/utilities.cpp
View File

@ -567,6 +567,25 @@ bool IsLaptop() {
#endif
}
QString UrlEncode(const ArgList& args, EncodedArgList* encoded_args) {
EncodedArgList temp_encoded_args;
QStringList query_items;
if (!encoded_args) {
encoded_args = &temp_encoded_args;
}
// Encode the arguments
foreach (const Arg& arg, args) {
EncodedArg encoded_arg(QUrl::toPercentEncoding(arg.first),
QUrl::toPercentEncoding(arg.second));
*encoded_args << encoded_arg;
query_items << QString(encoded_arg.first + "=" + encoded_arg.second);
}
return query_items.join("&");
}
} // namespace Utilities

11
src/core/utilities.h
View File

@ -95,6 +95,17 @@ namespace Utilities {
// Replaces some HTML entities with their normal characters.
QString DecodeHtmlEntities(const QString& text);
// URL encodes the key/value pairs and returns a string with them joined by
// & characters. Identical to Python's urllib.urlencode.
// If encoded_args is not NULL, appends the encoded arguments to that list as
// well so you can use them for generating signatures for web services.
typedef QPair<QString, QString> Arg;
typedef QList<Arg> ArgList;
typedef QPair<QByteArray, QByteArray> EncodedArg;
typedef QList<EncodedArg> EncodedArgList;
QString UrlEncode(const ArgList& args,
EncodedArgList* encoded_args = NULL);
// Shortcut for getting a Qt-aware enum value as a string.
// Pass in the QMetaObject of the class that owns the enum, the string name of
// the enum and a valid value from that enum.

41
src/covers/amazoncoverprovider.cpp
View File

@ -38,44 +38,33 @@ AmazonCoverProvider::AmazonCoverProvider(QObject* parent)
}
bool AmazonCoverProvider::StartSearch(const QString& artist, const QString& album, int id) {
typedef QPair<QString, QString> Arg;
typedef QList<Arg> ArgList;
typedef QPair<QByteArray, QByteArray> EncodedArg;
typedef QList<EncodedArg> EncodedArgList;
// Must be sorted by parameter name
ArgList args = ArgList()
<< Arg("AWSAccessKeyId", kAccessKey)
<< Arg("AssociateTag", kAssociateTag)
<< Arg("Keywords", artist + " " + album)
<< Arg("Operation", "ItemSearch")
<< Arg("ResponseGroup", "Images")
<< Arg("SearchIndex", "All")
<< Arg("Service", "AWSECommerceService")
<< Arg("Timestamp", QDateTime::currentDateTime().toString("yyyy-MM-ddThh:mm:ss.zzzZ"))
<< Arg("Version", "2009-11-01");
Utilities::ArgList args = Utilities::ArgList()
<< Utilities::Arg("AWSAccessKeyId", kAccessKey)
<< Utilities::Arg("AssociateTag", kAssociateTag)
<< Utilities::Arg("Keywords", artist + " " + album)
<< Utilities::Arg("Operation", "ItemSearch")
<< Utilities::Arg("ResponseGroup", "Images")
<< Utilities::Arg("SearchIndex", "All")
<< Utilities::Arg("Service", "AWSECommerceService")
<< Utilities::Arg("Timestamp", QDateTime::currentDateTime().toString("yyyy-MM-ddThh:mm:ss.zzzZ"))
<< Utilities::Arg("Version", "2009-11-01");
EncodedArgList encoded_args;
QStringList query_items;
Utilities::EncodedArgList encoded_args;
// Encode the arguments
foreach (const Arg& arg, args) {
EncodedArg encoded_arg(QUrl::toPercentEncoding(arg.first),
QUrl::toPercentEncoding(arg.second));
encoded_args << encoded_arg;
query_items << QString(encoded_arg.first + "=" + encoded_arg.second);
}
QString query_items = Utilities::UrlEncode(args, &encoded_args);
// Sign the request
QUrl url(kUrl);
const QByteArray data_to_sign = QString("GET\n%1\n%2\n%3").arg(
url.host(), url.path(), query_items.join("&")).toAscii();
url.host(), url.path(), query_items).toAscii();
const QByteArray signature(Utilities::HmacSha256(kSecretAccessKey, data_to_sign));
// Add the signature to the request
encoded_args << EncodedArg("Signature", QUrl::toPercentEncoding(signature.toBase64()));
encoded_args << Utilities::EncodedArg
("Signature", QUrl::toPercentEncoding(signature.toBase64()));
url.setEncodedQueryItems(encoded_args);
QNetworkReply* reply = network_->get(QNetworkRequest(url));