From 145f1efaf5c148de35d507fe3039396441a4594d Mon Sep 17 00:00:00 2001 From: John Maguire Date: Thu, 12 Jul 2012 14:09:20 +0200 Subject: [PATCH 01/20] Basic support for Google Drive & OAuth. --- data/data.qrc | 1 + data/providers/googledrive.png | Bin 0 -> 2148 bytes src/CMakeLists.txt | 4 + src/engines/gstenginepipeline.cpp | 15 +++ src/internet/googledriveservice.cpp | 75 ++++++++++++++ src/internet/googledriveservice.h | 35 +++++++ src/internet/internetmodel.cpp | 2 + src/internet/oauthenticator.cpp | 147 ++++++++++++++++++++++++++++ src/internet/oauthenticator.h | 38 +++++++ 9 files changed, 317 insertions(+) create mode 100644 data/providers/googledrive.png create mode 100644 src/internet/googledriveservice.cpp create mode 100644 src/internet/googledriveservice.h create mode 100644 src/internet/oauthenticator.cpp create mode 100644 src/internet/oauthenticator.h diff --git a/data/data.qrc b/data/data.qrc index 707ca59a3..06d0aa79d 100644 --- a/data/data.qrc +++ b/data/data.qrc @@ -284,6 +284,7 @@ providers/echonest.png providers/songkick.png providers/twitter.png + providers/googledrive.png lumberjacksong.txt schema/schema-18.sql star-off.png diff --git a/data/providers/googledrive.png b/data/providers/googledrive.png new file mode 100644 index 0000000000000000000000000000000000000000..590dde867f4bd46b2d836b9e5e8a8e2940b13ff0 GIT binary patch literal 2148 zcmV-q2%GnbP)Mwm4`kVS#<|WZ|&Ja z)7Sg^{(x6EwE!{Ccy2hdr(yVDgw5q`A5X6~gvQm&p}BteWj@L^9@9aOpn>K-nseX- zaB@{^fTG`^scsE^@yrHD6pEB_U=0@f=bgdA4eS|MZ+~9jy($in@im%8UBCsx$tzW8 zwkkp7A@@A%jLkxNKE7HRc!|bW0%ZLBWWCngg&RAdwZ@AKKP?(jPY!Q;Njx1 zFGLTmnhr5~LmjA2?p1}m#YJ9NMCpeMedTnMc#0l$_EYg-{opao9e_p37G;T;I z?wi`Hit03t8f&hA61_xFFMoCcI|`mMy3m%3e8y`i|}+ z*x*aRdXGR2m2?&wMyeh}>VRe&1*yw2f;VUT0x{c!l?e`xdowT#QECk_-AMHgXMuq3 zAQ%qA_M8lmon-0V1xcSVQBRT>lrxv+W4JKafw{|Fm^az+br3p2fAy=MY-8u4_Z9Kw@O5 zmJo^s^?VuD1b`xit$o{1wuFiJD7{e&BJ3&5r_{p7;nifJ8@FUPS;pRu9HWf2Y$np| z(Yb%U>NgkS zk=Ap^CYH03n*E@DfQ%khyBU%UOQgvL-O!Ip3<_kL0c2!XAn9JDLGxxT9~xA0;X`hC zj}zcE7zV%r=s&vyZaMd;(?EKWg66sag;F)3E(u-(-MjwVbUQ{x-9kw@AlKCJKG4&uqzrhw3vUjz$;uAD^a29}EgOX!)Er^(f8V1LceNIEu4Q zU{nH8?DLBst5gH!D#HCC@Sn5w<{%8bXI-zc(@R!@;{+%iftQy|w3SC9xIeH-7&D6gnHo_^J~nd>h3|2)l18IQQo?BVJe5 zLCGbBmlDnIUcY_kN_Xbcf?hwn4T{~{T;zial#(?*uV)fwn1)WmWJ40XFFAxbd^A4C zR=lxAP->Ru?>KSG%r4I~i$qa+y!vR`ArlWFYkU$yE0E`YjXXusZ(ein?dJbV+;rf7 a?!aG7`ph;%20c*#0000url().host().contains("googleusercontent.com") && + instance->url().hasFragment()) { + QByteArray authorization = QString("Bearer %1").arg( + instance->url().fragment()).toAscii(); + GstStructure* headers = gst_structure_new( + "extra-headers", + "Authorization", G_TYPE_STRING, + authorization.constData(), + NULL); + g_object_set(element, "extra-headers", headers, NULL); + gst_structure_free(headers); + } } void GstEnginePipeline::TransitionToNext() { diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp new file mode 100644 index 000000000..15d845bca --- /dev/null +++ b/src/internet/googledriveservice.cpp @@ -0,0 +1,75 @@ +#include "googledriveservice.h" + +#include + +#include "core/closure.h" +#include "internetmodel.h" +#include "oauthenticator.h" + +namespace { + +const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; + +} + +GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) + : InternetService("Google Drive", app, parent, parent), + root_(NULL), + oauth_(new OAuthenticator(this)) { + connect(oauth_, SIGNAL(AccessTokenAvailable(QString)), SLOT(AccessTokenAvailable(QString))); +} + +QStandardItem* GoogleDriveService::CreateRootItem() { + root_ = new QStandardItem(QIcon(":providers/googledrive.png"), "Google Drive"); + root_->setData(true, InternetModel::Role_CanLazyLoad); + return root_; +} + +void GoogleDriveService::LazyPopulate(QStandardItem* item) { + switch (item->data(InternetModel::Role_Type).toInt()) { + case InternetModel::Type_Service: + Connect(); + break; + default: + break; + } +} + +void GoogleDriveService::Connect() { + oauth_->StartAuthorisation(); +} + +void GoogleDriveService::AccessTokenAvailable(const QString& token) { + access_token_ = token; + QUrl url = QUrl(kGoogleDriveFiles); + url.addQueryItem("q", "mimeType = 'audio/mpeg'"); + + QNetworkRequest request = QNetworkRequest(url); + request.setRawHeader( + "Authorization", QString("Bearer %1").arg(token).toUtf8()); + QNetworkReply* reply = network_.get(request); + NewClosure(reply, SIGNAL(finished()), this, SLOT(ListFilesFinished(QNetworkReply*)), reply); +} + +void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { + reply->deleteLater(); + + QJson::Parser parser; + bool ok = false; + QVariantMap result = parser.parse(reply, &ok).toMap(); + if (!ok) { + qLog(Error) << "Failed to request files from Google Drive"; + return; + } + + QVariantList items = result["items"].toList(); + foreach (const QVariant& v, items) { + QVariantMap file = v.toMap(); + Song song; + song.set_title(file["title"].toString()); + QString url = file["downloadUrl"].toString() + "#" + access_token_; + song.set_url(url); + song.set_filesize(file["fileSize"].toInt()); + root_->appendRow(CreateSongItem(song)); + } +} diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h new file mode 100644 index 000000000..79ee1b68d --- /dev/null +++ b/src/internet/googledriveservice.h @@ -0,0 +1,35 @@ +#ifndef GOOGLEDRIVESERVICE_H +#define GOOGLEDRIVESERVICE_H + +#include "internetservice.h" + +#include "core/network.h" + +class QStandardItem; + +class OAuthenticator; + +class GoogleDriveService : public InternetService { + Q_OBJECT + public: + GoogleDriveService(Application* app, InternetModel* parent); + + QStandardItem* CreateRootItem(); + void LazyPopulate(QStandardItem* item); + + private slots: + void AccessTokenAvailable(const QString& token); + void ListFilesFinished(QNetworkReply* reply); + + private: + void Connect(); + + QStandardItem* root_; + OAuthenticator* oauth_; + + QString access_token_; + + NetworkAccessManager network_; +}; + +#endif diff --git a/src/internet/internetmodel.cpp b/src/internet/internetmodel.cpp index a12a91e8d..baf692033 100644 --- a/src/internet/internetmodel.cpp +++ b/src/internet/internetmodel.cpp @@ -17,6 +17,7 @@ #include "digitallyimportedservicebase.h" #include "icecastservice.h" +#include "googledriveservice.h" #include "jamendoservice.h" #include "magnatuneservice.h" #include "internetmimedata.h" @@ -64,6 +65,7 @@ InternetModel::InternetModel(Application* app, QObject* parent) #ifdef HAVE_LIBLASTFM AddService(new LastFMService(app, this)); #endif + AddService(new GoogleDriveService(app, this)); AddService(new GroovesharkService(app, this)); AddService(new MagnatuneService(app, this)); AddService(new PodcastService(app, this)); diff --git a/src/internet/oauthenticator.cpp b/src/internet/oauthenticator.cpp new file mode 100644 index 000000000..f3f297d6a --- /dev/null +++ b/src/internet/oauthenticator.cpp @@ -0,0 +1,147 @@ +#include "oauthenticator.h" + +#include +#include +#include +#include + +#include + +#include "core/closure.h" + +namespace { + +const char* kGoogleOAuthEndpoint = "https://accounts.google.com/o/oauth2/auth"; +const char* kGoogleOAuthTokenEndpoint = + "https://accounts.google.com/o/oauth2/token"; +const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; + +const char* kClientId = "679260893280.apps.googleusercontent.com"; +const char* kClientSecret = "l3cWb8efUZsrBI4wmY3uKl6i"; + +} // namespace + +OAuthenticator::OAuthenticator(QObject* parent) + : QObject(parent) { +} + +void OAuthenticator::StartAuthorisation() { + server_.listen(QHostAddress::LocalHost); + const quint16 port = server_.serverPort(); + + NewClosure(&server_, SIGNAL(newConnection()), this, SLOT(NewConnection())); + + QUrl url = QUrl(kGoogleOAuthEndpoint); + url.addQueryItem("response_type", "code"); + url.addQueryItem("client_id", kClientId); + url.addQueryItem("redirect_uri", QString("http://localhost:%1").arg(port)); + url.addQueryItem("scope", "https://www.googleapis.com/auth/drive.readonly"); + + QDesktopServices::openUrl(url); +} + +void OAuthenticator::NewConnection() { + QTcpSocket* socket = server_.nextPendingConnection(); + server_.close(); + + QByteArray buffer; + + NewClosure(socket, SIGNAL(readyRead()), + this, SLOT(RedirectArrived(QTcpSocket*, QByteArray)), socket, buffer); + + // Everything is bon. + socket->write("HTTP/1.0 200 OK\r\n"); + socket->flush(); +} + +void OAuthenticator::RedirectArrived(QTcpSocket* socket, QByteArray buffer) { + buffer.append(socket->readAll()); + + if (socket->atEnd() || buffer.endsWith("\r\n\r\n")) { + socket->deleteLater(); + const QByteArray& code = ParseHttpRequest(buffer); + qLog(Debug) << "Code:" << code; + RequestAccessToken(code, socket->localPort()); + } else { + NewClosure(socket, SIGNAL(readyReady()), + this, SLOT(RedirectArrived(QTcpSocket*, QByteArray)), socket, buffer); + } +} + +QByteArray OAuthenticator::ParseHttpRequest(const QByteArray& request) const { + QList split = request.split('\r'); + const QByteArray& request_line = split[0]; + QByteArray path = request_line.split(' ')[1]; + QByteArray code = path.split('=')[1]; + + return code; +} + +void OAuthenticator::RequestAccessToken(const QByteArray& code, quint16 port) { + typedef QPair Param; + QList parameters; + parameters << Param("code", code) + << Param("client_id", kClientId) + << Param("client_secret", kClientSecret) + << Param("grant_type", "authorization_code") + // Even though we don't use this URI anymore, it must match the + // original one. + << Param("redirect_uri", QString("http://localhost:%1").arg(port)); + + QStringList params; + foreach (const Param& p, parameters) { + params.append(QString("%1=%2").arg(p.first, QString(QUrl::toPercentEncoding(p.second)))); + } + QString post_data = params.join("&"); + qLog(Debug) << post_data; + + QNetworkRequest request = QNetworkRequest(QUrl(kGoogleOAuthTokenEndpoint)); + request.setHeader(QNetworkRequest::ContentTypeHeader, + "application/x-www-form-urlencoded"); + + QNetworkReply* reply = network_.post(request, post_data.toUtf8()); + NewClosure(reply, SIGNAL(finished()), this, + SLOT(FetchAccessTokenFinished(QNetworkReply*)), reply); +} + +void OAuthenticator::FetchAccessTokenFinished(QNetworkReply* reply) { + reply->deleteLater(); + + if (reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) != 200) { + qLog(Error) << "Failed to get access token" + << reply->readAll(); + return; + } + + QJson::Parser parser; + bool ok = false; + QVariantMap result = parser.parse(reply, &ok).toMap(); + if (!ok) { + qLog(Error) << "Failed to parse oauth reply"; + return; + } + + qLog(Debug) << result; + + access_token_ = result["access_token"].toString(); + refresh_token_ = result["refresh_token"].toString(); + + emit AccessTokenAvailable(access_token_); +} + +void OAuthenticator::ListFiles(const QString& access_token) { + QUrl url = QUrl(kGoogleDriveFiles); + url.addQueryItem("q", "mimeType = 'audio/mpeg'"); + QNetworkRequest request = QNetworkRequest(url); + request.setRawHeader( + "Authorization", QString("Bearer %1").arg(access_token).toUtf8()); + qLog(Debug) << "Header:" << request.rawHeader("Authorization"); + QNetworkReply* reply = network_.get(request); + NewClosure(reply, SIGNAL(finished()), this, SLOT(ListFilesResponse(QNetworkReply*)), reply); +} + +void OAuthenticator::ListFilesResponse(QNetworkReply* reply) { + reply->deleteLater(); + + qLog(Debug) << reply->readAll(); +} diff --git a/src/internet/oauthenticator.h b/src/internet/oauthenticator.h new file mode 100644 index 000000000..114197a80 --- /dev/null +++ b/src/internet/oauthenticator.h @@ -0,0 +1,38 @@ +#ifndef OAUTHENTICATOR_H +#define OAUTHENTICATOR_H + +#include +#include + +#include "core/network.h" + +class QTcpSocket; + +class OAuthenticator : public QObject { + Q_OBJECT + public: + explicit OAuthenticator(QObject* parent = 0); + void StartAuthorisation(); + + signals: + void AccessTokenAvailable(QString token); + + private slots: + void NewConnection(); + void RedirectArrived(QTcpSocket* socket, QByteArray buffer); + void FetchAccessTokenFinished(QNetworkReply* reply); + void ListFilesResponse(QNetworkReply* reply); + + private: + QByteArray ParseHttpRequest(const QByteArray& request) const; + void RequestAccessToken(const QByteArray& code, quint16 port); + void ListFiles(const QString& access_token); + + QTcpServer server_; + NetworkAccessManager network_; + + QString access_token_; + QString refresh_token_; +}; + +#endif From 0c861da7a906231a2c90054d73a1bfd46e5d8658 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Wed, 25 Jul 2012 17:57:50 +0200 Subject: [PATCH 02/20] Almost working tagging from stream. --- src/internet/googledriveservice.cpp | 145 ++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 15d845bca..e635b7fa0 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -1,7 +1,14 @@ #include "googledriveservice.h" +#include + #include +#include +#include +#include +using TagLib::ByteVector; + #include "core/closure.h" #include "internetmodel.h" #include "oauthenticator.h" @@ -12,6 +19,129 @@ const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; } + +class DriveStream : public TagLib::IOStream { + public: + DriveStream(const QUrl& url, + const QString& filename, + const long length, + const QString& auth, + QNetworkAccessManager* network) + : url_(url), + filename_(filename), + encoded_filename_(filename_.toUtf8()), + length_(length), + auth_(auth), + cursor_(0), + network_(network) { + qLog(Debug) << Q_FUNC_INFO; + } + + virtual TagLib::FileName name() const { + qLog(Debug) << Q_FUNC_INFO; + return encoded_filename_.data(); + } + + virtual TagLib::ByteVector readBlock(ulong length) { + qLog(Debug) << Q_FUNC_INFO; + QNetworkRequest request = QNetworkRequest(url_); + request.setRawHeader( + "Authorization", QString("Bearer %1").arg(auth_).toUtf8()); + + const int start = cursor_; + const int end = cursor_ + length; + request.setRawHeader( + "Range", QString("bytes=%1-%2").arg(start, end).toUtf8()); + + qLog(Debug) << "Requesting:" << start << "-" << end << "from:" << url_; + qLog(Debug) << request.rawHeaderList(); + + + QNetworkReply* reply = network_->get(request); + + QEventLoop loop; + QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); + + qLog(Debug) << "Starting loop"; + loop.exec(); + qLog(Debug) << "Finished loop"; + reply->deleteLater(); + + QByteArray data = reply->readAll(); + TagLib::ByteVector bytes(data.data(), data.size()); + return bytes; + } + + virtual void writeBlock(const ByteVector&) { + qLog(Debug) << Q_FUNC_INFO << "not implemented"; + } + + virtual void insert(const ByteVector&, ulong, ulong) { + qLog(Debug) << Q_FUNC_INFO << "not implemented"; + } + + virtual void removeBlock(ulong, ulong) { + qLog(Debug) << Q_FUNC_INFO << "not implemented"; + } + + virtual bool readOnly() const { + qLog(Debug) << Q_FUNC_INFO; + return true; + } + + virtual bool isOpen() const { + qLog(Debug) << Q_FUNC_INFO; + return true; + } + + virtual void seek(long offset, TagLib::IOStream::Position p) { + qLog(Debug) << Q_FUNC_INFO; + switch (p) { + case TagLib::IOStream::Beginning: + cursor_ = offset; + break; + + case TagLib::IOStream::Current: + cursor_ = qMin(cursor_ + offset, length_); + break; + + case TagLib::IOStream::End: + cursor_ = qMax(0L, length_ - offset); + break; + } + } + + virtual void clear() { + qLog(Debug) << Q_FUNC_INFO; + cursor_ = 0; + } + + virtual long tell() const { + qLog(Debug) << Q_FUNC_INFO; + return cursor_; + } + + virtual long length() { + qLog(Debug) << Q_FUNC_INFO; + return length_; + } + + virtual void truncate(long) { + qLog(Debug) << Q_FUNC_INFO << "not implemented"; + } + + private: + const QUrl url_; + const QString filename_; + const QByteArray encoded_filename_; + const long length_; + const QString auth_; + + int cursor_; + QNetworkAccessManager* network_; +}; + + GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) : InternetService("Google Drive", app, parent, parent), root_(NULL), @@ -71,5 +201,20 @@ void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { song.set_url(url); song.set_filesize(file["fileSize"].toInt()); root_->appendRow(CreateSongItem(song)); + + qLog(Debug) << "Creating stream"; + DriveStream stream( + file["downloadUrl"].toUrl(), + file["title"].toString(), + file["fileSize"].toUInt(), + access_token_, + &network_); + qLog(Debug) << "Creating tag"; + TagLib::MPEG::File tag( + &stream, + TagLib::ID3v2::FrameFactory::instance(), + TagLib::AudioProperties::Fast); + qLog(Debug) << "Tagging done"; + qLog(Debug) << tag.tag()->artist().toCString(); } } From bda3d0e0337d11b23d4526b7ec8816949b6b7236 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Thu, 26 Jul 2012 11:36:07 +0200 Subject: [PATCH 03/20] Hacky support for reading tags from MP3s on Google Drive. Requires TagLib 1.8 --- src/internet/googledriveservice.cpp | 41 ++++++++++++++++++++--------- src/playlist/playlistdelegates.cpp | 2 ++ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index e635b7fa0..179715ae0 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -34,7 +34,10 @@ class DriveStream : public TagLib::IOStream { auth_(auth), cursor_(0), network_(network) { - qLog(Debug) << Q_FUNC_INFO; + qLog(Debug) << Q_FUNC_INFO + << url_ + << filename_ + << length_; } virtual TagLib::FileName name() const { @@ -49,12 +52,13 @@ class DriveStream : public TagLib::IOStream { "Authorization", QString("Bearer %1").arg(auth_).toUtf8()); const int start = cursor_; - const int end = cursor_ + length; + const int end = cursor_ + length - 1; request.setRawHeader( - "Range", QString("bytes=%1-%2").arg(start, end).toUtf8()); + "Range", QString("bytes=%1-%2").arg(start).arg(end).toUtf8()); qLog(Debug) << "Requesting:" << start << "-" << end << "from:" << url_; qLog(Debug) << request.rawHeaderList(); + qLog(Debug) << request.rawHeader("Range"); QNetworkReply* reply = network_->get(request); @@ -67,8 +71,14 @@ class DriveStream : public TagLib::IOStream { qLog(Debug) << "Finished loop"; reply->deleteLater(); + qLog(Debug) << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) + << reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute) + << reply->error(); + QByteArray data = reply->readAll(); + qLog(Debug) << "Read:" << data.size(); TagLib::ByteVector bytes(data.data(), data.size()); + cursor_ += data.size(); return bytes; } @@ -195,15 +205,8 @@ void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { QVariantList items = result["items"].toList(); foreach (const QVariant& v, items) { QVariantMap file = v.toMap(); - Song song; - song.set_title(file["title"].toString()); - QString url = file["downloadUrl"].toString() + "#" + access_token_; - song.set_url(url); - song.set_filesize(file["fileSize"].toInt()); - root_->appendRow(CreateSongItem(song)); - qLog(Debug) << "Creating stream"; - DriveStream stream( + DriveStream* stream = new DriveStream( file["downloadUrl"].toUrl(), file["title"].toString(), file["fileSize"].toUInt(), @@ -211,10 +214,22 @@ void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { &network_); qLog(Debug) << "Creating tag"; TagLib::MPEG::File tag( - &stream, + stream, // Takes ownership. TagLib::ID3v2::FrameFactory::instance(), TagLib::AudioProperties::Fast); qLog(Debug) << "Tagging done"; - qLog(Debug) << tag.tag()->artist().toCString(); + if (tag.tag()) { + qLog(Debug) << tag.tag()->artist().toCString(); + Song song; + song.set_title(tag.tag()->title().toCString(true)); + song.set_artist(tag.tag()->artist().toCString(true)); + song.set_album(tag.tag()->album().toCString(true)); + QString url = file["downloadUrl"].toString() + "#" + access_token_; + song.set_url(url); + song.set_filesize(file["fileSize"].toInt()); + root_->appendRow(CreateSongItem(song)); + } else { + qLog(Debug) << "Tagging failed"; + } } } diff --git a/src/playlist/playlistdelegates.cpp b/src/playlist/playlistdelegates.cpp index 7221935b5..887a637dd 100644 --- a/src/playlist/playlistdelegates.cpp +++ b/src/playlist/playlistdelegates.cpp @@ -463,6 +463,8 @@ QPixmap SongSourceDelegate::LookupPixmap(const QUrl& url, const QSize& size) con icon = IconLoader::Load("folder-sound"); } else if (url.host() == "api.jamendo.com") { icon = QIcon(":/providers/jamendo.png"); + } else if (url.host().endsWith("googleusercontent.com")) { + icon = QIcon(":/providers/googledrive.png"); } } pixmap = icon.pixmap(size.height()); From 4a629e636b4503380a355c047f124a0c88dc50f6 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Thu, 26 Jul 2012 16:06:34 +0200 Subject: [PATCH 04/20] Cache taglib Drive requests in a google::sparsetable. Requires sparsehash. --- src/internet/googledriveservice.cpp | 71 ++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 179715ae0..c2f5d0e1e 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -4,6 +4,8 @@ #include +#include + #include #include #include @@ -15,7 +17,7 @@ using TagLib::ByteVector; namespace { -const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; +static const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; } @@ -33,7 +35,8 @@ class DriveStream : public TagLib::IOStream { length_(length), auth_(auth), cursor_(0), - network_(network) { + network_(network), + cache_(length) { qLog(Debug) << Q_FUNC_INFO << url_ << filename_ @@ -45,40 +48,64 @@ class DriveStream : public TagLib::IOStream { return encoded_filename_.data(); } + bool CheckCache(int start, int end) { + for (int i = start; i <= end; ++i) { + if (!cache_.test(i)) { + return false; + } + } + return true; + } + + void FillCache(int start, TagLib::ByteVector data) { + for (int i = 0; i < data.size(); ++i) { + cache_.set(start + i, data[i]); + } + } + + TagLib::ByteVector GetCached(int start, int end) { + const uint size = end - start + 1; + TagLib::ByteVector ret(size); + for (int i = 0; i < size; ++i) { + ret[i] = cache_.get(start + i); + } + return ret; + } + virtual TagLib::ByteVector readBlock(ulong length) { qLog(Debug) << Q_FUNC_INFO; + const uint start = cursor_; + const uint end = qMin(cursor_ + length - 1, length_ - 1); + + if (end <= start) { + return TagLib::ByteVector(); + } + + if (CheckCache(start, end)) { + qLog(Debug) << "Cache hit at:" << start << end; + TagLib::ByteVector cached = GetCached(start, end); + cursor_ += cached.size(); + return cached; + } + QNetworkRequest request = QNetworkRequest(url_); request.setRawHeader( "Authorization", QString("Bearer %1").arg(auth_).toUtf8()); - - const int start = cursor_; - const int end = cursor_ + length - 1; request.setRawHeader( "Range", QString("bytes=%1-%2").arg(start).arg(end).toUtf8()); - qLog(Debug) << "Requesting:" << start << "-" << end << "from:" << url_; - qLog(Debug) << request.rawHeaderList(); - qLog(Debug) << request.rawHeader("Range"); - - QNetworkReply* reply = network_->get(request); QEventLoop loop; QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); - - qLog(Debug) << "Starting loop"; loop.exec(); - qLog(Debug) << "Finished loop"; reply->deleteLater(); - qLog(Debug) << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute) - << reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute) - << reply->error(); - QByteArray data = reply->readAll(); - qLog(Debug) << "Read:" << data.size(); TagLib::ByteVector bytes(data.data(), data.size()); cursor_ += data.size(); + + FillCache(start, bytes); return bytes; } @@ -112,11 +139,11 @@ class DriveStream : public TagLib::IOStream { break; case TagLib::IOStream::Current: - cursor_ = qMin(cursor_ + offset, length_); + cursor_ = qMin(ulong(cursor_ + offset), length_); break; case TagLib::IOStream::End: - cursor_ = qMax(0L, length_ - offset); + cursor_ = qMax(0UL, length_ - offset); break; } } @@ -144,11 +171,13 @@ class DriveStream : public TagLib::IOStream { const QUrl url_; const QString filename_; const QByteArray encoded_filename_; - const long length_; + const ulong length_; const QString auth_; int cursor_; QNetworkAccessManager* network_; + + google::sparsetable cache_; }; From ecf7998f1cc170bfad42ab3d1dcd0cf2b6dacbcc Mon Sep 17 00:00:00 2001 From: John Maguire Date: Thu, 26 Jul 2012 16:35:57 +0200 Subject: [PATCH 05/20] Add URL handler for googledrive URLs and fetch a new temporary download URL for every play. --- src/CMakeLists.txt | 2 ++ src/internet/googledriveservice.cpp | 28 +++++++++++++++++++++++++- src/internet/googledriveservice.h | 2 ++ src/internet/googledriveurlhandler.cpp | 16 +++++++++++++++ src/internet/googledriveurlhandler.h | 21 +++++++++++++++++++ 5 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 src/internet/googledriveurlhandler.cpp create mode 100644 src/internet/googledriveurlhandler.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1bc7399b1..bb03e76e7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -161,6 +161,7 @@ set(SOURCES internet/digitallyimportedurlhandler.cpp internet/geolocator.cpp internet/googledriveservice.cpp + internet/googledriveurlhandler.cpp internet/groovesharkradio.cpp internet/groovesharkservice.cpp internet/groovesharksettingspage.cpp @@ -433,6 +434,7 @@ set(HEADERS internet/digitallyimportedsettingspage.h internet/geolocator.h internet/googledriveservice.h + internet/googledriveurlhandler.h internet/groovesharkservice.h internet/groovesharksettingspage.h internet/groovesharkurlhandler.h diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index c2f5d0e1e..854a912fd 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -11,13 +11,17 @@ #include using TagLib::ByteVector; +#include "core/application.h" #include "core/closure.h" +#include "core/player.h" +#include "googledriveurlhandler.h" #include "internetmodel.h" #include "oauthenticator.h" namespace { static const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; +static const char* kGoogleDriveFile = "https://www.googleapis.com/drive/v2/files/%1"; } @@ -186,6 +190,8 @@ GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) root_(NULL), oauth_(new OAuthenticator(this)) { connect(oauth_, SIGNAL(AccessTokenAvailable(QString)), SLOT(AccessTokenAvailable(QString))); + + app->player()->RegisterUrlHandler(new GoogleDriveUrlHandler(this, this)); } QStandardItem* GoogleDriveService::CreateRootItem() { @@ -253,8 +259,11 @@ void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { song.set_title(tag.tag()->title().toCString(true)); song.set_artist(tag.tag()->artist().toCString(true)); song.set_album(tag.tag()->album().toCString(true)); - QString url = file["downloadUrl"].toString() + "#" + access_token_; + + QString url = QString("googledrive:%1").arg(file["id"].toString()); song.set_url(url); + qLog(Debug) << "Set url to:" << url; + song.set_filesize(file["fileSize"].toInt()); root_->appendRow(CreateSongItem(song)); } else { @@ -262,3 +271,20 @@ void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { } } } + +QUrl GoogleDriveService::GetStreamingUrlFromSongId(const QString& id) { + QString url = QString(kGoogleDriveFile).arg(id); + QNetworkRequest request = QNetworkRequest(url); + request.setRawHeader( + "Authorization", QString("Bearer %1").arg(access_token_).toUtf8()); + QNetworkReply* reply = network_.get(request); + QEventLoop loop; + connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); + loop.exec(); + + QJson::Parser parser; + bool ok = false; + QVariantMap result = parser.parse(reply, &ok).toMap(); + QString download_url = result["downloadUrl"].toString() + "#" + access_token_; + return QUrl(download_url); +} diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index 79ee1b68d..f27624650 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -17,6 +17,8 @@ class GoogleDriveService : public InternetService { QStandardItem* CreateRootItem(); void LazyPopulate(QStandardItem* item); + QUrl GetStreamingUrlFromSongId(const QString& file_id); + private slots: void AccessTokenAvailable(const QString& token); void ListFilesFinished(QNetworkReply* reply); diff --git a/src/internet/googledriveurlhandler.cpp b/src/internet/googledriveurlhandler.cpp new file mode 100644 index 000000000..3c74e72dc --- /dev/null +++ b/src/internet/googledriveurlhandler.cpp @@ -0,0 +1,16 @@ +#include "googledriveurlhandler.h" + +#include "googledriveservice.h" + +GoogleDriveUrlHandler::GoogleDriveUrlHandler( + GoogleDriveService* service, + QObject* parent) + : UrlHandler(parent), + service_(service) { +} + +UrlHandler::LoadResult GoogleDriveUrlHandler::StartLoading(const QUrl& url) { + QString file_id = url.path(); + QUrl real_url = service_->GetStreamingUrlFromSongId(file_id); + return LoadResult(url, LoadResult::TrackAvailable, real_url); +} diff --git a/src/internet/googledriveurlhandler.h b/src/internet/googledriveurlhandler.h new file mode 100644 index 000000000..46b1bebea --- /dev/null +++ b/src/internet/googledriveurlhandler.h @@ -0,0 +1,21 @@ +#ifndef GOOGLEDRIVEURLHANDLER_H +#define GOOGLEDRIVEURLHANDLER_H + +#include "core/urlhandler.h" + +class GoogleDriveService; + +class GoogleDriveUrlHandler : public UrlHandler { + Q_OBJECT + public: + GoogleDriveUrlHandler(GoogleDriveService* service, QObject* parent = 0); + + QString scheme() const { return "googledrive"; } + QIcon icon() const { return QIcon(":providers/googledrive.png"); } + LoadResult StartLoading(const QUrl& url); + + private: + GoogleDriveService* service_; +}; + +#endif From 1031482024f40adb452d561cf957e2c7f30562ac Mon Sep 17 00:00:00 2001 From: John Maguire Date: Thu, 26 Jul 2012 16:55:59 +0200 Subject: [PATCH 06/20] Refresh oauth token when possible to avoid reauthenticating with Drive all the time. --- src/internet/googledriveservice.cpp | 22 +++++++++++++++- src/internet/googledriveservice.h | 2 ++ src/internet/oauthenticator.cpp | 41 ++++++++++++++++++++--------- src/internet/oauthenticator.h | 7 +++-- 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 854a912fd..3678184b0 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -22,6 +22,7 @@ namespace { static const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; static const char* kGoogleDriveFile = "https://www.googleapis.com/drive/v2/files/%1"; +static const char* kSettingsGroup = "GoogleDrive"; } @@ -190,6 +191,7 @@ GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) root_(NULL), oauth_(new OAuthenticator(this)) { connect(oauth_, SIGNAL(AccessTokenAvailable(QString)), SLOT(AccessTokenAvailable(QString))); + connect(oauth_, SIGNAL(RefreshTokenAvailable(QString)), SLOT(RefreshTokenAvailable(QString))); app->player()->RegisterUrlHandler(new GoogleDriveUrlHandler(this, this)); } @@ -211,7 +213,19 @@ void GoogleDriveService::LazyPopulate(QStandardItem* item) { } void GoogleDriveService::Connect() { - oauth_->StartAuthorisation(); + QSettings s; + s.beginGroup(kSettingsGroup); + + if (s.contains("refresh_token")) { + QString refresh_token = s.value("refresh_token").toString(); + RefreshAuthorisation(refresh_token); + } else { + oauth_->StartAuthorisation(); + } +} + +void GoogleDriveService::RefreshAuthorisation(const QString& refresh_token) { + oauth_->RefreshAuthorisation(refresh_token); } void GoogleDriveService::AccessTokenAvailable(const QString& token) { @@ -226,6 +240,12 @@ void GoogleDriveService::AccessTokenAvailable(const QString& token) { NewClosure(reply, SIGNAL(finished()), this, SLOT(ListFilesFinished(QNetworkReply*)), reply); } +void GoogleDriveService::RefreshTokenAvailable(const QString& token) { + QSettings s; + s.beginGroup(kSettingsGroup); + s.setValue("refresh_token", token); +} + void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { reply->deleteLater(); diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index f27624650..d19093b95 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -21,10 +21,12 @@ class GoogleDriveService : public InternetService { private slots: void AccessTokenAvailable(const QString& token); + void RefreshTokenAvailable(const QString& token); void ListFilesFinished(QNetworkReply* reply); private: void Connect(); + void RefreshAuthorisation(const QString& refresh_token); QStandardItem* root_; OAuthenticator* oauth_; diff --git a/src/internet/oauthenticator.cpp b/src/internet/oauthenticator.cpp index f3f297d6a..186b61dec 100644 --- a/src/internet/oauthenticator.cpp +++ b/src/internet/oauthenticator.cpp @@ -14,7 +14,6 @@ namespace { const char* kGoogleOAuthEndpoint = "https://accounts.google.com/o/oauth2/auth"; const char* kGoogleOAuthTokenEndpoint = "https://accounts.google.com/o/oauth2/token"; -const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; const char* kClientId = "679260893280.apps.googleusercontent.com"; const char* kClientSecret = "l3cWb8efUZsrBI4wmY3uKl6i"; @@ -127,21 +126,39 @@ void OAuthenticator::FetchAccessTokenFinished(QNetworkReply* reply) { refresh_token_ = result["refresh_token"].toString(); emit AccessTokenAvailable(access_token_); + emit RefreshTokenAvailable(refresh_token_); } -void OAuthenticator::ListFiles(const QString& access_token) { - QUrl url = QUrl(kGoogleDriveFiles); - url.addQueryItem("q", "mimeType = 'audio/mpeg'"); - QNetworkRequest request = QNetworkRequest(url); - request.setRawHeader( - "Authorization", QString("Bearer %1").arg(access_token).toUtf8()); - qLog(Debug) << "Header:" << request.rawHeader("Authorization"); - QNetworkReply* reply = network_.get(request); - NewClosure(reply, SIGNAL(finished()), this, SLOT(ListFilesResponse(QNetworkReply*)), reply); +void OAuthenticator::RefreshAuthorisation(const QString& refresh_token) { + QUrl url = QUrl(kGoogleOAuthTokenEndpoint); + + typedef QPair Param; + QList parameters; + parameters << Param("client_id", kClientId) + << Param("client_secret", kClientSecret) + << Param("grant_type", "refresh_token") + << Param("refresh_token", refresh_token); + QStringList params; + foreach (const Param& p, parameters) { + params.append(QString("%1=%2").arg(p.first, QString(QUrl::toPercentEncoding(p.second)))); + } + QString post_data = params.join("&"); + qLog(Debug) << "Refresh post data:" << post_data; + + QNetworkRequest request(url); + request.setHeader(QNetworkRequest::ContentTypeHeader, + "application/x-www-form-urlencoded"); + QNetworkReply* reply = network_.post(request, post_data.toUtf8()); + NewClosure(reply, SIGNAL(finished()), this, + SLOT(RefreshAccessTokenFinished(QNetworkReply*)), reply); } -void OAuthenticator::ListFilesResponse(QNetworkReply* reply) { +void OAuthenticator::RefreshAccessTokenFinished(QNetworkReply* reply) { reply->deleteLater(); + QJson::Parser parser; + bool ok = false; - qLog(Debug) << reply->readAll(); + QVariantMap result = parser.parse(reply, &ok).toMap(); + QString access_token = result["access_token"].toString(); + emit AccessTokenAvailable(access_token); } diff --git a/src/internet/oauthenticator.h b/src/internet/oauthenticator.h index 114197a80..16b05eac2 100644 --- a/src/internet/oauthenticator.h +++ b/src/internet/oauthenticator.h @@ -13,20 +13,23 @@ class OAuthenticator : public QObject { public: explicit OAuthenticator(QObject* parent = 0); void StartAuthorisation(); + void RefreshAuthorisation(const QString& refresh_token); signals: + // Token to use now. void AccessTokenAvailable(QString token); + // Token to use to get a new access token when it expires. + void RefreshTokenAvailable(QString token); private slots: void NewConnection(); void RedirectArrived(QTcpSocket* socket, QByteArray buffer); void FetchAccessTokenFinished(QNetworkReply* reply); - void ListFilesResponse(QNetworkReply* reply); + void RefreshAccessTokenFinished(QNetworkReply* reply); private: QByteArray ParseHttpRequest(const QByteArray& request) const; void RequestAccessToken(const QByteArray& code, quint16 port); - void ListFiles(const QString& access_token); QTcpServer server_; NetworkAccessManager network_; From 73062bce3dd4984fc0595692c7041b97249eca56 Mon Sep 17 00:00:00 2001 From: David Sansome Date: Thu, 26 Jul 2012 21:23:08 +0100 Subject: [PATCH 07/20] Make Google Drive support optional and add a cmake option for it --- CMakeLists.txt | 8 ++++++++ src/CMakeLists.txt | 15 +++++++++++---- src/internet/internetmodel.cpp | 6 +++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 488bedac0..cf0c879d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,8 @@ find_library(LASTFM_LIBRARIES lastfm) find_path(LASTFM_INCLUDE_DIRS lastfm/ws.h) find_path(LASTFM1_INCLUDE_DIRS lastfm/Track.h) +find_path(SPARSEHASH_INCLUDE_DIRS google/sparsetable) + if(LASTFM_INCLUDE_DIRS AND LASTFM1_INCLUDE_DIRS) set(HAVE_LIBLASTFM1 ON) endif() @@ -200,6 +202,7 @@ option(ENABLE_BREAKPAD "Enable crash reporting" OFF) option(ENABLE_SPOTIFY_BLOB "Build the spotify non-GPL binary" ON) option(ENABLE_SPOTIFY "Enable spotify support" ON) option(ENABLE_MOODBAR "Enable moodbar" ON) +option(ENABLE_GOOGLE_DRIVE "Enable Google Drive support" ON) if(WIN32) option(ENABLE_WIN32_CONSOLE "Show the windows console even outside Debug mode" OFF) @@ -266,6 +269,10 @@ if(ENABLE_MOODBAR AND FFTW3_FOUND) set(HAVE_MOODBAR ON) endif() +if(ENABLE_GOOGLE_DRIVE AND SPARSEHASH_INCLUDE_DIRS AND TAGLIB_VERSION VERSION_GREATER 1.7.999) + set(HAVE_GOOGLE_DRIVE ON) +endif() + if(ENABLE_VISUALISATIONS) # When/if upstream accepts our patches then these options can be used to link @@ -439,6 +446,7 @@ summary_add("Devices: iPod Touch, iPhone, iPad support" HAVE_IMOBILEDEVICE) summary_add("Devices: MTP support" HAVE_LIBMTP) summary_add("Devices: GIO backend" HAVE_GIO) summary_add("Gnome sound menu integration" HAVE_LIBINDICATE) +summary_add("Google Drive support" HAVE_GOOGLE_DRIVE) summary_add("Last.fm support" HAVE_LIBLASTFM) summary_add("Moodbar support" HAVE_MOODBAR) summary_add("Spotify support: core code" HAVE_SPOTIFY) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bb03e76e7..62f585c8c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -160,8 +160,6 @@ set(SOURCES internet/digitallyimportedsettingspage.cpp internet/digitallyimportedurlhandler.cpp internet/geolocator.cpp - internet/googledriveservice.cpp - internet/googledriveurlhandler.cpp internet/groovesharkradio.cpp internet/groovesharkservice.cpp internet/groovesharksettingspage.cpp @@ -433,8 +431,6 @@ set(HEADERS internet/digitallyimportedservicebase.h internet/digitallyimportedsettingspage.h internet/geolocator.h - internet/googledriveservice.h - internet/googledriveurlhandler.h internet/groovesharkservice.h internet/groovesharksettingspage.h internet/groovesharkurlhandler.h @@ -970,6 +966,7 @@ optional_source(HAVE_LIBMTP # Windows media lister optional_source(HAVE_SAC + INCLUDE_DIRECTORIES ${SPARSEHASH_INCLUDE_DIRS} SOURCES devices/wmdmdevice.cpp devices/wmdmlister.cpp @@ -999,6 +996,16 @@ optional_source(HAVE_MOODBAR moodbar/moodbarproxystyle.h ) +# Google Drive support +optional_source(HAVE_GOOGLE_DRIVE + SOURCES + internet/googledriveservice.cpp + internet/googledriveurlhandler.cpp + HEADERS + internet/googledriveservice.h + internet/googledriveurlhandler.h +) + # Hack to add Clementine to the Unity system tray whitelist optional_source(LINUX SOURCES core/ubuntuunityhack.cpp diff --git a/src/internet/internetmodel.cpp b/src/internet/internetmodel.cpp index baf692033..005c07e91 100644 --- a/src/internet/internetmodel.cpp +++ b/src/internet/internetmodel.cpp @@ -17,7 +17,6 @@ #include "digitallyimportedservicebase.h" #include "icecastservice.h" -#include "googledriveservice.h" #include "jamendoservice.h" #include "magnatuneservice.h" #include "internetmimedata.h" @@ -37,6 +36,9 @@ #ifdef HAVE_SPOTIFY #include "spotifyservice.h" #endif +#ifdef HAVE_GOOGLE_DRIVE + #include "googledriveservice.h" +#endif #include #include @@ -65,7 +67,9 @@ InternetModel::InternetModel(Application* app, QObject* parent) #ifdef HAVE_LIBLASTFM AddService(new LastFMService(app, this)); #endif +#ifdef HAVE_GOOGLE_DRIVE AddService(new GoogleDriveService(app, this)); +#endif AddService(new GroovesharkService(app, this)); AddService(new MagnatuneService(app, this)); AddService(new PodcastService(app, this)); From 49326981c3f5f0e10759bd79a7c220797a7ba060 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Fri, 27 Jul 2012 10:25:45 +0200 Subject: [PATCH 08/20] Make Google Drive option available as #define. --- src/config.h.in | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.h.in b/src/config.h.in index 9a6750420..cd0dc9fd4 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -26,6 +26,7 @@ #cmakedefine HAVE_DBUS #cmakedefine HAVE_DEVICEKIT #cmakedefine HAVE_GIO +#cmakedefine HAVE_GOOGLE_DRIVE #cmakedefine HAVE_IMOBILEDEVICE #cmakedefine HAVE_LIBARCHIVE #cmakedefine HAVE_LIBGPOD From f48383c73ea87d97fde6c53379a6389c4c02a47c Mon Sep 17 00:00:00 2001 From: John Maguire Date: Fri, 27 Jul 2012 16:04:12 +0200 Subject: [PATCH 09/20] Index Google Drive MP3s and write to local database. --- data/data.qrc | 1 + data/schema/schema-38.sql | 50 ++++++++++++ src/core/database.cpp | 2 +- src/core/song.cpp | 9 +- src/core/song.h | 3 + src/internet/googledriveservice.cpp | 122 +++++++++++++++++++--------- src/internet/googledriveservice.h | 10 +++ src/playlist/playlistdelegates.cpp | 2 - 8 files changed, 156 insertions(+), 43 deletions(-) create mode 100644 data/schema/schema-38.sql diff --git a/data/data.qrc b/data/data.qrc index d377d1588..7a71a36b6 100644 --- a/data/data.qrc +++ b/data/data.qrc @@ -325,6 +325,7 @@ schema/schema-35.sql schema/schema-36.sql schema/schema-37.sql + schema/schema-38.sql schema/schema-3.sql schema/schema-4.sql schema/schema-5.sql diff --git a/data/schema/schema-38.sql b/data/schema/schema-38.sql new file mode 100644 index 000000000..bfa88048e --- /dev/null +++ b/data/schema/schema-38.sql @@ -0,0 +1,50 @@ +ALTER TABLE %allsongstables ADD COLUMN etag TEXT; + +CREATE TABLE google_drive_songs( + title TEXT, + album TEXT, + artist TEXT, + albumartist TEXT, + composer TEXT, + track INTEGER, + disc INTEGER, + bpm REAL, + year INTEGER, + genre TEXT, + comment TEXT, + compilation INTEGER, + + length INTEGER, + bitrate INTEGER, + samplerate INTEGER, + + directory INTEGER NOT NULL, + filename TEXT NOT NULL, + mtime INTEGER NOT NULL, + ctime INTEGER NOT NULL, + filesize INTEGER NOT NULL, + sampler INTEGER NOT NULL DEFAULT 0, + art_automatic TEXT, + art_manual TEXT, + filetype INTEGER NOT NULL DEFAULT 0, + playcount INTEGER NOT NULL DEFAULT 0, + lastplayed INTEGER, + rating INTEGER, + forced_compilation_on INTEGER NOT NULL DEFAULT 0, + forced_compilation_off INTEGER NOT NULL DEFAULT 0, + effective_compilation NOT NULL DEFAULT 0, + skipcount INTEGER NOT NULL DEFAULT 0, + score INTEGER NOT NULL DEFAULT 0, + beginning INTEGER NOT NULL DEFAULT 0, + cue_path TEXT, + unavailable INTEGER DEFAULT 0, + effective_albumartist TEXT, + etag TEXT +); + +CREATE VIRTUAL TABLE google_drive_songs_fts USING fts3 ( + ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment, + tokenize=unicode +); + +UPDATE schema_version SET version=38; diff --git a/src/core/database.cpp b/src/core/database.cpp index 57191f94c..906e779a6 100644 --- a/src/core/database.cpp +++ b/src/core/database.cpp @@ -37,7 +37,7 @@ #include const char* Database::kDatabaseFilename = "clementine.db"; -const int Database::kSchemaVersion = 37; +const int Database::kSchemaVersion = 38; const char* Database::kMagicAllSongsTables = "%allsongstables"; int Database::sNextConnectionId = 1; diff --git a/src/core/song.cpp b/src/core/song.cpp index 1387c2232..23c757042 100644 --- a/src/core/song.cpp +++ b/src/core/song.cpp @@ -79,7 +79,7 @@ const QStringList Song::kColumns = QStringList() << "art_manual" << "filetype" << "playcount" << "lastplayed" << "rating" << "forced_compilation_on" << "forced_compilation_off" << "effective_compilation" << "skipcount" << "score" << "beginning" << "length" - << "cue_path" << "unavailable" << "effective_albumartist"; + << "cue_path" << "unavailable" << "effective_albumartist" << "etag"; const QString Song::kColumnSpec = Song::kColumns.join(", "); const QString Song::kBindSpec = Utilities::Prepend(":", Song::kColumns).join(", "); @@ -167,6 +167,8 @@ struct Song::Private : public QSharedData { // Whether the song does not exist on the file system anymore, but is still // stored in the database so as to remember the user's metadata. bool unavailable_; + + QString etag_; }; @@ -263,6 +265,7 @@ bool Song::is_stream() const { return d->filetype_ == Type_Stream; } bool Song::is_cdda() const { return d->filetype_ == Type_Cdda; } const QString& Song::art_automatic() const { return d->art_automatic_; } const QString& Song::art_manual() const { return d->art_manual_; } +const QString& Song::etag() const { return d->etag_; } bool Song::has_manually_unset_cover() const { return d->art_manual_ == kManuallyUnsetCover; } void Song::manually_unset_cover() { d->art_manual_ = kManuallyUnsetCover; } bool Song::has_embedded_cover() const { return d->art_automatic_ == kEmbeddedCover; } @@ -304,6 +307,7 @@ void Song::set_lastplayed(int v) { d->lastplayed_ = v; } void Song::set_score(int v) { d->score_ = qBound(0, v, 100); } void Song::set_cue_path(const QString& v) { d->cue_path_ = v; } void Song::set_unavailable(bool v) { d->unavailable_ = v; } +void Song::set_etag(const QString& etag) { d->etag_ = etag; } void Song::set_url(const QUrl& v) { d->url_ = v; } void Song::set_basefilename(const QString& v) { d->basefilename_ = v; } void Song::set_directory_id(int v) { d->directory_id_ = v; } @@ -1002,8 +1006,11 @@ void Song::BindToQuery(QSqlQuery *query) const { query->bindValue(":unavailable", d->unavailable_ ? 1 : 0); query->bindValue(":effective_albumartist", this->effective_albumartist()); + query->bindValue(":etag", strval(d->etag_)); + #undef intval #undef notnullintval + #undef strval } void Song::BindToFtsQuery(QSqlQuery *query) const { diff --git a/src/core/song.h b/src/core/song.h index 94f5a34ce..2a254ee54 100644 --- a/src/core/song.h +++ b/src/core/song.h @@ -187,6 +187,8 @@ class Song { const QString& art_automatic() const; const QString& art_manual() const; + const QString& etag() const; + // Returns true if this Song had it's cover manually unset by user. bool has_manually_unset_cover() const; // This method represents an explicit request to unset this song's @@ -250,6 +252,7 @@ class Song { void set_score(int v); void set_cue_path(const QString& v); void set_unavailable(bool v); + void set_etag(const QString& etag); // Setters that should only be used by tests void set_url(const QUrl& v); diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 3678184b0..284ca46bb 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -1,6 +1,7 @@ #include "googledriveservice.h" #include +#include #include @@ -13,7 +14,14 @@ using TagLib::ByteVector; #include "core/application.h" #include "core/closure.h" +#include "core/database.h" +#include "core/mergedproxymodel.h" #include "core/player.h" +#include "core/timeconstants.h" +#include "globalsearch/globalsearch.h" +#include "globalsearch/librarysearchprovider.h" +#include "library/librarybackend.h" +#include "library/librarymodel.h" #include "googledriveurlhandler.h" #include "internetmodel.h" #include "oauthenticator.h" @@ -24,6 +32,9 @@ static const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/file static const char* kGoogleDriveFile = "https://www.googleapis.com/drive/v2/files/%1"; static const char* kSettingsGroup = "GoogleDrive"; +static const char* kSongsTable = "google_drive_songs"; +static const char* kFtsTable = "google_drive_songs_fts"; + } @@ -42,14 +53,9 @@ class DriveStream : public TagLib::IOStream { cursor_(0), network_(network), cache_(length) { - qLog(Debug) << Q_FUNC_INFO - << url_ - << filename_ - << length_; } virtual TagLib::FileName name() const { - qLog(Debug) << Q_FUNC_INFO; return encoded_filename_.data(); } @@ -78,7 +84,6 @@ class DriveStream : public TagLib::IOStream { } virtual TagLib::ByteVector readBlock(ulong length) { - qLog(Debug) << Q_FUNC_INFO; const uint start = cursor_; const uint end = qMin(cursor_ + length - 1, length_ - 1); @@ -87,7 +92,6 @@ class DriveStream : public TagLib::IOStream { } if (CheckCache(start, end)) { - qLog(Debug) << "Cache hit at:" << start << end; TagLib::ByteVector cached = GetCached(start, end); cursor_ += cached.size(); return cached; @@ -132,12 +136,10 @@ class DriveStream : public TagLib::IOStream { } virtual bool isOpen() const { - qLog(Debug) << Q_FUNC_INFO; return true; } virtual void seek(long offset, TagLib::IOStream::Position p) { - qLog(Debug) << Q_FUNC_INFO; switch (p) { case TagLib::IOStream::Beginning: cursor_ = offset; @@ -154,17 +156,14 @@ class DriveStream : public TagLib::IOStream { } virtual void clear() { - qLog(Debug) << Q_FUNC_INFO; cursor_ = 0; } virtual long tell() const { - qLog(Debug) << Q_FUNC_INFO; return cursor_; } virtual long length() { - qLog(Debug) << Q_FUNC_INFO; return length_; } @@ -189,11 +188,29 @@ class DriveStream : public TagLib::IOStream { GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) : InternetService("Google Drive", app, parent, parent), root_(NULL), - oauth_(new OAuthenticator(this)) { + oauth_(new OAuthenticator(this)), + library_sort_model_(new QSortFilterProxyModel(this)) { connect(oauth_, SIGNAL(AccessTokenAvailable(QString)), SLOT(AccessTokenAvailable(QString))); connect(oauth_, SIGNAL(RefreshTokenAvailable(QString)), SLOT(RefreshTokenAvailable(QString))); + library_backend_ = new LibraryBackend; + library_backend_->moveToThread(app_->database()->thread()); + library_backend_->Init(app_->database(), kSongsTable, + QString::null, QString::null, kFtsTable); + library_model_ = new LibraryModel(library_backend_, app_, this); + + library_sort_model_->setSourceModel(library_model_); + library_sort_model_->setSortRole(LibraryModel::Role_SortText); + library_sort_model_->setDynamicSortFilter(true); + library_sort_model_->sort(0); + app->player()->RegisterUrlHandler(new GoogleDriveUrlHandler(this, this)); + app_->global_search()->AddProvider(new LibrarySearchProvider( + library_backend_, + tr("Google Drive"), + "google_drive", + QIcon(":/providers/googledrive.png"), + true, app_, this)); } QStandardItem* GoogleDriveService::CreateRootItem() { @@ -206,7 +223,10 @@ void GoogleDriveService::LazyPopulate(QStandardItem* item) { switch (item->data(InternetModel::Role_Type).toInt()) { case InternetModel::Type_Service: Connect(); + library_model_->Init(); + model()->merged_model()->AddSubModel(item->index(), library_sort_model_); break; + default: break; } @@ -260,35 +280,59 @@ void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { QVariantList items = result["items"].toList(); foreach (const QVariant& v, items) { QVariantMap file = v.toMap(); - qLog(Debug) << "Creating stream"; - DriveStream* stream = new DriveStream( - file["downloadUrl"].toUrl(), - file["title"].toString(), - file["fileSize"].toUInt(), - access_token_, - &network_); - qLog(Debug) << "Creating tag"; - TagLib::MPEG::File tag( - stream, // Takes ownership. - TagLib::ID3v2::FrameFactory::instance(), - TagLib::AudioProperties::Fast); - qLog(Debug) << "Tagging done"; - if (tag.tag()) { - qLog(Debug) << tag.tag()->artist().toCString(); - Song song; - song.set_title(tag.tag()->title().toCString(true)); - song.set_artist(tag.tag()->artist().toCString(true)); - song.set_album(tag.tag()->album().toCString(true)); + MaybeAddFileToDatabase(file); + } +} - QString url = QString("googledrive:%1").arg(file["id"].toString()); - song.set_url(url); - qLog(Debug) << "Set url to:" << url; +void GoogleDriveService::MaybeAddFileToDatabase(const QVariantMap& file) { + QString url = QString("googledrive:%1").arg(file["id"].toString()); + Song song = library_backend_->GetSongByUrl(QUrl(url)); + // Song already in index. + // TODO: Check etag and maybe update. + if (song.is_valid()) { + return; + } - song.set_filesize(file["fileSize"].toInt()); - root_->appendRow(CreateSongItem(song)); - } else { - qLog(Debug) << "Tagging failed"; + // Song not in index; tag and add. + DriveStream* stream = new DriveStream( + file["downloadUrl"].toUrl(), + file["title"].toString(), + file["fileSize"].toUInt(), + access_token_, + &network_); + TagLib::MPEG::File tag( + stream, // Takes ownership. + TagLib::ID3v2::FrameFactory::instance(), + TagLib::AudioProperties::Fast); + if (tag.tag()) { + Song song; + song.set_title(tag.tag()->title().toCString(true)); + song.set_artist(tag.tag()->artist().toCString(true)); + song.set_album(tag.tag()->album().toCString(true)); + + song.set_url(url); + song.set_filesize(file["fileSize"].toInt()); + song.set_etag(file["etag"].toString().remove('"')); + + QString modified_date = file["modifiedDate"].toString(); + QString created_date = file["createdDate"].toString(); + + song.set_mtime(QDateTime::fromString(modified_date, Qt::ISODate).toTime_t()); + song.set_ctime(QDateTime::fromString(created_date, Qt::ISODate).toTime_t()); + + song.set_filetype(Song::Type_Stream); + song.set_directory_id(0); + + if (tag.audioProperties()) { + song.set_length_nanosec(tag.audioProperties()->length() * kNsecPerSec); } + + SongList songs; + songs << song; + qLog(Debug) << "Adding song to db:" << song.title(); + library_backend_->AddOrUpdateSongs(songs); + } else { + qLog(Debug) << "Failed to tag:" << url; } } diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index d19093b95..cd12e2c08 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -7,7 +7,10 @@ class QStandardItem; +class LibraryBackend; +class LibraryModel; class OAuthenticator; +class QSortFilterProxyModel; class GoogleDriveService : public InternetService { Q_OBJECT @@ -27,6 +30,7 @@ class GoogleDriveService : public InternetService { private: void Connect(); void RefreshAuthorisation(const QString& refresh_token); + void MaybeAddFileToDatabase(const QVariantMap& file); QStandardItem* root_; OAuthenticator* oauth_; @@ -34,6 +38,12 @@ class GoogleDriveService : public InternetService { QString access_token_; NetworkAccessManager network_; + + LibraryBackend* library_backend_; + LibraryModel* library_model_; + QSortFilterProxyModel* library_sort_model_; + + int indexing_task_id_; }; #endif diff --git a/src/playlist/playlistdelegates.cpp b/src/playlist/playlistdelegates.cpp index 887a637dd..7221935b5 100644 --- a/src/playlist/playlistdelegates.cpp +++ b/src/playlist/playlistdelegates.cpp @@ -463,8 +463,6 @@ QPixmap SongSourceDelegate::LookupPixmap(const QUrl& url, const QSize& size) con icon = IconLoader::Load("folder-sound"); } else if (url.host() == "api.jamendo.com") { icon = QIcon(":/providers/jamendo.png"); - } else if (url.host().endsWith("googleusercontent.com")) { - icon = QIcon(":/providers/googledrive.png"); } } pixmap = icon.pixmap(size.height()); From 51631169fad82e58f60166e4bc94ce9ead84ffb2 Mon Sep 17 00:00:00 2001 From: David Sansome Date: Fri, 27 Jul 2012 18:59:03 +0100 Subject: [PATCH 10/20] Show a nicer success page when oauth is complete. --- data/data.qrc | 1 + data/oauthsuccess.html | 37 +++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 1 + src/internet/oauthenticator.cpp | 35 ++++++++++++++++++++++++++++++- src/internet/oauthenticator.h | 4 ++-- 5 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 data/oauthsuccess.html diff --git a/data/data.qrc b/data/data.qrc index 7a71a36b6..113826a3f 100644 --- a/data/data.qrc +++ b/data/data.qrc @@ -348,5 +348,6 @@ volumeslider-handle_glow.png volumeslider-handle.png volumeslider-inset.png + oauthsuccess.html diff --git a/data/oauthsuccess.html b/data/oauthsuccess.html new file mode 100644 index 000000000..4aa0bda46 --- /dev/null +++ b/data/oauthsuccess.html @@ -0,0 +1,37 @@ + + + tr("Return to Clementine") + + + + + +
+

tr("Success!")

+ +

tr("Please close your browser and return to Clementine.")

+
+ + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 62f585c8c..2a35a1475 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1026,6 +1026,7 @@ add_pot(POT ${CMAKE_CURRENT_SOURCE_DIR}/translations/header ${CMAKE_CURRENT_SOURCE_DIR}/translations/translations.pot ${SOURCES} ${MOC} ${UIC} ${OTHER_SOURCES} + ../data/oauthsuccess.html ) add_po(PO clementine_ LANGUAGES ${LANGUAGES} diff --git a/src/internet/oauthenticator.cpp b/src/internet/oauthenticator.cpp index 186b61dec..46e3e854c 100644 --- a/src/internet/oauthenticator.cpp +++ b/src/internet/oauthenticator.cpp @@ -1,7 +1,11 @@ #include "oauthenticator.h" +#include +#include #include +#include #include +#include #include #include @@ -48,8 +52,37 @@ void OAuthenticator::NewConnection() { NewClosure(socket, SIGNAL(readyRead()), this, SLOT(RedirectArrived(QTcpSocket*, QByteArray)), socket, buffer); - // Everything is bon. + // Everything is bon. Prepare and display the success page. + QFile page_file(":oauthsuccess.html"); + page_file.open(QIODevice::ReadOnly); + QString page_data = QString::fromLatin1(page_file.readAll()); + + // Translate the strings inside + QRegExp tr_regexp("tr\\(\"([^\"]+)\"\\)"); + int offset = 0; + forever { + offset = tr_regexp.indexIn(page_data, offset); + if (offset == -1) { + break; + } + + page_data.replace(offset, tr_regexp.matchedLength(), + tr(tr_regexp.cap(1).toAscii())); + offset += tr_regexp.matchedLength(); + } + + // Add the tick image. + QBuffer image_buffer; + image_buffer.open(QIODevice::ReadWrite); + QApplication::style()->standardIcon(QStyle::SP_DialogOkButton) + .pixmap(16).toImage().save(&image_buffer, "PNG"); + + page_data.replace("@IMAGE_DATA@", image_buffer.data().toBase64()); + socket->write("HTTP/1.0 200 OK\r\n"); + socket->write("Content-type: text/html;charset=UTF-8\r\n"); + socket->write("\r\n\r\n"); + socket->write(page_data.toUtf8()); socket->flush(); } diff --git a/src/internet/oauthenticator.h b/src/internet/oauthenticator.h index 16b05eac2..141017cd1 100644 --- a/src/internet/oauthenticator.h +++ b/src/internet/oauthenticator.h @@ -17,9 +17,9 @@ class OAuthenticator : public QObject { signals: // Token to use now. - void AccessTokenAvailable(QString token); + void AccessTokenAvailable(const QString& token); // Token to use to get a new access token when it expires. - void RefreshTokenAvailable(QString token); + void RefreshTokenAvailable(const QString& token); private slots: void NewConnection(); From 165cec1e863add376572ea06c2f858ad85deab29 Mon Sep 17 00:00:00 2001 From: David Sansome Date: Sat, 28 Jul 2012 17:18:03 +0100 Subject: [PATCH 11/20] Move some of the Google Drive bits out into a separate client class. --- src/CMakeLists.txt | 2 + src/internet/googledriveclient.cpp | 168 ++++++++++++++++++++++++++++ src/internet/googledriveclient.h | 145 ++++++++++++++++++++++++ src/internet/googledriveservice.cpp | 113 +++++++------------ src/internet/googledriveservice.h | 19 ++-- src/internet/oauthenticator.cpp | 9 +- src/internet/oauthenticator.h | 9 +- 7 files changed, 381 insertions(+), 84 deletions(-) create mode 100644 src/internet/googledriveclient.cpp create mode 100644 src/internet/googledriveclient.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2a35a1475..1b44f24b6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -999,9 +999,11 @@ optional_source(HAVE_MOODBAR # Google Drive support optional_source(HAVE_GOOGLE_DRIVE SOURCES + internet/googledriveclient.cpp internet/googledriveservice.cpp internet/googledriveurlhandler.cpp HEADERS + internet/googledriveclient.h internet/googledriveservice.h internet/googledriveurlhandler.h ) diff --git a/src/internet/googledriveclient.cpp b/src/internet/googledriveclient.cpp new file mode 100644 index 000000000..632f517f3 --- /dev/null +++ b/src/internet/googledriveclient.cpp @@ -0,0 +1,168 @@ +/* This file is part of Clementine. + Copyright 2012, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#include "googledriveclient.h" +#include "oauthenticator.h" +#include "core/closure.h" +#include "core/network.h" + +#include + +using namespace google_drive; + +namespace { + static const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; + static const char* kGoogleDriveFile = "https://www.googleapis.com/drive/v2/files/%1"; +} + +ConnectResponse::ConnectResponse(QObject* parent) + : QObject(parent) +{ +} + +ListFilesResponse::ListFilesResponse(const QString& query, QObject* parent) + : QObject(parent), + query_(query) +{ +} + +GetFileResponse::GetFileResponse(const QString& file_id, QObject* parent) + : QObject(parent), + file_id_(file_id) +{ +} + +Client::Client(QObject* parent) + : QObject(parent), + network_(new NetworkAccessManager(this)) +{ +} + +ConnectResponse* Client::Connect(const QString& refresh_token) { + ConnectResponse* ret = new ConnectResponse(this); + OAuthenticator* oauth = new OAuthenticator(this); + + if (refresh_token.isEmpty()) { + oauth->StartAuthorisation(); + } else { + oauth->RefreshAuthorisation(refresh_token); + } + + NewClosure(oauth, SIGNAL(Finished()), + this, SLOT(ConnectFinished(ConnectResponse*,OAuthenticator*)), + ret, oauth); + return ret; +} + +void Client::ConnectFinished(ConnectResponse* response, OAuthenticator* oauth) { + oauth->deleteLater(); + access_token_ = oauth->access_token(); + response->refresh_token_ = oauth->refresh_token(); + emit response->Finished(); +} + +void Client::AddAuthorizationHeader(QNetworkRequest* request) const { + request->setRawHeader( + "Authorization", QString("Bearer %1").arg(access_token_).toUtf8()); +} + +ListFilesResponse* Client::ListFiles(const QString& query) { + ListFilesResponse* ret = new ListFilesResponse(query, this); + MakeListFilesRequest(ret); + return ret; +} + +void Client::MakeListFilesRequest(ListFilesResponse* response, const QString& page_token) { + QUrl url = QUrl(kGoogleDriveFiles); + + if (!response->query_.isEmpty()) { + url.addQueryItem("q", response->query_); + } + + if (!page_token.isEmpty()) { + url.addQueryItem("pageToken", page_token); + } + + QNetworkRequest request = QNetworkRequest(url); + AddAuthorizationHeader(&request); + + QNetworkReply* reply = network_->get(request); + NewClosure(reply, SIGNAL(finished()), + this, SLOT(ListFilesFinished(ListFilesResponse*, QNetworkReply*)), + response, reply); +} + +void Client::ListFilesFinished(ListFilesResponse* response, QNetworkReply* reply) { + reply->deleteLater(); + + // Parse the response + QJson::Parser parser; + bool ok = false; + QVariantMap result = parser.parse(reply, &ok).toMap(); + if (!ok) { + qLog(Error) << "Failed to request files from Google Drive"; + emit response->Finished(); + return; + } + + // Emit the FilesFound signal for the files in the response. + FileList files; + foreach (const QVariant& v, result["items"].toList()) { + files << File(v.toMap()); + } + + emit response->FilesFound(files); + + // Get the next page of results if there is one. + if (result.contains("nextPageToken")) { + MakeListFilesRequest(response, result["nextPageToken"].toString()); + } else { + emit response->Finished(); + } +} + +GetFileResponse* Client::GetFile(const QString& file_id) { + GetFileResponse* ret = new GetFileResponse(file_id, this); + + QString url = QString(kGoogleDriveFile).arg(file_id); + + QNetworkRequest request = QNetworkRequest(url); + AddAuthorizationHeader(&request); + + QNetworkReply* reply = network_->get(request); + NewClosure(reply, SIGNAL(finished()), + this, SLOT(GetFileFinished(GetFileResponse*,QNetworkReply*)), + ret, reply); + + return ret; +} + +void Client::GetFileFinished(GetFileResponse* response, QNetworkReply* reply) { + reply->deleteLater(); + + QJson::Parser parser; + bool ok = false; + QVariantMap result = parser.parse(reply, &ok).toMap(); + if (!ok) { + qLog(Error) << "Failed to fetch file with ID" << response->file_id_; + emit response->Finished(); + return; + } + + response->file_ = File(result); + emit response->Finished(); +} diff --git a/src/internet/googledriveclient.h b/src/internet/googledriveclient.h new file mode 100644 index 000000000..25dca2838 --- /dev/null +++ b/src/internet/googledriveclient.h @@ -0,0 +1,145 @@ +/* This file is part of Clementine. + Copyright 2012, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#ifndef GOOGLEDRIVECLIENT_H +#define GOOGLEDRIVECLIENT_H + +#include +#include +#include +#include +#include + +class OAuthenticator; +class QNetworkAccessManager; +class QNetworkReply; +class QNetworkRequest; + + +namespace google_drive { + +class Client; + +// Holds the metadata for a file on Google Drive. +class File { +public: + File(const QVariantMap& data = QVariantMap()) : data_(data) {} + + QString id() const { return data_["id"].toString(); } + QString etag() const { return data_["etag"].toString(); } + QString title() const { return data_["title"].toString(); } + long size() const { return data_["fileSize"].toUInt(); } + QUrl download_url() const { return data_["downloadUrl"].toUrl(); } + + QDateTime modified_date() const { + return QDateTime::fromString(data_["modifiedDate"].toString(), Qt::ISODate); + } + + QDateTime created_date() const { + return QDateTime::fromString(data_["createdDate"].toString(), Qt::ISODate); + } + +private: + QVariantMap data_; +}; + +typedef QList FileList; + + +class ConnectResponse : public QObject { + Q_OBJECT + friend class Client; + +public: + const QString& refresh_token() const { return refresh_token_; } + +signals: + void Finished(); + +private: + ConnectResponse(QObject* parent); + QString refresh_token_; +}; + + +class ListFilesResponse : public QObject { + Q_OBJECT + friend class Client; + +public: + const QString& query() const { return query_; } + +signals: + void FilesFound(const QList& files); + void Finished(); + +private: + ListFilesResponse(const QString& query, QObject* parent); + QString query_; +}; + + +class GetFileResponse : public QObject { + Q_OBJECT + friend class Client; + +public: + const QString& file_id() const { return file_id_; } + const File& file() const { return file_; } + +signals: + void Finished(); + +private: + GetFileResponse(const QString& file_id, QObject* parent); + QString file_id_; + File file_; +}; + + +class Client : public QObject { + Q_OBJECT + +public: + Client(QObject* parent = 0); + + bool is_authenticated() const { return !access_token_.isEmpty(); } + const QString& access_token() const { return access_token_; } + + ConnectResponse* Connect(const QString& refresh_token = QString()); + ListFilesResponse* ListFiles(const QString& query); + GetFileResponse* GetFile(const QString& file_id); + +private slots: + void ConnectFinished(ConnectResponse* response, OAuthenticator* oauth); + void ListFilesFinished(ListFilesResponse* response, QNetworkReply* reply); + void GetFileFinished(GetFileResponse* response, QNetworkReply* reply); + +private: + void AddAuthorizationHeader(QNetworkRequest* request) const; + void MakeListFilesRequest(ListFilesResponse* response, + const QString& page_token = QString()); + +private: + QNetworkAccessManager* network_; + + QString access_token_; +}; + +} // namespace + +#endif // GOOGLEDRIVECLIENT_H diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 284ca46bb..1dd7256f8 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -1,10 +1,9 @@ #include "googledriveservice.h" #include +#include #include -#include - #include #include @@ -22,14 +21,12 @@ using TagLib::ByteVector; #include "globalsearch/librarysearchprovider.h" #include "library/librarybackend.h" #include "library/librarymodel.h" +#include "googledriveclient.h" #include "googledriveurlhandler.h" #include "internetmodel.h" -#include "oauthenticator.h" namespace { -static const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; -static const char* kGoogleDriveFile = "https://www.googleapis.com/drive/v2/files/%1"; static const char* kSettingsGroup = "GoogleDrive"; static const char* kSongsTable = "google_drive_songs"; @@ -188,11 +185,8 @@ class DriveStream : public TagLib::IOStream { GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) : InternetService("Google Drive", app, parent, parent), root_(NULL), - oauth_(new OAuthenticator(this)), + client_(new google_drive::Client(this)), library_sort_model_(new QSortFilterProxyModel(this)) { - connect(oauth_, SIGNAL(AccessTokenAvailable(QString)), SLOT(AccessTokenAvailable(QString))); - connect(oauth_, SIGNAL(RefreshTokenAvailable(QString)), SLOT(RefreshTokenAvailable(QString))); - library_backend_ = new LibraryBackend; library_backend_->moveToThread(app_->database()->thread()); library_backend_->Init(app_->database(), kSongsTable, @@ -236,56 +230,43 @@ void GoogleDriveService::Connect() { QSettings s; s.beginGroup(kSettingsGroup); - if (s.contains("refresh_token")) { - QString refresh_token = s.value("refresh_token").toString(); - RefreshAuthorisation(refresh_token); - } else { - oauth_->StartAuthorisation(); - } + google_drive::ConnectResponse* response = + client_->Connect(s.value("refresh_token").toString()); + NewClosure(response, SIGNAL(Finished()), + this, SLOT(ConnectFinished(google_drive::ConnectResponse*)), + response); } -void GoogleDriveService::RefreshAuthorisation(const QString& refresh_token) { - oauth_->RefreshAuthorisation(refresh_token); -} +void GoogleDriveService::ConnectFinished(google_drive::ConnectResponse* response) { + response->deleteLater(); -void GoogleDriveService::AccessTokenAvailable(const QString& token) { - access_token_ = token; - QUrl url = QUrl(kGoogleDriveFiles); - url.addQueryItem("q", "mimeType = 'audio/mpeg'"); - - QNetworkRequest request = QNetworkRequest(url); - request.setRawHeader( - "Authorization", QString("Bearer %1").arg(token).toUtf8()); - QNetworkReply* reply = network_.get(request); - NewClosure(reply, SIGNAL(finished()), this, SLOT(ListFilesFinished(QNetworkReply*)), reply); -} - -void GoogleDriveService::RefreshTokenAvailable(const QString& token) { + // Save the refresh token QSettings s; s.beginGroup(kSettingsGroup); - s.setValue("refresh_token", token); + s.setValue("refresh_token", response->refresh_token()); + + // Find any music files + google_drive::ListFilesResponse* list_response = + client_->ListFiles("mimeType = 'audio/mpeg'"); + connect(list_response, SIGNAL(FilesFound(QList)), + this, SLOT(FilesFound(QList))); + + NewClosure(list_response, SIGNAL(Finished()), + this, SLOT(ListFilesFinished(google_drive::ListFilesResponse*))); } -void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { - reply->deleteLater(); - - QJson::Parser parser; - bool ok = false; - QVariantMap result = parser.parse(reply, &ok).toMap(); - if (!ok) { - qLog(Error) << "Failed to request files from Google Drive"; - return; - } - - QVariantList items = result["items"].toList(); - foreach (const QVariant& v, items) { - QVariantMap file = v.toMap(); +void GoogleDriveService::FilesFound(const QList& files) { + foreach (const google_drive::File& file, files) { MaybeAddFileToDatabase(file); } } -void GoogleDriveService::MaybeAddFileToDatabase(const QVariantMap& file) { - QString url = QString("googledrive:%1").arg(file["id"].toString()); +void GoogleDriveService::ListFilesFinished(google_drive::ListFilesResponse* response) { + response->deleteLater(); +} + +void GoogleDriveService::MaybeAddFileToDatabase(const google_drive::File& file) { + QString url = QString("googledrive:%1").arg(file.id()); Song song = library_backend_->GetSongByUrl(QUrl(url)); // Song already in index. // TODO: Check etag and maybe update. @@ -295,10 +276,10 @@ void GoogleDriveService::MaybeAddFileToDatabase(const QVariantMap& file) { // Song not in index; tag and add. DriveStream* stream = new DriveStream( - file["downloadUrl"].toUrl(), - file["title"].toString(), - file["fileSize"].toUInt(), - access_token_, + file.download_url(), + file.title(), + file.size(), + client_->access_token(), &network_); TagLib::MPEG::File tag( stream, // Takes ownership. @@ -311,14 +292,11 @@ void GoogleDriveService::MaybeAddFileToDatabase(const QVariantMap& file) { song.set_album(tag.tag()->album().toCString(true)); song.set_url(url); - song.set_filesize(file["fileSize"].toInt()); - song.set_etag(file["etag"].toString().remove('"')); + song.set_filesize(file.size()); + song.set_etag(file.etag().remove('"')); - QString modified_date = file["modifiedDate"].toString(); - QString created_date = file["createdDate"].toString(); - - song.set_mtime(QDateTime::fromString(modified_date, Qt::ISODate).toTime_t()); - song.set_ctime(QDateTime::fromString(created_date, Qt::ISODate).toTime_t()); + song.set_mtime(file.modified_date().toTime_t()); + song.set_ctime(file.created_date().toTime_t()); song.set_filetype(Song::Type_Stream); song.set_directory_id(0); @@ -337,18 +315,13 @@ void GoogleDriveService::MaybeAddFileToDatabase(const QVariantMap& file) { } QUrl GoogleDriveService::GetStreamingUrlFromSongId(const QString& id) { - QString url = QString(kGoogleDriveFile).arg(id); - QNetworkRequest request = QNetworkRequest(url); - request.setRawHeader( - "Authorization", QString("Bearer %1").arg(access_token_).toUtf8()); - QNetworkReply* reply = network_.get(request); + QScopedPointer response(client_->GetFile(id)); + QEventLoop loop; - connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); + connect(response.data(), SIGNAL(Finished()), &loop, SLOT(quit())); loop.exec(); - QJson::Parser parser; - bool ok = false; - QVariantMap result = parser.parse(reply, &ok).toMap(); - QString download_url = result["downloadUrl"].toString() + "#" + access_token_; - return QUrl(download_url); + QUrl url(response->file().download_url()); + url.setFragment(client_->access_token()); + return url; } diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index cd12e2c08..89a326e75 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -9,9 +9,15 @@ class QStandardItem; class LibraryBackend; class LibraryModel; -class OAuthenticator; class QSortFilterProxyModel; +namespace google_drive { + class Client; + class ConnectResponse; + class File; + class ListFilesResponse; +} + class GoogleDriveService : public InternetService { Q_OBJECT public: @@ -23,19 +29,18 @@ class GoogleDriveService : public InternetService { QUrl GetStreamingUrlFromSongId(const QString& file_id); private slots: - void AccessTokenAvailable(const QString& token); - void RefreshTokenAvailable(const QString& token); - void ListFilesFinished(QNetworkReply* reply); + void ConnectFinished(google_drive::ConnectResponse* response); + void FilesFound(const QList& files); + void ListFilesFinished(google_drive::ListFilesResponse* response); private: void Connect(); void RefreshAuthorisation(const QString& refresh_token); - void MaybeAddFileToDatabase(const QVariantMap& file); + void MaybeAddFileToDatabase(const google_drive::File& file); QStandardItem* root_; - OAuthenticator* oauth_; - QString access_token_; + google_drive::Client* client_; NetworkAccessManager network_; diff --git a/src/internet/oauthenticator.cpp b/src/internet/oauthenticator.cpp index 46e3e854c..b2cf31e75 100644 --- a/src/internet/oauthenticator.cpp +++ b/src/internet/oauthenticator.cpp @@ -158,11 +158,12 @@ void OAuthenticator::FetchAccessTokenFinished(QNetworkReply* reply) { access_token_ = result["access_token"].toString(); refresh_token_ = result["refresh_token"].toString(); - emit AccessTokenAvailable(access_token_); - emit RefreshTokenAvailable(refresh_token_); + emit Finished(); } void OAuthenticator::RefreshAuthorisation(const QString& refresh_token) { + refresh_token_ = refresh_token; + QUrl url = QUrl(kGoogleOAuthTokenEndpoint); typedef QPair Param; @@ -192,6 +193,6 @@ void OAuthenticator::RefreshAccessTokenFinished(QNetworkReply* reply) { bool ok = false; QVariantMap result = parser.parse(reply, &ok).toMap(); - QString access_token = result["access_token"].toString(); - emit AccessTokenAvailable(access_token); + access_token_ = result["access_token"].toString(); + emit Finished(); } diff --git a/src/internet/oauthenticator.h b/src/internet/oauthenticator.h index 141017cd1..a073fb186 100644 --- a/src/internet/oauthenticator.h +++ b/src/internet/oauthenticator.h @@ -15,11 +15,14 @@ class OAuthenticator : public QObject { void StartAuthorisation(); void RefreshAuthorisation(const QString& refresh_token); - signals: // Token to use now. - void AccessTokenAvailable(const QString& token); + const QString& access_token() const { return access_token_; } + // Token to use to get a new access token when it expires. - void RefreshTokenAvailable(const QString& token); + const QString& refresh_token() const { return refresh_token_; } + + signals: + void Finished(); private slots: void NewConnection(); From 9653a45f667a54e774ddba598fae41e562b59285 Mon Sep 17 00:00:00 2001 From: David Sansome Date: Sat, 28 Jul 2012 19:35:12 +0100 Subject: [PATCH 12/20] Read Google Drive metadata in the tagreader worker process --- ext/clementine-tagreader/CMakeLists.txt | 7 + .../googledrivestream.cpp | 157 +++++++++++++ ext/clementine-tagreader/googledrivestream.h | 68 ++++++ ext/clementine-tagreader/tagreaderworker.cpp | 46 ++++ ext/clementine-tagreader/tagreaderworker.h | 14 ++ .../tagreadermessages.proto | 14 ++ src/CMakeLists.txt | 1 - src/core/tagreaderclient.cpp | 18 ++ src/core/tagreaderclient.h | 4 + src/internet/googledriveclient.h | 1 + src/internet/googledriveservice.cpp | 222 +++--------------- src/internet/googledriveservice.h | 4 + 12 files changed, 370 insertions(+), 186 deletions(-) create mode 100644 ext/clementine-tagreader/googledrivestream.cpp create mode 100644 ext/clementine-tagreader/googledrivestream.h diff --git a/ext/clementine-tagreader/CMakeLists.txt b/ext/clementine-tagreader/CMakeLists.txt index 77b07ae4a..796334137 100644 --- a/ext/clementine-tagreader/CMakeLists.txt +++ b/ext/clementine-tagreader/CMakeLists.txt @@ -4,6 +4,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) include_directories(${CMAKE_SOURCE_DIR}/ext/libclementine-common) include_directories(${CMAKE_BINARY_DIR}/ext/libclementine-tagreader) include_directories(${CMAKE_SOURCE_DIR}/src) +include_directories(${CMAKE_BINARY_DIR}/src) set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}) @@ -16,6 +17,12 @@ set(SOURCES set(HEADERS ) +optional_source(HAVE_GOOGLE_DRIVE + INCLUDE_DIRECTORIES ${SPARSEHASH_INCLUDE_DIRS} + SOURCES + googledrivestream.cpp +) + qt4_wrap_cpp(MOC ${HEADERS}) add_executable(clementine-tagreader diff --git a/ext/clementine-tagreader/googledrivestream.cpp b/ext/clementine-tagreader/googledrivestream.cpp new file mode 100644 index 000000000..6e869521a --- /dev/null +++ b/ext/clementine-tagreader/googledrivestream.cpp @@ -0,0 +1,157 @@ +/* This file is part of Clementine. + Copyright 2012, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#include "googledrivestream.h" +#include "core/logging.h" + +#include +#include +#include +#include + +#include +#include +using TagLib::ByteVector; + +GoogleDriveStream::GoogleDriveStream( + const QUrl& url, const QString& filename, const long length, + const QString& auth, QNetworkAccessManager* network) + : url_(url), + filename_(filename), + encoded_filename_(filename_.toUtf8()), + length_(length), + auth_(auth), + cursor_(0), + network_(network), + cache_(length) { +} + +TagLib::FileName GoogleDriveStream::name() const { + return encoded_filename_.data(); +} + +bool GoogleDriveStream::CheckCache(int start, int end) { + for (int i = start; i <= end; ++i) { + if (!cache_.test(i)) { + return false; + } + } + return true; +} + +void GoogleDriveStream::FillCache(int start, TagLib::ByteVector data) { + for (int i = 0; i < data.size(); ++i) { + cache_.set(start + i, data[i]); + } +} + +TagLib::ByteVector GoogleDriveStream::GetCached(int start, int end) { + const uint size = end - start + 1; + TagLib::ByteVector ret(size); + for (int i = 0; i < size; ++i) { + ret[i] = cache_.get(start + i); + } + return ret; +} + +TagLib::ByteVector GoogleDriveStream::readBlock(ulong length) { + const uint start = cursor_; + const uint end = qMin(cursor_ + length - 1, length_ - 1); + + if (end <= start) { + return TagLib::ByteVector(); + } + + if (CheckCache(start, end)) { + TagLib::ByteVector cached = GetCached(start, end); + cursor_ += cached.size(); + return cached; + } + + QNetworkRequest request = QNetworkRequest(url_); + request.setRawHeader( + "Authorization", QString("Bearer %1").arg(auth_).toUtf8()); + request.setRawHeader( + "Range", QString("bytes=%1-%2").arg(start).arg(end).toUtf8()); + + QNetworkReply* reply = network_->get(request); + + QEventLoop loop; + QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); + loop.exec(); + reply->deleteLater(); + + QByteArray data = reply->readAll(); + TagLib::ByteVector bytes(data.data(), data.size()); + cursor_ += data.size(); + + FillCache(start, bytes); + return bytes; +} + +void GoogleDriveStream::writeBlock(const ByteVector&) { + qLog(Debug) << Q_FUNC_INFO << "not implemented"; +} + +void GoogleDriveStream::insert(const ByteVector&, ulong, ulong) { + qLog(Debug) << Q_FUNC_INFO << "not implemented"; +} + +void GoogleDriveStream::removeBlock(ulong, ulong) { + qLog(Debug) << Q_FUNC_INFO << "not implemented"; +} + +bool GoogleDriveStream::readOnly() const { + qLog(Debug) << Q_FUNC_INFO; + return true; +} + +bool GoogleDriveStream::isOpen() const { + return true; +} + +void GoogleDriveStream::seek(long offset, TagLib::IOStream::Position p) { + switch (p) { + case TagLib::IOStream::Beginning: + cursor_ = offset; + break; + + case TagLib::IOStream::Current: + cursor_ = qMin(ulong(cursor_ + offset), length_); + break; + + case TagLib::IOStream::End: + cursor_ = qMax(0UL, length_ - offset); + break; + } +} + +void GoogleDriveStream::clear() { + cursor_ = 0; +} + +long GoogleDriveStream::tell() const { + return cursor_; +} + +long GoogleDriveStream::length() { + return length_; +} + +void GoogleDriveStream::truncate(long) { + qLog(Debug) << Q_FUNC_INFO << "not implemented"; +} diff --git a/ext/clementine-tagreader/googledrivestream.h b/ext/clementine-tagreader/googledrivestream.h new file mode 100644 index 000000000..5facd6163 --- /dev/null +++ b/ext/clementine-tagreader/googledrivestream.h @@ -0,0 +1,68 @@ +/* This file is part of Clementine. + Copyright 2012, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#ifndef GOOGLEDRIVESTREAM_H +#define GOOGLEDRIVESTREAM_H + +#include + +#include +#include + +class QNetworkAccessManager; + +class GoogleDriveStream : public TagLib::IOStream { + public: + GoogleDriveStream(const QUrl& url, + const QString& filename, + const long length, + const QString& auth, + QNetworkAccessManager* network); + + // Taglib::IOStream + virtual TagLib::FileName name() const; + virtual TagLib::ByteVector readBlock(ulong length); + virtual void writeBlock(const TagLib::ByteVector&); + virtual void insert(const TagLib::ByteVector&, ulong, ulong); + virtual void removeBlock(ulong, ulong); + virtual bool readOnly() const; + virtual bool isOpen() const; + virtual void seek(long offset, TagLib::IOStream::Position p); + virtual void clear(); + virtual long tell() const; + virtual long length(); + virtual void truncate(long); + + private: + bool CheckCache(int start, int end); + void FillCache(int start, TagLib::ByteVector data); + TagLib::ByteVector GetCached(int start, int end); + + private: + const QUrl url_; + const QString filename_; + const QByteArray encoded_filename_; + const ulong length_; + const QString auth_; + + int cursor_; + QNetworkAccessManager* network_; + + google::sparsetable cache_; +}; + +#endif // GOOGLEDRIVESTREAM_H diff --git a/ext/clementine-tagreader/tagreaderworker.cpp b/ext/clementine-tagreader/tagreaderworker.cpp index b87723a4d..185852b2e 100644 --- a/ext/clementine-tagreader/tagreaderworker.cpp +++ b/ext/clementine-tagreader/tagreaderworker.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -56,6 +57,10 @@ # define TAGLIB_HAS_FLAC_PICTURELIST #endif +#ifdef HAVE_GOOGLE_DRIVE +# include "googledrivestream.h" +#endif + using boost::scoped_ptr; @@ -93,6 +98,7 @@ TagLib::String QStringToTaglibString(const QString& s) { TagReaderWorker::TagReaderWorker(QIODevice* socket, QObject* parent) : AbstractMessageHandler(socket, parent), factory_(new TagLibFileRefFactory), + network_(new QNetworkAccessManager), kEmbeddedCover("(embedded)") { } @@ -123,6 +129,17 @@ void TagReaderWorker::MessageArrived(const pb::tagreader::Message& message) { QStringFromStdString(message.load_embedded_art_request().filename())); reply.mutable_load_embedded_art_response()->set_data( data.constData(), data.size()); + } else if (message.has_read_google_drive_request()) { +#ifdef HAVE_GOOGLE_DRIVE + const pb::tagreader::ReadGoogleDriveRequest& req = + message.read_google_drive_request(); + ReadGoogleDrive(QUrl::fromEncoded(QByteArray(req.download_url().data(), + req.download_url().size())), + QStringFromStdString(req.title()), + req.size(), + QStringFromStdString(req.access_token()), + reply.mutable_read_google_drive_response()->mutable_metadata()); +#endif } SendReply(message, &reply); @@ -588,3 +605,32 @@ void TagReaderWorker::DeviceClosed() { qApp->exit(); } + +#ifdef HAVE_GOOGLE_DRIVE +void TagReaderWorker::ReadGoogleDrive(const QUrl& download_url, + const QString& title, + int size, + const QString& access_token, + pb::tagreader::SongMetadata* song) const { + qLog(Debug) << "Loading tags from" << title; + + GoogleDriveStream* stream = new GoogleDriveStream( + download_url, title, size, access_token, network_); + TagLib::MPEG::File tag( + stream, // Takes ownership. + TagLib::ID3v2::FrameFactory::instance(), + TagLib::AudioProperties::Fast); + if (tag.tag()) { + song->set_title(tag.tag()->title().toCString(true)); + song->set_artist(tag.tag()->artist().toCString(true)); + song->set_album(tag.tag()->album().toCString(true)); + song->set_filesize(size); + + song->set_type(pb::tagreader::SongMetadata_Type_STREAM); + + if (tag.audioProperties()) { + song->set_length_nanosec(tag.audioProperties()->length() * kNsecPerSec); + } + } +} +#endif // HAVE_GOOGLE_DRIVE diff --git a/ext/clementine-tagreader/tagreaderworker.h b/ext/clementine-tagreader/tagreaderworker.h index 021600bb0..ead214dec 100644 --- a/ext/clementine-tagreader/tagreaderworker.h +++ b/ext/clementine-tagreader/tagreaderworker.h @@ -18,11 +18,16 @@ #ifndef TAGREADERWORKER_H #define TAGREADERWORKER_H +#include "config.h" #include "tagreadermessages.pb.h" #include "core/messagehandler.h" #include +#include + +class QNetworkAccessManager; + namespace TagLib { class FileRef; @@ -49,6 +54,14 @@ private: bool IsMediaFile(const QString& filename) const; QByteArray LoadEmbeddedArt(const QString& filename) const; + #ifdef HAVE_GOOGLE_DRIVE + void ReadGoogleDrive(const QUrl& download_url, + const QString& title, + int size, + const QString& access_token, + pb::tagreader::SongMetadata* song) const; + #endif // HAVE_GOOGLE_DRIVE + static void Decode(const TagLib::String& tag, const QTextCodec* codec, std::string* output); static void Decode(const QString& tag, const QTextCodec* codec, @@ -69,6 +82,7 @@ private: private: FileRefFactory* factory_; + QNetworkAccessManager* network_; const std::string kEmbeddedCover; }; diff --git a/ext/libclementine-tagreader/tagreadermessages.proto b/ext/libclementine-tagreader/tagreadermessages.proto index 505e0eba2..aca9c8ad3 100644 --- a/ext/libclementine-tagreader/tagreadermessages.proto +++ b/ext/libclementine-tagreader/tagreadermessages.proto @@ -85,6 +85,17 @@ message LoadEmbeddedArtResponse { optional bytes data = 1; } +message ReadGoogleDriveRequest { + optional string download_url = 1; + optional string title = 2; + optional int32 size = 3; + optional string access_token = 4; +} + +message ReadGoogleDriveResponse { + optional SongMetadata metadata = 1; +} + message Message { optional int32 id = 1; @@ -99,4 +110,7 @@ message Message { optional LoadEmbeddedArtRequest load_embedded_art_request = 8; optional LoadEmbeddedArtResponse load_embedded_art_response = 9; + + optional ReadGoogleDriveRequest read_google_drive_request = 10; + optional ReadGoogleDriveResponse read_google_drive_response = 11; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1b44f24b6..6fe1d3e1e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -966,7 +966,6 @@ optional_source(HAVE_LIBMTP # Windows media lister optional_source(HAVE_SAC - INCLUDE_DIRECTORIES ${SPARSEHASH_INCLUDE_DIRS} SOURCES devices/wmdmdevice.cpp devices/wmdmlister.cpp diff --git a/src/core/tagreaderclient.cpp b/src/core/tagreaderclient.cpp index 7aec6aeb5..1bdf1c918 100644 --- a/src/core/tagreaderclient.cpp +++ b/src/core/tagreaderclient.cpp @@ -21,6 +21,7 @@ #include #include #include +#include const char* TagReaderClient::kWorkerExecutableName = "clementine-tagreader"; @@ -83,6 +84,23 @@ TagReaderReply* TagReaderClient::LoadEmbeddedArt(const QString& filename) { return worker_pool_->SendMessageWithReply(&message); } +TagReaderReply* TagReaderClient::ReadGoogleDrive(const QUrl& download_url, + const QString& title, + int size, + const QString& access_token) { + pb::tagreader::Message message; + pb::tagreader::ReadGoogleDriveRequest* req = + message.mutable_read_google_drive_request(); + + const QString url_string = download_url.toEncoded(); + req->set_download_url(DataCommaSizeFromQString(url_string)); + req->set_title(DataCommaSizeFromQString(title)); + req->set_size(size); + req->set_access_token(DataCommaSizeFromQString(access_token)); + + return worker_pool_->SendMessageWithReply(&message); +} + void TagReaderClient::ReadFileBlocking(const QString& filename, Song* song) { Q_ASSERT(QThread::currentThread() != thread()); diff --git a/src/core/tagreaderclient.h b/src/core/tagreaderclient.h index 97d0b56cd..79183595a 100644 --- a/src/core/tagreaderclient.h +++ b/src/core/tagreaderclient.h @@ -45,6 +45,10 @@ public: ReplyType* SaveFile(const QString& filename, const Song& metadata); ReplyType* IsMediaFile(const QString& filename); ReplyType* LoadEmbeddedArt(const QString& filename); + ReplyType* ReadGoogleDrive(const QUrl& download_url, + const QString& title, + int size, + const QString& access_token); // Convenience functions that call the above functions and wait for a // response. These block the calling thread with a semaphore, and must NOT diff --git a/src/internet/googledriveclient.h b/src/internet/googledriveclient.h index 25dca2838..ddabd2023 100644 --- a/src/internet/googledriveclient.h +++ b/src/internet/googledriveclient.h @@ -42,6 +42,7 @@ public: QString id() const { return data_["id"].toString(); } QString etag() const { return data_["etag"].toString(); } QString title() const { return data_["title"].toString(); } + QString description() const { return data_["description"].toString(); } long size() const { return data_["fileSize"].toUInt(); } QUrl download_url() const { return data_["downloadUrl"].toUrl(); } diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 1dd7256f8..45810191d 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -4,13 +4,6 @@ #include #include -#include - -#include -#include -#include -using TagLib::ByteVector; - #include "core/application.h" #include "core/closure.h" #include "core/database.h" @@ -35,153 +28,6 @@ static const char* kFtsTable = "google_drive_songs_fts"; } -class DriveStream : public TagLib::IOStream { - public: - DriveStream(const QUrl& url, - const QString& filename, - const long length, - const QString& auth, - QNetworkAccessManager* network) - : url_(url), - filename_(filename), - encoded_filename_(filename_.toUtf8()), - length_(length), - auth_(auth), - cursor_(0), - network_(network), - cache_(length) { - } - - virtual TagLib::FileName name() const { - return encoded_filename_.data(); - } - - bool CheckCache(int start, int end) { - for (int i = start; i <= end; ++i) { - if (!cache_.test(i)) { - return false; - } - } - return true; - } - - void FillCache(int start, TagLib::ByteVector data) { - for (int i = 0; i < data.size(); ++i) { - cache_.set(start + i, data[i]); - } - } - - TagLib::ByteVector GetCached(int start, int end) { - const uint size = end - start + 1; - TagLib::ByteVector ret(size); - for (int i = 0; i < size; ++i) { - ret[i] = cache_.get(start + i); - } - return ret; - } - - virtual TagLib::ByteVector readBlock(ulong length) { - const uint start = cursor_; - const uint end = qMin(cursor_ + length - 1, length_ - 1); - - if (end <= start) { - return TagLib::ByteVector(); - } - - if (CheckCache(start, end)) { - TagLib::ByteVector cached = GetCached(start, end); - cursor_ += cached.size(); - return cached; - } - - QNetworkRequest request = QNetworkRequest(url_); - request.setRawHeader( - "Authorization", QString("Bearer %1").arg(auth_).toUtf8()); - request.setRawHeader( - "Range", QString("bytes=%1-%2").arg(start).arg(end).toUtf8()); - - QNetworkReply* reply = network_->get(request); - - QEventLoop loop; - QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); - loop.exec(); - reply->deleteLater(); - - QByteArray data = reply->readAll(); - TagLib::ByteVector bytes(data.data(), data.size()); - cursor_ += data.size(); - - FillCache(start, bytes); - return bytes; - } - - virtual void writeBlock(const ByteVector&) { - qLog(Debug) << Q_FUNC_INFO << "not implemented"; - } - - virtual void insert(const ByteVector&, ulong, ulong) { - qLog(Debug) << Q_FUNC_INFO << "not implemented"; - } - - virtual void removeBlock(ulong, ulong) { - qLog(Debug) << Q_FUNC_INFO << "not implemented"; - } - - virtual bool readOnly() const { - qLog(Debug) << Q_FUNC_INFO; - return true; - } - - virtual bool isOpen() const { - return true; - } - - virtual void seek(long offset, TagLib::IOStream::Position p) { - switch (p) { - case TagLib::IOStream::Beginning: - cursor_ = offset; - break; - - case TagLib::IOStream::Current: - cursor_ = qMin(ulong(cursor_ + offset), length_); - break; - - case TagLib::IOStream::End: - cursor_ = qMax(0UL, length_ - offset); - break; - } - } - - virtual void clear() { - cursor_ = 0; - } - - virtual long tell() const { - return cursor_; - } - - virtual long length() { - return length_; - } - - virtual void truncate(long) { - qLog(Debug) << Q_FUNC_INFO << "not implemented"; - } - - private: - const QUrl url_; - const QString filename_; - const QByteArray encoded_filename_; - const ulong length_; - const QString auth_; - - int cursor_; - QNetworkAccessManager* network_; - - google::sparsetable cache_; -}; - - GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) : InternetService("Google Drive", app, parent, parent), root_(NULL), @@ -199,7 +45,7 @@ GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) library_sort_model_->sort(0); app->player()->RegisterUrlHandler(new GoogleDriveUrlHandler(this, this)); - app_->global_search()->AddProvider(new LibrarySearchProvider( + app->global_search()->AddProvider(new LibrarySearchProvider( library_backend_, tr("Google Drive"), "google_drive", @@ -275,43 +121,49 @@ void GoogleDriveService::MaybeAddFileToDatabase(const google_drive::File& file) } // Song not in index; tag and add. - DriveStream* stream = new DriveStream( + TagReaderClient::ReplyType* reply = app_->tag_reader_client()->ReadGoogleDrive( file.download_url(), file.title(), file.size(), - client_->access_token(), - &network_); - TagLib::MPEG::File tag( - stream, // Takes ownership. - TagLib::ID3v2::FrameFactory::instance(), - TagLib::AudioProperties::Fast); - if (tag.tag()) { - Song song; - song.set_title(tag.tag()->title().toCString(true)); - song.set_artist(tag.tag()->artist().toCString(true)); - song.set_album(tag.tag()->album().toCString(true)); + client_->access_token()); - song.set_url(url); - song.set_filesize(file.size()); - song.set_etag(file.etag().remove('"')); + NewClosure(reply, SIGNAL(Finished(bool)), + this, SLOT(ReadTagsFinished(TagReaderClient::ReplyType*,google_drive::File,QString)), + reply, file, url); +} - song.set_mtime(file.modified_date().toTime_t()); - song.set_ctime(file.created_date().toTime_t()); +void GoogleDriveService::ReadTagsFinished(TagReaderClient::ReplyType* reply, + const google_drive::File& metadata, + const QString& url) { + reply->deleteLater(); - song.set_filetype(Song::Type_Stream); - song.set_directory_id(0); - - if (tag.audioProperties()) { - song.set_length_nanosec(tag.audioProperties()->length() * kNsecPerSec); - } - - SongList songs; - songs << song; - qLog(Debug) << "Adding song to db:" << song.title(); - library_backend_->AddOrUpdateSongs(songs); - } else { - qLog(Debug) << "Failed to tag:" << url; + const pb::tagreader::ReadGoogleDriveResponse& msg = + reply->message().read_google_drive_response(); + if (!msg.metadata().filesize()) { + qLog(Debug) << "Failed to tag:" << metadata.download_url(); + return; } + + // Read the Song metadata from the message. + Song song; + song.InitFromProtobuf(msg.metadata()); + + // Add some extra tags from the Google Drive metadata. + song.set_etag(metadata.etag().remove('"')); + song.set_mtime(metadata.modified_date().toTime_t()); + song.set_ctime(metadata.created_date().toTime_t()); + song.set_comment(metadata.description()); + song.set_directory_id(0); + song.set_url(url); + + // Use the Google Drive title if we couldn't read tags from the file. + if (song.title().isEmpty()) { + song.set_title(metadata.title()); + } + + // Add the song to the database + qLog(Debug) << "Adding song to db:" << song.title(); + library_backend_->AddOrUpdateSongs(SongList() << song); } QUrl GoogleDriveService::GetStreamingUrlFromSongId(const QString& id) { diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index 89a326e75..060e1f7fe 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -4,6 +4,7 @@ #include "internetservice.h" #include "core/network.h" +#include "core/tagreaderclient.h" class QStandardItem; @@ -32,6 +33,9 @@ class GoogleDriveService : public InternetService { void ConnectFinished(google_drive::ConnectResponse* response); void FilesFound(const QList& files); void ListFilesFinished(google_drive::ListFilesResponse* response); + void ReadTagsFinished(TagReaderClient::ReplyType* reply, + const google_drive::File& metadata, + const QString& url); private: void Connect(); From fd1d70c644e1c7101bec2945df77cb36d0342df7 Mon Sep 17 00:00:00 2001 From: David Sansome Date: Sun, 29 Jul 2012 15:06:23 +0100 Subject: [PATCH 13/20] Add a directory model for selecting a default upload directory on Google Drive. --- src/CMakeLists.txt | 6 ++ src/internet/googledriveclient.cpp | 20 ++++ src/internet/googledriveclient.h | 20 ++++ src/internet/googledrivefoldermodel.cpp | 115 +++++++++++++++++++++++ src/internet/googledrivefoldermodel.h | 66 +++++++++++++ src/internet/googledriveservice.cpp | 7 +- src/internet/googledriveservice.h | 5 + src/internet/googledrivesettingspage.cpp | 90 ++++++++++++++++++ src/internet/googledrivesettingspage.h | 57 +++++++++++ src/internet/googledrivesettingspage.ui | 72 ++++++++++++++ src/ui/settingsdialog.cpp | 8 ++ src/ui/settingsdialog.h | 1 + 12 files changed, 464 insertions(+), 3 deletions(-) create mode 100644 src/internet/googledrivefoldermodel.cpp create mode 100644 src/internet/googledrivefoldermodel.h create mode 100644 src/internet/googledrivesettingspage.cpp create mode 100644 src/internet/googledrivesettingspage.h create mode 100644 src/internet/googledrivesettingspage.ui diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6fe1d3e1e..291788f1f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -999,12 +999,18 @@ optional_source(HAVE_MOODBAR optional_source(HAVE_GOOGLE_DRIVE SOURCES internet/googledriveclient.cpp + internet/googledrivefoldermodel.cpp internet/googledriveservice.cpp + internet/googledrivesettingspage.cpp internet/googledriveurlhandler.cpp HEADERS internet/googledriveclient.h + internet/googledrivefoldermodel.h internet/googledriveservice.h + internet/googledrivesettingspage.h internet/googledriveurlhandler.h + UI + internet/googledrivesettingspage.ui ) # Hack to add Clementine to the Unity system tray whitelist diff --git a/src/internet/googledriveclient.cpp b/src/internet/googledriveclient.cpp index 632f517f3..b907a2b97 100644 --- a/src/internet/googledriveclient.cpp +++ b/src/internet/googledriveclient.cpp @@ -24,11 +24,29 @@ using namespace google_drive; +const char* File::kFolderMimeType = "application/vnd.google-apps.folder"; + namespace { static const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/files"; static const char* kGoogleDriveFile = "https://www.googleapis.com/drive/v2/files/%1"; } +QStringList File::parent_ids() const { + QStringList ret; + + foreach (const QVariant& var, data_["parents"].toList()) { + QVariantMap map(var.toMap()); + + if (map["isRoot"].toBool()) { + ret << QString(); + } else { + ret << map["id"].toString(); + } + } + + return ret; +} + ConnectResponse::ConnectResponse(QObject* parent) : QObject(parent) { @@ -73,6 +91,8 @@ void Client::ConnectFinished(ConnectResponse* response, OAuthenticator* oauth) { access_token_ = oauth->access_token(); response->refresh_token_ = oauth->refresh_token(); emit response->Finished(); + + emit Authenticated(); } void Client::AddAuthorizationHeader(QNetworkRequest* request) const { diff --git a/src/internet/googledriveclient.h b/src/internet/googledriveclient.h index ddabd2023..6944a373a 100644 --- a/src/internet/googledriveclient.h +++ b/src/internet/googledriveclient.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -39,9 +40,12 @@ class File { public: File(const QVariantMap& data = QVariantMap()) : data_(data) {} + static const char* kFolderMimeType; + QString id() const { return data_["id"].toString(); } QString etag() const { return data_["etag"].toString(); } QString title() const { return data_["title"].toString(); } + QString mime_type() const { return data_["mimeType"].toString(); } QString description() const { return data_["description"].toString(); } long size() const { return data_["fileSize"].toUInt(); } QUrl download_url() const { return data_["downloadUrl"].toUrl(); } @@ -54,6 +58,19 @@ public: return QDateTime::fromString(data_["createdDate"].toString(), Qt::ISODate); } + bool is_folder() const { return mime_type() == kFolderMimeType; } + QStringList parent_ids() const; + + bool has_label(const QString& name) const { + return data_["labels"].toMap()[name].toBool(); + } + + bool is_starred() const { return has_label("starred"); } + bool is_hidden() const { return has_label("hidden"); } + bool is_trashed() const { return has_label("trashed"); } + bool is_restricted() const { return has_label("restricted"); } + bool is_viewed() const { return has_label("viewed"); } + private: QVariantMap data_; }; @@ -125,6 +142,9 @@ public: ListFilesResponse* ListFiles(const QString& query); GetFileResponse* GetFile(const QString& file_id); +signals: + void Authenticated(); + private slots: void ConnectFinished(ConnectResponse* response, OAuthenticator* oauth); void ListFilesFinished(ListFilesResponse* response, QNetworkReply* reply); diff --git a/src/internet/googledrivefoldermodel.cpp b/src/internet/googledrivefoldermodel.cpp new file mode 100644 index 000000000..367e6edb4 --- /dev/null +++ b/src/internet/googledrivefoldermodel.cpp @@ -0,0 +1,115 @@ +/* This file is part of Clementine. + Copyright 2012, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#include "googledriveclient.h" +#include "googledrivefoldermodel.h" +#include "core/closure.h" +#include "ui/iconloader.h" + +using namespace google_drive; + +FolderModel::FolderModel(Client* client, QObject* parent) + : QStandardItemModel(parent), + client_(client) +{ + folder_icon_ = IconLoader::Load("folder"); + + root_ = new QStandardItem(tr("My Drive")); + item_by_id_[QString()] = root_; + invisibleRootItem()->appendRow(root_); + + connect(client, SIGNAL(Authenticated()), SLOT(Refresh())); + if (client->is_authenticated()) { + Refresh(); + } +} + +void FolderModel::Refresh() { + ListFilesResponse* reply = + client_->ListFiles(QString("mimeType = '%1'").arg(File::kFolderMimeType)); + connect(reply, SIGNAL(FilesFound(QList)), + this, SLOT(FilesFound(QList))); + NewClosure(reply, SIGNAL(Finished()), + this, SLOT(FindFilesFinished(ListFilesResponse*)), + reply); +} + +void FolderModel::FindFilesFinished(ListFilesResponse* reply) { + reply->deleteLater(); +} + +void FolderModel::FilesFound(const QList& files) { + foreach (const File& file, files) { + if (file.is_hidden() || file.is_trashed()) { + continue; + } + + const QString id(file.id()); + + // Does this file exist in the model already? + if (item_by_id_.contains(id)) { + // If it has the same etag ignore it, otherwise remove and recreate it. + QStandardItem* old_item = item_by_id_[id]; + if (old_item->data(Role_Etag).toString() == file.etag()) { + continue; + } else { + item_by_id_.remove(id); + old_item->parent()->removeRow(old_item->row()); + } + } + + // Get the first parent's ID + const QStringList parent_ids = file.parent_ids(); + if (parent_ids.isEmpty()) { + continue; + } + const QString parent_id = parent_ids.first(); + + // If the parent doesn't exist yet, remember this file for later. + if (!item_by_id_.contains(parent_id)) { + orphans_[parent_id] << file; + continue; + } + + // Find the item for the parent + QStandardItem* parent = item_by_id_[parent_id]; + + // Create the item + QStandardItem* item = new QStandardItem(file.title()); + item->setData(file.etag(), Role_Etag); + item->setData(id, Role_Id); + item_by_id_[id] = item; + parent->appendRow(item); + + // Add any children for this item that we saw before. + if (orphans_.contains(id)) { + FilesFound(orphans_.take(id)); + } + } +} + +QVariant FolderModel::data(const QModelIndex& index, int role) const { + if (role == Qt::DecorationRole) { + return folder_icon_; + } + + return QStandardItemModel::data(index, role); +} + +QStandardItem* FolderModel::ItemById(const QString& id) const { + return item_by_id_[id]; +} diff --git a/src/internet/googledrivefoldermodel.h b/src/internet/googledrivefoldermodel.h new file mode 100644 index 000000000..899274d14 --- /dev/null +++ b/src/internet/googledrivefoldermodel.h @@ -0,0 +1,66 @@ +/* This file is part of Clementine. + Copyright 2012, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#ifndef GOOGLEDRIVEFOLDERMODEL_H +#define GOOGLEDRIVEFOLDERMODEL_H + +#include "googledriveclient.h" + +#include + +namespace google_drive { + +class Client; + +class FolderModel : public QStandardItemModel { + Q_OBJECT + +public: + FolderModel(Client* client, QObject* parent = 0); + + enum Role { + Role_Etag = Qt::UserRole, + Role_Id + }; + + QIcon folder_icon() const { return folder_icon_; } + void set_folder_icon(const QIcon& icon) { folder_icon_ = icon; } + + QVariant data(const QModelIndex& index, int role) const; + + QStandardItem* ItemById(const QString& id) const; + +public slots: + void Refresh(); + +private slots: + void FilesFound(const QList& files); + void FindFilesFinished(ListFilesResponse* reply); + +private: + Client* client_; + QIcon folder_icon_; + + QStandardItem* root_; + + QMap item_by_id_; + QMap > orphans_; +}; + +} // namespace + +#endif // GOOGLEDRIVEFOLDERMODEL_H diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 45810191d..89453b9df 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -18,9 +18,10 @@ #include "googledriveurlhandler.h" #include "internetmodel.h" -namespace { +const char* GoogleDriveService::kServiceName = "Google Drive"; +const char* GoogleDriveService::kSettingsGroup = "GoogleDrive"; -static const char* kSettingsGroup = "GoogleDrive"; +namespace { static const char* kSongsTable = "google_drive_songs"; static const char* kFtsTable = "google_drive_songs_fts"; @@ -29,7 +30,7 @@ static const char* kFtsTable = "google_drive_songs_fts"; GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) - : InternetService("Google Drive", app, parent, parent), + : InternetService(kServiceName, app, parent, parent), root_(NULL), client_(new google_drive::Client(this)), library_sort_model_(new QSortFilterProxyModel(this)) { diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index 060e1f7fe..f71fc5918 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -24,6 +24,11 @@ class GoogleDriveService : public InternetService { public: GoogleDriveService(Application* app, InternetModel* parent); + static const char* kServiceName; + static const char* kSettingsGroup; + + google_drive::Client* client() const { return client_; } + QStandardItem* CreateRootItem(); void LazyPopulate(QStandardItem* item); diff --git a/src/internet/googledrivesettingspage.cpp b/src/internet/googledrivesettingspage.cpp new file mode 100644 index 000000000..cdffd0222 --- /dev/null +++ b/src/internet/googledrivesettingspage.cpp @@ -0,0 +1,90 @@ +/* This file is part of Clementine. + Copyright 2012, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#include "googledriveclient.h" +#include "googledrivefoldermodel.h" +#include "googledriveservice.h" +#include "googledrivesettingspage.h" +#include "ui_googledrivesettingspage.h" +#include "core/application.h" +#include "internet/internetmodel.h" +#include "ui/settingsdialog.h" + +#include + +GoogleDriveSettingsPage::GoogleDriveSettingsPage(SettingsDialog* parent) + : SettingsPage(parent), + ui_(new Ui::GoogleDriveSettingsPage), + model_(NULL), + proxy_model_(NULL), + item_needs_selecting_(false) +{ + ui_->setupUi(this); +} + +GoogleDriveSettingsPage::~GoogleDriveSettingsPage() { + delete ui_; +} + +void GoogleDriveSettingsPage::Load() { + QSettings s; + s.beginGroup(GoogleDriveService::kSettingsGroup); + + destination_folder_id_ = s.value("destination_folder_id").toString(); + item_needs_selecting_ = !destination_folder_id_.isEmpty(); + + if (!model_) { + GoogleDriveService* service = + dialog()->app()->internet_model()->Service(); + google_drive::Client* client = service->client(); + + model_ = new google_drive::FolderModel(client, this); + proxy_model_ = new QSortFilterProxyModel(this); + proxy_model_->setSourceModel(model_); + proxy_model_->setDynamicSortFilter(true); + proxy_model_->setSortCaseSensitivity(Qt::CaseInsensitive); + proxy_model_->sort(0); + + ui_->upload_destination->setModel(proxy_model_); + + connect(model_, SIGNAL(rowsInserted(QModelIndex,int,int)), + this, SLOT(DirectoryRowsInserted(QModelIndex))); + } +} + +void GoogleDriveSettingsPage::Save() { + QSettings s; + s.beginGroup(GoogleDriveService::kSettingsGroup); + + s.setValue("destination_folder_id", + ui_->upload_destination->currentIndex().data( + google_drive::FolderModel::Role_Id).toString()); +} + +void GoogleDriveSettingsPage::DirectoryRowsInserted(const QModelIndex& parent) { + ui_->upload_destination->expand(proxy_model_->mapFromSource(parent)); + + if (item_needs_selecting_) { + QStandardItem* item = model_->ItemById(destination_folder_id_); + if (item) { + ui_->upload_destination->selectionModel()->select( + proxy_model_->mapFromSource(item->index()), + QItemSelectionModel::ClearAndSelect); + item_needs_selecting_ = false; + } + } +} diff --git a/src/internet/googledrivesettingspage.h b/src/internet/googledrivesettingspage.h new file mode 100644 index 000000000..fb8ddf7b4 --- /dev/null +++ b/src/internet/googledrivesettingspage.h @@ -0,0 +1,57 @@ +/* This file is part of Clementine. + Copyright 2012, David Sansome + + Clementine is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Clementine is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Clementine. If not, see . +*/ + +#ifndef GOOGLEDRIVESETTINGSPAGE_H +#define GOOGLEDRIVESETTINGSPAGE_H + +#include "ui/settingspage.h" + +#include +#include + +class Ui_GoogleDriveSettingsPage; + +class QSortFilterProxyModel; + +namespace google_drive { + class FolderModel; +} + +class GoogleDriveSettingsPage : public SettingsPage { + Q_OBJECT + +public: + GoogleDriveSettingsPage(SettingsDialog* parent = 0); + ~GoogleDriveSettingsPage(); + + void Load(); + void Save(); + +private slots: + void DirectoryRowsInserted(const QModelIndex& parent); + +private: + Ui_GoogleDriveSettingsPage* ui_; + + google_drive::FolderModel* model_; + QSortFilterProxyModel* proxy_model_; + + QString destination_folder_id_; + bool item_needs_selecting_; +}; + +#endif // GOOGLEDRIVESETTINGSPAGE_H diff --git a/src/internet/googledrivesettingspage.ui b/src/internet/googledrivesettingspage.ui new file mode 100644 index 000000000..57c5b5b00 --- /dev/null +++ b/src/internet/googledrivesettingspage.ui @@ -0,0 +1,72 @@ + + + GoogleDriveSettingsPage + + + + 0 + 0 + 569 + 491 + + + + Google Drive + + + + :/providers/googledrive.png:/providers/googledrive.png + + + + + + Uploads + + + + + + You can upload songs to Google Drive by right clicking and using "Copy to device". + + + true + + + + + + + Upload new songs to + + + + + + + + 16777215 + 150 + + + + QAbstractItemView::NoEditTriggers + + + false + + + false + + + + + + + + + + + + + diff --git a/src/ui/settingsdialog.cpp b/src/ui/settingsdialog.cpp index fd7671376..868d06452 100644 --- a/src/ui/settingsdialog.cpp +++ b/src/ui/settingsdialog.cpp @@ -59,6 +59,10 @@ # include "internet/spotifysettingspage.h" #endif +#ifdef HAVE_GOOGLE_DRIVE +# include "internet/googledrivesettingspage.h" +#endif + #include #include #include @@ -139,6 +143,10 @@ SettingsDialog::SettingsDialog(Application* app, BackgroundStreams* streams, QWi AddPage(Page_Grooveshark, new GroovesharkSettingsPage(this), providers); +#ifdef HAVE_GOOGLE_DRIVE + AddPage(Page_GoogleDrive, new GoogleDriveSettingsPage(this), providers); +#endif + #ifdef HAVE_SPOTIFY AddPage(Page_Spotify, new SpotifySettingsPage(this), providers); #endif diff --git a/src/ui/settingsdialog.h b/src/ui/settingsdialog.h index b7db62d60..3d73b5691 100644 --- a/src/ui/settingsdialog.h +++ b/src/ui/settingsdialog.h @@ -76,6 +76,7 @@ public: Page_Remote, Page_Wiimotedev, Page_Podcasts, + Page_GoogleDrive, }; enum Role { From 08286102dad7ba318d065b901e36221c8cf8544e Mon Sep 17 00:00:00 2001 From: John Maguire Date: Mon, 30 Jul 2012 13:41:29 +0200 Subject: [PATCH 14/20] Precache the first and last parts of Google Drive MP3s to reduce the number of requests when using accurate tagging. --- .../googledrivestream.cpp | 39 +++++++++++++++---- ext/clementine-tagreader/googledrivestream.h | 12 ++++++ ext/clementine-tagreader/tagreaderworker.cpp | 7 +++- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/ext/clementine-tagreader/googledrivestream.cpp b/ext/clementine-tagreader/googledrivestream.cpp index 6e869521a..845b11db6 100644 --- a/ext/clementine-tagreader/googledrivestream.cpp +++ b/ext/clementine-tagreader/googledrivestream.cpp @@ -1,16 +1,16 @@ /* This file is part of Clementine. Copyright 2012, David Sansome - + Clementine is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Clementine is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with Clementine. If not, see . */ @@ -25,7 +25,11 @@ #include #include -using TagLib::ByteVector; + +namespace { + static const int kTaglibPrefixCacheBytes = 64 * 1024; // Should be enough. + static const int kTaglibSuffixCacheBytes = 2 * 1024; +} GoogleDriveStream::GoogleDriveStream( const QUrl& url, const QString& filename, const long length, @@ -37,7 +41,8 @@ GoogleDriveStream::GoogleDriveStream( auth_(auth), cursor_(0), network_(network), - cache_(length) { + cache_(length), + num_requests_(0) { } TagLib::FileName GoogleDriveStream::name() const { @@ -68,6 +73,25 @@ TagLib::ByteVector GoogleDriveStream::GetCached(int start, int end) { return ret; } +void GoogleDriveStream::Precache() { + // For reading the tags of an MP3, TagLib tends to request: + // 1. The first 1024 bytes + // 2. Somewhere between the first 2KB and first 60KB + // 3. The last KB or two. + // 4. Somewhere in the first 64KB again + // + // So, if we precache the first 64KB and the last 2KB we should be sorted :-) + // Ideally, we would use bytes=0-655364,-2048 but Google Drive does not seem + // to support multipart byte ranges yet so we have to make do with two + // requests. + + seek(0, TagLib::IOStream::Beginning); + readBlock(kTaglibPrefixCacheBytes); + seek(kTaglibSuffixCacheBytes, TagLib::IOStream::End); + readBlock(kTaglibSuffixCacheBytes); + clear(); +} + TagLib::ByteVector GoogleDriveStream::readBlock(ulong length) { const uint start = cursor_; const uint end = qMin(cursor_ + length - 1, length_ - 1); @@ -89,6 +113,7 @@ TagLib::ByteVector GoogleDriveStream::readBlock(ulong length) { "Range", QString("bytes=%1-%2").arg(start).arg(end).toUtf8()); QNetworkReply* reply = network_->get(request); + ++num_requests_; QEventLoop loop; QObject::connect(reply, SIGNAL(finished()), &loop, SLOT(quit())); @@ -103,11 +128,11 @@ TagLib::ByteVector GoogleDriveStream::readBlock(ulong length) { return bytes; } -void GoogleDriveStream::writeBlock(const ByteVector&) { +void GoogleDriveStream::writeBlock(const TagLib::ByteVector&) { qLog(Debug) << Q_FUNC_INFO << "not implemented"; } -void GoogleDriveStream::insert(const ByteVector&, ulong, ulong) { +void GoogleDriveStream::insert(const TagLib::ByteVector&, ulong, ulong) { qLog(Debug) << Q_FUNC_INFO << "not implemented"; } diff --git a/ext/clementine-tagreader/googledrivestream.h b/ext/clementine-tagreader/googledrivestream.h index 5facd6163..f11a34dc6 100644 --- a/ext/clementine-tagreader/googledrivestream.h +++ b/ext/clementine-tagreader/googledrivestream.h @@ -47,6 +47,17 @@ class GoogleDriveStream : public TagLib::IOStream { virtual long length(); virtual void truncate(long); + google::sparsetable::size_type cached_bytes() const { + return cache_.num_nonempty(); + } + + int num_requests() const { + return num_requests_; + } + + // Use educated guess to request the bytes that TagLib will probably want. + void Precache(); + private: bool CheckCache(int start, int end); void FillCache(int start, TagLib::ByteVector data); @@ -63,6 +74,7 @@ class GoogleDriveStream : public TagLib::IOStream { QNetworkAccessManager* network_; google::sparsetable cache_; + int num_requests_; }; #endif // GOOGLEDRIVESTREAM_H diff --git a/ext/clementine-tagreader/tagreaderworker.cpp b/ext/clementine-tagreader/tagreaderworker.cpp index 185852b2e..c86ff442c 100644 --- a/ext/clementine-tagreader/tagreaderworker.cpp +++ b/ext/clementine-tagreader/tagreaderworker.cpp @@ -616,10 +616,11 @@ void TagReaderWorker::ReadGoogleDrive(const QUrl& download_url, GoogleDriveStream* stream = new GoogleDriveStream( download_url, title, size, access_token, network_); + stream->Precache(); TagLib::MPEG::File tag( stream, // Takes ownership. TagLib::ID3v2::FrameFactory::instance(), - TagLib::AudioProperties::Fast); + TagLib::AudioProperties::Accurate); if (tag.tag()) { song->set_title(tag.tag()->title().toCString(true)); song->set_artist(tag.tag()->artist().toCString(true)); @@ -631,6 +632,10 @@ void TagReaderWorker::ReadGoogleDrive(const QUrl& download_url, if (tag.audioProperties()) { song->set_length_nanosec(tag.audioProperties()->length() * kNsecPerSec); } + qLog(Debug) << "Google Drive Tagging Stats for:" + << song->title().c_str(); + qLog(Debug) << "Downloaded bytes:" << stream->cached_bytes() + << "Number of requests:" << stream->num_requests(); } } #endif // HAVE_GOOGLE_DRIVE From 23a255d0cfd0b58e9a1949b1d8b0ad891ee8790c Mon Sep 17 00:00:00 2001 From: John Maguire Date: Mon, 30 Jul 2012 17:51:45 +0200 Subject: [PATCH 15/20] Ship some GIO modules on Mac so HTTPS works correctly. --- dist/macdeploy.py | 17 ++++++++++++++++- src/main.cpp | 5 +++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dist/macdeploy.py b/dist/macdeploy.py index 41dcedc0e..f1ef17452 100755 --- a/dist/macdeploy.py +++ b/dist/macdeploy.py @@ -106,6 +106,10 @@ QT_PLUGINS_SEARCH_PATH=[ '/Developer/Applications/Qt/plugins', ] +GIO_MODULES_SEARCH_PATH=[ + '/target/lib/gio/modules', +] + class Error(Exception): pass @@ -131,7 +135,7 @@ class CouldNotFindGstreamerPluginError(Error): pass -class CouldNotFindScriptPluginError(Error): +class CouldNotFindGioModuleError(Error): pass @@ -360,12 +364,23 @@ def FindGstreamerPlugin(name): raise CouldNotFindGstreamerPluginError(name) +def FindGioModule(name): + for path in GIO_MODULES_SEARCH_PATH: + if os.path.exists(path): + for dir, dirs, files in os.walk(path): + if name in files: + return os.path.join(dir, name) + raise CouldNotFindGioModuleError(name) + + FixBinary(binary) for plugin in GSTREAMER_PLUGINS: FixPlugin(FindGstreamerPlugin(plugin), 'gstreamer') FixPlugin(FindGstreamerPlugin('gst-plugin-scanner'), '.') +FixPlugin(FindGioModule('libgiognutls.so'), 'gio-modules') +FixPlugin(FindGioModule('libgiolibproxy.so'), 'gio-modules') try: FixPlugin('clementine-spotifyblob', '.') diff --git a/src/main.cpp b/src/main.cpp index 2a6b05381..0d907bad8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -184,6 +184,11 @@ void SetGstreamerEnvironment() { if (!registry_filename.isEmpty()) { SetEnv("GST_REGISTRY", registry_filename); } + +#ifdef Q_OS_DARWIN + SetEnv("GIO_EXTRA_MODULES", + QCoreApplication::applicationDirPath() + "/../PlugIns/gio-modules"); +#endif } #ifdef HAVE_GIO From e06ae8a28204542191a5b00b5fa6cf2074ac4ec6 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Tue, 31 Jul 2012 11:57:04 +0200 Subject: [PATCH 16/20] Support reading OGG & FLAC files from Google Drive. --- .../googledrivestream.cpp | 13 ++-- ext/clementine-tagreader/tagreaderworker.cpp | 71 +++++++++++++------ ext/clementine-tagreader/tagreaderworker.h | 3 +- .../tagreadermessages.proto | 1 + src/core/tagreaderclient.cpp | 2 + src/core/tagreaderclient.h | 1 + src/internet/googledriveservice.cpp | 25 ++++--- src/internet/googledriveservice.h | 1 + 8 files changed, 81 insertions(+), 36 deletions(-) diff --git a/ext/clementine-tagreader/googledrivestream.cpp b/ext/clementine-tagreader/googledrivestream.cpp index 845b11db6..7489fc116 100644 --- a/ext/clementine-tagreader/googledrivestream.cpp +++ b/ext/clementine-tagreader/googledrivestream.cpp @@ -28,7 +28,7 @@ namespace { static const int kTaglibPrefixCacheBytes = 64 * 1024; // Should be enough. - static const int kTaglibSuffixCacheBytes = 2 * 1024; + static const int kTaglibSuffixCacheBytes = 8 * 1024; } GoogleDriveStream::GoogleDriveStream( @@ -80,8 +80,10 @@ void GoogleDriveStream::Precache() { // 3. The last KB or two. // 4. Somewhere in the first 64KB again // - // So, if we precache the first 64KB and the last 2KB we should be sorted :-) - // Ideally, we would use bytes=0-655364,-2048 but Google Drive does not seem + // OGG Vorbis may read the last 4KB. + // + // So, if we precache the first 64KB and the last 8KB we should be sorted :-) + // Ideally, we would use bytes=0-655364,-8096 but Google Drive does not seem // to support multipart byte ranges yet so we have to make do with two // requests. @@ -96,7 +98,7 @@ TagLib::ByteVector GoogleDriveStream::readBlock(ulong length) { const uint start = cursor_; const uint end = qMin(cursor_ + length - 1, length_ - 1); - if (end <= start) { + if (end < start) { return TagLib::ByteVector(); } @@ -160,7 +162,8 @@ void GoogleDriveStream::seek(long offset, TagLib::IOStream::Position p) { break; case TagLib::IOStream::End: - cursor_ = qMax(0UL, length_ - offset); + // This should really not have qAbs(), but OGG reading needs it. + cursor_ = qMax(0UL, length_ - qAbs(offset)); break; } } diff --git a/ext/clementine-tagreader/tagreaderworker.cpp b/ext/clementine-tagreader/tagreaderworker.cpp index c86ff442c..54fac4af0 100644 --- a/ext/clementine-tagreader/tagreaderworker.cpp +++ b/ext/clementine-tagreader/tagreaderworker.cpp @@ -133,12 +133,16 @@ void TagReaderWorker::MessageArrived(const pb::tagreader::Message& message) { #ifdef HAVE_GOOGLE_DRIVE const pb::tagreader::ReadGoogleDriveRequest& req = message.read_google_drive_request(); - ReadGoogleDrive(QUrl::fromEncoded(QByteArray(req.download_url().data(), - req.download_url().size())), - QStringFromStdString(req.title()), - req.size(), - QStringFromStdString(req.access_token()), - reply.mutable_read_google_drive_response()->mutable_metadata()); + if (!ReadGoogleDrive( + QUrl::fromEncoded(QByteArray(req.download_url().data(), + req.download_url().size())), + QStringFromStdString(req.title()), + req.size(), + QStringFromStdString(req.mime_type()), + QStringFromStdString(req.access_token()), + reply.mutable_read_google_drive_response()->mutable_metadata())) { + reply.mutable_read_google_drive_response()->clear_metadata(); + } #endif } @@ -607,9 +611,10 @@ void TagReaderWorker::DeviceClosed() { } #ifdef HAVE_GOOGLE_DRIVE -void TagReaderWorker::ReadGoogleDrive(const QUrl& download_url, +bool TagReaderWorker::ReadGoogleDrive(const QUrl& download_url, const QString& title, int size, + const QString& mime_type, const QString& access_token, pb::tagreader::SongMetadata* song) const { qLog(Debug) << "Loading tags from" << title; @@ -617,25 +622,49 @@ void TagReaderWorker::ReadGoogleDrive(const QUrl& download_url, GoogleDriveStream* stream = new GoogleDriveStream( download_url, title, size, access_token, network_); stream->Precache(); - TagLib::MPEG::File tag( - stream, // Takes ownership. - TagLib::ID3v2::FrameFactory::instance(), - TagLib::AudioProperties::Accurate); - if (tag.tag()) { - song->set_title(tag.tag()->title().toCString(true)); - song->set_artist(tag.tag()->artist().toCString(true)); - song->set_album(tag.tag()->album().toCString(true)); + scoped_ptr tag; + if (mime_type == "audio/mpeg") { + tag.reset(new TagLib::MPEG::File( + stream, // Takes ownership. + TagLib::ID3v2::FrameFactory::instance(), + TagLib::AudioProperties::Accurate)); + } else if (mime_type == "application/ogg") { + tag.reset(new TagLib::Ogg::Vorbis::File( + stream, + true, + TagLib::AudioProperties::Accurate)); + } else if (mime_type == "application/x-flac") { + tag.reset(new TagLib::FLAC::File( + stream, + TagLib::ID3v2::FrameFactory::instance(), + true, + TagLib::AudioProperties::Accurate)); + } else { + qLog(Debug) << "Unknown mime type for tagging:" << mime_type; + return false; + } + + if (stream->num_requests() > 2) { + // Warn if pre-caching failed. + qLog(Warning) << "Total requests for file:" << title + << stream->num_requests() + << stream->cached_bytes(); + } + + if (tag->tag()) { + song->set_title(tag->tag()->title().toCString(true)); + song->set_artist(tag->tag()->artist().toCString(true)); + song->set_album(tag->tag()->album().toCString(true)); song->set_filesize(size); song->set_type(pb::tagreader::SongMetadata_Type_STREAM); - if (tag.audioProperties()) { - song->set_length_nanosec(tag.audioProperties()->length() * kNsecPerSec); + if (tag->audioProperties()) { + song->set_length_nanosec(tag->audioProperties()->length() * kNsecPerSec); } - qLog(Debug) << "Google Drive Tagging Stats for:" - << song->title().c_str(); - qLog(Debug) << "Downloaded bytes:" << stream->cached_bytes() - << "Number of requests:" << stream->num_requests(); + return true; } + + return false; } #endif // HAVE_GOOGLE_DRIVE diff --git a/ext/clementine-tagreader/tagreaderworker.h b/ext/clementine-tagreader/tagreaderworker.h index ead214dec..997a8aaef 100644 --- a/ext/clementine-tagreader/tagreaderworker.h +++ b/ext/clementine-tagreader/tagreaderworker.h @@ -55,9 +55,10 @@ private: QByteArray LoadEmbeddedArt(const QString& filename) const; #ifdef HAVE_GOOGLE_DRIVE - void ReadGoogleDrive(const QUrl& download_url, + bool ReadGoogleDrive(const QUrl& download_url, const QString& title, int size, + const QString& mime_type, const QString& access_token, pb::tagreader::SongMetadata* song) const; #endif // HAVE_GOOGLE_DRIVE diff --git a/ext/libclementine-tagreader/tagreadermessages.proto b/ext/libclementine-tagreader/tagreadermessages.proto index aca9c8ad3..4cc04c14a 100644 --- a/ext/libclementine-tagreader/tagreadermessages.proto +++ b/ext/libclementine-tagreader/tagreadermessages.proto @@ -90,6 +90,7 @@ message ReadGoogleDriveRequest { optional string title = 2; optional int32 size = 3; optional string access_token = 4; + optional string mime_type = 5; } message ReadGoogleDriveResponse { diff --git a/src/core/tagreaderclient.cpp b/src/core/tagreaderclient.cpp index 1bdf1c918..145d6dded 100644 --- a/src/core/tagreaderclient.cpp +++ b/src/core/tagreaderclient.cpp @@ -87,6 +87,7 @@ TagReaderReply* TagReaderClient::LoadEmbeddedArt(const QString& filename) { TagReaderReply* TagReaderClient::ReadGoogleDrive(const QUrl& download_url, const QString& title, int size, + const QString& mime_type, const QString& access_token) { pb::tagreader::Message message; pb::tagreader::ReadGoogleDriveRequest* req = @@ -96,6 +97,7 @@ TagReaderReply* TagReaderClient::ReadGoogleDrive(const QUrl& download_url, req->set_download_url(DataCommaSizeFromQString(url_string)); req->set_title(DataCommaSizeFromQString(title)); req->set_size(size); + req->set_mime_type(DataCommaSizeFromQString(mime_type)); req->set_access_token(DataCommaSizeFromQString(access_token)); return worker_pool_->SendMessageWithReply(&message); diff --git a/src/core/tagreaderclient.h b/src/core/tagreaderclient.h index 79183595a..864d8ba8f 100644 --- a/src/core/tagreaderclient.h +++ b/src/core/tagreaderclient.h @@ -48,6 +48,7 @@ public: ReplyType* ReadGoogleDrive(const QUrl& download_url, const QString& title, int size, + const QString& mime_type, const QString& access_token); // Convenience functions that call the above functions and wait for a diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 89453b9df..ad6438fbd 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -84,6 +84,16 @@ void GoogleDriveService::Connect() { response); } +void GoogleDriveService::ListFilesForMimeType(const QString& mime_type) { + google_drive::ListFilesResponse* list_response = client_->ListFiles( + QString("mimeType = '%1' and trashed = false").arg(mime_type)); + connect(list_response, SIGNAL(FilesFound(QList)), + this, SLOT(FilesFound(QList))); + NewClosure(list_response, SIGNAL(Finished()), + this, SLOT(ListFilesFinished(google_drive::ListFilesResponse*)), + list_response); +} + void GoogleDriveService::ConnectFinished(google_drive::ConnectResponse* response) { response->deleteLater(); @@ -93,13 +103,9 @@ void GoogleDriveService::ConnectFinished(google_drive::ConnectResponse* response s.setValue("refresh_token", response->refresh_token()); // Find any music files - google_drive::ListFilesResponse* list_response = - client_->ListFiles("mimeType = 'audio/mpeg'"); - connect(list_response, SIGNAL(FilesFound(QList)), - this, SLOT(FilesFound(QList))); - - NewClosure(list_response, SIGNAL(Finished()), - this, SLOT(ListFilesFinished(google_drive::ListFilesResponse*))); + ListFilesForMimeType("audio/mpeg"); // MP3 + ListFilesForMimeType("application/ogg"); // OGG + ListFilesForMimeType("application/x-flac"); // FLAC } void GoogleDriveService::FilesFound(const QList& files) { @@ -126,6 +132,7 @@ void GoogleDriveService::MaybeAddFileToDatabase(const google_drive::File& file) file.download_url(), file.title(), file.size(), + file.mime_type(), client_->access_token()); NewClosure(reply, SIGNAL(Finished(bool)), @@ -140,8 +147,8 @@ void GoogleDriveService::ReadTagsFinished(TagReaderClient::ReplyType* reply, const pb::tagreader::ReadGoogleDriveResponse& msg = reply->message().read_google_drive_response(); - if (!msg.metadata().filesize()) { - qLog(Debug) << "Failed to tag:" << metadata.download_url(); + if (!msg.has_metadata() || !msg.metadata().filesize()) { + qLog(Debug) << "Failed to tag:" << metadata.title(); return; } diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index f71fc5918..7c50d13a7 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -46,6 +46,7 @@ class GoogleDriveService : public InternetService { void Connect(); void RefreshAuthorisation(const QString& refresh_token); void MaybeAddFileToDatabase(const google_drive::File& file); + void ListFilesForMimeType(const QString& mime_type); QStandardItem* root_; From fc61f0627fb3cece38b8feb2a0f4b54b86ba1468 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Tue, 31 Jul 2012 15:25:49 +0200 Subject: [PATCH 17/20] Filter out trashed/hidden folders on the server. --- src/internet/googledrivefoldermodel.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/internet/googledrivefoldermodel.cpp b/src/internet/googledrivefoldermodel.cpp index 367e6edb4..cbf0b51aa 100644 --- a/src/internet/googledrivefoldermodel.cpp +++ b/src/internet/googledrivefoldermodel.cpp @@ -39,8 +39,9 @@ FolderModel::FolderModel(Client* client, QObject* parent) } void FolderModel::Refresh() { - ListFilesResponse* reply = - client_->ListFiles(QString("mimeType = '%1'").arg(File::kFolderMimeType)); + ListFilesResponse* reply = client_->ListFiles( + QString("mimeType = '%1' and trashed = false and hidden = false") + .arg(File::kFolderMimeType)); connect(reply, SIGNAL(FilesFound(QList)), this, SLOT(FilesFound(QList))); NewClosure(reply, SIGNAL(Finished()), @@ -54,10 +55,6 @@ void FolderModel::FindFilesFinished(ListFilesResponse* reply) { void FolderModel::FilesFound(const QList& files) { foreach (const File& file, files) { - if (file.is_hidden() || file.is_trashed()) { - continue; - } - const QString id(file.id()); // Does this file exist in the model already? From 94878b4bd238dac69df38ba1fd1a34ebd443d21b Mon Sep 17 00:00:00 2001 From: John Maguire Date: Tue, 31 Jul 2012 17:57:17 +0200 Subject: [PATCH 18/20] Slightly hacky support for AAC on Google Drive. --- ext/clementine-tagreader/tagreaderworker.cpp | 14 +++++++++++++- src/internet/googledriveservice.cpp | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ext/clementine-tagreader/tagreaderworker.cpp b/ext/clementine-tagreader/tagreaderworker.cpp index 54fac4af0..ab6b8a614 100644 --- a/ext/clementine-tagreader/tagreaderworker.cpp +++ b/ext/clementine-tagreader/tagreaderworker.cpp @@ -623,11 +623,16 @@ bool TagReaderWorker::ReadGoogleDrive(const QUrl& download_url, download_url, title, size, access_token, network_); stream->Precache(); scoped_ptr tag; - if (mime_type == "audio/mpeg") { + if (mime_type == "audio/mpeg" && title.endsWith(".mp3")) { tag.reset(new TagLib::MPEG::File( stream, // Takes ownership. TagLib::ID3v2::FrameFactory::instance(), TagLib::AudioProperties::Accurate)); + } else if (mime_type == "audio/mpeg" && title.endsWith(".m4a")) { + tag.reset(new TagLib::MP4::File( + stream, + true, + TagLib::AudioProperties::Accurate)); } else if (mime_type == "application/ogg") { tag.reset(new TagLib::Ogg::Vorbis::File( stream, @@ -657,6 +662,13 @@ bool TagReaderWorker::ReadGoogleDrive(const QUrl& download_url, song->set_album(tag->tag()->album().toCString(true)); song->set_filesize(size); + if (tag->tag()->track() != 0) { + song->set_track(tag->tag()->track()); + } + if (tag->tag()->year() != 0) { + song->set_year(tag->tag()->year()); + } + song->set_type(pb::tagreader::SongMetadata_Type_STREAM); if (tag->audioProperties()) { diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index ad6438fbd..1ea995f41 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -103,7 +103,7 @@ void GoogleDriveService::ConnectFinished(google_drive::ConnectResponse* response s.setValue("refresh_token", response->refresh_token()); // Find any music files - ListFilesForMimeType("audio/mpeg"); // MP3 + ListFilesForMimeType("audio/mpeg"); // MP3/AAC ListFilesForMimeType("application/ogg"); // OGG ListFilesForMimeType("application/x-flac"); // FLAC } From c5b74c832e94532b4e3895251346cc0b67a13dbd Mon Sep 17 00:00:00 2001 From: John Maguire Date: Mon, 6 Aug 2012 14:00:54 +0200 Subject: [PATCH 19/20] Add tasks for indexing Google Drive songs. --- src/core/taskmanager.h | 20 ++++++++++++++++++++ src/internet/googledriveservice.cpp | 14 +++++++++++--- src/internet/googledriveservice.h | 5 ++++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/core/taskmanager.h b/src/core/taskmanager.h index 3e2f753c1..4a240c1b5 100644 --- a/src/core/taskmanager.h +++ b/src/core/taskmanager.h @@ -36,6 +36,24 @@ public: bool blocks_library_scans; }; + class ScopedTask { + public: + ScopedTask(const int task_id, TaskManager* task_manager) + : task_id_(task_id), + task_manager_(task_manager) { + } + + ~ScopedTask() { + task_manager_->SetTaskFinished(task_id_); + } + + private: + const int task_id_; + TaskManager* task_manager_; + + Q_DISABLE_COPY(ScopedTask); + }; + // Everything here is thread safe QList GetTasks(); @@ -56,6 +74,8 @@ private: QMutex mutex_; QMap tasks_; int next_task_id_; + + Q_DISABLE_COPY(TaskManager); }; #endif // TASKMANAGER_H diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 1ea995f41..a6111988c 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -9,6 +9,7 @@ #include "core/database.h" #include "core/mergedproxymodel.h" #include "core/player.h" +#include "core/taskmanager.h" #include "core/timeconstants.h" #include "globalsearch/globalsearch.h" #include "globalsearch/librarysearchprovider.h" @@ -33,6 +34,7 @@ GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) : InternetService(kServiceName, app, parent, parent), root_(NULL), client_(new google_drive::Client(this)), + task_manager_(app->task_manager()), library_sort_model_(new QSortFilterProxyModel(this)) { library_backend_ = new LibraryBackend; library_backend_->moveToThread(app_->database()->thread()); @@ -127,6 +129,9 @@ void GoogleDriveService::MaybeAddFileToDatabase(const google_drive::File& file) return; } + const int task_id = task_manager_->StartTask( + tr("Indexing %1").arg(file.title())); + // Song not in index; tag and add. TagReaderClient::ReplyType* reply = app_->tag_reader_client()->ReadGoogleDrive( file.download_url(), @@ -136,15 +141,18 @@ void GoogleDriveService::MaybeAddFileToDatabase(const google_drive::File& file) client_->access_token()); NewClosure(reply, SIGNAL(Finished(bool)), - this, SLOT(ReadTagsFinished(TagReaderClient::ReplyType*,google_drive::File,QString)), - reply, file, url); + this, SLOT(ReadTagsFinished(TagReaderClient::ReplyType*,google_drive::File,QString,int)), + reply, file, url, task_id); } void GoogleDriveService::ReadTagsFinished(TagReaderClient::ReplyType* reply, const google_drive::File& metadata, - const QString& url) { + const QString& url, + const int task_id) { reply->deleteLater(); + TaskManager::ScopedTask(task_id, task_manager_); + const pb::tagreader::ReadGoogleDriveResponse& msg = reply->message().read_google_drive_response(); if (!msg.has_metadata() || !msg.metadata().filesize()) { diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index 7c50d13a7..a253e3fd8 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -10,6 +10,7 @@ class QStandardItem; class LibraryBackend; class LibraryModel; +class TaskManager; class QSortFilterProxyModel; namespace google_drive { @@ -40,7 +41,8 @@ class GoogleDriveService : public InternetService { void ListFilesFinished(google_drive::ListFilesResponse* response); void ReadTagsFinished(TagReaderClient::ReplyType* reply, const google_drive::File& metadata, - const QString& url); + const QString& url, + const int task_id); private: void Connect(); @@ -53,6 +55,7 @@ class GoogleDriveService : public InternetService { google_drive::Client* client_; NetworkAccessManager network_; + TaskManager* task_manager_; LibraryBackend* library_backend_; LibraryModel* library_model_; From 9ce8cf59d6d17e30ecaeb0f45cebb1bf58e5f81c Mon Sep 17 00:00:00 2001 From: John Maguire Date: Fri, 10 Aug 2012 11:46:26 -0700 Subject: [PATCH 20/20] Make sure Google Drive is connected when converting URLs. --- src/internet/googledriveservice.cpp | 16 +++++++++++++++- src/internet/googledriveservice.h | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index a6111988c..4c4086b60 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -65,7 +65,9 @@ QStandardItem* GoogleDriveService::CreateRootItem() { void GoogleDriveService::LazyPopulate(QStandardItem* item) { switch (item->data(InternetModel::Role_Type).toInt()) { case InternetModel::Type_Service: - Connect(); + if (!client_->is_authenticated()) { + Connect(); + } library_model_->Init(); model()->merged_model()->AddSubModel(item->index(), library_sort_model_); break; @@ -110,6 +112,17 @@ void GoogleDriveService::ConnectFinished(google_drive::ConnectResponse* response ListFilesForMimeType("application/x-flac"); // FLAC } +void GoogleDriveService::EnsureConnected() { + if (client_->is_authenticated()) { + return; + } + + QEventLoop loop; + connect(client_, SIGNAL(Authenticated()), &loop, SLOT(quit())); + Connect(); + loop.exec(); +} + void GoogleDriveService::FilesFound(const QList& files) { foreach (const google_drive::File& file, files) { MaybeAddFileToDatabase(file); @@ -183,6 +196,7 @@ void GoogleDriveService::ReadTagsFinished(TagReaderClient::ReplyType* reply, } QUrl GoogleDriveService::GetStreamingUrlFromSongId(const QString& id) { + EnsureConnected(); QScopedPointer response(client_->GetFile(id)); QEventLoop loop; diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index a253e3fd8..22fa562b0 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -46,6 +46,7 @@ class GoogleDriveService : public InternetService { private: void Connect(); + void EnsureConnected(); void RefreshAuthorisation(const QString& refresh_token); void MaybeAddFileToDatabase(const google_drive::File& file); void ListFilesForMimeType(const QString& mime_type);