From db586ca00e41c67172280890a1f12af273bacbc7 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Wed, 28 Nov 2012 14:43:03 +0100 Subject: [PATCH] Read tags from Ubuntu One files and add to local database. --- data/data.qrc | 1 + data/schema/schema-40.sql | 48 ++++++++++ ext/clementine-tagreader/CMakeLists.txt | 5 ++ ext/clementine-tagreader/data/data.qrc | 5 ++ .../data/godaddy-root.pem | 24 +++++ .../googledrivestream.cpp | 18 +++- ext/clementine-tagreader/googledrivestream.h | 15 +++- ext/clementine-tagreader/main.cpp | 4 + ext/clementine-tagreader/tagreaderworker.cpp | 6 +- ext/libclementine-common/core/messagereply.h | 6 +- .../tagreadermessages.proto | 2 +- src/core/database.cpp | 2 +- src/core/tagreaderclient.cpp | 4 +- src/core/tagreaderclient.h | 2 +- src/internet/googledriveservice.cpp | 3 +- src/internet/ubuntuoneservice.cpp | 90 ++++++++++++++++--- src/internet/ubuntuoneservice.h | 12 +++ 17 files changed, 217 insertions(+), 30 deletions(-) create mode 100644 data/schema/schema-40.sql create mode 100644 ext/clementine-tagreader/data/data.qrc create mode 100644 ext/clementine-tagreader/data/godaddy-root.pem diff --git a/data/data.qrc b/data/data.qrc index edda37da4..334f7a62e 100644 --- a/data/data.qrc +++ b/data/data.qrc @@ -332,6 +332,7 @@ schema/schema-38.sql schema/schema-39.sql schema/schema-3.sql + schema/schema-40.sql schema/schema-4.sql schema/schema-5.sql schema/schema-6.sql diff --git a/data/schema/schema-40.sql b/data/schema/schema-40.sql new file mode 100644 index 000000000..552b326c0 --- /dev/null +++ b/data/schema/schema-40.sql @@ -0,0 +1,48 @@ +CREATE TABLE ubuntu_one_songs( + title TEXT, + album TEXT, + artist TEXT, + albumartist TEXT, + composer TEXT, + track INTEGER, + disc INTEGER, + bpm REAL, + year INTEGER, + genre TEXT, + comment TEXT, + compilation INTEGER, + + length INTEGER, + bitrate INTEGER, + samplerate INTEGER, + + directory INTEGER NOT NULL, + filename TEXT NOT NULL, + mtime INTEGER NOT NULL, + ctime INTEGER NOT NULL, + filesize INTEGER NOT NULL, + sampler INTEGER NOT NULL DEFAULT 0, + art_automatic TEXT, + art_manual TEXT, + filetype INTEGER NOT NULL DEFAULT 0, + playcount INTEGER NOT NULL DEFAULT 0, + lastplayed INTEGER, + rating INTEGER, + forced_compilation_on INTEGER NOT NULL DEFAULT 0, + forced_compilation_off INTEGER NOT NULL DEFAULT 0, + effective_compilation NOT NULL DEFAULT 0, + skipcount INTEGER NOT NULL DEFAULT 0, + score INTEGER NOT NULL DEFAULT 0, + beginning INTEGER NOT NULL DEFAULT 0, + cue_path TEXT, + unavailable INTEGER DEFAULT 0, + effective_albumartist TEXT, + etag TEXT +); + +CREATE VIRTUAL TABLE ubuntu_one_songs_fts USING fts3 ( + ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment, + tokenize=unicode +); + +UPDATE schema_version SET version=40; diff --git a/ext/clementine-tagreader/CMakeLists.txt b/ext/clementine-tagreader/CMakeLists.txt index 2c03c5d96..21e20d9eb 100644 --- a/ext/clementine-tagreader/CMakeLists.txt +++ b/ext/clementine-tagreader/CMakeLists.txt @@ -6,6 +6,8 @@ include_directories(${CMAKE_BINARY_DIR}/ext/libclementine-tagreader) include_directories(${CMAKE_SOURCE_DIR}/src) include_directories(${CMAKE_BINARY_DIR}/src) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++0x -U__STRICT_ANSI__") + set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}) set(SOURCES @@ -15,6 +17,7 @@ set(SOURCES ) set(HEADERS + googledrivestream.h ) optional_source(HAVE_GOOGLE_DRIVE @@ -24,10 +27,12 @@ optional_source(HAVE_GOOGLE_DRIVE ) qt4_wrap_cpp(MOC ${HEADERS}) +qt4_add_resources(QRC data/data.qrc) add_executable(clementine-tagreader ${SOURCES} ${MOC} + ${QRC} ) target_link_libraries(clementine-tagreader diff --git a/ext/clementine-tagreader/data/data.qrc b/ext/clementine-tagreader/data/data.qrc new file mode 100644 index 000000000..8e2f501e7 --- /dev/null +++ b/ext/clementine-tagreader/data/data.qrc @@ -0,0 +1,5 @@ + + + godaddy-root.pem + + diff --git a/ext/clementine-tagreader/data/godaddy-root.pem b/ext/clementine-tagreader/data/godaddy-root.pem new file mode 100644 index 000000000..42e8d1eef --- /dev/null +++ b/ext/clementine-tagreader/data/godaddy-root.pem @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh +MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE +YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3 +MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo +ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg +MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN +ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA +PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w +wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi +EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY +avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+ +YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE +sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h +/t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5 +IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD +ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy +OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P +TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ +HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER +dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf +ReYNnyicsbkqWletNw+vHX/bvZ8= +-----END CERTIFICATE----- diff --git a/ext/clementine-tagreader/googledrivestream.cpp b/ext/clementine-tagreader/googledrivestream.cpp index 7489fc116..f964e82b5 100644 --- a/ext/clementine-tagreader/googledrivestream.cpp +++ b/ext/clementine-tagreader/googledrivestream.cpp @@ -109,12 +109,19 @@ TagLib::ByteVector GoogleDriveStream::readBlock(ulong length) { } QNetworkRequest request = QNetworkRequest(url_); - request.setRawHeader( - "Authorization", QString("Bearer %1").arg(auth_).toUtf8()); + request.setRawHeader("Authorization", auth_.toUtf8()); request.setRawHeader( "Range", QString("bytes=%1-%2").arg(start).arg(end).toUtf8()); + request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, + QNetworkRequest::AlwaysNetwork); + // The Ubuntu One server applies the byte range to the gzipped data, rather + // than the raw data so we must disable compression. + if (url_.host() == "files.one.ubuntu.com") { + request.setRawHeader("Accept-Encoding", "identity"); + } QNetworkReply* reply = network_->get(request); + connect(reply, SIGNAL(sslErrors(QList)), SLOT(SSLErrors(QList))); ++num_requests_; QEventLoop loop; @@ -183,3 +190,10 @@ long GoogleDriveStream::length() { void GoogleDriveStream::truncate(long) { qLog(Debug) << Q_FUNC_INFO << "not implemented"; } + +void GoogleDriveStream::SSLErrors(const QList& errors) { + for (const QSslError& error : errors) { + qLog(Debug) << error.error() << error.errorString(); + qLog(Debug) << error.certificate(); + } +} diff --git a/ext/clementine-tagreader/googledrivestream.h b/ext/clementine-tagreader/googledrivestream.h index f11a34dc6..df8e77967 100644 --- a/ext/clementine-tagreader/googledrivestream.h +++ b/ext/clementine-tagreader/googledrivestream.h @@ -1,16 +1,16 @@ /* This file is part of Clementine. Copyright 2012, David Sansome - + Clementine is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Clementine is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with Clementine. If not, see . */ @@ -18,6 +18,9 @@ #ifndef GOOGLEDRIVESTREAM_H #define GOOGLEDRIVESTREAM_H +#include +#include +#include #include #include @@ -25,7 +28,8 @@ class QNetworkAccessManager; -class GoogleDriveStream : public TagLib::IOStream { +class GoogleDriveStream : public QObject, public TagLib::IOStream { + Q_OBJECT public: GoogleDriveStream(const QUrl& url, const QString& filename, @@ -63,6 +67,9 @@ class GoogleDriveStream : public TagLib::IOStream { void FillCache(int start, TagLib::ByteVector data); TagLib::ByteVector GetCached(int start, int end); + private slots: + void SSLErrors(const QList& errors); + private: const QUrl url_; const QString filename_; diff --git a/ext/clementine-tagreader/main.cpp b/ext/clementine-tagreader/main.cpp index 37481a5d1..e2104e59c 100644 --- a/ext/clementine-tagreader/main.cpp +++ b/ext/clementine-tagreader/main.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -57,6 +58,9 @@ int main(int argc, char** argv) { return 1; } + QSslSocket::addDefaultCaCertificates( + QSslCertificate::fromPath(":/certs/godaddy-root.pem", QSsl::Pem)); + TagReaderWorker worker(&socket); return a.exec(); diff --git a/ext/clementine-tagreader/tagreaderworker.cpp b/ext/clementine-tagreader/tagreaderworker.cpp index ab6b8a614..f228a76cd 100644 --- a/ext/clementine-tagreader/tagreaderworker.cpp +++ b/ext/clementine-tagreader/tagreaderworker.cpp @@ -139,7 +139,7 @@ void TagReaderWorker::MessageArrived(const pb::tagreader::Message& message) { QStringFromStdString(req.title()), req.size(), QStringFromStdString(req.mime_type()), - QStringFromStdString(req.access_token()), + QStringFromStdString(req.authorisation_header()), reply.mutable_read_google_drive_response()->mutable_metadata())) { reply.mutable_read_google_drive_response()->clear_metadata(); } @@ -615,12 +615,12 @@ bool TagReaderWorker::ReadGoogleDrive(const QUrl& download_url, const QString& title, int size, const QString& mime_type, - const QString& access_token, + const QString& authorisation_header, pb::tagreader::SongMetadata* song) const { qLog(Debug) << "Loading tags from" << title; GoogleDriveStream* stream = new GoogleDriveStream( - download_url, title, size, access_token, network_); + download_url, title, size, authorisation_header, network_); stream->Precache(); scoped_ptr tag; if (mime_type == "audio/mpeg" && title.endsWith(".mp3")) { diff --git a/ext/libclementine-common/core/messagereply.h b/ext/libclementine-common/core/messagereply.h index 95a6b2707..ead5f38ba 100644 --- a/ext/libclementine-common/core/messagereply.h +++ b/ext/libclementine-common/core/messagereply.h @@ -1,16 +1,16 @@ /* This file is part of Clementine. Copyright 2011, David Sansome - + Clementine is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Clementine is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with Clementine. If not, see . */ diff --git a/ext/libclementine-tagreader/tagreadermessages.proto b/ext/libclementine-tagreader/tagreadermessages.proto index 0c3644ae9..5403f7e5d 100644 --- a/ext/libclementine-tagreader/tagreadermessages.proto +++ b/ext/libclementine-tagreader/tagreadermessages.proto @@ -86,7 +86,7 @@ message ReadGoogleDriveRequest { optional string download_url = 1; optional string title = 2; optional int32 size = 3; - optional string access_token = 4; + optional string authorisation_header = 4; optional string mime_type = 5; } diff --git a/src/core/database.cpp b/src/core/database.cpp index 7a6c61f2f..a8862d3b4 100644 --- a/src/core/database.cpp +++ b/src/core/database.cpp @@ -37,7 +37,7 @@ #include const char* Database::kDatabaseFilename = "clementine.db"; -const int Database::kSchemaVersion = 39; +const int Database::kSchemaVersion = 40; const char* Database::kMagicAllSongsTables = "%allsongstables"; int Database::sNextConnectionId = 1; diff --git a/src/core/tagreaderclient.cpp b/src/core/tagreaderclient.cpp index 145d6dded..9f2db4d95 100644 --- a/src/core/tagreaderclient.cpp +++ b/src/core/tagreaderclient.cpp @@ -88,7 +88,7 @@ TagReaderReply* TagReaderClient::ReadGoogleDrive(const QUrl& download_url, const QString& title, int size, const QString& mime_type, - const QString& access_token) { + const QString& authorisation_header) { pb::tagreader::Message message; pb::tagreader::ReadGoogleDriveRequest* req = message.mutable_read_google_drive_request(); @@ -98,7 +98,7 @@ TagReaderReply* TagReaderClient::ReadGoogleDrive(const QUrl& download_url, req->set_title(DataCommaSizeFromQString(title)); req->set_size(size); req->set_mime_type(DataCommaSizeFromQString(mime_type)); - req->set_access_token(DataCommaSizeFromQString(access_token)); + req->set_authorisation_header(DataCommaSizeFromQString(authorisation_header)); return worker_pool_->SendMessageWithReply(&message); } diff --git a/src/core/tagreaderclient.h b/src/core/tagreaderclient.h index 864d8ba8f..06c0de95f 100644 --- a/src/core/tagreaderclient.h +++ b/src/core/tagreaderclient.h @@ -49,7 +49,7 @@ public: const QString& title, int size, const QString& mime_type, - const QString& access_token); + const QString& authorisation_header); // Convenience functions that call the above functions and wait for a // response. These block the calling thread with a semaphore, and must NOT diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 4f69fa088..1a6d8dc6a 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -178,12 +178,13 @@ void GoogleDriveService::MaybeAddFileToDatabase(const google_drive::File& file) tr("Indexing %1").arg(file.title())); // Song not in index; tag and add. + QString authorisation_header = QString("Bearer %1").arg(client_->access_token()); TagReaderClient::ReplyType* reply = app_->tag_reader_client()->ReadGoogleDrive( file.download_url(), file.title(), file.size(), file.mime_type(), - client_->access_token()); + authorisation_header); NewClosure(reply, SIGNAL(Finished(bool)), this, SLOT(ReadTagsFinished(TagReaderClient::ReplyType*,google_drive::File,QString,int)), diff --git a/src/internet/ubuntuoneservice.cpp b/src/internet/ubuntuoneservice.cpp index 8eb8edf99..f5d371b86 100644 --- a/src/internet/ubuntuoneservice.cpp +++ b/src/internet/ubuntuoneservice.cpp @@ -2,19 +2,25 @@ #include #include +#include #include #include "core/application.h" #include "core/closure.h" +#include "core/database.h" #include "core/logging.h" +#include "core/mergedproxymodel.h" #include "core/network.h" #include "core/player.h" #include "core/timeconstants.h" #include "core/utilities.h" +#include "globalsearch/globalsearch.h" +#include "globalsearch/librarysearchprovider.h" #include "internet/internetmodel.h" #include "internet/ubuntuoneauthenticator.h" #include "internet/ubuntuoneurlhandler.h" +#include "library/librarybackend.h" const char* UbuntuOneService::kServiceName = "Ubuntu One"; const char* UbuntuOneService::kSettingsGroup = "Ubuntu One"; @@ -23,13 +29,33 @@ namespace { static const char* kFileStorageEndpoint = "https://one.ubuntu.com/api/file_storage/v1/~/Ubuntu One/"; static const char* kContentRoot = "https://files.one.ubuntu.com"; +static const char* kSongsTable = "ubuntu_one_songs"; +static const char* kFtsTable = "ubuntu_one_songs_fts"; } UbuntuOneService::UbuntuOneService(Application* app, InternetModel* parent) : InternetService(kServiceName, app, parent, parent), root_(nullptr), - network_(new NetworkAccessManager(this)) { + network_(new NetworkAccessManager(this)), + library_sort_model_(new QSortFilterProxyModel(this)) { + library_backend_ = new LibraryBackend; + library_backend_->moveToThread(app_->database()->thread()); + library_backend_->Init( + app->database(), kSongsTable, QString::null, QString::null, kFtsTable); + library_model_ = new LibraryModel(library_backend_, app_, this); + + library_sort_model_->setSourceModel(library_model_); + library_sort_model_->setSortRole(LibraryModel::Role_SortText); + library_sort_model_->setDynamicSortFilter(true); + library_sort_model_->sort(0); + app->player()->RegisterUrlHandler(new UbuntuOneUrlHandler(this, this)); + app->global_search()->AddProvider(new LibrarySearchProvider( + library_backend_, + kServiceName, + "ubuntu_one", + QIcon(":/providers/ubuntuone.png"), + true, app_, this)); } QStandardItem* UbuntuOneService::CreateRootItem() { @@ -42,6 +68,8 @@ void UbuntuOneService::LazyPopulate(QStandardItem* item) { switch (item->data(InternetModel::Role_Type).toInt()) { case InternetModel::Type_Service: Connect(); + library_model_->Init(); + model()->merged_model()->AddSubModel(item->index(), library_sort_model_); break; default: @@ -125,20 +153,58 @@ void UbuntuOneService::FileListRequestFinished(QNetworkReply* reply) { QVariantList children = result["children"].toList(); for (const QVariant& c : children) { QVariantMap child = c.toMap(); - QString content_path = child["content_path"].toString(); - - QUrl content_url; - content_url.setScheme("ubuntuonefile"); - content_url.setPath(content_path); - - Song song; - song.set_title(child["path"].toString().mid(1)); - song.set_url(content_url); - - root_->appendRow(CreateSongItem(song)); + MaybeAddFileToDatabase(child); } } +void UbuntuOneService::MaybeAddFileToDatabase(const QVariantMap& file) { + QString content_path = file["content_path"].toString(); + + QUrl service_url; + service_url.setScheme("ubuntuonefile"); + service_url.setPath(content_path); + Song song = library_backend_->GetSongByUrl(service_url); + if (song.is_valid()) { + return; + } + + QUrl content_url(kContentRoot); + content_url.setPath(content_path); + + TagReaderClient::ReplyType* reply = app_->tag_reader_client()->ReadGoogleDrive( + content_url, + file["path"].toString().mid(1), + file["size"].toInt(), + "audio/mpeg", + GenerateAuthorisationHeader()); + NewClosure( + reply, SIGNAL(Finished(bool)), + this, SLOT(ReadTagsFinished(TagReaderClient::ReplyType*,QVariantMap, QUrl)), + reply, file, service_url); +} + +void UbuntuOneService::ReadTagsFinished( + TagReaderClient::ReplyType* reply, const QVariantMap& file, const QUrl& url) { + qLog(Debug) << reply->message().DebugString().c_str(); + Song song; + song.InitFromProtobuf(reply->message().read_google_drive_response().metadata()); + song.set_directory_id(0); + song.set_etag(file["hash"].toString()); + song.set_mtime( + QDateTime::fromString(file["when_changed"].toString(), Qt::ISODate).toTime_t()); + song.set_ctime( + QDateTime::fromString(file["when_created"].toString(), Qt::ISODate).toTime_t()); + + song.set_url(url); + + if (song.title().isEmpty()) { + song.set_title(file["path"].toString().mid(1)); + } + + qLog(Debug) << "Adding song to db:" << song.title(); + library_backend_->AddOrUpdateSongs(SongList() << song); +} + QUrl UbuntuOneService::GetStreamingUrlFromSongId(const QString& song_id) { QUrl url(kContentRoot); url.setPath(song_id); diff --git a/src/internet/ubuntuoneservice.h b/src/internet/ubuntuoneservice.h index 756f8f51f..36bc3b343 100644 --- a/src/internet/ubuntuoneservice.h +++ b/src/internet/ubuntuoneservice.h @@ -3,8 +3,13 @@ #include "internet/internetservice.h" +#include "core/tagreaderclient.h" + +class LibraryBackend; +class LibraryModel; class NetworkAccessManager; class QNetworkReply; +class QSortFilterProxyModel; class UbuntuOneAuthenticator; class UbuntuOneService : public InternetService { @@ -24,10 +29,13 @@ class UbuntuOneService : public InternetService { private slots: void AuthenticationFinished(UbuntuOneAuthenticator* authenticator); void FileListRequestFinished(QNetworkReply* reply); + void ReadTagsFinished( + TagReaderClient::ReplyType* reply, const QVariantMap& file, const QUrl& url); private: void Connect(); void RequestFileList(); + void MaybeAddFileToDatabase(const QVariantMap& file); private: QByteArray GenerateAuthorisationHeader(); @@ -39,6 +47,10 @@ class UbuntuOneService : public InternetService { QString consumer_secret_; QString token_; QString token_secret_; + + LibraryBackend* library_backend_; + LibraryModel* library_model_; + QSortFilterProxyModel* library_sort_model_; }; #endif // UBUNTUONESERVICE_H