From f48383c73ea87d97fde6c53379a6389c4c02a47c Mon Sep 17 00:00:00 2001 From: John Maguire Date: Fri, 27 Jul 2012 16:04:12 +0200 Subject: [PATCH] Index Google Drive MP3s and write to local database. --- data/data.qrc | 1 + data/schema/schema-38.sql | 50 ++++++++++++ src/core/database.cpp | 2 +- src/core/song.cpp | 9 +- src/core/song.h | 3 + src/internet/googledriveservice.cpp | 122 +++++++++++++++++++--------- src/internet/googledriveservice.h | 10 +++ src/playlist/playlistdelegates.cpp | 2 - 8 files changed, 156 insertions(+), 43 deletions(-) create mode 100644 data/schema/schema-38.sql diff --git a/data/data.qrc b/data/data.qrc index d377d1588..7a71a36b6 100644 --- a/data/data.qrc +++ b/data/data.qrc @@ -325,6 +325,7 @@ schema/schema-35.sql schema/schema-36.sql schema/schema-37.sql + schema/schema-38.sql schema/schema-3.sql schema/schema-4.sql schema/schema-5.sql diff --git a/data/schema/schema-38.sql b/data/schema/schema-38.sql new file mode 100644 index 000000000..bfa88048e --- /dev/null +++ b/data/schema/schema-38.sql @@ -0,0 +1,50 @@ +ALTER TABLE %allsongstables ADD COLUMN etag TEXT; + +CREATE TABLE google_drive_songs( + title TEXT, + album TEXT, + artist TEXT, + albumartist TEXT, + composer TEXT, + track INTEGER, + disc INTEGER, + bpm REAL, + year INTEGER, + genre TEXT, + comment TEXT, + compilation INTEGER, + + length INTEGER, + bitrate INTEGER, + samplerate INTEGER, + + directory INTEGER NOT NULL, + filename TEXT NOT NULL, + mtime INTEGER NOT NULL, + ctime INTEGER NOT NULL, + filesize INTEGER NOT NULL, + sampler INTEGER NOT NULL DEFAULT 0, + art_automatic TEXT, + art_manual TEXT, + filetype INTEGER NOT NULL DEFAULT 0, + playcount INTEGER NOT NULL DEFAULT 0, + lastplayed INTEGER, + rating INTEGER, + forced_compilation_on INTEGER NOT NULL DEFAULT 0, + forced_compilation_off INTEGER NOT NULL DEFAULT 0, + effective_compilation NOT NULL DEFAULT 0, + skipcount INTEGER NOT NULL DEFAULT 0, + score INTEGER NOT NULL DEFAULT 0, + beginning INTEGER NOT NULL DEFAULT 0, + cue_path TEXT, + unavailable INTEGER DEFAULT 0, + effective_albumartist TEXT, + etag TEXT +); + +CREATE VIRTUAL TABLE google_drive_songs_fts USING fts3 ( + ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment, + tokenize=unicode +); + +UPDATE schema_version SET version=38; diff --git a/src/core/database.cpp b/src/core/database.cpp index 57191f94c..906e779a6 100644 --- a/src/core/database.cpp +++ b/src/core/database.cpp @@ -37,7 +37,7 @@ #include const char* Database::kDatabaseFilename = "clementine.db"; -const int Database::kSchemaVersion = 37; +const int Database::kSchemaVersion = 38; const char* Database::kMagicAllSongsTables = "%allsongstables"; int Database::sNextConnectionId = 1; diff --git a/src/core/song.cpp b/src/core/song.cpp index 1387c2232..23c757042 100644 --- a/src/core/song.cpp +++ b/src/core/song.cpp @@ -79,7 +79,7 @@ const QStringList Song::kColumns = QStringList() << "art_manual" << "filetype" << "playcount" << "lastplayed" << "rating" << "forced_compilation_on" << "forced_compilation_off" << "effective_compilation" << "skipcount" << "score" << "beginning" << "length" - << "cue_path" << "unavailable" << "effective_albumartist"; + << "cue_path" << "unavailable" << "effective_albumartist" << "etag"; const QString Song::kColumnSpec = Song::kColumns.join(", "); const QString Song::kBindSpec = Utilities::Prepend(":", Song::kColumns).join(", "); @@ -167,6 +167,8 @@ struct Song::Private : public QSharedData { // Whether the song does not exist on the file system anymore, but is still // stored in the database so as to remember the user's metadata. bool unavailable_; + + QString etag_; }; @@ -263,6 +265,7 @@ bool Song::is_stream() const { return d->filetype_ == Type_Stream; } bool Song::is_cdda() const { return d->filetype_ == Type_Cdda; } const QString& Song::art_automatic() const { return d->art_automatic_; } const QString& Song::art_manual() const { return d->art_manual_; } +const QString& Song::etag() const { return d->etag_; } bool Song::has_manually_unset_cover() const { return d->art_manual_ == kManuallyUnsetCover; } void Song::manually_unset_cover() { d->art_manual_ = kManuallyUnsetCover; } bool Song::has_embedded_cover() const { return d->art_automatic_ == kEmbeddedCover; } @@ -304,6 +307,7 @@ void Song::set_lastplayed(int v) { d->lastplayed_ = v; } void Song::set_score(int v) { d->score_ = qBound(0, v, 100); } void Song::set_cue_path(const QString& v) { d->cue_path_ = v; } void Song::set_unavailable(bool v) { d->unavailable_ = v; } +void Song::set_etag(const QString& etag) { d->etag_ = etag; } void Song::set_url(const QUrl& v) { d->url_ = v; } void Song::set_basefilename(const QString& v) { d->basefilename_ = v; } void Song::set_directory_id(int v) { d->directory_id_ = v; } @@ -1002,8 +1006,11 @@ void Song::BindToQuery(QSqlQuery *query) const { query->bindValue(":unavailable", d->unavailable_ ? 1 : 0); query->bindValue(":effective_albumartist", this->effective_albumartist()); + query->bindValue(":etag", strval(d->etag_)); + #undef intval #undef notnullintval + #undef strval } void Song::BindToFtsQuery(QSqlQuery *query) const { diff --git a/src/core/song.h b/src/core/song.h index 94f5a34ce..2a254ee54 100644 --- a/src/core/song.h +++ b/src/core/song.h @@ -187,6 +187,8 @@ class Song { const QString& art_automatic() const; const QString& art_manual() const; + const QString& etag() const; + // Returns true if this Song had it's cover manually unset by user. bool has_manually_unset_cover() const; // This method represents an explicit request to unset this song's @@ -250,6 +252,7 @@ class Song { void set_score(int v); void set_cue_path(const QString& v); void set_unavailable(bool v); + void set_etag(const QString& etag); // Setters that should only be used by tests void set_url(const QUrl& v); diff --git a/src/internet/googledriveservice.cpp b/src/internet/googledriveservice.cpp index 3678184b0..284ca46bb 100644 --- a/src/internet/googledriveservice.cpp +++ b/src/internet/googledriveservice.cpp @@ -1,6 +1,7 @@ #include "googledriveservice.h" #include +#include #include @@ -13,7 +14,14 @@ using TagLib::ByteVector; #include "core/application.h" #include "core/closure.h" +#include "core/database.h" +#include "core/mergedproxymodel.h" #include "core/player.h" +#include "core/timeconstants.h" +#include "globalsearch/globalsearch.h" +#include "globalsearch/librarysearchprovider.h" +#include "library/librarybackend.h" +#include "library/librarymodel.h" #include "googledriveurlhandler.h" #include "internetmodel.h" #include "oauthenticator.h" @@ -24,6 +32,9 @@ static const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/file static const char* kGoogleDriveFile = "https://www.googleapis.com/drive/v2/files/%1"; static const char* kSettingsGroup = "GoogleDrive"; +static const char* kSongsTable = "google_drive_songs"; +static const char* kFtsTable = "google_drive_songs_fts"; + } @@ -42,14 +53,9 @@ class DriveStream : public TagLib::IOStream { cursor_(0), network_(network), cache_(length) { - qLog(Debug) << Q_FUNC_INFO - << url_ - << filename_ - << length_; } virtual TagLib::FileName name() const { - qLog(Debug) << Q_FUNC_INFO; return encoded_filename_.data(); } @@ -78,7 +84,6 @@ class DriveStream : public TagLib::IOStream { } virtual TagLib::ByteVector readBlock(ulong length) { - qLog(Debug) << Q_FUNC_INFO; const uint start = cursor_; const uint end = qMin(cursor_ + length - 1, length_ - 1); @@ -87,7 +92,6 @@ class DriveStream : public TagLib::IOStream { } if (CheckCache(start, end)) { - qLog(Debug) << "Cache hit at:" << start << end; TagLib::ByteVector cached = GetCached(start, end); cursor_ += cached.size(); return cached; @@ -132,12 +136,10 @@ class DriveStream : public TagLib::IOStream { } virtual bool isOpen() const { - qLog(Debug) << Q_FUNC_INFO; return true; } virtual void seek(long offset, TagLib::IOStream::Position p) { - qLog(Debug) << Q_FUNC_INFO; switch (p) { case TagLib::IOStream::Beginning: cursor_ = offset; @@ -154,17 +156,14 @@ class DriveStream : public TagLib::IOStream { } virtual void clear() { - qLog(Debug) << Q_FUNC_INFO; cursor_ = 0; } virtual long tell() const { - qLog(Debug) << Q_FUNC_INFO; return cursor_; } virtual long length() { - qLog(Debug) << Q_FUNC_INFO; return length_; } @@ -189,11 +188,29 @@ class DriveStream : public TagLib::IOStream { GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent) : InternetService("Google Drive", app, parent, parent), root_(NULL), - oauth_(new OAuthenticator(this)) { + oauth_(new OAuthenticator(this)), + library_sort_model_(new QSortFilterProxyModel(this)) { connect(oauth_, SIGNAL(AccessTokenAvailable(QString)), SLOT(AccessTokenAvailable(QString))); connect(oauth_, SIGNAL(RefreshTokenAvailable(QString)), SLOT(RefreshTokenAvailable(QString))); + library_backend_ = new LibraryBackend; + library_backend_->moveToThread(app_->database()->thread()); + library_backend_->Init(app_->database(), kSongsTable, + QString::null, QString::null, kFtsTable); + library_model_ = new LibraryModel(library_backend_, app_, this); + + library_sort_model_->setSourceModel(library_model_); + library_sort_model_->setSortRole(LibraryModel::Role_SortText); + library_sort_model_->setDynamicSortFilter(true); + library_sort_model_->sort(0); + app->player()->RegisterUrlHandler(new GoogleDriveUrlHandler(this, this)); + app_->global_search()->AddProvider(new LibrarySearchProvider( + library_backend_, + tr("Google Drive"), + "google_drive", + QIcon(":/providers/googledrive.png"), + true, app_, this)); } QStandardItem* GoogleDriveService::CreateRootItem() { @@ -206,7 +223,10 @@ void GoogleDriveService::LazyPopulate(QStandardItem* item) { switch (item->data(InternetModel::Role_Type).toInt()) { case InternetModel::Type_Service: Connect(); + library_model_->Init(); + model()->merged_model()->AddSubModel(item->index(), library_sort_model_); break; + default: break; } @@ -260,35 +280,59 @@ void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) { QVariantList items = result["items"].toList(); foreach (const QVariant& v, items) { QVariantMap file = v.toMap(); - qLog(Debug) << "Creating stream"; - DriveStream* stream = new DriveStream( - file["downloadUrl"].toUrl(), - file["title"].toString(), - file["fileSize"].toUInt(), - access_token_, - &network_); - qLog(Debug) << "Creating tag"; - TagLib::MPEG::File tag( - stream, // Takes ownership. - TagLib::ID3v2::FrameFactory::instance(), - TagLib::AudioProperties::Fast); - qLog(Debug) << "Tagging done"; - if (tag.tag()) { - qLog(Debug) << tag.tag()->artist().toCString(); - Song song; - song.set_title(tag.tag()->title().toCString(true)); - song.set_artist(tag.tag()->artist().toCString(true)); - song.set_album(tag.tag()->album().toCString(true)); + MaybeAddFileToDatabase(file); + } +} - QString url = QString("googledrive:%1").arg(file["id"].toString()); - song.set_url(url); - qLog(Debug) << "Set url to:" << url; +void GoogleDriveService::MaybeAddFileToDatabase(const QVariantMap& file) { + QString url = QString("googledrive:%1").arg(file["id"].toString()); + Song song = library_backend_->GetSongByUrl(QUrl(url)); + // Song already in index. + // TODO: Check etag and maybe update. + if (song.is_valid()) { + return; + } - song.set_filesize(file["fileSize"].toInt()); - root_->appendRow(CreateSongItem(song)); - } else { - qLog(Debug) << "Tagging failed"; + // Song not in index; tag and add. + DriveStream* stream = new DriveStream( + file["downloadUrl"].toUrl(), + file["title"].toString(), + file["fileSize"].toUInt(), + access_token_, + &network_); + TagLib::MPEG::File tag( + stream, // Takes ownership. + TagLib::ID3v2::FrameFactory::instance(), + TagLib::AudioProperties::Fast); + if (tag.tag()) { + Song song; + song.set_title(tag.tag()->title().toCString(true)); + song.set_artist(tag.tag()->artist().toCString(true)); + song.set_album(tag.tag()->album().toCString(true)); + + song.set_url(url); + song.set_filesize(file["fileSize"].toInt()); + song.set_etag(file["etag"].toString().remove('"')); + + QString modified_date = file["modifiedDate"].toString(); + QString created_date = file["createdDate"].toString(); + + song.set_mtime(QDateTime::fromString(modified_date, Qt::ISODate).toTime_t()); + song.set_ctime(QDateTime::fromString(created_date, Qt::ISODate).toTime_t()); + + song.set_filetype(Song::Type_Stream); + song.set_directory_id(0); + + if (tag.audioProperties()) { + song.set_length_nanosec(tag.audioProperties()->length() * kNsecPerSec); } + + SongList songs; + songs << song; + qLog(Debug) << "Adding song to db:" << song.title(); + library_backend_->AddOrUpdateSongs(songs); + } else { + qLog(Debug) << "Failed to tag:" << url; } } diff --git a/src/internet/googledriveservice.h b/src/internet/googledriveservice.h index d19093b95..cd12e2c08 100644 --- a/src/internet/googledriveservice.h +++ b/src/internet/googledriveservice.h @@ -7,7 +7,10 @@ class QStandardItem; +class LibraryBackend; +class LibraryModel; class OAuthenticator; +class QSortFilterProxyModel; class GoogleDriveService : public InternetService { Q_OBJECT @@ -27,6 +30,7 @@ class GoogleDriveService : public InternetService { private: void Connect(); void RefreshAuthorisation(const QString& refresh_token); + void MaybeAddFileToDatabase(const QVariantMap& file); QStandardItem* root_; OAuthenticator* oauth_; @@ -34,6 +38,12 @@ class GoogleDriveService : public InternetService { QString access_token_; NetworkAccessManager network_; + + LibraryBackend* library_backend_; + LibraryModel* library_model_; + QSortFilterProxyModel* library_sort_model_; + + int indexing_task_id_; }; #endif diff --git a/src/playlist/playlistdelegates.cpp b/src/playlist/playlistdelegates.cpp index 887a637dd..7221935b5 100644 --- a/src/playlist/playlistdelegates.cpp +++ b/src/playlist/playlistdelegates.cpp @@ -463,8 +463,6 @@ QPixmap SongSourceDelegate::LookupPixmap(const QUrl& url, const QSize& size) con icon = IconLoader::Load("folder-sound"); } else if (url.host() == "api.jamendo.com") { icon = QIcon(":/providers/jamendo.png"); - } else if (url.host().endsWith("googleusercontent.com")) { - icon = QIcon(":/providers/googledrive.png"); } } pixmap = icon.pixmap(size.height());