Index Google Drive MP3s and write to local database.

data/data.qrc

@@ -325,6 +325,7 @@
         <file>schema/schema-35.sql</file>
         <file>schema/schema-36.sql</file>
         <file>schema/schema-37.sql</file>
+        <file>schema/schema-38.sql</file>
         <file>schema/schema-3.sql</file>
         <file>schema/schema-4.sql</file>
         <file>schema/schema-5.sql</file>

data/schema/schema-38.sql

@@ -0,0 +1,50 @@
+ALTER TABLE %allsongstables ADD COLUMN etag TEXT;
+
+CREATE TABLE google_drive_songs(
+  title TEXT,
+  album TEXT,
+  artist TEXT,
+  albumartist TEXT,
+  composer TEXT,
+  track INTEGER,
+  disc INTEGER,
+  bpm REAL,
+  year INTEGER,
+  genre TEXT,
+  comment TEXT,
+  compilation INTEGER,
+
+  length INTEGER,
+  bitrate INTEGER,
+  samplerate INTEGER,
+
+  directory INTEGER NOT NULL,
+  filename TEXT NOT NULL,
+  mtime INTEGER NOT NULL,
+  ctime INTEGER NOT NULL,
+  filesize INTEGER NOT NULL,
+  sampler INTEGER NOT NULL DEFAULT 0,
+  art_automatic TEXT,
+  art_manual TEXT,
+  filetype INTEGER NOT NULL DEFAULT 0,
+  playcount INTEGER NOT NULL DEFAULT 0,
+  lastplayed INTEGER,
+  rating INTEGER,
+  forced_compilation_on INTEGER NOT NULL DEFAULT 0,
+  forced_compilation_off INTEGER NOT NULL DEFAULT 0,
+  effective_compilation NOT NULL DEFAULT 0,
+  skipcount INTEGER NOT NULL DEFAULT 0,
+  score INTEGER NOT NULL DEFAULT 0,
+  beginning INTEGER NOT NULL DEFAULT 0,
+  cue_path TEXT,
+  unavailable INTEGER DEFAULT 0,
+  effective_albumartist TEXT,
+  etag TEXT
+);
+
+CREATE VIRTUAL TABLE google_drive_songs_fts USING fts3 (
+  ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment,
+  tokenize=unicode
+);
+
+UPDATE schema_version SET version=38;

src/core/database.cpp

@@ -37,7 +37,7 @@
 #include <QVariant>
 
 const char* Database::kDatabaseFilename = "clementine.db";
-const int Database::kSchemaVersion = 37;
+const int Database::kSchemaVersion = 38;
 const char* Database::kMagicAllSongsTables = "%allsongstables";
 
 int Database::sNextConnectionId = 1;

src/core/song.cpp

@@ -79,7 +79,7 @@ const QStringList Song::kColumns = QStringList()
     << "art_manual" << "filetype" << "playcount" << "lastplayed" << "rating"
     << "forced_compilation_on" << "forced_compilation_off"
     << "effective_compilation" << "skipcount" << "score" << "beginning" << "length"
-    << "cue_path" << "unavailable" << "effective_albumartist";
+    << "cue_path" << "unavailable" << "effective_albumartist" << "etag";
 
 const QString Song::kColumnSpec = Song::kColumns.join(", ");
 const QString Song::kBindSpec = Utilities::Prepend(":", Song::kColumns).join(", ");
@@ -167,6 +167,8 @@ struct Song::Private : public QSharedData {
   // Whether the song does not exist on the file system anymore, but is still
   // stored in the database so as to remember the user's metadata.
   bool unavailable_;
+
+  QString etag_;
 };
 
 
@@ -263,6 +265,7 @@ bool Song::is_stream() const { return d->filetype_ == Type_Stream; }
 bool Song::is_cdda() const { return d->filetype_ == Type_Cdda; }
 const QString& Song::art_automatic() const { return d->art_automatic_; }
 const QString& Song::art_manual() const { return d->art_manual_; }
+const QString& Song::etag() const { return d->etag_; }
 bool Song::has_manually_unset_cover() const { return d->art_manual_ == kManuallyUnsetCover; }
 void Song::manually_unset_cover() { d->art_manual_ = kManuallyUnsetCover; }
 bool Song::has_embedded_cover() const { return d->art_automatic_ == kEmbeddedCover; }
@@ -304,6 +307,7 @@ void Song::set_lastplayed(int v) { d->lastplayed_ = v; }
 void Song::set_score(int v) { d->score_ = qBound(0, v, 100); }
 void Song::set_cue_path(const QString& v) { d->cue_path_ = v; }
 void Song::set_unavailable(bool v) { d->unavailable_ = v; }
+void Song::set_etag(const QString& etag) { d->etag_ = etag; }
 void Song::set_url(const QUrl& v) { d->url_ = v; }
 void Song::set_basefilename(const QString& v) { d->basefilename_ = v; }
 void Song::set_directory_id(int v) { d->directory_id_ = v; }
@@ -1002,8 +1006,11 @@ void Song::BindToQuery(QSqlQuery *query) const {
   query->bindValue(":unavailable", d->unavailable_ ? 1 : 0);
   query->bindValue(":effective_albumartist", this->effective_albumartist());
 
+  query->bindValue(":etag", strval(d->etag_));
+
   #undef intval
   #undef notnullintval
+  #undef strval
 }
 
 void Song::BindToFtsQuery(QSqlQuery *query) const {

src/core/song.h

@@ -187,6 +187,8 @@ class Song {
   const QString& art_automatic() const;
   const QString& art_manual() const;
 
+  const QString& etag() const;
+
   // Returns true if this Song had it's cover manually unset by user.
   bool has_manually_unset_cover() const;
   // This method represents an explicit request to unset this song's
@@ -250,6 +252,7 @@ class Song {
   void set_score(int v);
   void set_cue_path(const QString& v);
   void set_unavailable(bool v);
+  void set_etag(const QString& etag);
 
   // Setters that should only be used by tests
   void set_url(const QUrl& v);

src/internet/googledriveservice.cpp

@@ -1,6 +1,7 @@
 #include "googledriveservice.h"
 
 #include <QEventLoop>
+#include <QSortFilterProxyModel>
 
 #include <qjson/parser.h>
 
@@ -13,7 +14,14 @@ using TagLib::ByteVector;
 
 #include "core/application.h"
 #include "core/closure.h"
+#include "core/database.h"
+#include "core/mergedproxymodel.h"
 #include "core/player.h"
+#include "core/timeconstants.h"
+#include "globalsearch/globalsearch.h"
+#include "globalsearch/librarysearchprovider.h"
+#include "library/librarybackend.h"
+#include "library/librarymodel.h"
 #include "googledriveurlhandler.h"
 #include "internetmodel.h"
 #include "oauthenticator.h"
@@ -24,6 +32,9 @@ static const char* kGoogleDriveFiles = "https://www.googleapis.com/drive/v2/file
 static const char* kGoogleDriveFile = "https://www.googleapis.com/drive/v2/files/%1";
 static const char* kSettingsGroup = "GoogleDrive";
 
+static const char* kSongsTable = "google_drive_songs";
+static const char* kFtsTable = "google_drive_songs_fts";
+
 }
 
 
@@ -42,14 +53,9 @@ class DriveStream : public TagLib::IOStream {
         cursor_(0),
         network_(network),
         cache_(length) {
-    qLog(Debug) << Q_FUNC_INFO
-                << url_
-                << filename_
-                << length_;
   }
 
   virtual TagLib::FileName name() const {
-    qLog(Debug) << Q_FUNC_INFO;
     return encoded_filename_.data();
   }
 
@@ -78,7 +84,6 @@ class DriveStream : public TagLib::IOStream {
   }
 
   virtual TagLib::ByteVector readBlock(ulong length) {
-    qLog(Debug) << Q_FUNC_INFO;
     const uint start = cursor_;
     const uint end = qMin(cursor_ + length - 1, length_ - 1);
 
@@ -87,7 +92,6 @@ class DriveStream : public TagLib::IOStream {
     }
 
     if (CheckCache(start, end)) {
-      qLog(Debug) << "Cache hit at:" << start << end;
       TagLib::ByteVector cached = GetCached(start, end);
       cursor_ += cached.size();
       return cached;
@@ -132,12 +136,10 @@ class DriveStream : public TagLib::IOStream {
   }
 
   virtual bool isOpen() const {
-    qLog(Debug) << Q_FUNC_INFO;
     return true;
   }
 
   virtual void seek(long offset, TagLib::IOStream::Position p) {
-    qLog(Debug) << Q_FUNC_INFO;
     switch (p) {
       case TagLib::IOStream::Beginning:
         cursor_ = offset;
@@ -154,17 +156,14 @@ class DriveStream : public TagLib::IOStream {
   }
 
   virtual void clear() {
-    qLog(Debug) << Q_FUNC_INFO;
     cursor_ = 0;
   }
 
   virtual long tell() const {
-    qLog(Debug) << Q_FUNC_INFO;
     return cursor_;
   }
 
   virtual long length() {
-    qLog(Debug) << Q_FUNC_INFO;
     return length_;
   }
 
@@ -189,11 +188,29 @@ class DriveStream : public TagLib::IOStream {
 GoogleDriveService::GoogleDriveService(Application* app, InternetModel* parent)
     : InternetService("Google Drive", app, parent, parent),
       root_(NULL),
-      oauth_(new OAuthenticator(this)) {
+      oauth_(new OAuthenticator(this)),
+      library_sort_model_(new QSortFilterProxyModel(this)) {
   connect(oauth_, SIGNAL(AccessTokenAvailable(QString)), SLOT(AccessTokenAvailable(QString)));
   connect(oauth_, SIGNAL(RefreshTokenAvailable(QString)), SLOT(RefreshTokenAvailable(QString)));
 
+  library_backend_ = new LibraryBackend;
+  library_backend_->moveToThread(app_->database()->thread());
+  library_backend_->Init(app_->database(), kSongsTable,
+                         QString::null, QString::null, kFtsTable);
+  library_model_ = new LibraryModel(library_backend_, app_, this);
+
+  library_sort_model_->setSourceModel(library_model_);
+  library_sort_model_->setSortRole(LibraryModel::Role_SortText);
+  library_sort_model_->setDynamicSortFilter(true);
+  library_sort_model_->sort(0);
+
   app->player()->RegisterUrlHandler(new GoogleDriveUrlHandler(this, this));
+  app_->global_search()->AddProvider(new LibrarySearchProvider(
+      library_backend_,
+      tr("Google Drive"),
+      "google_drive",
+      QIcon(":/providers/googledrive.png"),
+      true, app_, this));
 }
 
 QStandardItem* GoogleDriveService::CreateRootItem() {
@@ -206,7 +223,10 @@ void GoogleDriveService::LazyPopulate(QStandardItem* item) {
   switch (item->data(InternetModel::Role_Type).toInt()) {
     case InternetModel::Type_Service:
       Connect();
+      library_model_->Init();
+      model()->merged_model()->AddSubModel(item->index(), library_sort_model_);
       break;
+
     default:
       break;
   }
@@ -260,35 +280,59 @@ void GoogleDriveService::ListFilesFinished(QNetworkReply* reply) {
   QVariantList items = result["items"].toList();
   foreach (const QVariant& v, items) {
     QVariantMap file = v.toMap();
-    qLog(Debug) << "Creating stream";
-    DriveStream* stream = new DriveStream(
-        file["downloadUrl"].toUrl(),
-        file["title"].toString(),
-        file["fileSize"].toUInt(),
-        access_token_,
-        &network_);
-    qLog(Debug) << "Creating tag";
-    TagLib::MPEG::File tag(
-        stream,  // Takes ownership.
-        TagLib::ID3v2::FrameFactory::instance(),
-        TagLib::AudioProperties::Fast);
-    qLog(Debug) << "Tagging done";
-    if (tag.tag()) {
-      qLog(Debug) << tag.tag()->artist().toCString();
-      Song song;
-      song.set_title(tag.tag()->title().toCString(true));
-      song.set_artist(tag.tag()->artist().toCString(true));
-      song.set_album(tag.tag()->album().toCString(true));
+    MaybeAddFileToDatabase(file);
+  }
+}
 
-      QString url = QString("googledrive:%1").arg(file["id"].toString());
-      song.set_url(url);
-      qLog(Debug) << "Set url to:" << url;
+void GoogleDriveService::MaybeAddFileToDatabase(const QVariantMap& file) {
+  QString url = QString("googledrive:%1").arg(file["id"].toString());
+  Song song = library_backend_->GetSongByUrl(QUrl(url));
+  // Song already in index.
+  // TODO: Check etag and maybe update.
+  if (song.is_valid()) {
+    return;
+  }
 
-      song.set_filesize(file["fileSize"].toInt());
-      root_->appendRow(CreateSongItem(song));
-    } else {
-      qLog(Debug) << "Tagging failed";
+  // Song not in index; tag and add.
+  DriveStream* stream = new DriveStream(
+      file["downloadUrl"].toUrl(),
+      file["title"].toString(),
+      file["fileSize"].toUInt(),
+      access_token_,
+      &network_);
+  TagLib::MPEG::File tag(
+      stream,  // Takes ownership.
+      TagLib::ID3v2::FrameFactory::instance(),
+      TagLib::AudioProperties::Fast);
+  if (tag.tag()) {
+    Song song;
+    song.set_title(tag.tag()->title().toCString(true));
+    song.set_artist(tag.tag()->artist().toCString(true));
+    song.set_album(tag.tag()->album().toCString(true));
+
+    song.set_url(url);
+    song.set_filesize(file["fileSize"].toInt());
+    song.set_etag(file["etag"].toString().remove('"'));
+
+    QString modified_date = file["modifiedDate"].toString();
+    QString created_date = file["createdDate"].toString();
+
+    song.set_mtime(QDateTime::fromString(modified_date, Qt::ISODate).toTime_t());
+    song.set_ctime(QDateTime::fromString(created_date, Qt::ISODate).toTime_t());
+
+    song.set_filetype(Song::Type_Stream);
+    song.set_directory_id(0);
+
+    if (tag.audioProperties()) {
+      song.set_length_nanosec(tag.audioProperties()->length() * kNsecPerSec);
     }
+
+    SongList songs;
+    songs << song;
+    qLog(Debug) << "Adding song to db:" << song.title();
+    library_backend_->AddOrUpdateSongs(songs);
+  } else {
+    qLog(Debug) << "Failed to tag:" << url;
   }
 }
 

src/internet/googledriveservice.h

@@ -7,7 +7,10 @@
 
 class QStandardItem;
 
+class LibraryBackend;
+class LibraryModel;
 class OAuthenticator;
+class QSortFilterProxyModel;
 
 class GoogleDriveService : public InternetService {
   Q_OBJECT
@@ -27,6 +30,7 @@ class GoogleDriveService : public InternetService {
  private:
   void Connect();
   void RefreshAuthorisation(const QString& refresh_token);
+  void MaybeAddFileToDatabase(const QVariantMap& file);
 
   QStandardItem* root_;
   OAuthenticator* oauth_;
@@ -34,6 +38,12 @@ class GoogleDriveService : public InternetService {
   QString access_token_;
 
   NetworkAccessManager network_;
+
+  LibraryBackend* library_backend_;
+  LibraryModel* library_model_;
+  QSortFilterProxyModel* library_sort_model_;
+
+  int indexing_task_id_;
 };
 
 #endif

src/playlist/playlistdelegates.cpp

@@ -463,8 +463,6 @@ QPixmap SongSourceDelegate::LookupPixmap(const QUrl& url, const QSize& size) con
       icon = IconLoader::Load("folder-sound");
     } else if (url.host() == "api.jamendo.com") {
       icon = QIcon(":/providers/jamendo.png");
-    } else if (url.host().endsWith("googleusercontent.com")) {
-      icon = QIcon(":/providers/googledrive.png");
     }
   }
   pixmap = icon.pixmap(size.height());