Use sqlite's Full Text Search on the songs table

data/data.qrc

@@ -185,5 +185,6 @@
         <file>icons/32x32/view-fullscreen.png</file>
         <file>icons/48x48/view-fullscreen.png</file>
         <file>schema-12.sql</file>
+        <file>schema-13.sql</file>
     </qresource>
 </RCC>

data/schema-13.sql

@@ -0,0 +1,20 @@
+CREATE VIRTUAL TABLE songs_fts USING fts3(
+  ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment
+);
+
+CREATE VIRTUAL TABLE magnatune_songs_fts USING fts3(
+  ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment
+);
+
+INSERT INTO songs_fts (ROWID, ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment)
+    SELECT ROWID, title, album, artist, albumartist, composer, genre, comment
+    FROM songs;
+
+INSERT INTO magnatune_songs_fts (ROWID, ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment)
+    SELECT ROWID, title, album, artist, albumartist, composer, genre, comment
+    FROM magnatune_songs;
+
+CREATE INDEX idx_album ON songs (album);
+
+UPDATE schema_version SET version=13;
+

src/core/database.cpp

@@ -27,7 +27,7 @@
 #include <QVariant>
 
 const char* Database::kDatabaseFilename = "clementine.db";
-const int Database::kSchemaVersion = 12;
+const int Database::kSchemaVersion = 13;
 
 int (*Database::_sqlite3_create_function) (
     sqlite3*, const char*, int, int, void*,

src/core/song.cpp

@@ -84,6 +84,15 @@ const QString Song::kColumnSpec = Song::kColumns.join(", ");
 const QString Song::kBindSpec = Prepend(":", Song::kColumns).join(", ");
 const QString Song::kUpdateSpec = Updateify(Song::kColumns).join(", ");
 
+
+const QStringList Song::kFtsColumns = QStringList()
+    << "ftstitle" << "ftsalbum" << "ftsartist" << "ftsalbumartist"
+    << "ftscomposer" << "ftsgenre" << "ftscomment";
+
+const QString Song::kFtsColumnSpec = Song::kFtsColumns.join(", ");
+const QString Song::kFtsBindSpec = Prepend(":", Song::kFtsColumns).join(", ");
+const QString Song::kFtsUpdateSpec = Updateify(Song::kFtsColumns).join(", ");
+
 QString Song::JoinSpec(const QString& table) {
   return Prepend(table + ".", kColumns).join(", ");
 }
@@ -582,6 +591,16 @@ void Song::BindToQuery(QSqlQuery *query) const {
   #undef notnullintval
 }
 
+void Song::BindToFtsQuery(QSqlQuery *query) const {
+  query->bindValue(":ftstitle", d->title_);
+  query->bindValue(":ftsalbum", d->album_);
+  query->bindValue(":ftsartist", d->artist_);
+  query->bindValue(":ftsalbumartist", d->albumartist_);
+  query->bindValue(":ftscomposer", d->composer_);
+  query->bindValue(":ftsgenre", d->genre_);
+  query->bindValue(":ftscomment", d->comment_);
+}
+
 void Song::ToLastFM(lastfm::Track* track) const {
   lastfm::MutableTrack mtrack(*track);
 

src/core/song.h

@@ -85,10 +85,14 @@ class Song {
 
   static const QStringList kColumns;
   static const QString kColumnSpec;
-  static const QString kJoinSpec;
   static const QString kBindSpec;
   static const QString kUpdateSpec;
 
+  static const QStringList kFtsColumns;
+  static const QString kFtsColumnSpec;
+  static const QString kFtsBindSpec;
+  static const QString kFtsUpdateSpec;
+
   static QString JoinSpec(const QString& table);
 
   // Don't change these values - they're stored in the database
@@ -120,6 +124,7 @@ class Song {
 
   // Save
   void BindToQuery(QSqlQuery* query) const;
+  void BindToFtsQuery(QSqlQuery* query) const;
   void ToLastFM(lastfm::Track* track) const;
 
   // Simple accessors

src/library/library.cpp

@@ -22,6 +22,7 @@
 const char* Library::kSongsTable = "songs";
 const char* Library::kDirsTable = "directories";
 const char* Library::kSubdirsTable = "subdirectories";
+const char* Library::kFtsTable = "songs_fts";
 
 Library::Library(BackgroundThread<Database>* db_thread, QObject *parent)
   : QObject(parent),
@@ -31,7 +32,7 @@ Library::Library(BackgroundThread<Database>* db_thread, QObject *parent)
     watcher_(NULL)
 {
   backend_ = db_thread->CreateInThread<LibraryBackend>();
-  backend_->Init(db_thread->Worker(), kSongsTable, kDirsTable, kSubdirsTable);
+  backend_->Init(db_thread->Worker(), kSongsTable, kDirsTable, kSubdirsTable, kFtsTable);
 
   model_ = new LibraryModel(backend_, this);
 }

src/library/library.h

@@ -37,6 +37,7 @@ class Library : public QObject {
   static const char* kSongsTable;
   static const char* kDirsTable;
   static const char* kSubdirsTable;
+  static const char* kFtsTable;
 
   // Useful for tests.  The library takes ownership.
   void set_watcher_factory(BackgroundThreadFactory<LibraryWatcher>* factory);

src/library/librarybackend.cpp

@@ -31,11 +31,13 @@ LibraryBackend::LibraryBackend(QObject *parent)
 }
 
 void LibraryBackend::Init(boost::shared_ptr<Database> db, const QString& songs_table,
-                          const QString &dirs_table, const QString &subdirs_table) {
+                          const QString& dirs_table, const QString& subdirs_table,
+                          const QString& fts_table) {
   db_ = db;
   songs_table_ = songs_table;
   dirs_table_ = dirs_table;
   subdirs_table_ = subdirs_table;
+  fts_table_ = fts_table;
 }
 
 void LibraryBackend::LoadDirectoriesAsync() {
@@ -225,6 +227,11 @@ void LibraryBackend::AddOrUpdateSongs(const SongList& songs) {
                      .arg(songs_table_), db);
   QSqlQuery update_song(QString("UPDATE %1 SET " + Song::kUpdateSpec +
                                 " WHERE ROWID = :id").arg(songs_table_), db);
+  QSqlQuery add_song_fts(QString("INSERT INTO %1 (ROWID, " + Song::kFtsColumnSpec + ")"
+                                 " VALUES (:id, " + Song::kFtsBindSpec + ")")
+                         .arg(fts_table_), db);
+  QSqlQuery update_song_fts(QString("UPDATE %1 SET " + Song::kFtsUpdateSpec +
+                                    " WHERE ROWID = :id").arg(fts_table_), db);
 
   ScopedTransaction transaction(&db);
 
@@ -251,8 +258,15 @@ void LibraryBackend::AddOrUpdateSongs(const SongList& songs) {
       add_song.exec();
       if (db_->CheckErrors(add_song.lastError())) continue;
 
+      const int id = add_song.lastInsertId().toInt();
+
+      add_song_fts.bindValue(":id", id);
+      song.BindToFtsQuery(&add_song_fts);
+      add_song_fts.exec();
+      if (db_->CheckErrors(add_song_fts.lastError())) continue;
+
       Song copy(song);
-      copy.set_id(add_song.lastInsertId().toInt());
+      copy.set_id(id);
       added_songs << copy;
     } else {
       // Get the previous song data first
@@ -266,6 +280,11 @@ void LibraryBackend::AddOrUpdateSongs(const SongList& songs) {
       update_song.exec();
       if (db_->CheckErrors(update_song.lastError())) continue;
 
+      song.BindToFtsQuery(&update_song_fts);
+      update_song_fts.bindValue(":id", song.id());
+      update_song_fts.exec();
+      if (db_->CheckErrors(update_song_fts.lastError())) continue;
+
       deleted_songs << old_song;
       added_songs << song;
     }
@@ -303,14 +322,20 @@ void LibraryBackend::DeleteSongs(const SongList &songs) {
   QMutexLocker l(db_->Mutex());
   QSqlDatabase db(db_->Connect());
 
-  QSqlQuery q(QString("DELETE FROM %1 WHERE ROWID = :id")
+  QSqlQuery remove(QString("DELETE FROM %1 WHERE ROWID = :id")
                    .arg(songs_table_), db);
+  QSqlQuery remove_fts(QString("DELETE FROM %1 WHERE ROWID = :id")
+                       .arg(fts_table_), db);
 
   ScopedTransaction transaction(&db);
   foreach (const Song& song, songs) {
-    q.bindValue(":id", song.id());
+    remove.bindValue(":id", song.id());
-    q.exec();
+    remove.exec();
-    db_->CheckErrors(q.lastError());
+    db_->CheckErrors(remove.lastError());
+
+    remove_fts.bindValue(":id", song.id());
+    remove_fts.exec();
+    db_->CheckErrors(remove_fts.lastError());
   }
   transaction.Commit();
 
@@ -361,7 +386,7 @@ LibraryBackend::AlbumList LibraryBackend::GetAlbumsByArtist(const QString& artis
 
 SongList LibraryBackend::GetSongs(const QString& artist, const QString& album, const QueryOptions& opt) {
   LibraryQuery query(opt);
-  query.SetColumnSpec("ROWID, " + Song::kColumnSpec);
+  query.SetColumnSpec("%songs_table.ROWID, " + Song::kColumnSpec);
   query.AddCompilationRequirement(false);
   query.AddWhere("artist", artist);
   query.AddWhere("album", album);
@@ -397,8 +422,9 @@ Song LibraryBackend::GetSongById(int id) {
 
 bool LibraryBackend::HasCompilations(const QueryOptions& opt) {
   LibraryQuery query(opt);
-  query.SetColumnSpec("ROWID");
+  query.SetColumnSpec("%songs_table.ROWID");
   query.AddCompilationRequirement(true);
+  query.SetLimit(1);
 
   QMutexLocker l(db_->Mutex());
   if (!ExecQuery(&query)) return false;
@@ -412,7 +438,7 @@ LibraryBackend::AlbumList LibraryBackend::GetCompilationAlbums(const QueryOption
 
 SongList LibraryBackend::GetCompilationSongs(const QString& album, const QueryOptions& opt) {
   LibraryQuery query(opt);
-  query.SetColumnSpec("ROWID, " + Song::kColumnSpec);
+  query.SetColumnSpec("%songs_table.ROWID, " + Song::kColumnSpec);
   query.AddCompilationRequirement(true);
   query.AddWhere("album", album);
 
@@ -451,18 +477,13 @@ void LibraryBackend::UpdateCompilations() {
       continue;
 
     // Find the directory the song is in
-    QDir dir(filename);
+    int last_separator = filename.lastIndexOf('/');
-    QString path = QDir::toNativeSeparators(dir.canonicalPath());
-    int last_separator = path.lastIndexOf(QDir::separator());
-
     if (last_separator == -1)
       continue;
 
-    path = path.left(last_separator);
-
     CompilationInfo& info = compilation_info[album];
     info.artists.insert(artist);
-    info.directories.insert(path);
+    info.directories.insert(filename.left(last_separator));
     if (sampler) info.has_samplers = true;
     else         info.has_not_samplers = true;
   }
@@ -711,5 +732,5 @@ void LibraryBackend::ForceCompilation(const QString& artist, const QString& albu
 }
 
 bool LibraryBackend::ExecQuery(LibraryQuery *q) {
-  return !db_->CheckErrors(q->Exec(db_->Connect(), songs_table_));
+  return !db_->CheckErrors(q->Exec(db_->Connect(), songs_table_, fts_table_));
 }

src/library/librarybackend.h

@@ -34,7 +34,8 @@ class LibraryBackend : public QObject {
  public:
   Q_INVOKABLE LibraryBackend(QObject* parent = 0);
   void Init(boost::shared_ptr<Database> db, const QString& songs_table,
-            const QString& dirs_table, const QString& subdirs_table);
+            const QString& dirs_table, const QString& subdirs_table,
+            const QString& fts_table);
 
   boost::shared_ptr<Database> db() const { return db_; }
 
@@ -130,6 +131,7 @@ class LibraryBackend : public QObject {
   QString songs_table_;
   QString dirs_table_;
   QString subdirs_table_;
+  QString fts_table_;
 };
 
 #endif // LIBRARYBACKEND_H

src/library/librarymodel.cpp

@@ -433,7 +433,7 @@ void LibraryModel::InitQuery(GroupBy type, LibraryQuery* q) {
     q->SetColumnSpec("DISTINCT albumartist");
     break;
   case GroupBy_None:
-    q->SetColumnSpec("ROWID, " + Song::kColumnSpec);
+    q->SetColumnSpec("%songs_table.ROWID, " + Song::kColumnSpec);
     break;
   }
 }

src/library/libraryquery.cpp

@@ -27,21 +27,29 @@ QueryOptions::QueryOptions()
 }
 
 
-LibraryQuery::LibraryQuery(const QueryOptions& options) {
+LibraryQuery::LibraryQuery(const QueryOptions& options)
+  : join_with_fts_(false),
+    limit_(-1)
+{
   if (!options.filter.isEmpty()) {
-    where_clauses_ << "("
+    // We need to munge the filter text a little bit to get it to work as
-        "artist LIKE ? OR "
+    // expected with sqlite's FTS3:
-        "album LIKE ? OR "
+    //  1) Append * to all tokens.
-        "title LIKE ? OR "
+    //  2) Prefix "fts" to column names.
-        "composer LIKE ? OR "
+
-        "genre LIKE ? OR "
+    // Split on whitespace
-        "albumartist LIKE ?)";
+    QStringList tokens(options.filter.split(QRegExp("\\s+")));
-    bound_values_ << "%" + options.filter + "%";
+    QString query;
-    bound_values_ << "%" + options.filter + "%";
+    foreach (const QString& token, tokens) {
-    bound_values_ << "%" + options.filter + "%";
+      if (token.contains(':'))
-    bound_values_ << "%" + options.filter + "%";
+        query += "fts" + token + "* ";
-    bound_values_ << "%" + options.filter + "%";
+      else
-    bound_values_ << "%" + options.filter + "%";
+        query += token + "* ";
+    }
+
+    where_clauses_ << "fts.%fts_table MATCH ?";
+    bound_values_ << query;
+    join_with_fts_ = true;
   }
 
   if (options.max_age != -1) {
@@ -64,17 +72,20 @@ void LibraryQuery::AddWhere(const QString& column, const QVariant& value, const
   }
 }
 
-void LibraryQuery::AddWhereLike(const QString& column, const QVariant& value) {
-  where_clauses_ << QString("%1 LIKE ?").arg(column);
-  bound_values_ << value;
-}
-
 void LibraryQuery::AddCompilationRequirement(bool compilation) {
   where_clauses_ << QString("effective_compilation = %1").arg(compilation ? 1 : 0);
 }
 
-QSqlError LibraryQuery::Exec(QSqlDatabase db, const QString& table) {
+QSqlError LibraryQuery::Exec(QSqlDatabase db, const QString& songs_table,
-  QString sql = QString("SELECT %1 FROM %2").arg(column_spec_, table);
+                             const QString& fts_table) {
+  QString sql;
+  if (join_with_fts_) {
+    sql = QString("SELECT %1 FROM %2 INNER JOIN %3 AS fts ON %2.ROWID = fts.ROWID")
+          .arg(column_spec_, songs_table, fts_table);
+  } else {
+    sql = QString("SELECT %1 FROM %2")
+          .arg(column_spec_, songs_table);
+  }
 
   if (!where_clauses_.isEmpty())
     sql += " WHERE " + where_clauses_.join(" AND ");
@@ -82,6 +93,11 @@ QSqlError LibraryQuery::Exec(QSqlDatabase db, const QString& table) {
   if (!order_by_.isEmpty())
     sql += " ORDER BY " + order_by_;
 
+  if (limit_ != -1)
+    sql += " LIMIT " + QString::number(limit_);
+
+  sql.replace("%songs_table", songs_table);
+  sql.replace("%fts_table", fts_table);
   query_ = QSqlQuery(sql, db);
 
   // Bind values

src/library/libraryquery.h

@@ -42,20 +42,22 @@ class LibraryQuery {
   void SetColumnSpec(const QString& spec) { column_spec_ = spec; }
   void SetOrderBy(const QString& order_by) { order_by_ = order_by; }
   void AddWhere(const QString& column, const QVariant& value, const QString& op = "=");
-  void AddWhereLike(const QString& column, const QVariant& value);
   void AddCompilationRequirement(bool compilation);
+  void SetLimit(int limit) { limit_ = limit; }
 
-  QSqlError Exec(QSqlDatabase db, const QString& table);
+  QSqlError Exec(QSqlDatabase db, const QString& songs_table, const QString& fts_table);
   bool Next();
   QVariant Value(int column) const;
 
   operator const QSqlQuery& () const { return query_; }
 
  private:
+  bool join_with_fts_;
   QString column_spec_;
   QString order_by_;
   QStringList where_clauses_;
   QVariantList bound_values_;
+  int limit_;
 
   QSqlQuery query_;
 };

src/radio/magnatuneservice.cpp

@@ -45,6 +45,7 @@ using boost::shared_ptr;
 const char* MagnatuneService::kServiceName = "Magnatune";
 const char* MagnatuneService::kSettingsGroup = "Magnatune";
 const char* MagnatuneService::kSongsTable = "magnatune_songs";
+const char* MagnatuneService::kFtsTable = "magnatune_songs_fts";
 
 const char* MagnatuneService::kHomepage = "http://magnatune.com";
 const char* MagnatuneService::kDatabaseUrl = "http://magnatune.com/info/song_info_xml.gz";
@@ -72,7 +73,7 @@ MagnatuneService::MagnatuneService(RadioModel* parent)
   // Create the library backend in the database thread
   library_backend_ = parent->db_thread()->CreateInThread<LibraryBackend>();
   library_backend_->Init(parent->db_thread()->Worker(), kSongsTable,
-                         QString::null, QString::null);
+                         QString::null, QString::null, kFtsTable);
   library_model_ = new LibraryModel(library_backend_, this);
 
   connect(library_backend_, SIGNAL(TotalSongCountUpdated(int)),

src/radio/magnatuneservice.h

@@ -57,6 +57,7 @@ class MagnatuneService : public RadioService {
   static const char* kSettingsGroup;
   static const char* kDatabaseUrl;
   static const char* kSongsTable;
+  static const char* kFtsTable;
   static const char* kHomepage;
   static const char* kStreamingHostname;
   static const char* kDownloadHostname;

tests/librarybackend_test.cpp

@@ -36,7 +36,8 @@ class LibraryBackendTest : public ::testing::Test {
     database_.reset(new MemoryDatabase);
     backend_.reset(new LibraryBackend);
     backend_->Init(database_, Library::kSongsTable,
-                   Library::kDirsTable, Library::kSubdirsTable);
+                   Library::kDirsTable, Library::kSubdirsTable,
+                   Library::kFtsTable);
   }
 
   Song MakeDummySong(int directory_id) {

tests/librarymodel_test.cpp

@@ -35,7 +35,7 @@ class LibraryModelTest : public ::testing::Test {
     database_.reset(new MemoryDatabase);
     backend_.reset(new LibraryBackend());
     backend_->Init(database_, Library::kSongsTable,
-                   Library::kDirsTable, Library::kSubdirsTable);
+                   Library::kDirsTable, Library::kSubdirsTable, Library::kFtsTable);
     model_.reset(new LibraryModel(backend_.get()));
 
     added_dir_ = false;