From 492d8fec87c67a6cbf76e3ca6e7ba58676d540be Mon Sep 17 00:00:00 2001 From: David Sansome Date: Sun, 20 Jun 2010 16:30:10 +0000 Subject: [PATCH] Use sqlite's Full Text Search on the songs table --- data/data.qrc | 1 + data/schema-13.sql | 20 ++++++++++++ src/core/database.cpp | 2 +- src/core/song.cpp | 19 +++++++++++ src/core/song.h | 7 +++- src/library/library.cpp | 3 +- src/library/library.h | 1 + src/library/librarybackend.cpp | 59 +++++++++++++++++++++++----------- src/library/librarybackend.h | 4 ++- src/library/librarymodel.cpp | 2 +- src/library/libraryquery.cpp | 58 +++++++++++++++++++++------------ src/library/libraryquery.h | 6 ++-- src/radio/magnatuneservice.cpp | 3 +- src/radio/magnatuneservice.h | 1 + tests/librarybackend_test.cpp | 3 +- tests/librarymodel_test.cpp | 2 +- 16 files changed, 141 insertions(+), 50 deletions(-) create mode 100644 data/schema-13.sql diff --git a/data/data.qrc b/data/data.qrc index 3aa1c3729..0a8a8c1f9 100644 --- a/data/data.qrc +++ b/data/data.qrc @@ -185,5 +185,6 @@ icons/32x32/view-fullscreen.png icons/48x48/view-fullscreen.png schema-12.sql + schema-13.sql diff --git a/data/schema-13.sql b/data/schema-13.sql new file mode 100644 index 000000000..700cf3d06 --- /dev/null +++ b/data/schema-13.sql @@ -0,0 +1,20 @@ +CREATE VIRTUAL TABLE songs_fts USING fts3( + ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment +); + +CREATE VIRTUAL TABLE magnatune_songs_fts USING fts3( + ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment +); + +INSERT INTO songs_fts (ROWID, ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment) + SELECT ROWID, title, album, artist, albumartist, composer, genre, comment + FROM songs; + +INSERT INTO magnatune_songs_fts (ROWID, ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment) + SELECT ROWID, title, album, artist, albumartist, composer, genre, comment + FROM magnatune_songs; + +CREATE INDEX idx_album ON songs (album); + +UPDATE schema_version SET version=13; + diff --git a/src/core/database.cpp b/src/core/database.cpp index 9c1f952b3..aecfd5888 100644 --- a/src/core/database.cpp +++ b/src/core/database.cpp @@ -27,7 +27,7 @@ #include const char* Database::kDatabaseFilename = "clementine.db"; -const int Database::kSchemaVersion = 12; +const int Database::kSchemaVersion = 13; int (*Database::_sqlite3_create_function) ( sqlite3*, const char*, int, int, void*, diff --git a/src/core/song.cpp b/src/core/song.cpp index 5c1a8be29..3cbdcbab5 100644 --- a/src/core/song.cpp +++ b/src/core/song.cpp @@ -84,6 +84,15 @@ const QString Song::kColumnSpec = Song::kColumns.join(", "); const QString Song::kBindSpec = Prepend(":", Song::kColumns).join(", "); const QString Song::kUpdateSpec = Updateify(Song::kColumns).join(", "); + +const QStringList Song::kFtsColumns = QStringList() + << "ftstitle" << "ftsalbum" << "ftsartist" << "ftsalbumartist" + << "ftscomposer" << "ftsgenre" << "ftscomment"; + +const QString Song::kFtsColumnSpec = Song::kFtsColumns.join(", "); +const QString Song::kFtsBindSpec = Prepend(":", Song::kFtsColumns).join(", "); +const QString Song::kFtsUpdateSpec = Updateify(Song::kFtsColumns).join(", "); + QString Song::JoinSpec(const QString& table) { return Prepend(table + ".", kColumns).join(", "); } @@ -582,6 +591,16 @@ void Song::BindToQuery(QSqlQuery *query) const { #undef notnullintval } +void Song::BindToFtsQuery(QSqlQuery *query) const { + query->bindValue(":ftstitle", d->title_); + query->bindValue(":ftsalbum", d->album_); + query->bindValue(":ftsartist", d->artist_); + query->bindValue(":ftsalbumartist", d->albumartist_); + query->bindValue(":ftscomposer", d->composer_); + query->bindValue(":ftsgenre", d->genre_); + query->bindValue(":ftscomment", d->comment_); +} + void Song::ToLastFM(lastfm::Track* track) const { lastfm::MutableTrack mtrack(*track); diff --git a/src/core/song.h b/src/core/song.h index 17e2fb092..8b6a98198 100644 --- a/src/core/song.h +++ b/src/core/song.h @@ -85,10 +85,14 @@ class Song { static const QStringList kColumns; static const QString kColumnSpec; - static const QString kJoinSpec; static const QString kBindSpec; static const QString kUpdateSpec; + static const QStringList kFtsColumns; + static const QString kFtsColumnSpec; + static const QString kFtsBindSpec; + static const QString kFtsUpdateSpec; + static QString JoinSpec(const QString& table); // Don't change these values - they're stored in the database @@ -120,6 +124,7 @@ class Song { // Save void BindToQuery(QSqlQuery* query) const; + void BindToFtsQuery(QSqlQuery* query) const; void ToLastFM(lastfm::Track* track) const; // Simple accessors diff --git a/src/library/library.cpp b/src/library/library.cpp index 0b288cac5..a63226918 100644 --- a/src/library/library.cpp +++ b/src/library/library.cpp @@ -22,6 +22,7 @@ const char* Library::kSongsTable = "songs"; const char* Library::kDirsTable = "directories"; const char* Library::kSubdirsTable = "subdirectories"; +const char* Library::kFtsTable = "songs_fts"; Library::Library(BackgroundThread* db_thread, QObject *parent) : QObject(parent), @@ -31,7 +32,7 @@ Library::Library(BackgroundThread* db_thread, QObject *parent) watcher_(NULL) { backend_ = db_thread->CreateInThread(); - backend_->Init(db_thread->Worker(), kSongsTable, kDirsTable, kSubdirsTable); + backend_->Init(db_thread->Worker(), kSongsTable, kDirsTable, kSubdirsTable, kFtsTable); model_ = new LibraryModel(backend_, this); } diff --git a/src/library/library.h b/src/library/library.h index 857eab45e..a6aca8a5e 100644 --- a/src/library/library.h +++ b/src/library/library.h @@ -37,6 +37,7 @@ class Library : public QObject { static const char* kSongsTable; static const char* kDirsTable; static const char* kSubdirsTable; + static const char* kFtsTable; // Useful for tests. The library takes ownership. void set_watcher_factory(BackgroundThreadFactory* factory); diff --git a/src/library/librarybackend.cpp b/src/library/librarybackend.cpp index 849d3c511..6536d6509 100644 --- a/src/library/librarybackend.cpp +++ b/src/library/librarybackend.cpp @@ -30,12 +30,14 @@ LibraryBackend::LibraryBackend(QObject *parent) { } -void LibraryBackend::Init(boost::shared_ptr db, const QString &songs_table, - const QString &dirs_table, const QString &subdirs_table) { +void LibraryBackend::Init(boost::shared_ptr db, const QString& songs_table, + const QString& dirs_table, const QString& subdirs_table, + const QString& fts_table) { db_ = db; songs_table_ = songs_table; dirs_table_ = dirs_table; subdirs_table_ = subdirs_table; + fts_table_ = fts_table; } void LibraryBackend::LoadDirectoriesAsync() { @@ -225,6 +227,11 @@ void LibraryBackend::AddOrUpdateSongs(const SongList& songs) { .arg(songs_table_), db); QSqlQuery update_song(QString("UPDATE %1 SET " + Song::kUpdateSpec + " WHERE ROWID = :id").arg(songs_table_), db); + QSqlQuery add_song_fts(QString("INSERT INTO %1 (ROWID, " + Song::kFtsColumnSpec + ")" + " VALUES (:id, " + Song::kFtsBindSpec + ")") + .arg(fts_table_), db); + QSqlQuery update_song_fts(QString("UPDATE %1 SET " + Song::kFtsUpdateSpec + + " WHERE ROWID = :id").arg(fts_table_), db); ScopedTransaction transaction(&db); @@ -251,8 +258,15 @@ void LibraryBackend::AddOrUpdateSongs(const SongList& songs) { add_song.exec(); if (db_->CheckErrors(add_song.lastError())) continue; + const int id = add_song.lastInsertId().toInt(); + + add_song_fts.bindValue(":id", id); + song.BindToFtsQuery(&add_song_fts); + add_song_fts.exec(); + if (db_->CheckErrors(add_song_fts.lastError())) continue; + Song copy(song); - copy.set_id(add_song.lastInsertId().toInt()); + copy.set_id(id); added_songs << copy; } else { // Get the previous song data first @@ -266,6 +280,11 @@ void LibraryBackend::AddOrUpdateSongs(const SongList& songs) { update_song.exec(); if (db_->CheckErrors(update_song.lastError())) continue; + song.BindToFtsQuery(&update_song_fts); + update_song_fts.bindValue(":id", song.id()); + update_song_fts.exec(); + if (db_->CheckErrors(update_song_fts.lastError())) continue; + deleted_songs << old_song; added_songs << song; } @@ -303,14 +322,20 @@ void LibraryBackend::DeleteSongs(const SongList &songs) { QMutexLocker l(db_->Mutex()); QSqlDatabase db(db_->Connect()); - QSqlQuery q(QString("DELETE FROM %1 WHERE ROWID = :id") - .arg(songs_table_), db); + QSqlQuery remove(QString("DELETE FROM %1 WHERE ROWID = :id") + .arg(songs_table_), db); + QSqlQuery remove_fts(QString("DELETE FROM %1 WHERE ROWID = :id") + .arg(fts_table_), db); ScopedTransaction transaction(&db); foreach (const Song& song, songs) { - q.bindValue(":id", song.id()); - q.exec(); - db_->CheckErrors(q.lastError()); + remove.bindValue(":id", song.id()); + remove.exec(); + db_->CheckErrors(remove.lastError()); + + remove_fts.bindValue(":id", song.id()); + remove_fts.exec(); + db_->CheckErrors(remove_fts.lastError()); } transaction.Commit(); @@ -361,7 +386,7 @@ LibraryBackend::AlbumList LibraryBackend::GetAlbumsByArtist(const QString& artis SongList LibraryBackend::GetSongs(const QString& artist, const QString& album, const QueryOptions& opt) { LibraryQuery query(opt); - query.SetColumnSpec("ROWID, " + Song::kColumnSpec); + query.SetColumnSpec("%songs_table.ROWID, " + Song::kColumnSpec); query.AddCompilationRequirement(false); query.AddWhere("artist", artist); query.AddWhere("album", album); @@ -397,8 +422,9 @@ Song LibraryBackend::GetSongById(int id) { bool LibraryBackend::HasCompilations(const QueryOptions& opt) { LibraryQuery query(opt); - query.SetColumnSpec("ROWID"); + query.SetColumnSpec("%songs_table.ROWID"); query.AddCompilationRequirement(true); + query.SetLimit(1); QMutexLocker l(db_->Mutex()); if (!ExecQuery(&query)) return false; @@ -412,7 +438,7 @@ LibraryBackend::AlbumList LibraryBackend::GetCompilationAlbums(const QueryOption SongList LibraryBackend::GetCompilationSongs(const QString& album, const QueryOptions& opt) { LibraryQuery query(opt); - query.SetColumnSpec("ROWID, " + Song::kColumnSpec); + query.SetColumnSpec("%songs_table.ROWID, " + Song::kColumnSpec); query.AddCompilationRequirement(true); query.AddWhere("album", album); @@ -451,18 +477,13 @@ void LibraryBackend::UpdateCompilations() { continue; // Find the directory the song is in - QDir dir(filename); - QString path = QDir::toNativeSeparators(dir.canonicalPath()); - int last_separator = path.lastIndexOf(QDir::separator()); - + int last_separator = filename.lastIndexOf('/'); if (last_separator == -1) continue; - path = path.left(last_separator); - CompilationInfo& info = compilation_info[album]; info.artists.insert(artist); - info.directories.insert(path); + info.directories.insert(filename.left(last_separator)); if (sampler) info.has_samplers = true; else info.has_not_samplers = true; } @@ -711,5 +732,5 @@ void LibraryBackend::ForceCompilation(const QString& artist, const QString& albu } bool LibraryBackend::ExecQuery(LibraryQuery *q) { - return !db_->CheckErrors(q->Exec(db_->Connect(), songs_table_)); + return !db_->CheckErrors(q->Exec(db_->Connect(), songs_table_, fts_table_)); } diff --git a/src/library/librarybackend.h b/src/library/librarybackend.h index fd9ed3a8f..ad63561c7 100644 --- a/src/library/librarybackend.h +++ b/src/library/librarybackend.h @@ -34,7 +34,8 @@ class LibraryBackend : public QObject { public: Q_INVOKABLE LibraryBackend(QObject* parent = 0); void Init(boost::shared_ptr db, const QString& songs_table, - const QString& dirs_table, const QString& subdirs_table); + const QString& dirs_table, const QString& subdirs_table, + const QString& fts_table); boost::shared_ptr db() const { return db_; } @@ -130,6 +131,7 @@ class LibraryBackend : public QObject { QString songs_table_; QString dirs_table_; QString subdirs_table_; + QString fts_table_; }; #endif // LIBRARYBACKEND_H diff --git a/src/library/librarymodel.cpp b/src/library/librarymodel.cpp index 03eb3fb0a..1fa9a3060 100644 --- a/src/library/librarymodel.cpp +++ b/src/library/librarymodel.cpp @@ -433,7 +433,7 @@ void LibraryModel::InitQuery(GroupBy type, LibraryQuery* q) { q->SetColumnSpec("DISTINCT albumartist"); break; case GroupBy_None: - q->SetColumnSpec("ROWID, " + Song::kColumnSpec); + q->SetColumnSpec("%songs_table.ROWID, " + Song::kColumnSpec); break; } } diff --git a/src/library/libraryquery.cpp b/src/library/libraryquery.cpp index 23c4b1ad5..a420485d4 100644 --- a/src/library/libraryquery.cpp +++ b/src/library/libraryquery.cpp @@ -27,21 +27,29 @@ QueryOptions::QueryOptions() } -LibraryQuery::LibraryQuery(const QueryOptions& options) { +LibraryQuery::LibraryQuery(const QueryOptions& options) + : join_with_fts_(false), + limit_(-1) +{ if (!options.filter.isEmpty()) { - where_clauses_ << "(" - "artist LIKE ? OR " - "album LIKE ? OR " - "title LIKE ? OR " - "composer LIKE ? OR " - "genre LIKE ? OR " - "albumartist LIKE ?)"; - bound_values_ << "%" + options.filter + "%"; - bound_values_ << "%" + options.filter + "%"; - bound_values_ << "%" + options.filter + "%"; - bound_values_ << "%" + options.filter + "%"; - bound_values_ << "%" + options.filter + "%"; - bound_values_ << "%" + options.filter + "%"; + // We need to munge the filter text a little bit to get it to work as + // expected with sqlite's FTS3: + // 1) Append * to all tokens. + // 2) Prefix "fts" to column names. + + // Split on whitespace + QStringList tokens(options.filter.split(QRegExp("\\s+"))); + QString query; + foreach (const QString& token, tokens) { + if (token.contains(':')) + query += "fts" + token + "* "; + else + query += token + "* "; + } + + where_clauses_ << "fts.%fts_table MATCH ?"; + bound_values_ << query; + join_with_fts_ = true; } if (options.max_age != -1) { @@ -64,17 +72,20 @@ void LibraryQuery::AddWhere(const QString& column, const QVariant& value, const } } -void LibraryQuery::AddWhereLike(const QString& column, const QVariant& value) { - where_clauses_ << QString("%1 LIKE ?").arg(column); - bound_values_ << value; -} - void LibraryQuery::AddCompilationRequirement(bool compilation) { where_clauses_ << QString("effective_compilation = %1").arg(compilation ? 1 : 0); } -QSqlError LibraryQuery::Exec(QSqlDatabase db, const QString& table) { - QString sql = QString("SELECT %1 FROM %2").arg(column_spec_, table); +QSqlError LibraryQuery::Exec(QSqlDatabase db, const QString& songs_table, + const QString& fts_table) { + QString sql; + if (join_with_fts_) { + sql = QString("SELECT %1 FROM %2 INNER JOIN %3 AS fts ON %2.ROWID = fts.ROWID") + .arg(column_spec_, songs_table, fts_table); + } else { + sql = QString("SELECT %1 FROM %2") + .arg(column_spec_, songs_table); + } if (!where_clauses_.isEmpty()) sql += " WHERE " + where_clauses_.join(" AND "); @@ -82,6 +93,11 @@ QSqlError LibraryQuery::Exec(QSqlDatabase db, const QString& table) { if (!order_by_.isEmpty()) sql += " ORDER BY " + order_by_; + if (limit_ != -1) + sql += " LIMIT " + QString::number(limit_); + + sql.replace("%songs_table", songs_table); + sql.replace("%fts_table", fts_table); query_ = QSqlQuery(sql, db); // Bind values diff --git a/src/library/libraryquery.h b/src/library/libraryquery.h index 37ce7226d..d1a41fa7b 100644 --- a/src/library/libraryquery.h +++ b/src/library/libraryquery.h @@ -42,20 +42,22 @@ class LibraryQuery { void SetColumnSpec(const QString& spec) { column_spec_ = spec; } void SetOrderBy(const QString& order_by) { order_by_ = order_by; } void AddWhere(const QString& column, const QVariant& value, const QString& op = "="); - void AddWhereLike(const QString& column, const QVariant& value); void AddCompilationRequirement(bool compilation); + void SetLimit(int limit) { limit_ = limit; } - QSqlError Exec(QSqlDatabase db, const QString& table); + QSqlError Exec(QSqlDatabase db, const QString& songs_table, const QString& fts_table); bool Next(); QVariant Value(int column) const; operator const QSqlQuery& () const { return query_; } private: + bool join_with_fts_; QString column_spec_; QString order_by_; QStringList where_clauses_; QVariantList bound_values_; + int limit_; QSqlQuery query_; }; diff --git a/src/radio/magnatuneservice.cpp b/src/radio/magnatuneservice.cpp index d9564cd44..9f9a58128 100644 --- a/src/radio/magnatuneservice.cpp +++ b/src/radio/magnatuneservice.cpp @@ -45,6 +45,7 @@ using boost::shared_ptr; const char* MagnatuneService::kServiceName = "Magnatune"; const char* MagnatuneService::kSettingsGroup = "Magnatune"; const char* MagnatuneService::kSongsTable = "magnatune_songs"; +const char* MagnatuneService::kFtsTable = "magnatune_songs_fts"; const char* MagnatuneService::kHomepage = "http://magnatune.com"; const char* MagnatuneService::kDatabaseUrl = "http://magnatune.com/info/song_info_xml.gz"; @@ -72,7 +73,7 @@ MagnatuneService::MagnatuneService(RadioModel* parent) // Create the library backend in the database thread library_backend_ = parent->db_thread()->CreateInThread(); library_backend_->Init(parent->db_thread()->Worker(), kSongsTable, - QString::null, QString::null); + QString::null, QString::null, kFtsTable); library_model_ = new LibraryModel(library_backend_, this); connect(library_backend_, SIGNAL(TotalSongCountUpdated(int)), diff --git a/src/radio/magnatuneservice.h b/src/radio/magnatuneservice.h index 9020d3c4a..13b68aada 100644 --- a/src/radio/magnatuneservice.h +++ b/src/radio/magnatuneservice.h @@ -57,6 +57,7 @@ class MagnatuneService : public RadioService { static const char* kSettingsGroup; static const char* kDatabaseUrl; static const char* kSongsTable; + static const char* kFtsTable; static const char* kHomepage; static const char* kStreamingHostname; static const char* kDownloadHostname; diff --git a/tests/librarybackend_test.cpp b/tests/librarybackend_test.cpp index f8d987d3b..41093bd58 100644 --- a/tests/librarybackend_test.cpp +++ b/tests/librarybackend_test.cpp @@ -36,7 +36,8 @@ class LibraryBackendTest : public ::testing::Test { database_.reset(new MemoryDatabase); backend_.reset(new LibraryBackend); backend_->Init(database_, Library::kSongsTable, - Library::kDirsTable, Library::kSubdirsTable); + Library::kDirsTable, Library::kSubdirsTable, + Library::kFtsTable); } Song MakeDummySong(int directory_id) { diff --git a/tests/librarymodel_test.cpp b/tests/librarymodel_test.cpp index 296b66d2a..2781d30ed 100644 --- a/tests/librarymodel_test.cpp +++ b/tests/librarymodel_test.cpp @@ -35,7 +35,7 @@ class LibraryModelTest : public ::testing::Test { database_.reset(new MemoryDatabase); backend_.reset(new LibraryBackend()); backend_->Init(database_, Library::kSongsTable, - Library::kDirsTable, Library::kSubdirsTable); + Library::kDirsTable, Library::kSubdirsTable, Library::kFtsTable); model_.reset(new LibraryModel(backend_.get())); added_dir_ = false;