Funkwhale
/
Clementine-audio-player-Mac-Linux-Windows
mirror of https://github.com/clementine-player/Clementine
1
0
Fork 0
Code Issues Releases Wiki Activity

Use sqlite's Full Text Search on the songs table

This commit is contained in:
David Sansome 2010-06-20 16:30:10 +00:00
parent 5da00151c9
commit 492d8fec87
16 changed files with 141 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
data/data.qrc
View File

@ -185,5 +185,6 @@
<file>icons/32x32/view-fullscreen.png</file>
<file>icons/48x48/view-fullscreen.png</file>
<file>schema-12.sql</file>
<file>schema-13.sql</file>
</qresource>
</RCC>

20
data/schema-13.sql Normal file
View File

@ -0,0 +1,20 @@
CREATE VIRTUAL TABLE songs_fts USING fts3(
ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment
);
CREATE VIRTUAL TABLE magnatune_songs_fts USING fts3(
ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment
);
INSERT INTO songs_fts (ROWID, ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment)
SELECT ROWID, title, album, artist, albumartist, composer, genre, comment
FROM songs;
INSERT INTO magnatune_songs_fts (ROWID, ftstitle, ftsalbum, ftsartist, ftsalbumartist, ftscomposer, ftsgenre, ftscomment)
SELECT ROWID, title, album, artist, albumartist, composer, genre, comment
FROM magnatune_songs;
CREATE INDEX idx_album ON songs (album);
UPDATE schema_version SET version=13;

2
src/core/database.cpp
View File

@ -27,7 +27,7 @@
#include <QVariant>
const char* Database::kDatabaseFilename = "clementine.db";
const int Database::kSchemaVersion = 12;
const int Database::kSchemaVersion = 13;
int (*Database::_sqlite3_create_function) (
sqlite3*, const char*, int, int, void*,

19
src/core/song.cpp
View File

@ -84,6 +84,15 @@ const QString Song::kColumnSpec = Song::kColumns.join(", ");
const QString Song::kBindSpec = Prepend(":", Song::kColumns).join(", ");
const QString Song::kUpdateSpec = Updateify(Song::kColumns).join(", ");
const QStringList Song::kFtsColumns = QStringList()
<< "ftstitle" << "ftsalbum" << "ftsartist" << "ftsalbumartist"
<< "ftscomposer" << "ftsgenre" << "ftscomment";
const QString Song::kFtsColumnSpec = Song::kFtsColumns.join(", ");
const QString Song::kFtsBindSpec = Prepend(":", Song::kFtsColumns).join(", ");
const QString Song::kFtsUpdateSpec = Updateify(Song::kFtsColumns).join(", ");
QString Song::JoinSpec(const QString& table) {
return Prepend(table + ".", kColumns).join(", ");
}
@ -582,6 +591,16 @@ void Song::BindToQuery(QSqlQuery *query) const {
#undef notnullintval
}
void Song::BindToFtsQuery(QSqlQuery *query) const {
query->bindValue(":ftstitle", d->title_);
query->bindValue(":ftsalbum", d->album_);
query->bindValue(":ftsartist", d->artist_);
query->bindValue(":ftsalbumartist", d->albumartist_);
query->bindValue(":ftscomposer", d->composer_);
query->bindValue(":ftsgenre", d->genre_);
query->bindValue(":ftscomment", d->comment_);
}
void Song::ToLastFM(lastfm::Track* track) const {
lastfm::MutableTrack mtrack(*track);

7
src/core/song.h
View File

@ -85,10 +85,14 @@ class Song {
static const QStringList kColumns;
static const QString kColumnSpec;
static const QString kJoinSpec;
static const QString kBindSpec;
static const QString kUpdateSpec;
static const QStringList kFtsColumns;
static const QString kFtsColumnSpec;
static const QString kFtsBindSpec;
static const QString kFtsUpdateSpec;
static QString JoinSpec(const QString& table);
// Don't change these values - they're stored in the database
@ -120,6 +124,7 @@ class Song {
// Save
void BindToQuery(QSqlQuery* query) const;
void BindToFtsQuery(QSqlQuery* query) const;
void ToLastFM(lastfm::Track* track) const;
// Simple accessors

3
src/library/library.cpp
View File

@ -22,6 +22,7 @@
const char* Library::kSongsTable = "songs";
const char* Library::kDirsTable = "directories";
const char* Library::kSubdirsTable = "subdirectories";
const char* Library::kFtsTable = "songs_fts";
Library::Library(BackgroundThread<Database>* db_thread, QObject *parent)
: QObject(parent),
@ -31,7 +32,7 @@ Library::Library(BackgroundThread<Database>* db_thread, QObject *parent)
watcher_(NULL)
{
backend_ = db_thread->CreateInThread<LibraryBackend>();
backend_->Init(db_thread->Worker(), kSongsTable, kDirsTable, kSubdirsTable);
backend_->Init(db_thread->Worker(), kSongsTable, kDirsTable, kSubdirsTable, kFtsTable);
model_ = new LibraryModel(backend_, this);
}

1
src/library/library.h
View File

@ -37,6 +37,7 @@ class Library : public QObject {
static const char* kSongsTable;
static const char* kDirsTable;
static const char* kSubdirsTable;
static const char* kFtsTable;
// Useful for tests. The library takes ownership.
void set_watcher_factory(BackgroundThreadFactory<LibraryWatcher>* factory);

59
src/library/librarybackend.cpp
View File

@ -30,12 +30,14 @@ LibraryBackend::LibraryBackend(QObject *parent)
{
}
void LibraryBackend::Init(boost::shared_ptr<Database> db, const QString &songs_table,
const QString &dirs_table, const QString &subdirs_table) {
void LibraryBackend::Init(boost::shared_ptr<Database> db, const QString& songs_table,
const QString& dirs_table, const QString& subdirs_table,
const QString& fts_table) {
db_ = db;
songs_table_ = songs_table;
dirs_table_ = dirs_table;
subdirs_table_ = subdirs_table;
fts_table_ = fts_table;
}
void LibraryBackend::LoadDirectoriesAsync() {
@ -225,6 +227,11 @@ void LibraryBackend::AddOrUpdateSongs(const SongList& songs) {
.arg(songs_table_), db);
QSqlQuery update_song(QString("UPDATE %1 SET " + Song::kUpdateSpec +
" WHERE ROWID = :id").arg(songs_table_), db);
QSqlQuery add_song_fts(QString("INSERT INTO %1 (ROWID, " + Song::kFtsColumnSpec + ")"
" VALUES (:id, " + Song::kFtsBindSpec + ")")
.arg(fts_table_), db);
QSqlQuery update_song_fts(QString("UPDATE %1 SET " + Song::kFtsUpdateSpec +
" WHERE ROWID = :id").arg(fts_table_), db);
ScopedTransaction transaction(&db);
@ -251,8 +258,15 @@ void LibraryBackend::AddOrUpdateSongs(const SongList& songs) {
add_song.exec();
if (db_->CheckErrors(add_song.lastError())) continue;
const int id = add_song.lastInsertId().toInt();
add_song_fts.bindValue(":id", id);
song.BindToFtsQuery(&add_song_fts);
add_song_fts.exec();
if (db_->CheckErrors(add_song_fts.lastError())) continue;
Song copy(song);
copy.set_id(add_song.lastInsertId().toInt());
copy.set_id(id);
added_songs << copy;
} else {
// Get the previous song data first
@ -266,6 +280,11 @@ void LibraryBackend::AddOrUpdateSongs(const SongList& songs) {
update_song.exec();
if (db_->CheckErrors(update_song.lastError())) continue;
song.BindToFtsQuery(&update_song_fts);
update_song_fts.bindValue(":id", song.id());
update_song_fts.exec();
if (db_->CheckErrors(update_song_fts.lastError())) continue;
deleted_songs << old_song;
added_songs << song;
}
@ -303,14 +322,20 @@ void LibraryBackend::DeleteSongs(const SongList &songs) {
QMutexLocker l(db_->Mutex());
QSqlDatabase db(db_->Connect());
QSqlQuery q(QString("DELETE FROM %1 WHERE ROWID = :id")
.arg(songs_table_), db);
QSqlQuery remove(QString("DELETE FROM %1 WHERE ROWID = :id")
.arg(songs_table_), db);
QSqlQuery remove_fts(QString("DELETE FROM %1 WHERE ROWID = :id")
.arg(fts_table_), db);
ScopedTransaction transaction(&db);
foreach (const Song& song, songs) {
q.bindValue(":id", song.id());
q.exec();
db_->CheckErrors(q.lastError());
remove.bindValue(":id", song.id());
remove.exec();
db_->CheckErrors(remove.lastError());
remove_fts.bindValue(":id", song.id());
remove_fts.exec();
db_->CheckErrors(remove_fts.lastError());
}
transaction.Commit();
@ -361,7 +386,7 @@ LibraryBackend::AlbumList LibraryBackend::GetAlbumsByArtist(const QString& artis
SongList LibraryBackend::GetSongs(const QString& artist, const QString& album, const QueryOptions& opt) {
LibraryQuery query(opt);
query.SetColumnSpec("ROWID, " + Song::kColumnSpec);
query.SetColumnSpec("%songs_table.ROWID, " + Song::kColumnSpec);
query.AddCompilationRequirement(false);
query.AddWhere("artist", artist);
query.AddWhere("album", album);
@ -397,8 +422,9 @@ Song LibraryBackend::GetSongById(int id) {
bool LibraryBackend::HasCompilations(const QueryOptions& opt) {
LibraryQuery query(opt);
query.SetColumnSpec("ROWID");
query.SetColumnSpec("%songs_table.ROWID");
query.AddCompilationRequirement(true);
query.SetLimit(1);
QMutexLocker l(db_->Mutex());
if (!ExecQuery(&query)) return false;
@ -412,7 +438,7 @@ LibraryBackend::AlbumList LibraryBackend::GetCompilationAlbums(const QueryOption
SongList LibraryBackend::GetCompilationSongs(const QString& album, const QueryOptions& opt) {
LibraryQuery query(opt);
query.SetColumnSpec("ROWID, " + Song::kColumnSpec);
query.SetColumnSpec("%songs_table.ROWID, " + Song::kColumnSpec);
query.AddCompilationRequirement(true);
query.AddWhere("album", album);
@ -451,18 +477,13 @@ void LibraryBackend::UpdateCompilations() {
continue;
// Find the directory the song is in
QDir dir(filename);
QString path = QDir::toNativeSeparators(dir.canonicalPath());
int last_separator = path.lastIndexOf(QDir::separator());
int last_separator = filename.lastIndexOf('/');
if (last_separator == -1)
continue;
path = path.left(last_separator);
CompilationInfo& info = compilation_info[album];
info.artists.insert(artist);
info.directories.insert(path);
info.directories.insert(filename.left(last_separator));
if (sampler) info.has_samplers = true;
else info.has_not_samplers = true;
}
@ -711,5 +732,5 @@ void LibraryBackend::ForceCompilation(const QString& artist, const QString& albu
}
bool LibraryBackend::ExecQuery(LibraryQuery *q) {
return !db_->CheckErrors(q->Exec(db_->Connect(), songs_table_));
return !db_->CheckErrors(q->Exec(db_->Connect(), songs_table_, fts_table_));
}

4
src/library/librarybackend.h
View File

@ -34,7 +34,8 @@ class LibraryBackend : public QObject {
public:
Q_INVOKABLE LibraryBackend(QObject* parent = 0);
void Init(boost::shared_ptr<Database> db, const QString& songs_table,
const QString& dirs_table, const QString& subdirs_table);
const QString& dirs_table, const QString& subdirs_table,
const QString& fts_table);
boost::shared_ptr<Database> db() const { return db_; }
@ -130,6 +131,7 @@ class LibraryBackend : public QObject {
QString songs_table_;
QString dirs_table_;
QString subdirs_table_;
QString fts_table_;
};
#endif // LIBRARYBACKEND_H

2
src/library/librarymodel.cpp
View File

@ -433,7 +433,7 @@ void LibraryModel::InitQuery(GroupBy type, LibraryQuery* q) {
q->SetColumnSpec("DISTINCT albumartist");
break;
case GroupBy_None:
q->SetColumnSpec("ROWID, " + Song::kColumnSpec);
q->SetColumnSpec("%songs_table.ROWID, " + Song::kColumnSpec);
break;
}
}

58
src/library/libraryquery.cpp
View File

@ -27,21 +27,29 @@ QueryOptions::QueryOptions()
}
LibraryQuery::LibraryQuery(const QueryOptions& options) {
LibraryQuery::LibraryQuery(const QueryOptions& options)
: join_with_fts_(false),
limit_(-1)
{
if (!options.filter.isEmpty()) {
where_clauses_ << "("
"artist LIKE ? OR "
"album LIKE ? OR "
"title LIKE ? OR "
"composer LIKE ? OR "
"genre LIKE ? OR "
"albumartist LIKE ?)";
bound_values_ << "%" + options.filter + "%";
bound_values_ << "%" + options.filter + "%";
bound_values_ << "%" + options.filter + "%";
bound_values_ << "%" + options.filter + "%";
bound_values_ << "%" + options.filter + "%";
bound_values_ << "%" + options.filter + "%";
// We need to munge the filter text a little bit to get it to work as
// expected with sqlite's FTS3:
// 1) Append * to all tokens.
// 2) Prefix "fts" to column names.
// Split on whitespace
QStringList tokens(options.filter.split(QRegExp("\\s+")));
QString query;
foreach (const QString& token, tokens) {
if (token.contains(':'))
query += "fts" + token + "* ";
else
query += token + "* ";
}
where_clauses_ << "fts.%fts_table MATCH ?";
bound_values_ << query;
join_with_fts_ = true;
}
if (options.max_age != -1) {
@ -64,17 +72,20 @@ void LibraryQuery::AddWhere(const QString& column, const QVariant& value, const
}
}
void LibraryQuery::AddWhereLike(const QString& column, const QVariant& value) {
where_clauses_ << QString("%1 LIKE ?").arg(column);
bound_values_ << value;
}
void LibraryQuery::AddCompilationRequirement(bool compilation) {
where_clauses_ << QString("effective_compilation = %1").arg(compilation ? 1 : 0);
}
QSqlError LibraryQuery::Exec(QSqlDatabase db, const QString& table) {
QString sql = QString("SELECT %1 FROM %2").arg(column_spec_, table);
QSqlError LibraryQuery::Exec(QSqlDatabase db, const QString& songs_table,
const QString& fts_table) {
QString sql;
if (join_with_fts_) {
sql = QString("SELECT %1 FROM %2 INNER JOIN %3 AS fts ON %2.ROWID = fts.ROWID")
.arg(column_spec_, songs_table, fts_table);
} else {
sql = QString("SELECT %1 FROM %2")
.arg(column_spec_, songs_table);
}
if (!where_clauses_.isEmpty())
sql += " WHERE " + where_clauses_.join(" AND ");
@ -82,6 +93,11 @@ QSqlError LibraryQuery::Exec(QSqlDatabase db, const QString& table) {
if (!order_by_.isEmpty())
sql += " ORDER BY " + order_by_;
if (limit_ != -1)
sql += " LIMIT " + QString::number(limit_);
sql.replace("%songs_table", songs_table);
sql.replace("%fts_table", fts_table);
query_ = QSqlQuery(sql, db);
// Bind values

6
src/library/libraryquery.h
View File

@ -42,20 +42,22 @@ class LibraryQuery {
void SetColumnSpec(const QString& spec) { column_spec_ = spec; }
void SetOrderBy(const QString& order_by) { order_by_ = order_by; }
void AddWhere(const QString& column, const QVariant& value, const QString& op = "=");
void AddWhereLike(const QString& column, const QVariant& value);
void AddCompilationRequirement(bool compilation);
void SetLimit(int limit) { limit_ = limit; }
QSqlError Exec(QSqlDatabase db, const QString& table);
QSqlError Exec(QSqlDatabase db, const QString& songs_table, const QString& fts_table);
bool Next();
QVariant Value(int column) const;
operator const QSqlQuery& () const { return query_; }
private:
bool join_with_fts_;
QString column_spec_;
QString order_by_;
QStringList where_clauses_;
QVariantList bound_values_;
int limit_;
QSqlQuery query_;
};

3
src/radio/magnatuneservice.cpp
View File

@ -45,6 +45,7 @@ using boost::shared_ptr;
const char* MagnatuneService::kServiceName = "Magnatune";
const char* MagnatuneService::kSettingsGroup = "Magnatune";
const char* MagnatuneService::kSongsTable = "magnatune_songs";
const char* MagnatuneService::kFtsTable = "magnatune_songs_fts";
const char* MagnatuneService::kHomepage = "http://magnatune.com";
const char* MagnatuneService::kDatabaseUrl = "http://magnatune.com/info/song_info_xml.gz";
@ -72,7 +73,7 @@ MagnatuneService::MagnatuneService(RadioModel* parent)
// Create the library backend in the database thread
library_backend_ = parent->db_thread()->CreateInThread<LibraryBackend>();
library_backend_->Init(parent->db_thread()->Worker(), kSongsTable,
QString::null, QString::null);
QString::null, QString::null, kFtsTable);
library_model_ = new LibraryModel(library_backend_, this);
connect(library_backend_, SIGNAL(TotalSongCountUpdated(int)),

1
src/radio/magnatuneservice.h
View File

@ -57,6 +57,7 @@ class MagnatuneService : public RadioService {
static const char* kSettingsGroup;
static const char* kDatabaseUrl;
static const char* kSongsTable;
static const char* kFtsTable;
static const char* kHomepage;
static const char* kStreamingHostname;
static const char* kDownloadHostname;

3
tests/librarybackend_test.cpp
View File

@ -36,7 +36,8 @@ class LibraryBackendTest : public ::testing::Test {
database_.reset(new MemoryDatabase);
backend_.reset(new LibraryBackend);
backend_->Init(database_, Library::kSongsTable,
Library::kDirsTable, Library::kSubdirsTable);
Library::kDirsTable, Library::kSubdirsTable,
Library::kFtsTable);
}
Song MakeDummySong(int directory_id) {

2
tests/librarymodel_test.cpp
View File

@ -35,7 +35,7 @@ class LibraryModelTest : public ::testing::Test {
database_.reset(new MemoryDatabase);
backend_.reset(new LibraryBackend());
backend_->Init(database_, Library::kSongsTable,
Library::kDirsTable, Library::kSubdirsTable);
Library::kDirsTable, Library::kSubdirsTable, Library::kFtsTable);
model_.reset(new LibraryModel(backend_.get()));
added_dir_ = false;