Reworked the library watcher to be much more efficient - the backend now holds a list of subdirectories within each library directory, and only the mtimes of these are checked on startup.

data/data.qrc

@@ -75,5 +75,6 @@
         <file>schema-5.sql</file>
         <file>osd_shadow_corner.png</file>
         <file>osd_shadow_edge.png</file>
+        <file>schema-6.sql</file>
     </qresource>
 </RCC>

data/schema-6.sql

@@ -0,0 +1,8 @@
+CREATE TABLE subdirectories (
+  directory INTEGER NOT NULL,
+  path TEXT NOT NULL,
+  mtime INTEGER NOT NULL
+);
+
+UPDATE schema_version SET version=6;
+

src/directory.h

@@ -24,6 +24,8 @@
 class QSqlQuery;
 
 struct Directory {
+  Directory() : id(-1) {}
+
   QString path;
   int id;
 };
@@ -32,4 +34,17 @@ Q_DECLARE_METATYPE(Directory);
 typedef QList<Directory> DirectoryList;
 Q_DECLARE_METATYPE(DirectoryList);
 
+
+struct Subdirectory {
+  Subdirectory() : directory_id(-1), mtime(0) {}
+
+  int directory_id;
+  QString path;
+  uint mtime;
+};
+Q_DECLARE_METATYPE(Subdirectory);
+
+typedef QList<Subdirectory> SubdirectoryList;
+Q_DECLARE_METATYPE(SubdirectoryList);
+
 #endif // DIRECTORY_H

src/library.cpp

@@ -108,14 +108,20 @@ void Library::Initialise() {
   // connect them together and start everything off.
   watcher_->Worker()->SetBackend(backend_->Worker());
 
-  connect(backend_->Worker().get(), SIGNAL(DirectoriesDiscovered(DirectoryList)),
+  connect(backend_->Worker().get(), SIGNAL(DirectoryDiscovered(Directory,SubdirectoryList)),
-          watcher_->Worker().get(), SLOT(AddDirectories(DirectoryList)));
+          watcher_->Worker().get(), SLOT(AddDirectory(Directory,SubdirectoryList)));
+  connect(backend_->Worker().get(), SIGNAL(DirectoryDeleted(Directory)),
+          watcher_->Worker().get(), SLOT(RemoveDirectory(Directory)));
   connect(watcher_->Worker().get(), SIGNAL(NewOrUpdatedSongs(SongList)),
           backend_->Worker().get(), SLOT(AddOrUpdateSongs(SongList)));
   connect(watcher_->Worker().get(), SIGNAL(SongsMTimeUpdated(SongList)),
           backend_->Worker().get(), SLOT(UpdateMTimesOnly(SongList)));
   connect(watcher_->Worker().get(), SIGNAL(SongsDeleted(SongList)),
           backend_->Worker().get(), SLOT(DeleteSongs(SongList)));
+  connect(watcher_->Worker().get(), SIGNAL(SubdirsDiscovered(SubdirectoryList)),
+          backend_->Worker().get(), SLOT(AddSubdirs(SubdirectoryList)));
+  connect(watcher_->Worker().get(), SIGNAL(SubdirsMTimeUpdated(SubdirectoryList)),
+          backend_->Worker().get(), SLOT(UpdateSubdirMTimes(SubdirectoryList)));
 
   // This will start the watcher checking for updates
   backend_->Worker()->LoadDirectoriesAsync();
@@ -132,6 +138,10 @@ void Library::SongsDiscovered(const SongList& songs) {
     if (!query_options_.Matches(song))
       continue;
 
+    // Hey, we've already got that one!
+    if (song_nodes_.contains(song.id()))
+      continue;
+
     // Before we can add each song we need to make sure the required container
     // items already exist in the tree.  These depend on which "group by"
     // settings the user has on the library.  Eg. if the user grouped by
@@ -170,8 +180,7 @@ void Library::SongsDiscovered(const SongList& songs) {
         if (!container_nodes_[i].contains(key)) {
           // Create the container
           container_nodes_[i][key] =
-              ItemFromSong(type, true, i == 0, container, song);
+              ItemFromSong(type, true, i == 0, container, song, i);
-          container_nodes_[i][key]->container_level = i;
         }
         container = container_nodes_[i][key];
       }
@@ -188,7 +197,7 @@ void Library::SongsDiscovered(const SongList& songs) {
     // We've gone all the way down to the deepest level and everything was
     // already lazy loaded, so now we have to create the song in the container.
     song_nodes_[song.id()] =
-        ItemFromSong(GroupBy_None, true, false, container, song);
+        ItemFromSong(GroupBy_None, true, false, container, song, -1);
   }
 }
 
@@ -429,8 +438,7 @@ void Library::LazyPopulate(LibraryItem* parent, bool signal) {
   while (q.Next()) {
     // Create the item - it will get inserted into the model here
     LibraryItem* item =
-        ItemFromQuery(child_type, signal, child_level == 0, parent, q);
+        ItemFromQuery(child_type, signal, child_level == 0, parent, q, child_level);
-    item->container_level = child_level;
 
     // Save a pointer to it for later
     if (child_type == GroupBy_None)
@@ -526,7 +534,8 @@ void Library::FilterQuery(GroupBy type, LibraryItem* item, LibraryQuery* q) {
   }
 }
 
-LibraryItem* Library::InitItem(GroupBy type, bool signal, LibraryItem *parent) {
+LibraryItem* Library::InitItem(GroupBy type, bool signal, LibraryItem *parent,
+                               int container_level) {
   LibraryItem::Type item_type =
       type == GroupBy_None ? LibraryItem::Type_Song :
       LibraryItem::Type_Container;
@@ -536,13 +545,16 @@ LibraryItem* Library::InitItem(GroupBy type, bool signal, LibraryItem *parent) {
                     parent->children.count(),parent->children.count());
 
   // Initialise the item depending on what type it's meant to be
-  return new LibraryItem(item_type, parent);
+  LibraryItem* item = new LibraryItem(item_type, parent);
+  item->container_level = container_level;
+  return item;
 }
 
 LibraryItem* Library::ItemFromQuery(GroupBy type,
                                     bool signal, bool create_divider,
-                                    LibraryItem* parent, const LibraryQuery& q) {
+                                    LibraryItem* parent, const LibraryQuery& q,
-  LibraryItem* item = InitItem(type, signal, parent);
+                                    int container_level) {
+  LibraryItem* item = InitItem(type, signal, parent, container_level);
   int year = 0;
 
   switch (type) {
@@ -587,8 +599,9 @@ LibraryItem* Library::ItemFromQuery(GroupBy type,
 
 LibraryItem* Library::ItemFromSong(GroupBy type,
                                    bool signal, bool create_divider,
-                                   LibraryItem* parent, const Song& s) {
+                                   LibraryItem* parent, const Song& s,
-  LibraryItem* item = InitItem(type, signal, parent);
+                                   int container_level) {
+  LibraryItem* item = InitItem(type, signal, parent, container_level);
   int year = 0;
 
   switch (type) {

src/library.h

@@ -132,20 +132,23 @@ class Library : public SimpleTreeModel<LibraryItem> {
   // for each parent item, restricting the songs returned to a particular
   // album or artist for example.
   void InitQuery(GroupBy type, LibraryQuery* q);
-  void FilterQuery(GroupBy type, LibraryItem* item,LibraryQuery* q);
+  void FilterQuery(GroupBy type, LibraryItem* item, LibraryQuery* q);
 
   // Items can be created either from a query that's been run to populate a
   // node, or by a spontaneous SongsDiscovered emission from the backend.
   LibraryItem* ItemFromQuery(GroupBy type, bool signal, bool create_divider,
-                             LibraryItem* parent, const LibraryQuery& q);
+                             LibraryItem* parent, const LibraryQuery& q,
+                             int container_level);
   LibraryItem* ItemFromSong(GroupBy type, bool signal, bool create_divider,
-                            LibraryItem* parent, const Song& s);
+                            LibraryItem* parent, const Song& s,
+                            int container_level);
 
   // The "Various Artists" node is an annoying special case.
   LibraryItem* CreateCompilationArtistNode(bool signal, LibraryItem* parent);
 
   // Helpers for ItemFromQuery and ItemFromSong
-  LibraryItem* InitItem(GroupBy type, bool signal, LibraryItem* parent);
+  LibraryItem* InitItem(GroupBy type, bool signal, LibraryItem* parent,
+                        int container_level);
   void FinishItem(GroupBy type, bool signal, bool create_divider,
                   LibraryItem* parent, LibraryItem* item);
 

src/librarybackend.cpp

@@ -32,7 +32,7 @@
 
 
 const char* LibraryBackend::kDatabaseName = "clementine.db";
-const int LibraryBackend::kSchemaVersion = 5;
+const int LibraryBackend::kSchemaVersion = 6;
 
 int (*LibraryBackend::_sqlite3_create_function) (
     sqlite3*, const char*, int, int, void*,
@@ -288,19 +288,36 @@ void LibraryBackend::UpdateCompilationsAsync() {
 void LibraryBackend::LoadDirectories() {
   QSqlDatabase db(Connect());
 
-  QSqlQuery q("SELECT ROWID, path"
+  QSqlQuery q("SELECT ROWID, path FROM directories", db);
-              " FROM directories", db);
   q.exec();
   if (CheckErrors(q.lastError())) return;
 
-  DirectoryList directories;
   while (q.next()) {
     Directory dir;
     dir.id = q.value(0).toInt();
     dir.path = q.value(1).toString();
-    directories << dir;
+
+    emit DirectoryDiscovered(dir, SubdirsInDirectory(dir.id, db));
   }
-  emit DirectoriesDiscovered(directories);
+}
+
+SubdirectoryList LibraryBackend::SubdirsInDirectory(int id, QSqlDatabase &db) {
+  QSqlQuery q("SELECT path, mtime FROM subdirectories"
+              " WHERE directory = :dir", db);
+  q.bindValue(":dir", id);
+  q.exec();
+  if (CheckErrors(q.lastError())) return SubdirectoryList();
+
+  SubdirectoryList subdirs;
+  while (q.next()) {
+    Subdirectory subdir;
+    subdir.directory_id = id;
+    subdir.path = q.value(0).toString();
+    subdir.mtime = q.value(1).toUInt();
+    subdirs << subdir;
+  }
+
+  return subdirs;
 }
 
 void LibraryBackend::UpdateTotalSongCount() {
@@ -327,7 +344,7 @@ void LibraryBackend::AddDirectory(const QString &path) {
   dir.path = path;
   dir.id = q.lastInsertId().toInt();
 
-  emit DirectoriesDiscovered(DirectoryList() << dir);
+  emit DirectoryDiscovered(dir, SubdirectoryList());
 }
 
 void LibraryBackend::RemoveDirectory(const Directory& dir) {
@@ -336,13 +353,23 @@ void LibraryBackend::RemoveDirectory(const Directory& dir) {
   // Remove songs first
   DeleteSongs(FindSongsInDirectory(dir.id));
 
-  // Now remove the directory
+  db.transaction();
-  QSqlQuery q("DELETE FROM directories WHERE ROWID = :id", db);
+
+  // Delete the subdirs that were in this directory
+  QSqlQuery q("DELETE FROM subdirectories WHERE directory = :id", db);
   q.bindValue(":id", dir.id);
   q.exec();
   if (CheckErrors(q.lastError())) return;
 
-  emit DirectoriesDeleted(DirectoryList() << dir);
+  // Now remove the directory itself
+  q = QSqlQuery("DELETE FROM directories WHERE ROWID = :id", db);
+  q.bindValue(":id", dir.id);
+  q.exec();
+  if (CheckErrors(q.lastError())) return;
+
+  emit DirectoryDeleted(dir);
+
+  db.commit();
 }
 
 SongList LibraryBackend::FindSongsInDirectory(int id) {
@@ -363,6 +390,38 @@ SongList LibraryBackend::FindSongsInDirectory(int id) {
   return ret;
 }
 
+void LibraryBackend::AddSubdirs(const SubdirectoryList& subdirs) {
+  QSqlDatabase db(Connect());
+  QSqlQuery q("INSERT INTO subdirectories (directory, path, mtime)"
+              " VALUES (:id, :path, :mtime)", db);
+
+  db.transaction();
+  foreach (const Subdirectory& subdir, subdirs) {
+    q.bindValue(":id", subdir.directory_id);
+    q.bindValue(":path", subdir.path);
+    q.bindValue(":mtime", subdir.mtime);
+    q.exec();
+    if (CheckErrors(q.lastError())) continue;
+  }
+  db.commit();
+}
+
+void LibraryBackend::UpdateSubdirMTimes(const SubdirectoryList& subdirs) {
+  QSqlDatabase db(Connect());
+  QSqlQuery q("UPDATE subdirectories SET mtime = :mtime"
+              " WHERE directory = :id AND path = :path", db);
+
+  db.transaction();
+  foreach (const Subdirectory& subdir, subdirs) {
+    q.bindValue(":mtime", subdir.mtime);
+    q.bindValue(":id", subdir.directory_id);
+    q.bindValue(":path", subdir.path);
+    q.exec();
+    CheckErrors(q.lastError());
+  }
+  db.commit();
+}
+
 void LibraryBackend::AddOrUpdateSongs(const SongList& songs) {
   QSqlDatabase db(Connect());
 

src/librarybackend.h

@@ -90,6 +90,8 @@ class LibraryBackendInterface : public QObject {
   virtual void AddOrUpdateSongs(const SongList& songs) = 0;
   virtual void UpdateMTimesOnly(const SongList& songs) = 0;
   virtual void DeleteSongs(const SongList& songs) = 0;
+  virtual void AddSubdirs(const SubdirectoryList& subdirs) = 0;
+  virtual void UpdateSubdirMTimes(const SubdirectoryList& subdirs) = 0;
   virtual void UpdateCompilations() = 0;
   virtual void UpdateManualAlbumArt(const QString& artist, const QString& album, const QString& art) = 0;
   virtual void ForceCompilation(const QString& artist, const QString& album, bool on) = 0;
@@ -97,8 +99,8 @@ class LibraryBackendInterface : public QObject {
  signals:
   void Error(const QString& message);
 
-  void DirectoriesDiscovered(const DirectoryList& directories);
+  void DirectoryDiscovered(const Directory& dir, const SubdirectoryList& subdirs);
-  void DirectoriesDeleted(const DirectoryList& directories);
+  void DirectoryDeleted(const Directory& dir);
 
   void SongsDiscovered(const SongList& songs);
   void SongsDeleted(const SongList& songs);
@@ -154,6 +156,8 @@ class LibraryBackend : public LibraryBackendInterface {
   void AddOrUpdateSongs(const SongList& songs);
   void UpdateMTimesOnly(const SongList& songs);
   void DeleteSongs(const SongList& songs);
+  void AddSubdirs(const SubdirectoryList& subdirs);
+  void UpdateSubdirMTimes(const SubdirectoryList& subdirs);
   void UpdateCompilations();
   void UpdateManualAlbumArt(const QString& artist, const QString& album, const QString& art);
   void ForceCompilation(const QString& artist, const QString& album, bool on);
@@ -178,6 +182,7 @@ class LibraryBackend : public LibraryBackendInterface {
                           const QString& album, int sampler);
   AlbumList GetAlbums(const QString& artist, bool compilation = false,
                       const QueryOptions& opt = QueryOptions());
+  SubdirectoryList SubdirsInDirectory(int id, QSqlDatabase& db);
 
  private:
   static const char* kDatabaseName;

src/librarydirectorymodel.cpp

@@ -29,28 +29,24 @@ void LibraryDirectoryModel::SetBackend(boost::shared_ptr<LibraryBackendInterface
 
   backend_ = backend;
 
-  connect(backend_.get(), SIGNAL(DirectoriesDiscovered(DirectoryList)), SLOT(DirectoriesDiscovered(DirectoryList)));
+  connect(backend_.get(), SIGNAL(DirectoryDiscovered(Directory, SubdirectoryList)), SLOT(DirectoryDiscovered(Directory)));
-  connect(backend_.get(), SIGNAL(DirectoriesDeleted(DirectoryList)), SLOT(DirectoriesDeleted(DirectoryList)));
+  connect(backend_.get(), SIGNAL(DirectoryDeleted(Directory)), SLOT(DirectoryDeleted(Directory)));
 
   emit BackendReady();
 }
 
-void LibraryDirectoryModel::DirectoriesDiscovered(const DirectoryList &directories) {
+void LibraryDirectoryModel::DirectoryDiscovered(const Directory &dir) {
-  foreach (const Directory& dir, directories) {
+  QStandardItem* item = new QStandardItem(dir.path);
-    QStandardItem* item = new QStandardItem(dir.path);
+  item->setData(dir.id, kIdRole);
-    item->setData(dir.id, kIdRole);
+  item->setIcon(dir_icon_);
-    item->setIcon(dir_icon_);
+  appendRow(item);
-    appendRow(item);
-  }
 }
 
-void LibraryDirectoryModel::DirectoriesDeleted(const DirectoryList &directories) {
+void LibraryDirectoryModel::DirectoryDeleted(const Directory &dir) {
-  foreach (const Directory& dir, directories) {
+  for (int i=0 ; i<rowCount() ; ++i) {
-    for (int i=0 ; i<rowCount() ; ++i) {
+    if (item(i, 0)->data(kIdRole).toInt() == dir.id) {
-      if (item(i, 0)->data(kIdRole).toInt() == dir.id) {
+      removeRow(i);
-        removeRow(i);
+      break;
-        break;
-      }
     }
   }
 }

src/librarydirectorymodel.h

@@ -44,8 +44,8 @@ class LibraryDirectoryModel : public QStandardItemModel {
 
  private slots:
   // To be called by the backend
-  void DirectoriesDiscovered(const DirectoryList& directories);
+  void DirectoryDiscovered(const Directory& directories);
-  void DirectoriesDeleted(const DirectoryList& directories);
+  void DirectoryDeleted(const Directory& directories);
 
  private:
   static const int kIdRole = Qt::UserRole + 1;

src/librarywatcher.cpp

@@ -28,109 +28,154 @@
 #include <taglib/fileref.h>
 #include <taglib/tag.h>
 
+QStringList LibraryWatcher::sValidImages;
+QStringList LibraryWatcher::sValidPlaylists;
+
 
 LibraryWatcher::LibraryWatcher(QObject* parent)
   : QObject(parent),
     stop_requested_(false),
-    fs_watcher_(new QFileSystemWatcher(this)),
     rescan_timer_(new QTimer(this)),
     total_watches_(0)
 {
   rescan_timer_->setInterval(1000);
   rescan_timer_->setSingleShot(true);
 
-  connect(fs_watcher_, SIGNAL(directoryChanged(QString)), SLOT(DirectoryChanged(QString)));
+  if (sValidImages.isEmpty()) {
+    sValidImages << "jpg" << "png" << "gif" << "jpeg";
+    sValidPlaylists << "m3u" << "pls";
+  }
+
+  //connect(fs_watcher_, SIGNAL(directoryChanged(QString)), SLOT(DirectoryChanged(QString)));
   connect(rescan_timer_, SIGNAL(timeout()), SLOT(RescanPathsNow()));
 }
 
-void LibraryWatcher::AddDirectories(const DirectoryList& directories) {
+LibraryWatcher::ScanTransaction::ScanTransaction(LibraryWatcher* watcher,
-  // Iterate through each directory to find a list of files that look like they
+                                                 int dir, bool incremental)
-  // could be music.
+  : dir_(dir),
+    incremental_(incremental),
+    watcher_(watcher),
+    cached_songs_dirty_(true)
+{
+  emit watcher_->ScanStarted();
+}
 
-  foreach (const Directory& dir, directories) {
+LibraryWatcher::ScanTransaction::~ScanTransaction() {
-    if (stop_requested_) return;
+  if (!new_songs.isEmpty())
-    paths_watched_[dir.path] = dir;
+    emit watcher_->NewOrUpdatedSongs(new_songs);
-    ScanDirectory(dir.path);
-    if (stop_requested_) return;
 
-    // Start monitoring this directory for more changes
+  if (!touched_songs.isEmpty())
-    fs_watcher_->addPath(dir.path);
+    emit watcher_->SongsMTimeUpdated(touched_songs);
-    ++total_watches_;
 
-    // And all the subdirectories
+  if (!deleted_songs.isEmpty())
-    QDirIterator it(dir.path,
+    emit watcher_->SongsDeleted(deleted_songs);
-                    QDir::NoDotAndDotDot | QDir::Dirs,
+
-                    QDirIterator::Subdirectories);
+  if (!new_subdirs.isEmpty())
-    while (it.hasNext()) {
+    emit watcher_->SubdirsDiscovered(new_subdirs);
-      QString subdir(it.next());
+
-      fs_watcher_->addPath(subdir);
+  if (!touched_subdirs.isEmpty())
-      paths_watched_[subdir] = dir;
+    emit watcher_->SubdirsMTimeUpdated(touched_subdirs);
-      #ifdef Q_OS_DARWIN
+
-      if (++total_watches_ > kMaxWatches) {
+  emit watcher_->ScanFinished();
-        qWarning() << "Trying to watch more files than we can manage";
+}
-        return;
+
-      }
+SongList LibraryWatcher::ScanTransaction::FindSongsInSubdirectory(const QString &path) {
-      #endif
+  if (cached_songs_dirty_) {
+    cached_songs_ = watcher_->backend_->FindSongsInDirectory(dir_);
+    cached_songs_dirty_ = false;
+  }
+
+  // TODO: Make this faster
+  SongList ret;
+  foreach (const Song& song, cached_songs_) {
+    if (song.filename().section('/', 0, -2) == path)
+      ret << song;
+  }
+  return ret;
+}
+
+void LibraryWatcher::AddDirectory(const Directory& dir, const SubdirectoryList& subdirs) {
+  DirData data;
+  data.dir = dir;
+  data.known_subdirs = subdirs;
+  data.watcher = new QFileSystemWatcher(this);
+  connect(data.watcher, SIGNAL(directoryChanged(QString)), SLOT(DirectoryChanged(QString)));
+  watched_dirs_[dir.id] = data;
+
+  if (subdirs.isEmpty()) {
+    // This is a new directory that we've never seen before.
+    // Scan it fully.
+    ScanTransaction transaction(this, dir.id, false);
+    ScanSubdirectory(dir.path, Subdirectory(), &transaction);
+    AddWatch(data.watcher, dir.path);
+  } else {
+    // We can do an incremental scan - looking at the mtimes of each
+    // subdirectory and only rescan if the directory has changed.
+    ScanTransaction transaction(this, dir.id, true);
+    foreach (const Subdirectory& subdir, subdirs) {
+      ScanSubdirectory(subdir.path, subdir, &transaction);
+      AddWatch(data.watcher, subdir.path);
     }
   }
 
-  qDebug() << "Updating compilations...";
+  backend_->UpdateCompilationsAsync();
-  backend_.get()->UpdateCompilationsAsync();
 }
 
-void LibraryWatcher::RemoveDirectories(const DirectoryList &directories) {
+bool LibraryWatcher::HasSeenSubdir(int id, const QString& path) const {
-  foreach (const Directory& dir, directories) {
+  foreach (const Subdirectory& subdir, watched_dirs_[id].known_subdirs) {
-    fs_watcher_->removePath(dir.path);
+    if (subdir.path == path)
-    paths_watched_.remove(dir.path);
+      return true;
-    paths_needing_rescan_.removeAll(dir.path);
-
-    // And all the subdirectories
-    QDirIterator it(dir.path,
-                    QDir::NoDotAndDotDot | QDir::Dirs,
-                    QDirIterator::Subdirectories);
-    while (it.hasNext()) {
-      QString subdir(it.next());
-      fs_watcher_->removePath(subdir);
-      paths_watched_.remove(subdir);
-    }
   }
+  return false;
 }
 
-void LibraryWatcher::ScanDirectory(const QString& path) {
+void LibraryWatcher::ScanSubdirectory(
-  const Directory& dir = paths_watched_[path];
+    const QString& path, const Subdirectory& subdir, ScanTransaction* t) {
-  qDebug() << "Scanning" << path;
+  QFileInfo path_info(path);
-  emit ScanStarted();
+  if (t->is_incremental() && subdir.mtime == path_info.lastModified().toTime_t()) {
+    // The directory hasn't changed since last time
+    return;
+  }
 
-  QStringList valid_images = QStringList() << "jpg" << "png" << "gif" << "jpeg";
-  QStringList valid_playlists = QStringList() << "m3u" << "pls";
-
-  // Map from canonical directory name to list of possible filenames for cover
-  // art
   QMap<QString, QStringList> album_art;
-
   QStringList files_on_disk;
-  QDirIterator it(dir.path,
+  SubdirectoryList my_new_subdirs;
-                  QDir::Files | QDir::NoDotAndDotDot | QDir::Readable,
+
-                  QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);
+  // First we "quickly" get a list of the files in the directory that we
+  // think might be music.  While we're here, we also look for new subdirectories
+  // and possible album artwork.
+  QDirIterator it(path, QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot);
   while (it.hasNext()) {
-    QString path(it.next());
+    QString child(it.next());
-    QString ext(ExtensionPart(path));
+    QFileInfo child_info(child);
-    QString dir(DirectoryPart(path));
 
-    if (valid_images.contains(ext))
+    if (child_info.isDir()) {
-      album_art[dir] << path;
+      if (!HasSeenSubdir(t->dir(), child)) {
-    else if (engine_->canDecode(QUrl::fromLocalFile(path)))
+        // We haven't seen this subdirectory before - add it to a list and
-      files_on_disk << path;
+        // later we'll tell the backend about it and scan it.
+        Subdirectory new_subdir;
+        new_subdir.directory_id = t->dir();
+        new_subdir.path = child;
+        new_subdir.mtime = child_info.lastModified().toTime_t();
+        my_new_subdirs << new_subdir;
+      }
+    } else {
+      QString ext_part(ExtensionPart(child));
+      QString dir_part(DirectoryPart(child));
 
-    if (stop_requested_) return;
+      if (sValidImages.contains(ext_part))
+        album_art[dir_part] << child;
+      else if (engine_->canDecode(QUrl::fromLocalFile(child)))
+        files_on_disk << child;
+    }
   }
 
+  if (stop_requested_) return;
+
   // Ask the database for a list of files in this directory
-  SongList songs_in_db = backend_->FindSongsInDirectory(dir.id);
+  SongList songs_in_db = t->FindSongsInSubdirectory(path);
 
   // Now compare the list from the database with the list of files on disk
-  SongList new_songs;
-  SongList touched_songs;
   foreach (const QString& file, files_on_disk) {
     if (stop_requested_) return;
 
@@ -161,7 +206,7 @@ void LibraryWatcher::ScanDirectory(const QString& path) {
 
         // It's changed - reread the metadata from the file
         Song song_on_disk;
-        song_on_disk.InitFromFile(file, dir.id);
+        song_on_disk.InitFromFile(file, t->dir());
         if (!song_on_disk.is_valid())
           continue;
         song_on_disk.set_id(matching_song.id());
@@ -170,17 +215,17 @@ void LibraryWatcher::ScanDirectory(const QString& path) {
         if (!matching_song.IsMetadataEqual(song_on_disk)) {
           qDebug() << file << "metadata changed";
           // Update the song in the DB
-          new_songs << song_on_disk;
+          t->new_songs << song_on_disk;
         } else {
           // Only the metadata changed
-          touched_songs << song_on_disk;
+          t->touched_songs << song_on_disk;
         }
       }
     } else {
       // The song is on disk but not in the DB
 
       Song song;
-      song.InitFromFile(file, dir.id);
+      song.InitFromFile(file, t->dir());
       if (!song.is_valid())
         continue;
       qDebug() << file << "created";
@@ -188,37 +233,57 @@ void LibraryWatcher::ScanDirectory(const QString& path) {
       // Choose an image for the song
       song.set_art_automatic(ImageForSong(file, album_art));
 
-      new_songs << song;
+      t->new_songs << song;
     }
   }
 
-  if (stop_requested_) return;
-
-  if (!new_songs.isEmpty())
-    emit NewOrUpdatedSongs(new_songs);
-
-  if (!touched_songs.isEmpty())
-    emit SongsMTimeUpdated(touched_songs);
-
   // Look for deleted songs
-  SongList deleted_songs;
   foreach (const Song& song, songs_in_db) {
     if (!files_on_disk.contains(song.filename())) {
       qDebug() << "Song deleted from disk:" << song.filename();
-      deleted_songs << song;
+      t->deleted_songs << song;
     }
   }
 
-  if (stop_requested_) return;
+  // Add this subdir to the new or touched list
+  Subdirectory updated_subdir;
+  updated_subdir.directory_id = t->dir();
+  updated_subdir.mtime = path_info.lastModified().toTime_t();
+  updated_subdir.path = path;
 
-  if (!deleted_songs.isEmpty())
+  if (subdir.directory_id == -1)
-    emit SongsDeleted(deleted_songs);
+    t->new_subdirs << updated_subdir;
+  else
+    t->touched_subdirs << updated_subdir;
 
-  qDebug() << "Finished scanning" << path;
+  // Recurse into the new subdirs that we found
-  emit ScanFinished();
+  foreach (const Subdirectory& my_new_subdir, my_new_subdirs) {
+    ScanSubdirectory(my_new_subdir.path, my_new_subdir, t);
+  }
+  t->new_subdirs << my_new_subdirs;
+}
+
+void LibraryWatcher::AddWatch(QFileSystemWatcher* w, const QString& path) {
+#ifdef Q_OS_DARWIN
+  if (++total_watches_ > kMaxWatches) {
+    qWarning() << "Trying to watch more files than we can manage";
+    return;
+  }
+#endif
+  w->addPath(path);
+}
+
+void LibraryWatcher::RemoveDirectory(const Directory& dir) {
+  if (watched_dirs_.contains(dir.id)) {
+    delete watched_dirs_[dir.id].watcher;
+  }
+
+  rescan_queue_.remove(dir.id);
+  watched_dirs_.remove(dir.id);
 }
 
 bool LibraryWatcher::FindSongByPath(const SongList& list, const QString& path, Song* out) {
+  // TODO: Make this faster
   foreach (const Song& song, list) {
     if (song.filename() == path) {
       *out = song;
@@ -228,24 +293,45 @@ bool LibraryWatcher::FindSongByPath(const SongList& list, const QString& path, S
   return false;
 }
 
-void LibraryWatcher::DirectoryChanged(const QString &path) {
+void LibraryWatcher::DirectoryChanged(const QString &subdir) {
-  qDebug() << path;
+  // Find what dir it was in
-  if (!paths_needing_rescan_.contains(path))
+  QFileSystemWatcher* watcher = qobject_cast<QFileSystemWatcher*>(sender());
-    paths_needing_rescan_ << path;
+  if (!watcher)
+    return;
+
+  Directory dir;
+  foreach (const DirData& info, watched_dirs_) {
+    if (info.watcher == watcher)
+      dir = info.dir;
+  }
+
+  qDebug() << "Subdir" << subdir << "changed under directory" << dir.path << "id" << dir.id;
+
+  // Queue the subdir for rescanning
+  if (!rescan_queue_[dir.id].contains(subdir))
+    rescan_queue_[dir.id] << subdir;
 
   rescan_timer_->start();
 }
 
 void LibraryWatcher::RescanPathsNow() {
-  foreach (const QString& path, paths_needing_rescan_) {
+  foreach (int dir, rescan_queue_.keys()) {
     if (stop_requested_) return;
-    ScanDirectory(path);
+    ScanTransaction transaction(this, dir, false);
+
+    foreach (const QString& path, rescan_queue_[dir]) {
+      if (stop_requested_) return;
+      Subdirectory subdir;
+      subdir.directory_id = dir;
+      subdir.mtime = 0;
+      subdir.path = path;
+      ScanSubdirectory(path, subdir, &transaction);
+    }
   }
 
-  paths_needing_rescan_.clear();
+  rescan_queue_.clear();
 
-  qDebug() << "Updating compilations...";
+  backend_->UpdateCompilationsAsync();
-  backend_.get()->UpdateCompilationsAsync();
 }
 
 QString LibraryWatcher::PickBestImage(const QStringList& images) {

src/librarywatcher.h

@@ -47,18 +47,58 @@ class LibraryWatcher : public QObject {
   void NewOrUpdatedSongs(const SongList& songs);
   void SongsMTimeUpdated(const SongList& songs);
   void SongsDeleted(const SongList& songs);
+  void SubdirsDiscovered(const SubdirectoryList& subdirs);
+  void SubdirsMTimeUpdated(const SubdirectoryList& subdirs);
 
   void ScanStarted();
   void ScanFinished();
 
  public slots:
-  void AddDirectories(const DirectoryList& directories);
+  void AddDirectory(const Directory& dir, const SubdirectoryList& subdirs);
-  void RemoveDirectories(const DirectoryList& directories);
+  void RemoveDirectory(const Directory& dir);
+
+ private:
+  // This class encapsulates a full or partial scan of a directory.
+  // Each directory has one or more subdirectories, and any number of
+  // subdirectories can be scanned during one transaction.  ScanSubdirectory()
+  // adds its results to the members of this transaction class, and they are
+  // "committed" through calls to the LibraryBackend in the transaction's dtor.
+  // The transaction also caches the list of songs in this directory according
+  // to the library.  Multiple calls to FindSongsInSubdirectory during one
+  // transaction will only result in one call to
+  // LibraryBackend::FindSongsInDirectory.
+  class ScanTransaction {
+   public:
+    ScanTransaction(LibraryWatcher* watcher, int dir, bool incremental);
+    ~ScanTransaction();
+
+    SongList FindSongsInSubdirectory(const QString& path);
+
+    int dir() const { return dir_; }
+    bool is_incremental() const { return incremental_; }
+
+    SongList deleted_songs;
+    SongList new_songs;
+    SongList touched_songs;
+    SubdirectoryList new_subdirs;
+    SubdirectoryList touched_subdirs;
+
+   private:
+    ScanTransaction(const ScanTransaction&) {}
+    ScanTransaction& operator =(const ScanTransaction&) { return *this; }
+
+    int dir_;
+    bool incremental_;
+    LibraryWatcher* watcher_;
+    SongList cached_songs_;
+    bool cached_songs_dirty_;
+  };
 
  private slots:
   void DirectoryChanged(const QString& path);
   void RescanPathsNow();
-  void ScanDirectory(const QString& path);
+  void ScanSubdirectory(const QString& path, const Subdirectory& subdir,
+                        ScanTransaction* t);
 
  private:
   static bool FindSongByPath(const SongList& list, const QString& path, Song* out);
@@ -66,20 +106,30 @@ class LibraryWatcher : public QObject {
   inline static QString DirectoryPart( const QString &fileName );
   static QString PickBestImage(const QStringList& images);
   static QString ImageForSong(const QString& path, QMap<QString, QStringList>& album_art);
+  void AddWatch(QFileSystemWatcher* w, const QString& path);
+  bool HasSeenSubdir(int id, const QString& path) const;
 
  private:
+  // One of these gets stored for each Directory we're watching
+  struct DirData {
+    Directory dir;
+    SubdirectoryList known_subdirs;
+    QFileSystemWatcher* watcher;
+  };
+
   EngineBase* engine_;
   boost::shared_ptr<LibraryBackendInterface> backend_;
   bool stop_requested_;
 
-  QFileSystemWatcher* fs_watcher_;
+  QMap<int, DirData> watched_dirs_;
   QTimer* rescan_timer_;
-
+  QMap<int, QStringList> rescan_queue_; // dir id -> list of subdirs to be scanned
-  QMap<QString, Directory> paths_watched_;
-  QStringList paths_needing_rescan_;
 
   int total_watches_;
 
+  static QStringList sValidImages;
+  static QStringList sValidPlaylists;
+
   #ifdef Q_OS_DARWIN
   static const int kMaxWatches = 100;
   #endif

src/main.cpp

@@ -69,7 +69,10 @@ int main(int argc, char *argv[]) {
   QCoreApplication::setOrganizationName("Clementine");
   QCoreApplication::setOrganizationDomain("davidsansome.com");
 
+  qRegisterMetaType<Directory>("Directory");
   qRegisterMetaType<DirectoryList>("DirectoryList");
+  qRegisterMetaType<Subdirectory>("Subdirectory");
+  qRegisterMetaType<SubdirectoryList>("SubdirectoryList");
   qRegisterMetaType<SongList>("SongList");
 
 

src/mainwindow.cpp

@@ -395,6 +395,7 @@ void MainWindow::QueueFiles(const QList<QUrl>& urls) {
 }
 
 void MainWindow::ReportError(const QString& message) {
+  // TODO: rate limiting
   QMessageBox::warning(this, "Error", message);
 }