Reworked the library watcher to be much more efficient - the backend now holds a list of subdirectories within each library directory, and only the mtimes of these are checked on startup.

2025-01-15 03:06:56 +01:00 · 2010-04-01 16:59:32 +00:00 · 2010-04-01 16:59:32 +00:00 · 2443ce6585
commit 2443ce6585
parent 1f2220ac63
13 changed files with 391 additions and 151 deletions
--- a/data/data.qrc
+++ b/data/data.qrc
@ -75,5 +75,6 @@
        <file>schema-5.sql</file>
        <file>osd_shadow_corner.png</file>
        <file>osd_shadow_edge.png</file>
+        <file>schema-6.sql</file>
    </qresource>
 </RCC>
--- a/data/schema-6.sql
+++ b/data/schema-6.sql
@ -0,0 +1,8 @@
+CREATE TABLE subdirectories (
+  directory INTEGER NOT NULL,
+  path TEXT NOT NULL,
+  mtime INTEGER NOT NULL
+);
+
+UPDATE schema_version SET version=6;
+
--- a/src/directory.h
+++ b/src/directory.h
@ -24,6 +24,8 @@
 class QSqlQuery;

 struct Directory {
+  Directory() : id(-1) {}
+
  QString path;
  int id;
 };
@ -32,4 +34,17 @@ Q_DECLARE_METATYPE(Directory);
 typedef QList<Directory> DirectoryList;
 Q_DECLARE_METATYPE(DirectoryList);

+
+struct Subdirectory {
+  Subdirectory() : directory_id(-1), mtime(0) {}
+
+  int directory_id;
+  QString path;
+  uint mtime;
+};
+Q_DECLARE_METATYPE(Subdirectory);
+
+typedef QList<Subdirectory> SubdirectoryList;
+Q_DECLARE_METATYPE(SubdirectoryList);
+
 #endif // DIRECTORY_H
--- a/src/library.cpp
+++ b/src/library.cpp
@ -108,14 +108,20 @@ void Library::Initialise() {
  // connect them together and start everything off.
  watcher_->Worker()->SetBackend(backend_->Worker());

-  connect(backend_->Worker().get(), SIGNAL(DirectoriesDiscovered(DirectoryList)),
-          watcher_->Worker().get(), SLOT(AddDirectories(DirectoryList)));
+  connect(backend_->Worker().get(), SIGNAL(DirectoryDiscovered(Directory,SubdirectoryList)),
+          watcher_->Worker().get(), SLOT(AddDirectory(Directory,SubdirectoryList)));
+  connect(backend_->Worker().get(), SIGNAL(DirectoryDeleted(Directory)),
+          watcher_->Worker().get(), SLOT(RemoveDirectory(Directory)));
  connect(watcher_->Worker().get(), SIGNAL(NewOrUpdatedSongs(SongList)),
          backend_->Worker().get(), SLOT(AddOrUpdateSongs(SongList)));
  connect(watcher_->Worker().get(), SIGNAL(SongsMTimeUpdated(SongList)),
          backend_->Worker().get(), SLOT(UpdateMTimesOnly(SongList)));
  connect(watcher_->Worker().get(), SIGNAL(SongsDeleted(SongList)),
          backend_->Worker().get(), SLOT(DeleteSongs(SongList)));
+  connect(watcher_->Worker().get(), SIGNAL(SubdirsDiscovered(SubdirectoryList)),
+          backend_->Worker().get(), SLOT(AddSubdirs(SubdirectoryList)));
+  connect(watcher_->Worker().get(), SIGNAL(SubdirsMTimeUpdated(SubdirectoryList)),
+          backend_->Worker().get(), SLOT(UpdateSubdirMTimes(SubdirectoryList)));

  // This will start the watcher checking for updates
  backend_->Worker()->LoadDirectoriesAsync();
@ -132,6 +138,10 @@ void Library::SongsDiscovered(const SongList& songs) {
    if (!query_options_.Matches(song))
      continue;

+    // Hey, we've already got that one!
+    if (song_nodes_.contains(song.id()))
+      continue;
+
    // Before we can add each song we need to make sure the required container
    // items already exist in the tree.  These depend on which "group by"
    // settings the user has on the library.  Eg. if the user grouped by
@ -170,8 +180,7 @@ void Library::SongsDiscovered(const SongList& songs) {
        if (!container_nodes_[i].contains(key)) {
          // Create the container
          container_nodes_[i][key] =
-              ItemFromSong(type, true, i == 0, container, song);
-          container_nodes_[i][key]->container_level = i;
+              ItemFromSong(type, true, i == 0, container, song, i);
        }
        container = container_nodes_[i][key];
      }
@ -188,7 +197,7 @@ void Library::SongsDiscovered(const SongList& songs) {
    // We've gone all the way down to the deepest level and everything was
    // already lazy loaded, so now we have to create the song in the container.
    song_nodes_[song.id()] =
-        ItemFromSong(GroupBy_None, true, false, container, song);
+        ItemFromSong(GroupBy_None, true, false, container, song, -1);
  }
 }

@ -429,8 +438,7 @@ void Library::LazyPopulate(LibraryItem* parent, bool signal) {
  while (q.Next()) {
    // Create the item - it will get inserted into the model here
    LibraryItem* item =
-        ItemFromQuery(child_type, signal, child_level == 0, parent, q);
-    item->container_level = child_level;
+        ItemFromQuery(child_type, signal, child_level == 0, parent, q, child_level);

    // Save a pointer to it for later
    if (child_type == GroupBy_None)
@ -526,7 +534,8 @@ void Library::FilterQuery(GroupBy type, LibraryItem* item, LibraryQuery* q) {
  }
 }

-LibraryItem* Library::InitItem(GroupBy type, bool signal, LibraryItem *parent) {
+LibraryItem* Library::InitItem(GroupBy type, bool signal, LibraryItem *parent,
+                               int container_level) {
  LibraryItem::Type item_type =
      type == GroupBy_None ? LibraryItem::Type_Song :
      LibraryItem::Type_Container;
@ -536,13 +545,16 @@ LibraryItem* Library::InitItem(GroupBy type, bool signal, LibraryItem *parent) {
                    parent->children.count(),parent->children.count());

  // Initialise the item depending on what type it's meant to be
-  return new LibraryItem(item_type, parent);
+  LibraryItem* item = new LibraryItem(item_type, parent);
+  item->container_level = container_level;
+  return item;
 }

 LibraryItem* Library::ItemFromQuery(GroupBy type,
                                    bool signal, bool create_divider,
-                                    LibraryItem* parent, const LibraryQuery& q) {
-  LibraryItem* item = InitItem(type, signal, parent);
+                                    LibraryItem* parent, const LibraryQuery& q,
+                                    int container_level) {
+  LibraryItem* item = InitItem(type, signal, parent, container_level);
  int year = 0;

  switch (type) {
@ -587,8 +599,9 @@ LibraryItem* Library::ItemFromQuery(GroupBy type,

 LibraryItem* Library::ItemFromSong(GroupBy type,
                                   bool signal, bool create_divider,
-                                   LibraryItem* parent, const Song& s) {
-  LibraryItem* item = InitItem(type, signal, parent);
+                                   LibraryItem* parent, const Song& s,
+                                   int container_level) {
+  LibraryItem* item = InitItem(type, signal, parent, container_level);
  int year = 0;

  switch (type) {
--- a/src/library.h
+++ b/src/library.h
@ -132,20 +132,23 @@ class Library : public SimpleTreeModel<LibraryItem> {
  // for each parent item, restricting the songs returned to a particular
  // album or artist for example.
  void InitQuery(GroupBy type, LibraryQuery* q);
-  void FilterQuery(GroupBy type, LibraryItem* item,LibraryQuery* q);
+  void FilterQuery(GroupBy type, LibraryItem* item, LibraryQuery* q);

  // Items can be created either from a query that's been run to populate a
  // node, or by a spontaneous SongsDiscovered emission from the backend.
  LibraryItem* ItemFromQuery(GroupBy type, bool signal, bool create_divider,
-                             LibraryItem* parent, const LibraryQuery& q);
+                             LibraryItem* parent, const LibraryQuery& q,
+                             int container_level);
  LibraryItem* ItemFromSong(GroupBy type, bool signal, bool create_divider,
-                            LibraryItem* parent, const Song& s);
+                            LibraryItem* parent, const Song& s,
+                            int container_level);

  // The "Various Artists" node is an annoying special case.
  LibraryItem* CreateCompilationArtistNode(bool signal, LibraryItem* parent);

  // Helpers for ItemFromQuery and ItemFromSong
-  LibraryItem* InitItem(GroupBy type, bool signal, LibraryItem* parent);
+  LibraryItem* InitItem(GroupBy type, bool signal, LibraryItem* parent,
+                        int container_level);
  void FinishItem(GroupBy type, bool signal, bool create_divider,
                  LibraryItem* parent, LibraryItem* item);

--- a/src/librarybackend.cpp
+++ b/src/librarybackend.cpp
@ -32,7 +32,7 @@


 const char* LibraryBackend::kDatabaseName = "clementine.db";
-const int LibraryBackend::kSchemaVersion = 5;
+const int LibraryBackend::kSchemaVersion = 6;

 int (*LibraryBackend::_sqlite3_create_function) (
    sqlite3*, const char*, int, int, void*,
@ -288,19 +288,36 @@ void LibraryBackend::UpdateCompilationsAsync() {
 void LibraryBackend::LoadDirectories() {
  QSqlDatabase db(Connect());

-  QSqlQuery q("SELECT ROWID, path"
-              " FROM directories", db);
+  QSqlQuery q("SELECT ROWID, path FROM directories", db);
  q.exec();
  if (CheckErrors(q.lastError())) return;

-  DirectoryList directories;
  while (q.next()) {
    Directory dir;
    dir.id = q.value(0).toInt();
    dir.path = q.value(1).toString();
-    directories << dir;
+
+    emit DirectoryDiscovered(dir, SubdirsInDirectory(dir.id, db));
  }
-  emit DirectoriesDiscovered(directories);
+}
+
+SubdirectoryList LibraryBackend::SubdirsInDirectory(int id, QSqlDatabase &db) {
+  QSqlQuery q("SELECT path, mtime FROM subdirectories"
+              " WHERE directory = :dir", db);
+  q.bindValue(":dir", id);
+  q.exec();
+  if (CheckErrors(q.lastError())) return SubdirectoryList();
+
+  SubdirectoryList subdirs;
+  while (q.next()) {
+    Subdirectory subdir;
+    subdir.directory_id = id;
+    subdir.path = q.value(0).toString();
+    subdir.mtime = q.value(1).toUInt();
+    subdirs << subdir;
+  }
+
+  return subdirs;
 }

 void LibraryBackend::UpdateTotalSongCount() {
@ -327,7 +344,7 @@ void LibraryBackend::AddDirectory(const QString &path) {
  dir.path = path;
  dir.id = q.lastInsertId().toInt();

-  emit DirectoriesDiscovered(DirectoryList() << dir);
+  emit DirectoryDiscovered(dir, SubdirectoryList());
 }

 void LibraryBackend::RemoveDirectory(const Directory& dir) {
@ -336,13 +353,23 @@ void LibraryBackend::RemoveDirectory(const Directory& dir) {
  // Remove songs first
  DeleteSongs(FindSongsInDirectory(dir.id));

-  // Now remove the directory
-  QSqlQuery q("DELETE FROM directories WHERE ROWID = :id", db);
+  db.transaction();
+
+  // Delete the subdirs that were in this directory
+  QSqlQuery q("DELETE FROM subdirectories WHERE directory = :id", db);
  q.bindValue(":id", dir.id);
  q.exec();
  if (CheckErrors(q.lastError())) return;

-  emit DirectoriesDeleted(DirectoryList() << dir);
+  // Now remove the directory itself
+  q = QSqlQuery("DELETE FROM directories WHERE ROWID = :id", db);
+  q.bindValue(":id", dir.id);
+  q.exec();
+  if (CheckErrors(q.lastError())) return;
+
+  emit DirectoryDeleted(dir);
+
+  db.commit();
 }

 SongList LibraryBackend::FindSongsInDirectory(int id) {
@ -363,6 +390,38 @@ SongList LibraryBackend::FindSongsInDirectory(int id) {
  return ret;
 }

+void LibraryBackend::AddSubdirs(const SubdirectoryList& subdirs) {
+  QSqlDatabase db(Connect());
+  QSqlQuery q("INSERT INTO subdirectories (directory, path, mtime)"
+              " VALUES (:id, :path, :mtime)", db);
+
+  db.transaction();
+  foreach (const Subdirectory& subdir, subdirs) {
+    q.bindValue(":id", subdir.directory_id);
+    q.bindValue(":path", subdir.path);
+    q.bindValue(":mtime", subdir.mtime);
+    q.exec();
+    if (CheckErrors(q.lastError())) continue;
+  }
+  db.commit();
+}
+
+void LibraryBackend::UpdateSubdirMTimes(const SubdirectoryList& subdirs) {
+  QSqlDatabase db(Connect());
+  QSqlQuery q("UPDATE subdirectories SET mtime = :mtime"
+              " WHERE directory = :id AND path = :path", db);
+
+  db.transaction();
+  foreach (const Subdirectory& subdir, subdirs) {
+    q.bindValue(":mtime", subdir.mtime);
+    q.bindValue(":id", subdir.directory_id);
+    q.bindValue(":path", subdir.path);
+    q.exec();
+    CheckErrors(q.lastError());
+  }
+  db.commit();
+}
+
 void LibraryBackend::AddOrUpdateSongs(const SongList& songs) {
  QSqlDatabase db(Connect());

--- a/src/librarybackend.h
+++ b/src/librarybackend.h
@ -90,6 +90,8 @@ class LibraryBackendInterface : public QObject {
  virtual void AddOrUpdateSongs(const SongList& songs) = 0;
  virtual void UpdateMTimesOnly(const SongList& songs) = 0;
  virtual void DeleteSongs(const SongList& songs) = 0;
+  virtual void AddSubdirs(const SubdirectoryList& subdirs) = 0;
+  virtual void UpdateSubdirMTimes(const SubdirectoryList& subdirs) = 0;
  virtual void UpdateCompilations() = 0;
  virtual void UpdateManualAlbumArt(const QString& artist, const QString& album, const QString& art) = 0;
  virtual void ForceCompilation(const QString& artist, const QString& album, bool on) = 0;
@ -97,8 +99,8 @@ class LibraryBackendInterface : public QObject {
 signals:
  void Error(const QString& message);

-  void DirectoriesDiscovered(const DirectoryList& directories);
-  void DirectoriesDeleted(const DirectoryList& directories);
+  void DirectoryDiscovered(const Directory& dir, const SubdirectoryList& subdirs);
+  void DirectoryDeleted(const Directory& dir);

  void SongsDiscovered(const SongList& songs);
  void SongsDeleted(const SongList& songs);
@ -154,6 +156,8 @@ class LibraryBackend : public LibraryBackendInterface {
  void AddOrUpdateSongs(const SongList& songs);
  void UpdateMTimesOnly(const SongList& songs);
  void DeleteSongs(const SongList& songs);
+  void AddSubdirs(const SubdirectoryList& subdirs);
+  void UpdateSubdirMTimes(const SubdirectoryList& subdirs);
  void UpdateCompilations();
  void UpdateManualAlbumArt(const QString& artist, const QString& album, const QString& art);
  void ForceCompilation(const QString& artist, const QString& album, bool on);
@ -178,6 +182,7 @@ class LibraryBackend : public LibraryBackendInterface {
                          const QString& album, int sampler);
  AlbumList GetAlbums(const QString& artist, bool compilation = false,
                      const QueryOptions& opt = QueryOptions());
+  SubdirectoryList SubdirsInDirectory(int id, QSqlDatabase& db);

 private:
  static const char* kDatabaseName;
--- a/src/librarydirectorymodel.cpp
+++ b/src/librarydirectorymodel.cpp
@ -29,28 +29,24 @@ void LibraryDirectoryModel::SetBackend(boost::shared_ptr<LibraryBackendInterface

  backend_ = backend;

-  connect(backend_.get(), SIGNAL(DirectoriesDiscovered(DirectoryList)), SLOT(DirectoriesDiscovered(DirectoryList)));
-  connect(backend_.get(), SIGNAL(DirectoriesDeleted(DirectoryList)), SLOT(DirectoriesDeleted(DirectoryList)));
+  connect(backend_.get(), SIGNAL(DirectoryDiscovered(Directory, SubdirectoryList)), SLOT(DirectoryDiscovered(Directory)));
+  connect(backend_.get(), SIGNAL(DirectoryDeleted(Directory)), SLOT(DirectoryDeleted(Directory)));

  emit BackendReady();
 }

-void LibraryDirectoryModel::DirectoriesDiscovered(const DirectoryList &directories) {
-  foreach (const Directory& dir, directories) {
-    QStandardItem* item = new QStandardItem(dir.path);
-    item->setData(dir.id, kIdRole);
-    item->setIcon(dir_icon_);
-    appendRow(item);
-  }
+void LibraryDirectoryModel::DirectoryDiscovered(const Directory &dir) {
+  QStandardItem* item = new QStandardItem(dir.path);
+  item->setData(dir.id, kIdRole);
+  item->setIcon(dir_icon_);
+  appendRow(item);
 }

-void LibraryDirectoryModel::DirectoriesDeleted(const DirectoryList &directories) {
-  foreach (const Directory& dir, directories) {
-    for (int i=0 ; i<rowCount() ; ++i) {
-      if (item(i, 0)->data(kIdRole).toInt() == dir.id) {
-        removeRow(i);
-        break;
-      }
+void LibraryDirectoryModel::DirectoryDeleted(const Directory &dir) {
+  for (int i=0 ; i<rowCount() ; ++i) {
+    if (item(i, 0)->data(kIdRole).toInt() == dir.id) {
+      removeRow(i);
+      break;
    }
  }
 }
--- a/src/librarydirectorymodel.h
+++ b/src/librarydirectorymodel.h
@ -44,8 +44,8 @@ class LibraryDirectoryModel : public QStandardItemModel {

 private slots:
  // To be called by the backend
-  void DirectoriesDiscovered(const DirectoryList& directories);
-  void DirectoriesDeleted(const DirectoryList& directories);
+  void DirectoryDiscovered(const Directory& directories);
+  void DirectoryDeleted(const Directory& directories);

 private:
  static const int kIdRole = Qt::UserRole + 1;
--- a/src/librarywatcher.cpp
+++ b/src/librarywatcher.cpp
@ -28,109 +28,154 @@
 #include <taglib/fileref.h>
 #include <taglib/tag.h>

+QStringList LibraryWatcher::sValidImages;
+QStringList LibraryWatcher::sValidPlaylists;
+

 LibraryWatcher::LibraryWatcher(QObject* parent)
  : QObject(parent),
    stop_requested_(false),
-    fs_watcher_(new QFileSystemWatcher(this)),
    rescan_timer_(new QTimer(this)),
    total_watches_(0)
 {
  rescan_timer_->setInterval(1000);
  rescan_timer_->setSingleShot(true);

-  connect(fs_watcher_, SIGNAL(directoryChanged(QString)), SLOT(DirectoryChanged(QString)));
+  if (sValidImages.isEmpty()) {
+    sValidImages << "jpg" << "png" << "gif" << "jpeg";
+    sValidPlaylists << "m3u" << "pls";
+  }
+
+  //connect(fs_watcher_, SIGNAL(directoryChanged(QString)), SLOT(DirectoryChanged(QString)));
  connect(rescan_timer_, SIGNAL(timeout()), SLOT(RescanPathsNow()));
 }

-void LibraryWatcher::AddDirectories(const DirectoryList& directories) {
-  // Iterate through each directory to find a list of files that look like they
-  // could be music.
+LibraryWatcher::ScanTransaction::ScanTransaction(LibraryWatcher* watcher,
+                                                 int dir, bool incremental)
+  : dir_(dir),
+    incremental_(incremental),
+    watcher_(watcher),
+    cached_songs_dirty_(true)
+{
+  emit watcher_->ScanStarted();
+}

-  foreach (const Directory& dir, directories) {
-    if (stop_requested_) return;
-    paths_watched_[dir.path] = dir;
-    ScanDirectory(dir.path);
-    if (stop_requested_) return;
+LibraryWatcher::ScanTransaction::~ScanTransaction() {
+  if (!new_songs.isEmpty())
+    emit watcher_->NewOrUpdatedSongs(new_songs);

-    // Start monitoring this directory for more changes
-    fs_watcher_->addPath(dir.path);
-    ++total_watches_;
+  if (!touched_songs.isEmpty())
+    emit watcher_->SongsMTimeUpdated(touched_songs);

-    // And all the subdirectories
-    QDirIterator it(dir.path,
-                    QDir::NoDotAndDotDot | QDir::Dirs,
-                    QDirIterator::Subdirectories);
-    while (it.hasNext()) {
-      QString subdir(it.next());
-      fs_watcher_->addPath(subdir);
-      paths_watched_[subdir] = dir;
-      #ifdef Q_OS_DARWIN
-      if (++total_watches_ > kMaxWatches) {
-        qWarning() << "Trying to watch more files than we can manage";
-        return;
-      }
-      #endif
+  if (!deleted_songs.isEmpty())
+    emit watcher_->SongsDeleted(deleted_songs);
+
+  if (!new_subdirs.isEmpty())
+    emit watcher_->SubdirsDiscovered(new_subdirs);
+
+  if (!touched_subdirs.isEmpty())
+    emit watcher_->SubdirsMTimeUpdated(touched_subdirs);
+
+  emit watcher_->ScanFinished();
+}
+
+SongList LibraryWatcher::ScanTransaction::FindSongsInSubdirectory(const QString &path) {
+  if (cached_songs_dirty_) {
+    cached_songs_ = watcher_->backend_->FindSongsInDirectory(dir_);
+    cached_songs_dirty_ = false;
+  }
+
+  // TODO: Make this faster
+  SongList ret;
+  foreach (const Song& song, cached_songs_) {
+    if (song.filename().section('/', 0, -2) == path)
+      ret << song;
+  }
+  return ret;
+}
+
+void LibraryWatcher::AddDirectory(const Directory& dir, const SubdirectoryList& subdirs) {
+  DirData data;
+  data.dir = dir;
+  data.known_subdirs = subdirs;
+  data.watcher = new QFileSystemWatcher(this);
+  connect(data.watcher, SIGNAL(directoryChanged(QString)), SLOT(DirectoryChanged(QString)));
+  watched_dirs_[dir.id] = data;
+
+  if (subdirs.isEmpty()) {
+    // This is a new directory that we've never seen before.
+    // Scan it fully.
+    ScanTransaction transaction(this, dir.id, false);
+    ScanSubdirectory(dir.path, Subdirectory(), &transaction);
+    AddWatch(data.watcher, dir.path);
+  } else {
+    // We can do an incremental scan - looking at the mtimes of each
+    // subdirectory and only rescan if the directory has changed.
+    ScanTransaction transaction(this, dir.id, true);
+    foreach (const Subdirectory& subdir, subdirs) {
+      ScanSubdirectory(subdir.path, subdir, &transaction);
+      AddWatch(data.watcher, subdir.path);
    }
  }

-  qDebug() << "Updating compilations...";
-  backend_.get()->UpdateCompilationsAsync();
+  backend_->UpdateCompilationsAsync();
 }

-void LibraryWatcher::RemoveDirectories(const DirectoryList &directories) {
-  foreach (const Directory& dir, directories) {
-    fs_watcher_->removePath(dir.path);
-    paths_watched_.remove(dir.path);
-    paths_needing_rescan_.removeAll(dir.path);
-
-    // And all the subdirectories
-    QDirIterator it(dir.path,
-                    QDir::NoDotAndDotDot | QDir::Dirs,
-                    QDirIterator::Subdirectories);
-    while (it.hasNext()) {
-      QString subdir(it.next());
-      fs_watcher_->removePath(subdir);
-      paths_watched_.remove(subdir);
-    }
+bool LibraryWatcher::HasSeenSubdir(int id, const QString& path) const {
+  foreach (const Subdirectory& subdir, watched_dirs_[id].known_subdirs) {
+    if (subdir.path == path)
+      return true;
  }
+  return false;
 }

-void LibraryWatcher::ScanDirectory(const QString& path) {
-  const Directory& dir = paths_watched_[path];
-  qDebug() << "Scanning" << path;
-  emit ScanStarted();
+void LibraryWatcher::ScanSubdirectory(
+    const QString& path, const Subdirectory& subdir, ScanTransaction* t) {
+  QFileInfo path_info(path);
+  if (t->is_incremental() && subdir.mtime == path_info.lastModified().toTime_t()) {
+    // The directory hasn't changed since last time
+    return;
+  }

-  QStringList valid_images = QStringList() << "jpg" << "png" << "gif" << "jpeg";
-  QStringList valid_playlists = QStringList() << "m3u" << "pls";
-
-  // Map from canonical directory name to list of possible filenames for cover
-  // art
  QMap<QString, QStringList> album_art;
-
  QStringList files_on_disk;
-  QDirIterator it(dir.path,
-                  QDir::Files | QDir::NoDotAndDotDot | QDir::Readable,
-                  QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);
+  SubdirectoryList my_new_subdirs;
+
+  // First we "quickly" get a list of the files in the directory that we
+  // think might be music.  While we're here, we also look for new subdirectories
+  // and possible album artwork.
+  QDirIterator it(path, QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot);
  while (it.hasNext()) {
-    QString path(it.next());
-    QString ext(ExtensionPart(path));
-    QString dir(DirectoryPart(path));
+    QString child(it.next());
+    QFileInfo child_info(child);

-    if (valid_images.contains(ext))
-      album_art[dir] << path;
-    else if (engine_->canDecode(QUrl::fromLocalFile(path)))
-      files_on_disk << path;
+    if (child_info.isDir()) {
+      if (!HasSeenSubdir(t->dir(), child)) {
+        // We haven't seen this subdirectory before - add it to a list and
+        // later we'll tell the backend about it and scan it.
+        Subdirectory new_subdir;
+        new_subdir.directory_id = t->dir();
+        new_subdir.path = child;
+        new_subdir.mtime = child_info.lastModified().toTime_t();
+        my_new_subdirs << new_subdir;
+      }
+    } else {
+      QString ext_part(ExtensionPart(child));
+      QString dir_part(DirectoryPart(child));

-    if (stop_requested_) return;
+      if (sValidImages.contains(ext_part))
+        album_art[dir_part] << child;
+      else if (engine_->canDecode(QUrl::fromLocalFile(child)))
+        files_on_disk << child;
+    }
  }

+  if (stop_requested_) return;
+
  // Ask the database for a list of files in this directory
-  SongList songs_in_db = backend_->FindSongsInDirectory(dir.id);
+  SongList songs_in_db = t->FindSongsInSubdirectory(path);

  // Now compare the list from the database with the list of files on disk
-  SongList new_songs;
-  SongList touched_songs;
  foreach (const QString& file, files_on_disk) {
    if (stop_requested_) return;

@ -161,7 +206,7 @@ void LibraryWatcher::ScanDirectory(const QString& path) {

        // It's changed - reread the metadata from the file
        Song song_on_disk;
-        song_on_disk.InitFromFile(file, dir.id);
+        song_on_disk.InitFromFile(file, t->dir());
        if (!song_on_disk.is_valid())
          continue;
        song_on_disk.set_id(matching_song.id());
@ -170,17 +215,17 @@ void LibraryWatcher::ScanDirectory(const QString& path) {
        if (!matching_song.IsMetadataEqual(song_on_disk)) {
          qDebug() << file << "metadata changed";
          // Update the song in the DB
-          new_songs << song_on_disk;
+          t->new_songs << song_on_disk;
        } else {
          // Only the metadata changed
-          touched_songs << song_on_disk;
+          t->touched_songs << song_on_disk;
        }
      }
    } else {
      // The song is on disk but not in the DB

      Song song;
-      song.InitFromFile(file, dir.id);
+      song.InitFromFile(file, t->dir());
      if (!song.is_valid())
        continue;
      qDebug() << file << "created";
@ -188,37 +233,57 @@ void LibraryWatcher::ScanDirectory(const QString& path) {
      // Choose an image for the song
      song.set_art_automatic(ImageForSong(file, album_art));

-      new_songs << song;
+      t->new_songs << song;
    }
  }

-  if (stop_requested_) return;
-
-  if (!new_songs.isEmpty())
-    emit NewOrUpdatedSongs(new_songs);
-
-  if (!touched_songs.isEmpty())
-    emit SongsMTimeUpdated(touched_songs);
-
  // Look for deleted songs
-  SongList deleted_songs;
  foreach (const Song& song, songs_in_db) {
    if (!files_on_disk.contains(song.filename())) {
      qDebug() << "Song deleted from disk:" << song.filename();
-      deleted_songs << song;
+      t->deleted_songs << song;
    }
  }

-  if (stop_requested_) return;
+  // Add this subdir to the new or touched list
+  Subdirectory updated_subdir;
+  updated_subdir.directory_id = t->dir();
+  updated_subdir.mtime = path_info.lastModified().toTime_t();
+  updated_subdir.path = path;

-  if (!deleted_songs.isEmpty())
-    emit SongsDeleted(deleted_songs);
+  if (subdir.directory_id == -1)
+    t->new_subdirs << updated_subdir;
+  else
+    t->touched_subdirs << updated_subdir;

-  qDebug() << "Finished scanning" << path;
-  emit ScanFinished();
+  // Recurse into the new subdirs that we found
+  foreach (const Subdirectory& my_new_subdir, my_new_subdirs) {
+    ScanSubdirectory(my_new_subdir.path, my_new_subdir, t);
+  }
+  t->new_subdirs << my_new_subdirs;
+}
+
+void LibraryWatcher::AddWatch(QFileSystemWatcher* w, const QString& path) {
+#ifdef Q_OS_DARWIN
+  if (++total_watches_ > kMaxWatches) {
+    qWarning() << "Trying to watch more files than we can manage";
+    return;
+  }
+#endif
+  w->addPath(path);
+}
+
+void LibraryWatcher::RemoveDirectory(const Directory& dir) {
+  if (watched_dirs_.contains(dir.id)) {
+    delete watched_dirs_[dir.id].watcher;
+  }
+
+  rescan_queue_.remove(dir.id);
+  watched_dirs_.remove(dir.id);
 }

 bool LibraryWatcher::FindSongByPath(const SongList& list, const QString& path, Song* out) {
+  // TODO: Make this faster
  foreach (const Song& song, list) {
    if (song.filename() == path) {
      *out = song;
@ -228,24 +293,45 @@ bool LibraryWatcher::FindSongByPath(const SongList& list, const QString& path, S
  return false;
 }

-void LibraryWatcher::DirectoryChanged(const QString &path) {
-  qDebug() << path;
-  if (!paths_needing_rescan_.contains(path))
-    paths_needing_rescan_ << path;
+void LibraryWatcher::DirectoryChanged(const QString &subdir) {
+  // Find what dir it was in
+  QFileSystemWatcher* watcher = qobject_cast<QFileSystemWatcher*>(sender());
+  if (!watcher)
+    return;
+
+  Directory dir;
+  foreach (const DirData& info, watched_dirs_) {
+    if (info.watcher == watcher)
+      dir = info.dir;
+  }
+
+  qDebug() << "Subdir" << subdir << "changed under directory" << dir.path << "id" << dir.id;
+
+  // Queue the subdir for rescanning
+  if (!rescan_queue_[dir.id].contains(subdir))
+    rescan_queue_[dir.id] << subdir;

  rescan_timer_->start();
 }

 void LibraryWatcher::RescanPathsNow() {
-  foreach (const QString& path, paths_needing_rescan_) {
+  foreach (int dir, rescan_queue_.keys()) {
    if (stop_requested_) return;
-    ScanDirectory(path);
+    ScanTransaction transaction(this, dir, false);
+
+    foreach (const QString& path, rescan_queue_[dir]) {
+      if (stop_requested_) return;
+      Subdirectory subdir;
+      subdir.directory_id = dir;
+      subdir.mtime = 0;
+      subdir.path = path;
+      ScanSubdirectory(path, subdir, &transaction);
+    }
  }

-  paths_needing_rescan_.clear();
+  rescan_queue_.clear();

-  qDebug() << "Updating compilations...";
-  backend_.get()->UpdateCompilationsAsync();
+  backend_->UpdateCompilationsAsync();
 }

 QString LibraryWatcher::PickBestImage(const QStringList& images) {
--- a/src/librarywatcher.h
+++ b/src/librarywatcher.h
@ -47,18 +47,58 @@ class LibraryWatcher : public QObject {
  void NewOrUpdatedSongs(const SongList& songs);
  void SongsMTimeUpdated(const SongList& songs);
  void SongsDeleted(const SongList& songs);
+  void SubdirsDiscovered(const SubdirectoryList& subdirs);
+  void SubdirsMTimeUpdated(const SubdirectoryList& subdirs);

  void ScanStarted();
  void ScanFinished();

 public slots:
-  void AddDirectories(const DirectoryList& directories);
-  void RemoveDirectories(const DirectoryList& directories);
+  void AddDirectory(const Directory& dir, const SubdirectoryList& subdirs);
+  void RemoveDirectory(const Directory& dir);
+
+ private:
+  // This class encapsulates a full or partial scan of a directory.
+  // Each directory has one or more subdirectories, and any number of
+  // subdirectories can be scanned during one transaction.  ScanSubdirectory()
+  // adds its results to the members of this transaction class, and they are
+  // "committed" through calls to the LibraryBackend in the transaction's dtor.
+  // The transaction also caches the list of songs in this directory according
+  // to the library.  Multiple calls to FindSongsInSubdirectory during one
+  // transaction will only result in one call to
+  // LibraryBackend::FindSongsInDirectory.
+  class ScanTransaction {
+   public:
+    ScanTransaction(LibraryWatcher* watcher, int dir, bool incremental);
+    ~ScanTransaction();
+
+    SongList FindSongsInSubdirectory(const QString& path);
+
+    int dir() const { return dir_; }
+    bool is_incremental() const { return incremental_; }
+
+    SongList deleted_songs;
+    SongList new_songs;
+    SongList touched_songs;
+    SubdirectoryList new_subdirs;
+    SubdirectoryList touched_subdirs;
+
+   private:
+    ScanTransaction(const ScanTransaction&) {}
+    ScanTransaction& operator =(const ScanTransaction&) { return *this; }
+
+    int dir_;
+    bool incremental_;
+    LibraryWatcher* watcher_;
+    SongList cached_songs_;
+    bool cached_songs_dirty_;
+  };

 private slots:
  void DirectoryChanged(const QString& path);
  void RescanPathsNow();
-  void ScanDirectory(const QString& path);
+  void ScanSubdirectory(const QString& path, const Subdirectory& subdir,
+                        ScanTransaction* t);

 private:
  static bool FindSongByPath(const SongList& list, const QString& path, Song* out);
@ -66,20 +106,30 @@ class LibraryWatcher : public QObject {
  inline static QString DirectoryPart( const QString &fileName );
  static QString PickBestImage(const QStringList& images);
  static QString ImageForSong(const QString& path, QMap<QString, QStringList>& album_art);
+  void AddWatch(QFileSystemWatcher* w, const QString& path);
+  bool HasSeenSubdir(int id, const QString& path) const;

 private:
+  // One of these gets stored for each Directory we're watching
+  struct DirData {
+    Directory dir;
+    SubdirectoryList known_subdirs;
+    QFileSystemWatcher* watcher;
+  };
+
  EngineBase* engine_;
  boost::shared_ptr<LibraryBackendInterface> backend_;
  bool stop_requested_;

-  QFileSystemWatcher* fs_watcher_;
+  QMap<int, DirData> watched_dirs_;
  QTimer* rescan_timer_;
-
-  QMap<QString, Directory> paths_watched_;
-  QStringList paths_needing_rescan_;
+  QMap<int, QStringList> rescan_queue_; // dir id -> list of subdirs to be scanned

  int total_watches_;

+  static QStringList sValidImages;
+  static QStringList sValidPlaylists;
+
  #ifdef Q_OS_DARWIN
  static const int kMaxWatches = 100;
  #endif
--- a/src/main.cpp
+++ b/src/main.cpp
@ -69,7 +69,10 @@ int main(int argc, char *argv[]) {
  QCoreApplication::setOrganizationName("Clementine");
  QCoreApplication::setOrganizationDomain("davidsansome.com");

+  qRegisterMetaType<Directory>("Directory");
  qRegisterMetaType<DirectoryList>("DirectoryList");
+  qRegisterMetaType<Subdirectory>("Subdirectory");
+  qRegisterMetaType<SubdirectoryList>("SubdirectoryList");
  qRegisterMetaType<SongList>("SongList");


--- a/src/mainwindow.cpp
+++ b/src/mainwindow.cpp
@ -395,6 +395,7 @@ void MainWindow::QueueFiles(const QList<QUrl>& urls) {
 }

 void MainWindow::ReportError(const QString& message) {
+  // TODO: rate limiting
  QMessageBox::warning(this, "Error", message);
 }