diff --git a/src/playlistview.cpp b/src/playlistview.cpp
index cb6021c2d..b90cfe255 100644
--- a/src/playlistview.cpp
+++ b/src/playlistview.cpp
@@ -41,6 +41,7 @@ PlaylistView::PlaylistView(QWidget *parent)
     glow_intensity_step_(0),
     inhibit_autoscroll_timer_(new QTimer(this)),
     inhibit_autoscroll_(false),
+    currently_autoscrolling_(false),
     row_height_(-1),
     currenttrack_play_(":currenttrack_play.png"),
     currenttrack_pause_(":currenttrack_pause.png")
@@ -344,8 +345,12 @@ void PlaylistView::mousePressEvent(QMouseEvent *event) {
 
 void PlaylistView::scrollContentsBy(int dx, int dy) {
   QTreeView::scrollContentsBy(dx, dy);
-  inhibit_autoscroll_ = true;
-  inhibit_autoscroll_timer_->start();
+
+  if (!currently_autoscrolling_) {
+    // We only want to do this if the scroll was initiated by the user
+    inhibit_autoscroll_ = true;
+    inhibit_autoscroll_timer_->start();
+  }
 }
 
 void PlaylistView::InhibitAutoscrollTimeout() {
@@ -365,5 +370,7 @@ void PlaylistView::MaybeAutoscroll() {
     return;
 
   QModelIndex current = playlist->index(playlist->current_index(), 0);
+  currently_autoscrolling_ = true;
   scrollTo(current, QAbstractItemView::PositionAtCenter);
+  currently_autoscrolling_ = false;
 }
diff --git a/src/playlistview.h b/src/playlistview.h
index a438f9e3f..35c8ab9ba 100644
--- a/src/playlistview.h
+++ b/src/playlistview.h
@@ -84,6 +84,7 @@ class PlaylistView : public QTreeView {
 
   QTimer* inhibit_autoscroll_timer_;
   bool inhibit_autoscroll_;
+  bool currently_autoscrolling_;
 
   int row_height_; // Used to invalidate the currenttrack_bar pixmaps
   QList<QPixmap> currenttrack_bar_left_;