Delete removed files in local feeds

app/src/androidTest/java/de/test/antennapod/storage/DBTasksTest.java

@@ -29,6 +29,7 @@ import static java.util.Collections.singletonList;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -66,7 +67,7 @@ public class DBTasksTest {
         for (int i = 0; i < NUM_ITEMS; i++) {
             feed.getItems().add(new FeedItem(0, "item " + i, "id " + i, "link " + i, new Date(), FeedItem.UNPLAYED, feed));
         }
-        Feed newFeed = DBTasks.updateFeed(context, feed)[0];
+        Feed newFeed = DBTasks.updateFeed(context, feed, false);
 
         assertTrue(newFeed == feed);
         assertTrue(feed.getId() != 0);
@@ -86,8 +87,8 @@ public class DBTasksTest {
         feed1.setItems(new ArrayList<>());
         feed2.setItems(new ArrayList<>());
 
-        Feed savedFeed1 = DBTasks.updateFeed(context, feed1)[0];
+        Feed savedFeed1 = DBTasks.updateFeed(context, feed1, false);
-        Feed savedFeed2 = DBTasks.updateFeed(context, feed2)[0];
+        Feed savedFeed2 = DBTasks.updateFeed(context, feed2, false);
 
         assertTrue(savedFeed1.getId() != savedFeed2.getId());
     }
@@ -122,7 +123,7 @@ public class DBTasksTest {
             feed.getItems().add(0, new FeedItem(0, "item " + i, "id " + i, "link " + i, new Date(i), FeedItem.UNPLAYED, feed));
         }
 
-        final Feed newFeed = DBTasks.updateFeed(context, feed)[0];
+        final Feed newFeed = DBTasks.updateFeed(context, feed, false);
         assertTrue(feed != newFeed);
 
         updatedFeedTest(newFeed, feedID, itemIDs, NUM_ITEMS_OLD, NUM_ITEMS_NEW);
@@ -154,7 +155,7 @@ public class DBTasksTest {
         list.add(item);
         feed.setItems(list);
 
-        final Feed newFeed = DBTasks.updateFeed(context, feed)[0];
+        final Feed newFeed = DBTasks.updateFeed(context, feed, false);
         assertTrue(feed != newFeed);
 
         final Feed feedFromDB = DBReader.getFeed(newFeed.getId());
@@ -162,6 +163,27 @@ public class DBTasksTest {
         assertTrue("state: " + feedItemFromDB.getState(), feedItemFromDB.isNew());
     }
 
+    @Test
+    public void testUpdateFeedRemoveUnlistedItems() {
+        final Feed feed = new Feed("url", null, "title");
+        feed.setItems(new ArrayList<>());
+        for (int i = 0; i < 10; i++) {
+            feed.getItems().add(new FeedItem(0, "item " + i, "id " + i, "link " + i, new Date(i), FeedItem.PLAYED, feed));
+        }
+        PodDBAdapter adapter = PodDBAdapter.getInstance();
+        adapter.open();
+        adapter.setCompleteFeed(feed);
+        adapter.close();
+
+        // delete some items
+        feed.getItems().subList(0, 2).clear();
+        Feed newFeed = DBTasks.updateFeed(context, feed, true);
+        assertEquals(8, newFeed.getItems().size()); // 10 - 2 = 8 items
+
+        Feed feedFromDB = DBReader.getFeed(newFeed.getId());
+        assertEquals(8, feedFromDB.getItems().size()); // 10 - 2 = 8 items
+    }
+
     private void updatedFeedTest(final Feed newFeed, long feedID, List<Long> itemIDs, final int NUM_ITEMS_OLD, final int NUM_ITEMS_NEW) {
         assertTrue(newFeed.getId() == feedID);
         assertTrue(newFeed.getItems().size() == NUM_ITEMS_NEW + NUM_ITEMS_OLD);

app/src/androidTest/java/de/test/antennapod/storage/DBWriterTest.java

@@ -422,6 +422,41 @@ public class DBWriterTest {
         adapter.close();
     }
 
+    @Test
+    public void testDeleteFeedItems() throws Exception {
+        Feed feed = new Feed("url", null, "title");
+        feed.setItems(new ArrayList<>());
+        feed.setImageUrl("url");
+
+        // create items
+        for (int i = 0; i < 10; i++) {
+            FeedItem item = new FeedItem(0, "Item " + i, "Item" + i, "url", new Date(), FeedItem.PLAYED, feed);
+            feed.getItems().add(item);
+        }
+
+        PodDBAdapter adapter = PodDBAdapter.getInstance();
+        adapter.open();
+        adapter.setCompleteFeed(feed);
+        adapter.close();
+
+        List<FeedItem> itemsToDelete = feed.getItems().subList(0, 2);
+        DBWriter.deleteFeedItems(getInstrumentation().getTargetContext(), itemsToDelete).get(TIMEOUT, TimeUnit.SECONDS);
+
+        adapter = PodDBAdapter.getInstance();
+        adapter.open();
+        for (int i = 0; i < feed.getItems().size(); i++) {
+            FeedItem feedItem = feed.getItems().get(i);
+            Cursor c = adapter.getFeedItemCursor(String.valueOf(feedItem.getId()));
+            if (i < 2) {
+                assertEquals(0, c.getCount());
+            } else {
+                assertEquals(1, c.getCount());
+            }
+            c.close();
+        }
+        adapter.close();
+    }
+
     private FeedMedia playbackHistorySetup(Date playbackCompletionDate) {
         Feed feed = new Feed("url", null, "title");
         feed.setItems(new ArrayList<>());

core/src/main/java/de/danoeh/antennapod/core/feed/LocalFeedUpdater.java

@@ -15,7 +15,10 @@ import de.danoeh.antennapod.core.util.DownloadError;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 
 public class LocalFeedUpdater {
@@ -36,16 +39,20 @@ public class LocalFeedUpdater {
             feed.setItems(new ArrayList<>());
         }
         //make sure it is the latest 'version' of this feed from the db (all items etc)
-        feed = DBTasks.updateFeed(context, feed)[0];
+        feed = DBTasks.updateFeed(context, feed, false);
 
+        // list files in feed folder
         List<DocumentFile> mediaFiles = new ArrayList<>();
+        Set<String> mediaFileNames = new HashSet<>();
         for (DocumentFile file : documentFolder.listFiles()) {
             String mime = file.getType();
             if (mime != null && (mime.startsWith("audio/") || mime.startsWith("video/"))) {
                 mediaFiles.add(file);
+                mediaFileNames.add(file.getName());
             }
         }
 
+        // add new files to feed and update item data
         List<FeedItem> newItems = feed.getItems();
         for (DocumentFile f : mediaFiles) {
             FeedItem oldItem = feedContainsFile(feed, f.getName());
@@ -57,6 +64,15 @@ public class LocalFeedUpdater {
             }
         }
 
+        // remove feed items without corresponding file
+        Iterator<FeedItem> it = newItems.iterator();
+        while (it.hasNext()) {
+            FeedItem feedItem = it.next();
+            if (!mediaFileNames.contains(feedItem.getLink())) {
+                it.remove();
+            }
+        }
+
         List<String> iconLocations = Arrays.asList("folder.jpg", "Folder.jpg", "folder.png", "Folder.png");
         for (String iconLocation : iconLocations) {
             DocumentFile image = documentFolder.findFile(iconLocation);
@@ -66,7 +82,11 @@ public class LocalFeedUpdater {
             }
         }
 
-        DBTasks.updateFeed(context, feed);
+        // update items, delete items without existing file;
+        // only delete items if the folder contains at least one element to avoid accidentally
+        // deleting played state or position in case the folder is temporarily unavailable.
+        boolean removeUnlistedItems = (newItems.size() >= 1);
+        DBTasks.updateFeed(context, feed, removeUnlistedItems);
     }
 
     private static FeedItem feedContainsFile(Feed feed, String filename) {

core/src/main/java/de/danoeh/antennapod/core/service/download/handler/FeedSyncTask.java

@@ -30,8 +30,7 @@ public class FeedSyncTask {
             return false;
         }
 
-        Feed[] savedFeeds = DBTasks.updateFeed(context, result.feed);
+        Feed savedFeed = DBTasks.updateFeed(context, result.feed, false);
-        Feed savedFeed = savedFeeds[0];
         // If loadAllPages=true, check if another page is available and queue it for download
         final boolean loadAllPages = request.getArguments().getBoolean(DownloadRequester.REQUEST_ARG_LOAD_ALL_PAGES);
         final Feed feed = result.feed;

core/src/main/java/de/danoeh/antennapod/core/storage/DBTasks.java

@@ -373,118 +373,133 @@ public final class DBTasks {
      * <p/>
      * This method should NOT be executed on the GUI thread.
      *
-     * @param context  Used for accessing the DB.
+     * @param context Used for accessing the DB.
-     * @param newFeeds The new Feed objects.
+     * @param newFeed The new Feed object.
-     * @return The updated Feeds from the database if it already existed, or the new Feed from the parameters otherwise.
+     * @param removeUnlistedItems The item list in the new Feed object is considered to be exhaustive.
+     *                            I.e. items are removed from the database if they are not in this item list.
+     * @return The updated Feed from the database if it already existed, or the new Feed from the parameters otherwise.
      */
-    public static synchronized Feed[] updateFeed(final Context context,
+    public static synchronized Feed updateFeed(Context context, Feed newFeed, boolean removeUnlistedItems) {
-                                                 final Feed... newFeeds) {
+        Feed resultFeed;
-        List<Feed> newFeedsList = new ArrayList<>();
+        List<FeedItem> unlistedItems = new ArrayList<>();
-        List<Feed> updatedFeedsList = new ArrayList<>();
+
-        Feed[] resultFeeds = new Feed[newFeeds.length];
         PodDBAdapter adapter = PodDBAdapter.getInstance();
         adapter.open();
 
-        for (int feedIdx = 0; feedIdx < newFeeds.length; feedIdx++) {
+        // Look up feed in the feedslist
+        final Feed savedFeed = searchFeedByIdentifyingValueOrID(adapter, newFeed);
+        if (savedFeed == null) {
+            Log.d(TAG, "Found no existing Feed with title "
+                            + newFeed.getTitle() + ". Adding as new one.");
 
-            final Feed newFeed = newFeeds[feedIdx];
+            // Add a new Feed
+            // all new feeds will have the most recent item marked as unplayed
+            FeedItem mostRecent = newFeed.getMostRecentItem();
+            if (mostRecent != null) {
+                mostRecent.setNew();
+            }
 
-            // Look up feed in the feedslist
+            resultFeed = newFeed;
-            final Feed savedFeed = searchFeedByIdentifyingValueOrID(adapter,
+        } else {
-                    newFeed);
+            Log.d(TAG, "Feed with title " + newFeed.getTitle()
-            if (savedFeed == null) {
+                        + " already exists. Syncing new with existing one.");
-                Log.d(TAG, "Found no existing Feed with title "
-                                + newFeed.getTitle() + ". Adding as new one.");
 
-                // Add a new Feed
+            Collections.sort(newFeed.getItems(), new FeedItemPubdateComparator());
-                // all new feeds will have the most recent item marked as unplayed
+
-                FeedItem mostRecent = newFeed.getMostRecentItem();
+            if (newFeed.getPageNr() == savedFeed.getPageNr()) {
-                if (mostRecent != null) {
+                if (savedFeed.compareWithOther(newFeed)) {
-                    mostRecent.setNew();
+                    Log.d(TAG, "Feed has updated attribute values. Updating old feed's attributes");
+                    savedFeed.updateFromOther(newFeed);
                 }
-
-                newFeedsList.add(newFeed);
-                resultFeeds[feedIdx] = newFeed;
             } else {
-                Log.d(TAG, "Feed with title " + newFeed.getTitle()
+                Log.d(TAG, "New feed has a higher page number.");
-                            + " already exists. Syncing new with existing one.");
+                savedFeed.setNextPageLink(newFeed.getNextPageLink());
+            }
+            if (savedFeed.getPreferences().compareWithOther(newFeed.getPreferences())) {
+                Log.d(TAG, "Feed has updated preferences. Updating old feed's preferences");
+                savedFeed.getPreferences().updateFromOther(newFeed.getPreferences());
+            }
 
-                Collections.sort(newFeed.getItems(), new FeedItemPubdateComparator());
+            // get the most recent date now, before we start changing the list
+            FeedItem priorMostRecent = savedFeed.getMostRecentItem();
+            Date priorMostRecentDate = null;
+            if (priorMostRecent != null) {
+                priorMostRecentDate = priorMostRecent.getPubDate();
+            }
 
-                if (newFeed.getPageNr() == savedFeed.getPageNr()) {
+            // Look for new or updated Items
-                    if (savedFeed.compareWithOther(newFeed)) {
+            for (int idx = 0; idx < newFeed.getItems().size(); idx++) {
-                        Log.d(TAG, "Feed has updated attribute values. Updating old feed's attributes");
+                final FeedItem item = newFeed.getItems().get(idx);
-                        savedFeed.updateFromOther(newFeed);
+                FeedItem oldItem = searchFeedItemByIdentifyingValue(savedFeed, item.getIdentifyingValue());
+                if (oldItem == null) {
+                    // item is new
+                    item.setFeed(savedFeed);
+                    item.setAutoDownload(savedFeed.getPreferences().getAutoDownload());
+
+                    if (idx >= savedFeed.getItems().size()) {
+                        savedFeed.getItems().add(item);
+                    } else {
+                        savedFeed.getItems().add(idx, item);
+                    }
+
+                    // only mark the item new if it was published after or at the same time
+                    // as the most recent item
+                    // (if the most recent date is null then we can assume there are no items
+                    // and this is the first, hence 'new')
+                    if (priorMostRecentDate == null
+                            || priorMostRecentDate.before(item.getPubDate())
+                            || priorMostRecentDate.equals(item.getPubDate())) {
+                        Log.d(TAG, "Marking item published on " + item.getPubDate()
+                                + " new, prior most recent date = " + priorMostRecentDate);
+                        item.setNew();
                     }
                 } else {
-                    Log.d(TAG, "New feed has a higher page number.");
+                    oldItem.updateFromOther(item);
-                    savedFeed.setNextPageLink(newFeed.getNextPageLink());
-                }
-                if (savedFeed.getPreferences().compareWithOther(newFeed.getPreferences())) {
-                    Log.d(TAG, "Feed has updated preferences. Updating old feed's preferences");
-                    savedFeed.getPreferences().updateFromOther(newFeed.getPreferences());
                 }
+            }
 
-                // get the most recent date now, before we start changing the list
+            // identify items to be removed
-                FeedItem priorMostRecent = savedFeed.getMostRecentItem();
+            if (removeUnlistedItems) {
-                Date priorMostRecentDate = null;
+                Iterator<FeedItem> it = savedFeed.getItems().iterator();
-                if (priorMostRecent != null) {
+                while (it.hasNext()) {
-                    priorMostRecentDate = priorMostRecent.getPubDate();
+                    FeedItem feedItem = it.next();
-                }
+                    if (searchFeedItemByIdentifyingValue(newFeed, feedItem.getIdentifyingValue()) == null) {
-
+                        unlistedItems.add(feedItem);
-                // Look for new or updated Items
+                        it.remove();
-                for (int idx = 0; idx < newFeed.getItems().size(); idx++) {
-                    final FeedItem item = newFeed.getItems().get(idx);
-                    FeedItem oldItem = searchFeedItemByIdentifyingValue(savedFeed,
-                            item.getIdentifyingValue());
-                    if (oldItem == null) {
-                        // item is new
-                        item.setFeed(savedFeed);
-                        item.setAutoDownload(savedFeed.getPreferences().getAutoDownload());
-
-                        if (idx >= savedFeed.getItems().size()) {
-                            savedFeed.getItems().add(item);
-                        } else {
-                            savedFeed.getItems().add(idx, item);
-                        }
-
-                        // only mark the item new if it was published after or at the same time
-                        // as the most recent item
-                        // (if the most recent date is null then we can assume there are no items
-                        // and this is the first, hence 'new')
-                        if (priorMostRecentDate == null
-                                || priorMostRecentDate.before(item.getPubDate())
-                                || priorMostRecentDate.equals(item.getPubDate())) {
-                            Log.d(TAG, "Marking item published on " + item.getPubDate()
-                                    + " new, prior most recent date = " + priorMostRecentDate);
-                            item.setNew();
-                        }
-                    } else {
-                        oldItem.updateFromOther(item);
                     }
                 }
-                // update attributes
-                savedFeed.setLastUpdate(newFeed.getLastUpdate());
-                savedFeed.setType(newFeed.getType());
-                savedFeed.setLastUpdateFailed(false);
-
-                updatedFeedsList.add(savedFeed);
-                resultFeeds[feedIdx] = savedFeed;
             }
+
+            // update attributes
+            savedFeed.setLastUpdate(newFeed.getLastUpdate());
+            savedFeed.setType(newFeed.getType());
+            savedFeed.setLastUpdateFailed(false);
+
+            resultFeed = savedFeed;
         }
 
         adapter.close();
 
         try {
-            DBWriter.addNewFeed(context, newFeedsList.toArray(new Feed[0])).get();
+            if (savedFeed == null) {
-            DBWriter.setCompleteFeed(updatedFeedsList.toArray(new Feed[0])).get();
+                DBWriter.addNewFeed(context, newFeed).get();
+            } else {
+                DBWriter.setCompleteFeed(savedFeed).get();
+            }
+            if (removeUnlistedItems) {
+                DBWriter.deleteFeedItems(context, unlistedItems).get();
+            }
         } catch (InterruptedException | ExecutionException e) {
             e.printStackTrace();
         }
 
-        EventBus.getDefault().post(new FeedListUpdateEvent(updatedFeedsList));
+        if (savedFeed != null) {
+            EventBus.getDefault().post(new FeedListUpdateEvent(savedFeed));
+        } else {
+            EventBus.getDefault().post(new FeedListUpdateEvent(Collections.emptyList()));
+        }
 
-        return resultFeeds;
+        return resultFeed;
     }
 
     /**

core/src/main/java/de/danoeh/antennapod/core/storage/DBWriter.java

@@ -141,50 +141,76 @@ public class DBWriter {
         return dbExec.submit(() -> {
             DownloadRequester requester = DownloadRequester.getInstance();
             final Feed feed = DBReader.getFeed(feedId);
-
+            if (feed == null) {
-            if (feed != null) {
+                return;
-                // delete stored media files and mark them as read
-                List<FeedItem> queue = DBReader.getQueue();
-                List<FeedItem> removed = new ArrayList<>();
-                if (feed.getItems() == null) {
-                    DBReader.getFeedItemList(feed);
-                }
-
-                for (FeedItem item : feed.getItems()) {
-                    if (queue.remove(item)) {
-                        removed.add(item);
-                    }
-                    if (item.getMedia() != null && item.getMedia().isDownloaded()) {
-                        deleteFeedMediaSynchronous(context, item.getMedia());
-                    } else if (item.getMedia() != null && requester.isDownloadingFile(item.getMedia())) {
-                        requester.cancelDownload(context, item.getMedia());
-                    }
-                }
-                PodDBAdapter adapter = PodDBAdapter.getInstance();
-                adapter.open();
-                if (removed.size() > 0) {
-                    adapter.setQueue(queue);
-                    for (FeedItem item : removed) {
-                        EventBus.getDefault().post(QueueEvent.irreversibleRemoved(item));
-                    }
-                }
-                adapter.removeFeed(feed);
-                adapter.close();
-
-                SyncService.enqueueFeedRemoved(context, feed.getDownload_url());
-                EventBus.getDefault().post(new FeedListUpdateEvent(feed));
-
-                // we assume we also removed download log entries for the feed or its media files.
-                // especially important if download or refresh failed, as the user should not be able
-                // to retry these
-                EventBus.getDefault().post(DownloadLogEvent.listUpdated());
-
-                BackupManager backupManager = new BackupManager(context);
-                backupManager.dataChanged();
             }
+
+            // delete stored media files and mark them as read
+            if (feed.getItems() == null) {
+                DBReader.getFeedItemList(feed);
+            }
+            deleteFeedItemsSynchronous(context, feed.getItems());
+
+            // delete feed
+            PodDBAdapter adapter = PodDBAdapter.getInstance();
+            adapter.open();
+            adapter.removeFeed(feed);
+            adapter.close();
+
+            SyncService.enqueueFeedRemoved(context, feed.getDownload_url());
+            EventBus.getDefault().post(new FeedListUpdateEvent(feed));
         });
     }
 
+    /**
+     * Remove the listed items and their FeedMedia entries.
+     * Deleting media also removes the download log entries.
+     */
+    @NonNull
+    public static Future<?> deleteFeedItems(@NonNull Context context, @NonNull List<FeedItem> items) {
+        return dbExec.submit(() -> deleteFeedItemsSynchronous(context, items) );
+    }
+
+    /**
+     * Remove the listed items and their FeedMedia entries.
+     * Deleting media also removes the download log entries.
+     */
+    private static void deleteFeedItemsSynchronous(@NonNull Context context, @NonNull List<FeedItem> items) {
+        DownloadRequester requester = DownloadRequester.getInstance();
+        List<FeedItem> queue = DBReader.getQueue();
+        List<FeedItem> removedFromQueue = new ArrayList<>();
+        for (FeedItem item : items) {
+            if (queue.remove(item)) {
+                removedFromQueue.add(item);
+            }
+            if (item.getMedia() != null && item.getMedia().isDownloaded()) {
+                deleteFeedMediaSynchronous(context, item.getMedia());
+            } else if (item.getMedia() != null && requester.isDownloadingFile(item.getMedia())) {
+                requester.cancelDownload(context, item.getMedia());
+            }
+        }
+
+        PodDBAdapter adapter = PodDBAdapter.getInstance();
+        adapter.open();
+        if (!removedFromQueue.isEmpty()) {
+            adapter.setQueue(queue);
+        }
+        adapter.removeFeedItems(items);
+        adapter.close();
+
+        for (FeedItem item : removedFromQueue) {
+            EventBus.getDefault().post(QueueEvent.irreversibleRemoved(item));
+        }
+
+        // we assume we also removed download log entries for the feed or its media files.
+        // especially important if download or refresh failed, as the user should not be able
+        // to retry these
+        EventBus.getDefault().post(DownloadLogEvent.listUpdated());
+
+        BackupManager backupManager = new BackupManager(context);
+        backupManager.dataChanged();
+    }
+
     /**
      * Deletes the entire playback history.
      */

core/src/main/java/de/danoeh/antennapod/core/storage/PodDBAdapter.java

@@ -14,6 +14,7 @@ import android.database.sqlite.SQLiteOpenHelper;
 import android.text.TextUtils;
 import android.util.Log;
 
+import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 
 import org.apache.commons.io.FileUtils;
@@ -851,6 +852,23 @@ public class PodDBAdapter {
                 new String[]{String.valueOf(item.getId())});
     }
 
+    /**
+     * Remove the listed items and their FeedMedia entries.
+     */
+    public void removeFeedItems(@NonNull List<FeedItem> items) {
+        try {
+            db.beginTransactionNonExclusive();
+            for (FeedItem item : items) {
+                removeFeedItem(item);
+            }
+            db.setTransactionSuccessful();
+        } catch (SQLException e) {
+            Log.e(TAG, Log.getStackTraceString(e));
+        } finally {
+            db.endTransaction();
+        }
+    }
+
     /**
      * Remove a feed with all its FeedItems and Media entries.
      */