diff --git a/app/src/androidTest/java/de/test/antennapod/storage/DBTasksTest.java b/app/src/androidTest/java/de/test/antennapod/storage/DBTasksTest.java index 090cd2213..63f76e5dd 100644 --- a/app/src/androidTest/java/de/test/antennapod/storage/DBTasksTest.java +++ b/app/src/androidTest/java/de/test/antennapod/storage/DBTasksTest.java @@ -29,6 +29,7 @@ import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; /** @@ -66,7 +67,7 @@ public class DBTasksTest { for (int i = 0; i < NUM_ITEMS; i++) { feed.getItems().add(new FeedItem(0, "item " + i, "id " + i, "link " + i, new Date(), FeedItem.UNPLAYED, feed)); } - Feed newFeed = DBTasks.updateFeed(context, feed)[0]; + Feed newFeed = DBTasks.updateFeed(context, feed, false); assertTrue(newFeed == feed); assertTrue(feed.getId() != 0); @@ -86,8 +87,8 @@ public class DBTasksTest { feed1.setItems(new ArrayList<>()); feed2.setItems(new ArrayList<>()); - Feed savedFeed1 = DBTasks.updateFeed(context, feed1)[0]; - Feed savedFeed2 = DBTasks.updateFeed(context, feed2)[0]; + Feed savedFeed1 = DBTasks.updateFeed(context, feed1, false); + Feed savedFeed2 = DBTasks.updateFeed(context, feed2, false); assertTrue(savedFeed1.getId() != savedFeed2.getId()); } @@ -122,7 +123,7 @@ public class DBTasksTest { feed.getItems().add(0, new FeedItem(0, "item " + i, "id " + i, "link " + i, new Date(i), FeedItem.UNPLAYED, feed)); } - final Feed newFeed = DBTasks.updateFeed(context, feed)[0]; + final Feed newFeed = DBTasks.updateFeed(context, feed, false); assertTrue(feed != newFeed); updatedFeedTest(newFeed, feedID, itemIDs, NUM_ITEMS_OLD, NUM_ITEMS_NEW); @@ -154,7 +155,7 @@ public class DBTasksTest { list.add(item); feed.setItems(list); - final Feed newFeed = DBTasks.updateFeed(context, feed)[0]; + final Feed newFeed = DBTasks.updateFeed(context, feed, false); assertTrue(feed != newFeed); final Feed feedFromDB = DBReader.getFeed(newFeed.getId()); @@ -162,6 +163,27 @@ public class DBTasksTest { assertTrue("state: " + feedItemFromDB.getState(), feedItemFromDB.isNew()); } + @Test + public void testUpdateFeedRemoveUnlistedItems() { + final Feed feed = new Feed("url", null, "title"); + feed.setItems(new ArrayList<>()); + for (int i = 0; i < 10; i++) { + feed.getItems().add(new FeedItem(0, "item " + i, "id " + i, "link " + i, new Date(i), FeedItem.PLAYED, feed)); + } + PodDBAdapter adapter = PodDBAdapter.getInstance(); + adapter.open(); + adapter.setCompleteFeed(feed); + adapter.close(); + + // delete some items + feed.getItems().subList(0, 2).clear(); + Feed newFeed = DBTasks.updateFeed(context, feed, true); + assertEquals(8, newFeed.getItems().size()); // 10 - 2 = 8 items + + Feed feedFromDB = DBReader.getFeed(newFeed.getId()); + assertEquals(8, feedFromDB.getItems().size()); // 10 - 2 = 8 items + } + private void updatedFeedTest(final Feed newFeed, long feedID, List itemIDs, final int NUM_ITEMS_OLD, final int NUM_ITEMS_NEW) { assertTrue(newFeed.getId() == feedID); assertTrue(newFeed.getItems().size() == NUM_ITEMS_NEW + NUM_ITEMS_OLD); diff --git a/app/src/androidTest/java/de/test/antennapod/storage/DBWriterTest.java b/app/src/androidTest/java/de/test/antennapod/storage/DBWriterTest.java index d82e366da..5d18619a7 100644 --- a/app/src/androidTest/java/de/test/antennapod/storage/DBWriterTest.java +++ b/app/src/androidTest/java/de/test/antennapod/storage/DBWriterTest.java @@ -422,6 +422,41 @@ public class DBWriterTest { adapter.close(); } + @Test + public void testDeleteFeedItems() throws Exception { + Feed feed = new Feed("url", null, "title"); + feed.setItems(new ArrayList<>()); + feed.setImageUrl("url"); + + // create items + for (int i = 0; i < 10; i++) { + FeedItem item = new FeedItem(0, "Item " + i, "Item" + i, "url", new Date(), FeedItem.PLAYED, feed); + feed.getItems().add(item); + } + + PodDBAdapter adapter = PodDBAdapter.getInstance(); + adapter.open(); + adapter.setCompleteFeed(feed); + adapter.close(); + + List itemsToDelete = feed.getItems().subList(0, 2); + DBWriter.deleteFeedItems(getInstrumentation().getTargetContext(), itemsToDelete).get(TIMEOUT, TimeUnit.SECONDS); + + adapter = PodDBAdapter.getInstance(); + adapter.open(); + for (int i = 0; i < feed.getItems().size(); i++) { + FeedItem feedItem = feed.getItems().get(i); + Cursor c = adapter.getFeedItemCursor(String.valueOf(feedItem.getId())); + if (i < 2) { + assertEquals(0, c.getCount()); + } else { + assertEquals(1, c.getCount()); + } + c.close(); + } + adapter.close(); + } + private FeedMedia playbackHistorySetup(Date playbackCompletionDate) { Feed feed = new Feed("url", null, "title"); feed.setItems(new ArrayList<>()); diff --git a/core/src/main/java/de/danoeh/antennapod/core/feed/LocalFeedUpdater.java b/core/src/main/java/de/danoeh/antennapod/core/feed/LocalFeedUpdater.java index 04d2afb0b..2d58b7b52 100644 --- a/core/src/main/java/de/danoeh/antennapod/core/feed/LocalFeedUpdater.java +++ b/core/src/main/java/de/danoeh/antennapod/core/feed/LocalFeedUpdater.java @@ -15,7 +15,10 @@ import de.danoeh.antennapod.core.util.DownloadError; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.UUID; public class LocalFeedUpdater { @@ -36,16 +39,20 @@ public class LocalFeedUpdater { feed.setItems(new ArrayList<>()); } //make sure it is the latest 'version' of this feed from the db (all items etc) - feed = DBTasks.updateFeed(context, feed)[0]; + feed = DBTasks.updateFeed(context, feed, false); + // list files in feed folder List mediaFiles = new ArrayList<>(); + Set mediaFileNames = new HashSet<>(); for (DocumentFile file : documentFolder.listFiles()) { String mime = file.getType(); if (mime != null && (mime.startsWith("audio/") || mime.startsWith("video/"))) { mediaFiles.add(file); + mediaFileNames.add(file.getName()); } } + // add new files to feed and update item data List newItems = feed.getItems(); for (DocumentFile f : mediaFiles) { FeedItem oldItem = feedContainsFile(feed, f.getName()); @@ -57,6 +64,15 @@ public class LocalFeedUpdater { } } + // remove feed items without corresponding file + Iterator it = newItems.iterator(); + while (it.hasNext()) { + FeedItem feedItem = it.next(); + if (!mediaFileNames.contains(feedItem.getLink())) { + it.remove(); + } + } + List iconLocations = Arrays.asList("folder.jpg", "Folder.jpg", "folder.png", "Folder.png"); for (String iconLocation : iconLocations) { DocumentFile image = documentFolder.findFile(iconLocation); @@ -66,7 +82,11 @@ public class LocalFeedUpdater { } } - DBTasks.updateFeed(context, feed); + // update items, delete items without existing file; + // only delete items if the folder contains at least one element to avoid accidentally + // deleting played state or position in case the folder is temporarily unavailable. + boolean removeUnlistedItems = (newItems.size() >= 1); + DBTasks.updateFeed(context, feed, removeUnlistedItems); } private static FeedItem feedContainsFile(Feed feed, String filename) { diff --git a/core/src/main/java/de/danoeh/antennapod/core/service/download/handler/FeedSyncTask.java b/core/src/main/java/de/danoeh/antennapod/core/service/download/handler/FeedSyncTask.java index 8be3d2980..483a2aa56 100644 --- a/core/src/main/java/de/danoeh/antennapod/core/service/download/handler/FeedSyncTask.java +++ b/core/src/main/java/de/danoeh/antennapod/core/service/download/handler/FeedSyncTask.java @@ -30,8 +30,7 @@ public class FeedSyncTask { return false; } - Feed[] savedFeeds = DBTasks.updateFeed(context, result.feed); - Feed savedFeed = savedFeeds[0]; + Feed savedFeed = DBTasks.updateFeed(context, result.feed, false); // If loadAllPages=true, check if another page is available and queue it for download final boolean loadAllPages = request.getArguments().getBoolean(DownloadRequester.REQUEST_ARG_LOAD_ALL_PAGES); final Feed feed = result.feed; diff --git a/core/src/main/java/de/danoeh/antennapod/core/storage/DBTasks.java b/core/src/main/java/de/danoeh/antennapod/core/storage/DBTasks.java index 323e34b6a..9359774e9 100644 --- a/core/src/main/java/de/danoeh/antennapod/core/storage/DBTasks.java +++ b/core/src/main/java/de/danoeh/antennapod/core/storage/DBTasks.java @@ -373,118 +373,133 @@ public final class DBTasks { *

* This method should NOT be executed on the GUI thread. * - * @param context Used for accessing the DB. - * @param newFeeds The new Feed objects. - * @return The updated Feeds from the database if it already existed, or the new Feed from the parameters otherwise. + * @param context Used for accessing the DB. + * @param newFeed The new Feed object. + * @param removeUnlistedItems The item list in the new Feed object is considered to be exhaustive. + * I.e. items are removed from the database if they are not in this item list. + * @return The updated Feed from the database if it already existed, or the new Feed from the parameters otherwise. */ - public static synchronized Feed[] updateFeed(final Context context, - final Feed... newFeeds) { - List newFeedsList = new ArrayList<>(); - List updatedFeedsList = new ArrayList<>(); - Feed[] resultFeeds = new Feed[newFeeds.length]; + public static synchronized Feed updateFeed(Context context, Feed newFeed, boolean removeUnlistedItems) { + Feed resultFeed; + List unlistedItems = new ArrayList<>(); + PodDBAdapter adapter = PodDBAdapter.getInstance(); adapter.open(); - for (int feedIdx = 0; feedIdx < newFeeds.length; feedIdx++) { + // Look up feed in the feedslist + final Feed savedFeed = searchFeedByIdentifyingValueOrID(adapter, newFeed); + if (savedFeed == null) { + Log.d(TAG, "Found no existing Feed with title " + + newFeed.getTitle() + ". Adding as new one."); - final Feed newFeed = newFeeds[feedIdx]; + // Add a new Feed + // all new feeds will have the most recent item marked as unplayed + FeedItem mostRecent = newFeed.getMostRecentItem(); + if (mostRecent != null) { + mostRecent.setNew(); + } - // Look up feed in the feedslist - final Feed savedFeed = searchFeedByIdentifyingValueOrID(adapter, - newFeed); - if (savedFeed == null) { - Log.d(TAG, "Found no existing Feed with title " - + newFeed.getTitle() + ". Adding as new one."); + resultFeed = newFeed; + } else { + Log.d(TAG, "Feed with title " + newFeed.getTitle() + + " already exists. Syncing new with existing one."); - // Add a new Feed - // all new feeds will have the most recent item marked as unplayed - FeedItem mostRecent = newFeed.getMostRecentItem(); - if (mostRecent != null) { - mostRecent.setNew(); + Collections.sort(newFeed.getItems(), new FeedItemPubdateComparator()); + + if (newFeed.getPageNr() == savedFeed.getPageNr()) { + if (savedFeed.compareWithOther(newFeed)) { + Log.d(TAG, "Feed has updated attribute values. Updating old feed's attributes"); + savedFeed.updateFromOther(newFeed); } - - newFeedsList.add(newFeed); - resultFeeds[feedIdx] = newFeed; } else { - Log.d(TAG, "Feed with title " + newFeed.getTitle() - + " already exists. Syncing new with existing one."); + Log.d(TAG, "New feed has a higher page number."); + savedFeed.setNextPageLink(newFeed.getNextPageLink()); + } + if (savedFeed.getPreferences().compareWithOther(newFeed.getPreferences())) { + Log.d(TAG, "Feed has updated preferences. Updating old feed's preferences"); + savedFeed.getPreferences().updateFromOther(newFeed.getPreferences()); + } - Collections.sort(newFeed.getItems(), new FeedItemPubdateComparator()); + // get the most recent date now, before we start changing the list + FeedItem priorMostRecent = savedFeed.getMostRecentItem(); + Date priorMostRecentDate = null; + if (priorMostRecent != null) { + priorMostRecentDate = priorMostRecent.getPubDate(); + } - if (newFeed.getPageNr() == savedFeed.getPageNr()) { - if (savedFeed.compareWithOther(newFeed)) { - Log.d(TAG, "Feed has updated attribute values. Updating old feed's attributes"); - savedFeed.updateFromOther(newFeed); + // Look for new or updated Items + for (int idx = 0; idx < newFeed.getItems().size(); idx++) { + final FeedItem item = newFeed.getItems().get(idx); + FeedItem oldItem = searchFeedItemByIdentifyingValue(savedFeed, item.getIdentifyingValue()); + if (oldItem == null) { + // item is new + item.setFeed(savedFeed); + item.setAutoDownload(savedFeed.getPreferences().getAutoDownload()); + + if (idx >= savedFeed.getItems().size()) { + savedFeed.getItems().add(item); + } else { + savedFeed.getItems().add(idx, item); + } + + // only mark the item new if it was published after or at the same time + // as the most recent item + // (if the most recent date is null then we can assume there are no items + // and this is the first, hence 'new') + if (priorMostRecentDate == null + || priorMostRecentDate.before(item.getPubDate()) + || priorMostRecentDate.equals(item.getPubDate())) { + Log.d(TAG, "Marking item published on " + item.getPubDate() + + " new, prior most recent date = " + priorMostRecentDate); + item.setNew(); } } else { - Log.d(TAG, "New feed has a higher page number."); - savedFeed.setNextPageLink(newFeed.getNextPageLink()); - } - if (savedFeed.getPreferences().compareWithOther(newFeed.getPreferences())) { - Log.d(TAG, "Feed has updated preferences. Updating old feed's preferences"); - savedFeed.getPreferences().updateFromOther(newFeed.getPreferences()); + oldItem.updateFromOther(item); } + } - // get the most recent date now, before we start changing the list - FeedItem priorMostRecent = savedFeed.getMostRecentItem(); - Date priorMostRecentDate = null; - if (priorMostRecent != null) { - priorMostRecentDate = priorMostRecent.getPubDate(); - } - - // Look for new or updated Items - for (int idx = 0; idx < newFeed.getItems().size(); idx++) { - final FeedItem item = newFeed.getItems().get(idx); - FeedItem oldItem = searchFeedItemByIdentifyingValue(savedFeed, - item.getIdentifyingValue()); - if (oldItem == null) { - // item is new - item.setFeed(savedFeed); - item.setAutoDownload(savedFeed.getPreferences().getAutoDownload()); - - if (idx >= savedFeed.getItems().size()) { - savedFeed.getItems().add(item); - } else { - savedFeed.getItems().add(idx, item); - } - - // only mark the item new if it was published after or at the same time - // as the most recent item - // (if the most recent date is null then we can assume there are no items - // and this is the first, hence 'new') - if (priorMostRecentDate == null - || priorMostRecentDate.before(item.getPubDate()) - || priorMostRecentDate.equals(item.getPubDate())) { - Log.d(TAG, "Marking item published on " + item.getPubDate() - + " new, prior most recent date = " + priorMostRecentDate); - item.setNew(); - } - } else { - oldItem.updateFromOther(item); + // identify items to be removed + if (removeUnlistedItems) { + Iterator it = savedFeed.getItems().iterator(); + while (it.hasNext()) { + FeedItem feedItem = it.next(); + if (searchFeedItemByIdentifyingValue(newFeed, feedItem.getIdentifyingValue()) == null) { + unlistedItems.add(feedItem); + it.remove(); } } - // update attributes - savedFeed.setLastUpdate(newFeed.getLastUpdate()); - savedFeed.setType(newFeed.getType()); - savedFeed.setLastUpdateFailed(false); - - updatedFeedsList.add(savedFeed); - resultFeeds[feedIdx] = savedFeed; } + + // update attributes + savedFeed.setLastUpdate(newFeed.getLastUpdate()); + savedFeed.setType(newFeed.getType()); + savedFeed.setLastUpdateFailed(false); + + resultFeed = savedFeed; } adapter.close(); try { - DBWriter.addNewFeed(context, newFeedsList.toArray(new Feed[0])).get(); - DBWriter.setCompleteFeed(updatedFeedsList.toArray(new Feed[0])).get(); + if (savedFeed == null) { + DBWriter.addNewFeed(context, newFeed).get(); + } else { + DBWriter.setCompleteFeed(savedFeed).get(); + } + if (removeUnlistedItems) { + DBWriter.deleteFeedItems(context, unlistedItems).get(); + } } catch (InterruptedException | ExecutionException e) { e.printStackTrace(); } - EventBus.getDefault().post(new FeedListUpdateEvent(updatedFeedsList)); + if (savedFeed != null) { + EventBus.getDefault().post(new FeedListUpdateEvent(savedFeed)); + } else { + EventBus.getDefault().post(new FeedListUpdateEvent(Collections.emptyList())); + } - return resultFeeds; + return resultFeed; } /** diff --git a/core/src/main/java/de/danoeh/antennapod/core/storage/DBWriter.java b/core/src/main/java/de/danoeh/antennapod/core/storage/DBWriter.java index e33b67719..9f917b44e 100644 --- a/core/src/main/java/de/danoeh/antennapod/core/storage/DBWriter.java +++ b/core/src/main/java/de/danoeh/antennapod/core/storage/DBWriter.java @@ -141,50 +141,76 @@ public class DBWriter { return dbExec.submit(() -> { DownloadRequester requester = DownloadRequester.getInstance(); final Feed feed = DBReader.getFeed(feedId); - - if (feed != null) { - // delete stored media files and mark them as read - List queue = DBReader.getQueue(); - List removed = new ArrayList<>(); - if (feed.getItems() == null) { - DBReader.getFeedItemList(feed); - } - - for (FeedItem item : feed.getItems()) { - if (queue.remove(item)) { - removed.add(item); - } - if (item.getMedia() != null && item.getMedia().isDownloaded()) { - deleteFeedMediaSynchronous(context, item.getMedia()); - } else if (item.getMedia() != null && requester.isDownloadingFile(item.getMedia())) { - requester.cancelDownload(context, item.getMedia()); - } - } - PodDBAdapter adapter = PodDBAdapter.getInstance(); - adapter.open(); - if (removed.size() > 0) { - adapter.setQueue(queue); - for (FeedItem item : removed) { - EventBus.getDefault().post(QueueEvent.irreversibleRemoved(item)); - } - } - adapter.removeFeed(feed); - adapter.close(); - - SyncService.enqueueFeedRemoved(context, feed.getDownload_url()); - EventBus.getDefault().post(new FeedListUpdateEvent(feed)); - - // we assume we also removed download log entries for the feed or its media files. - // especially important if download or refresh failed, as the user should not be able - // to retry these - EventBus.getDefault().post(DownloadLogEvent.listUpdated()); - - BackupManager backupManager = new BackupManager(context); - backupManager.dataChanged(); + if (feed == null) { + return; } + + // delete stored media files and mark them as read + if (feed.getItems() == null) { + DBReader.getFeedItemList(feed); + } + deleteFeedItemsSynchronous(context, feed.getItems()); + + // delete feed + PodDBAdapter adapter = PodDBAdapter.getInstance(); + adapter.open(); + adapter.removeFeed(feed); + adapter.close(); + + SyncService.enqueueFeedRemoved(context, feed.getDownload_url()); + EventBus.getDefault().post(new FeedListUpdateEvent(feed)); }); } + /** + * Remove the listed items and their FeedMedia entries. + * Deleting media also removes the download log entries. + */ + @NonNull + public static Future deleteFeedItems(@NonNull Context context, @NonNull List items) { + return dbExec.submit(() -> deleteFeedItemsSynchronous(context, items) ); + } + + /** + * Remove the listed items and their FeedMedia entries. + * Deleting media also removes the download log entries. + */ + private static void deleteFeedItemsSynchronous(@NonNull Context context, @NonNull List items) { + DownloadRequester requester = DownloadRequester.getInstance(); + List queue = DBReader.getQueue(); + List removedFromQueue = new ArrayList<>(); + for (FeedItem item : items) { + if (queue.remove(item)) { + removedFromQueue.add(item); + } + if (item.getMedia() != null && item.getMedia().isDownloaded()) { + deleteFeedMediaSynchronous(context, item.getMedia()); + } else if (item.getMedia() != null && requester.isDownloadingFile(item.getMedia())) { + requester.cancelDownload(context, item.getMedia()); + } + } + + PodDBAdapter adapter = PodDBAdapter.getInstance(); + adapter.open(); + if (!removedFromQueue.isEmpty()) { + adapter.setQueue(queue); + } + adapter.removeFeedItems(items); + adapter.close(); + + for (FeedItem item : removedFromQueue) { + EventBus.getDefault().post(QueueEvent.irreversibleRemoved(item)); + } + + // we assume we also removed download log entries for the feed or its media files. + // especially important if download or refresh failed, as the user should not be able + // to retry these + EventBus.getDefault().post(DownloadLogEvent.listUpdated()); + + BackupManager backupManager = new BackupManager(context); + backupManager.dataChanged(); + } + /** * Deletes the entire playback history. */ diff --git a/core/src/main/java/de/danoeh/antennapod/core/storage/PodDBAdapter.java b/core/src/main/java/de/danoeh/antennapod/core/storage/PodDBAdapter.java index e6d47b32a..a2247a3db 100644 --- a/core/src/main/java/de/danoeh/antennapod/core/storage/PodDBAdapter.java +++ b/core/src/main/java/de/danoeh/antennapod/core/storage/PodDBAdapter.java @@ -14,6 +14,7 @@ import android.database.sqlite.SQLiteOpenHelper; import android.text.TextUtils; import android.util.Log; +import androidx.annotation.NonNull; import androidx.annotation.Nullable; import org.apache.commons.io.FileUtils; @@ -851,6 +852,23 @@ public class PodDBAdapter { new String[]{String.valueOf(item.getId())}); } + /** + * Remove the listed items and their FeedMedia entries. + */ + public void removeFeedItems(@NonNull List items) { + try { + db.beginTransactionNonExclusive(); + for (FeedItem item : items) { + removeFeedItem(item); + } + db.setTransactionSuccessful(); + } catch (SQLException e) { + Log.e(TAG, Log.getStackTraceString(e)); + } finally { + db.endTransaction(); + } + } + /** * Remove a feed with all its FeedItems and Media entries. */