Castopod/AntennaPod
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Delete removed files in local feeds

Browse Source
This commit is contained in:
Herbert Reiter 2020-07-08 21:07:51 +02:00 committed by GitHub
parent b1ef9f424f
commit 984233d1d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 268 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
app/src/androidTest/java/de/test/antennapod/storage/DBTasksTest.java
View File

@ -29,6 +29,7 @@ import static java.util.Collections.singletonList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertTrue;
/**
@ -66,7 +67,7 @@ public class DBTasksTest {
for (int i = 0; i < NUM_ITEMS; i++) {
feed.getItems().add(new FeedItem(0, "item " + i, "id " + i, "link " + i, new Date(), FeedItem.UNPLAYED, feed));
}
Feed newFeed = DBTasks.updateFeed(context, feed)[0];
Feed newFeed = DBTasks.updateFeed(context, feed, false);
assertTrue(newFeed == feed);
assertTrue(feed.getId() != 0);
@ -86,8 +87,8 @@ public class DBTasksTest {
feed1.setItems(new ArrayList<>());
feed2.setItems(new ArrayList<>());
Feed savedFeed1 = DBTasks.updateFeed(context, feed1)[0];
Feed savedFeed2 = DBTasks.updateFeed(context, feed2)[0];
Feed savedFeed1 = DBTasks.updateFeed(context, feed1, false);
Feed savedFeed2 = DBTasks.updateFeed(context, feed2, false);
assertTrue(savedFeed1.getId() != savedFeed2.getId());
}
@ -122,7 +123,7 @@ public class DBTasksTest {
feed.getItems().add(0, new FeedItem(0, "item " + i, "id " + i, "link " + i, new Date(i), FeedItem.UNPLAYED, feed));
}
final Feed newFeed = DBTasks.updateFeed(context, feed)[0];
final Feed newFeed = DBTasks.updateFeed(context, feed, false);
assertTrue(feed != newFeed);
updatedFeedTest(newFeed, feedID, itemIDs, NUM_ITEMS_OLD, NUM_ITEMS_NEW);
@ -154,7 +155,7 @@ public class DBTasksTest {
list.add(item);
feed.setItems(list);
final Feed newFeed = DBTasks.updateFeed(context, feed)[0];
final Feed newFeed = DBTasks.updateFeed(context, feed, false);
assertTrue(feed != newFeed);
final Feed feedFromDB = DBReader.getFeed(newFeed.getId());
@ -162,6 +163,27 @@ public class DBTasksTest {
assertTrue("state: " + feedItemFromDB.getState(), feedItemFromDB.isNew());
}
@Test
public void testUpdateFeedRemoveUnlistedItems() {
final Feed feed = new Feed("url", null, "title");
feed.setItems(new ArrayList<>());
for (int i = 0; i < 10; i++) {
feed.getItems().add(new FeedItem(0, "item " + i, "id " + i, "link " + i, new Date(i), FeedItem.PLAYED, feed));
}
PodDBAdapter adapter = PodDBAdapter.getInstance();
adapter.open();
adapter.setCompleteFeed(feed);
adapter.close();
// delete some items
feed.getItems().subList(0, 2).clear();
Feed newFeed = DBTasks.updateFeed(context, feed, true);
assertEquals(8, newFeed.getItems().size()); // 10 - 2 = 8 items
Feed feedFromDB = DBReader.getFeed(newFeed.getId());
assertEquals(8, feedFromDB.getItems().size()); // 10 - 2 = 8 items
}
private void updatedFeedTest(final Feed newFeed, long feedID, List<Long> itemIDs, final int NUM_ITEMS_OLD, final int NUM_ITEMS_NEW) {
assertTrue(newFeed.getId() == feedID);
assertTrue(newFeed.getItems().size() == NUM_ITEMS_NEW + NUM_ITEMS_OLD);

35
app/src/androidTest/java/de/test/antennapod/storage/DBWriterTest.java
View File

@ -422,6 +422,41 @@ public class DBWriterTest {
adapter.close();
}
@Test
public void testDeleteFeedItems() throws Exception {
Feed feed = new Feed("url", null, "title");
feed.setItems(new ArrayList<>());
feed.setImageUrl("url");
// create items
for (int i = 0; i < 10; i++) {
FeedItem item = new FeedItem(0, "Item " + i, "Item" + i, "url", new Date(), FeedItem.PLAYED, feed);
feed.getItems().add(item);
}
PodDBAdapter adapter = PodDBAdapter.getInstance();
adapter.open();
adapter.setCompleteFeed(feed);
adapter.close();
List<FeedItem> itemsToDelete = feed.getItems().subList(0, 2);
DBWriter.deleteFeedItems(getInstrumentation().getTargetContext(), itemsToDelete).get(TIMEOUT, TimeUnit.SECONDS);
adapter = PodDBAdapter.getInstance();
adapter.open();
for (int i = 0; i < feed.getItems().size(); i++) {
FeedItem feedItem = feed.getItems().get(i);
Cursor c = adapter.getFeedItemCursor(String.valueOf(feedItem.getId()));
if (i < 2) {
assertEquals(0, c.getCount());
} else {
assertEquals(1, c.getCount());
}
c.close();
}
adapter.close();
}
private FeedMedia playbackHistorySetup(Date playbackCompletionDate) {
Feed feed = new Feed("url", null, "title");
feed.setItems(new ArrayList<>());

24
core/src/main/java/de/danoeh/antennapod/core/feed/LocalFeedUpdater.java
View File

@ -15,7 +15,10 @@ import de.danoeh.antennapod.core.util.DownloadError;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.UUID;
public class LocalFeedUpdater {
@ -36,16 +39,20 @@ public class LocalFeedUpdater {
feed.setItems(new ArrayList<>());
}
//make sure it is the latest 'version' of this feed from the db (all items etc)
feed = DBTasks.updateFeed(context, feed)[0];
feed = DBTasks.updateFeed(context, feed, false);
// list files in feed folder
List<DocumentFile> mediaFiles = new ArrayList<>();
Set<String> mediaFileNames = new HashSet<>();
for (DocumentFile file : documentFolder.listFiles()) {
String mime = file.getType();
if (mime != null && (mime.startsWith("audio/") || mime.startsWith("video/"))) {
mediaFiles.add(file);
mediaFileNames.add(file.getName());
}
}
// add new files to feed and update item data
List<FeedItem> newItems = feed.getItems();
for (DocumentFile f : mediaFiles) {
FeedItem oldItem = feedContainsFile(feed, f.getName());
@ -57,6 +64,15 @@ public class LocalFeedUpdater {
}
}
// remove feed items without corresponding file
Iterator<FeedItem> it = newItems.iterator();
while (it.hasNext()) {
FeedItem feedItem = it.next();
if (!mediaFileNames.contains(feedItem.getLink())) {
it.remove();
}
}
List<String> iconLocations = Arrays.asList("folder.jpg", "Folder.jpg", "folder.png", "Folder.png");
for (String iconLocation : iconLocations) {
DocumentFile image = documentFolder.findFile(iconLocation);
@ -66,7 +82,11 @@ public class LocalFeedUpdater {
}
}
DBTasks.updateFeed(context, feed);
// update items, delete items without existing file;
// only delete items if the folder contains at least one element to avoid accidentally
// deleting played state or position in case the folder is temporarily unavailable.
boolean removeUnlistedItems = (newItems.size() >= 1);
DBTasks.updateFeed(context, feed, removeUnlistedItems);
}
private static FeedItem feedContainsFile(Feed feed, String filename) {

3
core/src/main/java/de/danoeh/antennapod/core/service/download/handler/FeedSyncTask.java
View File

@ -30,8 +30,7 @@ public class FeedSyncTask {
return false;
}
Feed[] savedFeeds = DBTasks.updateFeed(context, result.feed);
Feed savedFeed = savedFeeds[0];
Feed savedFeed = DBTasks.updateFeed(context, result.feed, false);
// If loadAllPages=true, check if another page is available and queue it for download
final boolean loadAllPages = request.getArguments().getBoolean(DownloadRequester.REQUEST_ARG_LOAD_ALL_PAGES);
final Feed feed = result.feed;

183
core/src/main/java/de/danoeh/antennapod/core/storage/DBTasks.java
View File

@ -373,118 +373,133 @@ public final class DBTasks {
* <p/>
* This method should NOT be executed on the GUI thread.
*
* @param context Used for accessing the DB.
* @param newFeeds The new Feed objects.
* @return The updated Feeds from the database if it already existed, or the new Feed from the parameters otherwise.
* @param context Used for accessing the DB.
* @param newFeed The new Feed object.
* @param removeUnlistedItems The item list in the new Feed object is considered to be exhaustive.
* I.e. items are removed from the database if they are not in this item list.
* @return The updated Feed from the database if it already existed, or the new Feed from the parameters otherwise.
*/
public static synchronized Feed[] updateFeed(final Context context,
final Feed... newFeeds) {
List<Feed> newFeedsList = new ArrayList<>();
List<Feed> updatedFeedsList = new ArrayList<>();
Feed[] resultFeeds = new Feed[newFeeds.length];
public static synchronized Feed updateFeed(Context context, Feed newFeed, boolean removeUnlistedItems) {
Feed resultFeed;
List<FeedItem> unlistedItems = new ArrayList<>();
PodDBAdapter adapter = PodDBAdapter.getInstance();
adapter.open();
for (int feedIdx = 0; feedIdx < newFeeds.length; feedIdx++) {
// Look up feed in the feedslist
final Feed savedFeed = searchFeedByIdentifyingValueOrID(adapter, newFeed);
if (savedFeed == null) {
Log.d(TAG, "Found no existing Feed with title "
+ newFeed.getTitle() + ". Adding as new one.");
final Feed newFeed = newFeeds[feedIdx];
// Add a new Feed
// all new feeds will have the most recent item marked as unplayed
FeedItem mostRecent = newFeed.getMostRecentItem();
if (mostRecent != null) {
mostRecent.setNew();
}
// Look up feed in the feedslist
final Feed savedFeed = searchFeedByIdentifyingValueOrID(adapter,
newFeed);
if (savedFeed == null) {
Log.d(TAG, "Found no existing Feed with title "
+ newFeed.getTitle() + ". Adding as new one.");
resultFeed = newFeed;
} else {
Log.d(TAG, "Feed with title " + newFeed.getTitle()
+ " already exists. Syncing new with existing one.");
// Add a new Feed
// all new feeds will have the most recent item marked as unplayed
FeedItem mostRecent = newFeed.getMostRecentItem();
if (mostRecent != null) {
mostRecent.setNew();
Collections.sort(newFeed.getItems(), new FeedItemPubdateComparator());
if (newFeed.getPageNr() == savedFeed.getPageNr()) {
if (savedFeed.compareWithOther(newFeed)) {
Log.d(TAG, "Feed has updated attribute values. Updating old feed's attributes");
savedFeed.updateFromOther(newFeed);
}
newFeedsList.add(newFeed);
resultFeeds[feedIdx] = newFeed;
} else {
Log.d(TAG, "Feed with title " + newFeed.getTitle()
+ " already exists. Syncing new with existing one.");
Log.d(TAG, "New feed has a higher page number.");
savedFeed.setNextPageLink(newFeed.getNextPageLink());
}
if (savedFeed.getPreferences().compareWithOther(newFeed.getPreferences())) {
Log.d(TAG, "Feed has updated preferences. Updating old feed's preferences");
savedFeed.getPreferences().updateFromOther(newFeed.getPreferences());
}
Collections.sort(newFeed.getItems(), new FeedItemPubdateComparator());
// get the most recent date now, before we start changing the list
FeedItem priorMostRecent = savedFeed.getMostRecentItem();
Date priorMostRecentDate = null;
if (priorMostRecent != null) {
priorMostRecentDate = priorMostRecent.getPubDate();
}
if (newFeed.getPageNr() == savedFeed.getPageNr()) {
if (savedFeed.compareWithOther(newFeed)) {
Log.d(TAG, "Feed has updated attribute values. Updating old feed's attributes");
savedFeed.updateFromOther(newFeed);
// Look for new or updated Items
for (int idx = 0; idx < newFeed.getItems().size(); idx++) {
final FeedItem item = newFeed.getItems().get(idx);
FeedItem oldItem = searchFeedItemByIdentifyingValue(savedFeed, item.getIdentifyingValue());
if (oldItem == null) {
// item is new
item.setFeed(savedFeed);
item.setAutoDownload(savedFeed.getPreferences().getAutoDownload());
if (idx >= savedFeed.getItems().size()) {
savedFeed.getItems().add(item);
} else {
savedFeed.getItems().add(idx, item);
}
// only mark the item new if it was published after or at the same time
// as the most recent item
// (if the most recent date is null then we can assume there are no items
// and this is the first, hence 'new')
if (priorMostRecentDate == null
|| priorMostRecentDate.before(item.getPubDate())
|| priorMostRecentDate.equals(item.getPubDate())) {
Log.d(TAG, "Marking item published on " + item.getPubDate()
+ " new, prior most recent date = " + priorMostRecentDate);
item.setNew();
}
} else {
Log.d(TAG, "New feed has a higher page number.");
savedFeed.setNextPageLink(newFeed.getNextPageLink());
}
if (savedFeed.getPreferences().compareWithOther(newFeed.getPreferences())) {
Log.d(TAG, "Feed has updated preferences. Updating old feed's preferences");
savedFeed.getPreferences().updateFromOther(newFeed.getPreferences());
oldItem.updateFromOther(item);
}
}
// get the most recent date now, before we start changing the list
FeedItem priorMostRecent = savedFeed.getMostRecentItem();
Date priorMostRecentDate = null;
if (priorMostRecent != null) {
priorMostRecentDate = priorMostRecent.getPubDate();
}
// Look for new or updated Items
for (int idx = 0; idx < newFeed.getItems().size(); idx++) {
final FeedItem item = newFeed.getItems().get(idx);
FeedItem oldItem = searchFeedItemByIdentifyingValue(savedFeed,
item.getIdentifyingValue());
if (oldItem == null) {
// item is new
item.setFeed(savedFeed);
item.setAutoDownload(savedFeed.getPreferences().getAutoDownload());
if (idx >= savedFeed.getItems().size()) {
savedFeed.getItems().add(item);
} else {
savedFeed.getItems().add(idx, item);
}
// only mark the item new if it was published after or at the same time
// as the most recent item
// (if the most recent date is null then we can assume there are no items
// and this is the first, hence 'new')
if (priorMostRecentDate == null
|| priorMostRecentDate.before(item.getPubDate())
|| priorMostRecentDate.equals(item.getPubDate())) {
Log.d(TAG, "Marking item published on " + item.getPubDate()
+ " new, prior most recent date = " + priorMostRecentDate);
item.setNew();
}
} else {
oldItem.updateFromOther(item);
// identify items to be removed
if (removeUnlistedItems) {
Iterator<FeedItem> it = savedFeed.getItems().iterator();
while (it.hasNext()) {
FeedItem feedItem = it.next();
if (searchFeedItemByIdentifyingValue(newFeed, feedItem.getIdentifyingValue()) == null) {
unlistedItems.add(feedItem);
it.remove();
}
}
// update attributes
savedFeed.setLastUpdate(newFeed.getLastUpdate());
savedFeed.setType(newFeed.getType());
savedFeed.setLastUpdateFailed(false);
updatedFeedsList.add(savedFeed);
resultFeeds[feedIdx] = savedFeed;
}
// update attributes
savedFeed.setLastUpdate(newFeed.getLastUpdate());
savedFeed.setType(newFeed.getType());
savedFeed.setLastUpdateFailed(false);
resultFeed = savedFeed;
}
adapter.close();
try {
DBWriter.addNewFeed(context, newFeedsList.toArray(new Feed[0])).get();
DBWriter.setCompleteFeed(updatedFeedsList.toArray(new Feed[0])).get();
if (savedFeed == null) {
DBWriter.addNewFeed(context, newFeed).get();
} else {
DBWriter.setCompleteFeed(savedFeed).get();
}
if (removeUnlistedItems) {
DBWriter.deleteFeedItems(context, unlistedItems).get();
}
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
EventBus.getDefault().post(new FeedListUpdateEvent(updatedFeedsList));
if (savedFeed != null) {
EventBus.getDefault().post(new FeedListUpdateEvent(savedFeed));
} else {
EventBus.getDefault().post(new FeedListUpdateEvent(Collections.emptyList()));
}
return resultFeeds;
return resultFeed;
}
/**

106
core/src/main/java/de/danoeh/antennapod/core/storage/DBWriter.java
View File

@ -141,50 +141,76 @@ public class DBWriter {
return dbExec.submit(() -> {
DownloadRequester requester = DownloadRequester.getInstance();
final Feed feed = DBReader.getFeed(feedId);
if (feed != null) {
// delete stored media files and mark them as read
List<FeedItem> queue = DBReader.getQueue();
List<FeedItem> removed = new ArrayList<>();
if (feed.getItems() == null) {
DBReader.getFeedItemList(feed);
}
for (FeedItem item : feed.getItems()) {
if (queue.remove(item)) {
removed.add(item);
}
if (item.getMedia() != null && item.getMedia().isDownloaded()) {
deleteFeedMediaSynchronous(context, item.getMedia());
} else if (item.getMedia() != null && requester.isDownloadingFile(item.getMedia())) {
requester.cancelDownload(context, item.getMedia());
}
}
PodDBAdapter adapter = PodDBAdapter.getInstance();
adapter.open();
if (removed.size() > 0) {
adapter.setQueue(queue);
for (FeedItem item : removed) {
EventBus.getDefault().post(QueueEvent.irreversibleRemoved(item));
}
}
adapter.removeFeed(feed);
adapter.close();
SyncService.enqueueFeedRemoved(context, feed.getDownload_url());
EventBus.getDefault().post(new FeedListUpdateEvent(feed));
// we assume we also removed download log entries for the feed or its media files.
// especially important if download or refresh failed, as the user should not be able
// to retry these
EventBus.getDefault().post(DownloadLogEvent.listUpdated());
BackupManager backupManager = new BackupManager(context);
backupManager.dataChanged();
if (feed == null) {
return;
}
// delete stored media files and mark them as read
if (feed.getItems() == null) {
DBReader.getFeedItemList(feed);
}
deleteFeedItemsSynchronous(context, feed.getItems());
// delete feed
PodDBAdapter adapter = PodDBAdapter.getInstance();
adapter.open();
adapter.removeFeed(feed);
adapter.close();
SyncService.enqueueFeedRemoved(context, feed.getDownload_url());
EventBus.getDefault().post(new FeedListUpdateEvent(feed));
});
}
/**
* Remove the listed items and their FeedMedia entries.
* Deleting media also removes the download log entries.
*/
@NonNull
public static Future<?> deleteFeedItems(@NonNull Context context, @NonNull List<FeedItem> items) {
return dbExec.submit(() -> deleteFeedItemsSynchronous(context, items) );
}
/**
* Remove the listed items and their FeedMedia entries.
* Deleting media also removes the download log entries.
*/
private static void deleteFeedItemsSynchronous(@NonNull Context context, @NonNull List<FeedItem> items) {
DownloadRequester requester = DownloadRequester.getInstance();
List<FeedItem> queue = DBReader.getQueue();
List<FeedItem> removedFromQueue = new ArrayList<>();
for (FeedItem item : items) {
if (queue.remove(item)) {
removedFromQueue.add(item);
}
if (item.getMedia() != null && item.getMedia().isDownloaded()) {
deleteFeedMediaSynchronous(context, item.getMedia());
} else if (item.getMedia() != null && requester.isDownloadingFile(item.getMedia())) {
requester.cancelDownload(context, item.getMedia());
}
}
PodDBAdapter adapter = PodDBAdapter.getInstance();
adapter.open();
if (!removedFromQueue.isEmpty()) {
adapter.setQueue(queue);
}
adapter.removeFeedItems(items);
adapter.close();
for (FeedItem item : removedFromQueue) {
EventBus.getDefault().post(QueueEvent.irreversibleRemoved(item));
}
// we assume we also removed download log entries for the feed or its media files.
// especially important if download or refresh failed, as the user should not be able
// to retry these
EventBus.getDefault().post(DownloadLogEvent.listUpdated());
BackupManager backupManager = new BackupManager(context);
backupManager.dataChanged();
}
/**
* Deletes the entire playback history.
*/

18
core/src/main/java/de/danoeh/antennapod/core/storage/PodDBAdapter.java
View File

@ -14,6 +14,7 @@ import android.database.sqlite.SQLiteOpenHelper;
import android.text.TextUtils;
import android.util.Log;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.apache.commons.io.FileUtils;
@ -851,6 +852,23 @@ public class PodDBAdapter {
new String[]{String.valueOf(item.getId())});
}
/**
* Remove the listed items and their FeedMedia entries.
*/
public void removeFeedItems(@NonNull List<FeedItem> items) {
try {
db.beginTransactionNonExclusive();
for (FeedItem item : items) {
removeFeedItem(item);
}
db.setTransactionSuccessful();
} catch (SQLException e) {
Log.e(TAG, Log.getStackTraceString(e));
} finally {
db.endTransaction();
}
}
/**
* Remove a feed with all its FeedItems and Media entries.
*/