Improve sync speed

Put feed and folder insertion/update logic in transactions. Get only the feed id for an item insertion instead of getting the full feed

app/src/main/java/com/readrops/app/database/dao/AccountDao.java

@@ -2,42 +2,30 @@ package com.readrops.app.database.dao;
 
 import androidx.lifecycle.LiveData;
 import androidx.room.Dao;
-import androidx.room.Delete;
-import androidx.room.Insert;
 import androidx.room.Query;
-import androidx.room.Update;
 
 import com.readrops.app.database.entities.Account;
 
 import java.util.List;
 
 @Dao
-public interface AccountDao {
+public abstract class AccountDao implements BaseDao<Account> {
 
     @Query("Select * from Account")
-    LiveData<List<Account>> selectAll();
+    public abstract LiveData<List<Account>> selectAll();
-
-    @Insert
-    long insert(Account account);
-
-    @Update
-    void update(Account account);
-
-    @Delete
-    void delete(Account account);
 
     @Query("Update Account set last_modified = :lastModified Where id = :accountId")
-    void updateLastModified(int accountId, long lastModified);
+    public abstract void updateLastModified(int accountId, long lastModified);
 
     @Query("Update Account set current_account = 0 Where id Not In (:accountId)")
-    void deselectOldCurrentAccount(int accountId);
+    public abstract void deselectOldCurrentAccount(int accountId);
 
     @Query("Update Account set current_account = 1 Where id = :accountId")
-    void setCurrentAccount(int accountId);
+    public abstract void setCurrentAccount(int accountId);
 
     @Query("Select count(*) From Account Where account_type = :accountType")
-    Integer getAccountCountByType(int accountType);
+    public abstract Integer getAccountCountByType(int accountType);
 
     @Query("Select count(*) From Account")
-    Integer getAccountCount();
+    public abstract Integer getAccountCount();
 }

app/src/main/java/com/readrops/app/database/dao/BaseDao.java

@@ -0,0 +1,28 @@
+package com.readrops.app.database.dao;
+
+import androidx.room.Delete;
+import androidx.room.Insert;
+import androidx.room.Update;
+
+import java.util.List;
+
+public interface BaseDao<T> {
+
+    @Insert
+    long insert(T entity);
+
+    @Insert
+    List<Long> insert(List<T> entities);
+
+    @Update
+    void update(T entity);
+
+    @Update
+    void update(List<T> entities);
+
+    @Delete
+    void delete(T entity);
+
+    @Delete
+    void delete(List<T> entities);
+}

app/src/main/java/com/readrops/app/database/dao/FeedDao.java

@@ -3,74 +3,107 @@ package com.readrops.app.database.dao;
 
 import androidx.lifecycle.LiveData;
 import androidx.room.Dao;
-import androidx.room.Insert;
 import androidx.room.Query;
+import androidx.room.Transaction;
 
+import com.readrops.app.database.entities.Account;
 import com.readrops.app.database.entities.Feed;
 import com.readrops.app.database.pojo.FeedWithFolder;
+import com.readrops.app.utils.FeedMatcher;
+import com.readrops.readropslibrary.services.nextcloudnews.json.NextNewsFeed;
 
+import java.util.ArrayList;
 import java.util.List;
 
 @Dao
-public interface FeedDao {
+public abstract class FeedDao implements BaseDao<Feed> {
 
     @Query("Select * from Feed Where account_id = :accountId order by name ASC")
-    List<Feed> getAllFeeds(int accountId);
+    public abstract List<Feed> getAllFeeds(int accountId);
-
-    @Insert
-    long insert(Feed feed);
-
-    @Insert
-    long[] insert(List<Feed> feeds);
 
     @Query("Delete From Feed Where id = :feedId")
-    void delete(int feedId);
+    public abstract void delete(int feedId);
 
     @Query("Select case When :feedUrl In (Select url from Feed Where account_id = :accountId) Then 1 else 0 end")
-    boolean feedExists(String feedUrl, int accountId);
+    public abstract boolean feedExists(String feedUrl, int accountId);
 
     @Query("Select case When :remoteId In (Select remoteId from Feed Where account_id = :accountId) Then 1 else 0 end")
-    boolean remoteFeedExists(int remoteId, int accountId);
+    public abstract boolean remoteFeedExists(int remoteId, int accountId);
 
     @Query("Select count(*) from Feed Where account_id = :accountId")
-    int getFeedCount(int accountId);
+    public abstract int getFeedCount(int accountId);
 
     @Query("Select * from Feed Where url = :feedUrl")
-    Feed getFeedByUrl(String feedUrl);
+    public abstract Feed getFeedByUrl(String feedUrl);
 
-    @Query("Select * from Feed Where remoteId = :remoteId And account_id = :accountId")
+    @Query("Select id from Feed Where remoteId = :remoteId And account_id = :accountId")
-    Feed getFeedByRemoteId(int remoteId, int accountId);
+    public abstract int getFeedIdByRemoteId(int remoteId, int accountId);
 
     @Query("Select * from Feed Where folder_id = :folderId")
-    List<Feed> getFeedsByFolder(int folderId);
+    public abstract List<Feed> getFeedsByFolder(int folderId);
 
     @Query("Select * from Feed Where account_id = :accountId And folder_id is null")
-    List<Feed> getFeedsWithoutFolder(int accountId);
+    public abstract List<Feed> getFeedsWithoutFolder(int accountId);
 
     @Query("Update Feed set etag = :etag, last_modified = :lastModified Where id = :feedId")
-    void updateHeaders(String etag, String lastModified, int feedId);
+    public abstract void updateHeaders(String etag, String lastModified, int feedId);
 
     @Query("Update Feed set name = :feedName, url = :feedUrl, folder_id = :folderId Where id = :feedId")
-    void updateFeedFields(int feedId, String feedName, String feedUrl, Integer folderId);
+    public abstract void updateFeedFields(int feedId, String feedName, String feedUrl, Integer folderId);
 
     @Query("Update Feed set name = :name, folder_id = :folderId Where remoteId = :remoteFeedId And account_id = :accountId")
-    void updateNameAndFolder(int remoteFeedId, int accountId, String name, Integer folderId);
+    public abstract void updateNameAndFolder(int remoteFeedId, int accountId, String name, Integer folderId);
 
     @Query("Update Feed set text_color = :textColor, background_color = :bgColor Where id = :feedId")
-    void updateColors(int feedId, int textColor, int bgColor);
+    public abstract void updateColors(int feedId, int textColor, int bgColor);
 
     @Query("Select Feed.name as feed_name, Feed.id as feed_id, Folder.name as folder_name, Folder.id as folder_id, Folder.remoteId as folder_remoteId," +
             "Feed.description as feed_description, Feed.icon_url as feed_icon_url, Feed.url as feed_url, Feed.folder_id as feed_folder_id" +
             ", Feed.siteUrl as feed_siteUrl, Feed.remoteId as feed_remoteId from Feed Left Join Folder on Feed.folder_id = Folder.id Where Feed.account_id = :accountId Order by Feed.name")
-    LiveData<List<FeedWithFolder>> getAllFeedsWithFolder(int accountId);
+    public abstract LiveData<List<FeedWithFolder>> getAllFeedsWithFolder(int accountId);
 
     @Query("Select * From Feed Where id in (:ids)")
-    List<Feed> selectFromIdList(long[] ids);
+    public abstract List<Feed> selectFromIdList(List<Long> ids);
 
     @Query("Select remoteId From Feed Where account_id = :accountId")
-    List<Long> getFeedRemoteIdsOfAccount(int accountId);
+    public abstract List<Long> getFeedRemoteIdsOfAccount(int accountId);
 
     @Query("Delete from Feed Where id in (:ids)")
-    void deleteByIds(List<Long> ids);
+    public abstract void deleteByIds(List<Long> ids);
+
+    @Query("Select id From Folder Where remoteId = :remoteId And account_id = :accountId")
+    abstract int getRemoteFolderLocalId(int remoteId, int accountId);
+
+    /**
+     * Insert, update and delete feeds, by account
+     * @param nextNewsFeeds feeds to insert or update
+     * @param account owner of the feeds
+     * @return the list of the inserted feeds ids
+     */
+    @Transaction
+    public List<Long> upsert(List<NextNewsFeed> nextNewsFeeds, Account account) {
+        List<Long> accountFeedIds = getFeedRemoteIdsOfAccount(account.getId());
+        List<Feed> feedsToInsert = new ArrayList<>();
+
+        for (NextNewsFeed nextNewsFeed : nextNewsFeeds) {
+            Feed feed = FeedMatcher.nextNewsFeedToFeed(nextNewsFeed, account);
+            Integer folderId = nextNewsFeed.getId() == 0 ? null : getRemoteFolderLocalId(nextNewsFeed.getFolderId(), account.getId());
+
+            if (remoteFeedExists(nextNewsFeed.getId(), account.getId())) {
+                updateNameAndFolder(nextNewsFeed.getId(), account.getId(), nextNewsFeed.getTitle(), folderId);
+
+                accountFeedIds.remove((long) nextNewsFeed.getId());
+            } else {
+                feed.setFolderId(folderId);
+
+                feedsToInsert.add(feed);
+            }
+        }
+
+        if (!accountFeedIds.isEmpty())
+            deleteByIds(accountFeedIds);
+
+        return insert(feedsToInsert);
+    }
 }
 

app/src/main/java/com/readrops/app/database/dao/FolderDao.java

@@ -2,51 +2,68 @@ package com.readrops.app.database.dao;
 
 import androidx.lifecycle.LiveData;
 import androidx.room.Dao;
-import androidx.room.Delete;
-import androidx.room.Insert;
 import androidx.room.Query;
-import androidx.room.Update;
+import androidx.room.Transaction;
 
+import com.readrops.app.database.entities.Account;
 import com.readrops.app.database.entities.Folder;
+import com.readrops.readropslibrary.services.nextcloudnews.json.NextNewsFolder;
 
+import java.util.ArrayList;
 import java.util.List;
 
 @Dao
-public interface FolderDao {
+public abstract class FolderDao implements BaseDao<Folder> {
 
     @Query("Select * from Folder Where account_id = :accountId Order By name ASC")
-    LiveData<List<Folder>> getAllFolders(int accountId);
+    public abstract LiveData<List<Folder>> getAllFolders(int accountId);
 
     @Query("Select * from Folder Where account_id = :accountId Order By name ASC")
-    List<Folder> getFolders(int accountId);
+    public abstract List<Folder> getFolders(int accountId);
-
-    @Insert
-    long insert(Folder folder);
-
-    @Insert
-    long[] insert(List<Folder> folders);
-
-    @Update
-    void update(Folder folder);
 
     @Query("Update Folder set name = :name Where remoteId = :remoteFolderId And account_id = :accountId")
-    void updateName(int remoteFolderId, int accountId, String name);
+    public abstract void updateName(int remoteFolderId, int accountId, String name);
-
-    @Delete
-    void delete(Folder folder);
-
-    @Query("Select id From Folder Where remoteId = :remoteId And account_id = :accountId")
-    int getRemoteFolderLocalId(int remoteId, int accountId);
 
     @Query("Select case When :remoteId In (Select remoteId From Folder Where account_id = :accountId) Then 1 else 0 end")
-    boolean remoteFolderExists(int remoteId, int accountId);
+    public abstract boolean remoteFolderExists(int remoteId, int accountId);
 
     @Query("Select * from Folder Where id = :folderId")
-    Folder select(int folderId);
+    public abstract Folder select(int folderId);
 
     @Query("Select remoteId From Folder Where account_id = :accountId")
-    List<Long> getFolderRemoteIdsOfAccount(int accountId);
+    public abstract List<Long> getFolderRemoteIdsOfAccount(int accountId);
 
     @Query("Delete From Folder Where id in (:ids)")
-    void deleteByIds(List<Long> ids);
+    public abstract void deleteByIds(List<Long> ids);
+
+    /**
+     * Insert, update and delete folders
+     * @param nextNewsFolders folders to insert or update
+     * @param account owner of the feeds
+     * @return the list of the inserted feeds ids
+     */
+    @Transaction
+    public List<Long> upsert(List<NextNewsFolder> nextNewsFolders, Account account) {
+        List<Long> accountFolderIds = getFolderRemoteIdsOfAccount(account.getId());
+        List<Folder> foldersToInsert = new ArrayList<>();
+
+        for (NextNewsFolder nextNewsFolder : nextNewsFolders) {
+            if (remoteFolderExists(nextNewsFolder.getId(), account.getId())) {
+                updateName(nextNewsFolder.getId(), account.getId(), nextNewsFolder.getName());
+
+                accountFolderIds.remove((long) nextNewsFolder.getId());
+            } else {
+                Folder folder = new Folder(nextNewsFolder.getName());
+                folder.setRemoteId(nextNewsFolder.getId());
+                folder.setAccountId(account.getId());
+
+                foldersToInsert.add(folder);
+            }
+        }
+
+        if (!accountFolderIds.isEmpty())
+            deleteByIds(accountFolderIds);
+
+        return insert(foldersToInsert);
+    }
 }

app/src/main/java/com/readrops/app/database/dao/ItemDao.java

@@ -4,7 +4,6 @@ package com.readrops.app.database.dao;
 import androidx.lifecycle.LiveData;
 import androidx.paging.PageKeyedDataSource;
 import androidx.room.Dao;
-import androidx.room.Insert;
 import androidx.room.Query;
 import androidx.room.RawQuery;
 import androidx.sqlite.db.SupportSQLiteQuery;
@@ -17,22 +16,16 @@ import com.readrops.app.database.pojo.ItemWithFeed;
 import java.util.List;
 
 @Dao
-public interface ItemDao {
+public abstract class ItemDao implements BaseDao<Item> {
 
     @RawQuery(observedEntities = {Item.class, Folder.class, Feed.class})
-    PageKeyedDataSource.Factory<Integer, ItemWithFeed> selectAll(SupportSQLiteQuery query);
+    public abstract PageKeyedDataSource.Factory<Integer, ItemWithFeed> selectAll(SupportSQLiteQuery query);
 
     @Query("Select case When :guid In (Select guid From Item Inner Join Feed on Item.feed_id = Feed.id and account_id = :accountId) Then 1 else 0 end")
-    boolean itemExists(String guid, int accountId);
+    public abstract boolean itemExists(String guid, int accountId);
 
     @Query("Select case When :remoteId In (Select remoteId from Item) And :feedId In (Select feed_id From Item) Then 1 else 0 end")
-    boolean remoteItemExists(int remoteId, int feedId);
+    public abstract boolean remoteItemExists(int remoteId, int feedId);
-
-    @Insert
-    long insert(Item item);
-
-    @Insert
-    long[] insert(List<Item> items);
 
     /**
      * Set an item read or unread
@@ -42,31 +35,31 @@ public interface ItemDao {
      * @param readChanged
      */
     @Query("Update Item Set read_changed = :readChanged, read = :readState Where id = :itemId")
-    void setReadState(int itemId, int readState, int readChanged);
+    public abstract void setReadState(int itemId, int readState, int readChanged);
 
     @Query("Update Item set read_changed = 1, read = :readState")
-    void setAllItemsReadState(int readState);
+    public abstract void setAllItemsReadState(int readState);
 
     @Query("Update Item set read_changed = 1, read = :readState Where feed_id = :feedId")
-    void setAllItemsReadState(int feedId, int readState);
+    public abstract void setAllItemsReadState(int feedId, int readState);
 
     @Query("Update Item set read_it_later = 1 Where id = :itemId")
-    void setReadItLater(int itemId);
+    public abstract void setReadItLater(int itemId);
 
     @Query("Select count(*) From Item Where feed_id = :feedId And read = 0")
-    int getUnreadCount(int feedId);
+    public abstract int getUnreadCount(int feedId);
 
     @Query("Select title, Item.description, content, link, pub_date, image_link, author, read, text_color, " +
             "background_color, read_time, Feed.name, Feed.id as feedId, siteUrl, Folder.id as folder_id, " +
             "Folder.name as folder_name from Item Inner Join Feed On Item.feed_id = Feed.id Left Join Folder on Folder.id = Feed.folder_id Where Item.id = :id")
-    LiveData<ItemWithFeed> getItemById(int id);
+    public abstract LiveData<ItemWithFeed> getItemById(int id);
 
     @Query("Select remoteId From Item Where read_changed = 1 And read = 1")
-    List<Integer> getReadChanges();
+    public abstract List<Integer> getReadChanges();
 
     @Query("Select remoteId From Item Where read_changed = 1 And read = 0")
-    List<Integer> getUnreadChanges();
+    public abstract List<Integer> getUnreadChanges();
 
     @Query("Update Item set read_changed = 0")
-    void resetReadChanges();
+    public abstract void resetReadChanges();
 }

app/src/main/java/com/readrops/app/repositories/NextNewsRepository.java

@@ -9,7 +9,6 @@ import com.readrops.app.database.entities.Feed;
 import com.readrops.app.database.entities.Folder;
 import com.readrops.app.database.entities.Item;
 import com.readrops.app.utils.FeedInsertionResult;
-import com.readrops.app.utils.FeedMatcher;
 import com.readrops.app.utils.ItemMatcher;
 import com.readrops.app.utils.ParsingResult;
 import com.readrops.app.utils.Utils;
@@ -83,10 +82,12 @@ public class NextNewsRepository extends ARepository {
                     syncData.setUnreadItems(database.itemDao().getUnreadChanges());
                 }
 
+                TimingLogger timings = new TimingLogger(TAG, "nextcloud news " + syncType.name().toLowerCase());
                 SyncResult syncResult = newsAPI.sync(account.toCredentials(), syncType, syncData);
+                timings.addSplit("server queries");
 
                 if (!syncResult.isError()) {
-                    TimingLogger timings = new TimingLogger(TAG, "sync");
+
                     insertFolders(syncResult.getFolders(), account);
                     timings.addSplit("insert folders");
 
@@ -262,92 +263,33 @@ public class NextNewsRepository extends ARepository {
     }
 
     private List<Feed> insertFeeds(List<NextNewsFeed> feeds, Account account) {
-        List<Feed> newFeeds = new ArrayList<>();
+        List<Long> insertedFeedsIds = database.feedDao().upsert(feeds, account);
-
-        // get remote feeds ids in local
-        List<Long> accountFeedIds = database.feedDao().getFeedRemoteIdsOfAccount(account.getId());
-
-        for (NextNewsFeed nextNewsFeed : feeds) {
-            Feed feed = FeedMatcher.nextNewsFeedToFeed(nextNewsFeed, account);
-
-            if (!database.feedDao().remoteFeedExists(nextNewsFeed.getId(), account.getId())) {
-
-                // if the Nextcloud feed has a folder, it is already inserted, so we have to get its local id
-                if (nextNewsFeed.getFolderId() != 0) {
-                    int folderId = database.folderDao().getRemoteFolderLocalId(nextNewsFeed.getFolderId(), account.getId());
-
-                    if (folderId != 0)
-                        feed.setFolderId(folderId);
-                } else
-                    feed.setFolderId(null);
-
-                newFeeds.add(feed);
-            } else { // update feed
-                Integer folderId = nextNewsFeed.getId() == 0 ? null :
-                        database.folderDao().getRemoteFolderLocalId(nextNewsFeed.getFolderId(), account.getId());
-                database.feedDao().updateNameAndFolder(nextNewsFeed.getId(), account.getId(),
-                        nextNewsFeed.getTitle(), folderId);
-
-                accountFeedIds.remove((long) nextNewsFeed.getId()); // the feeds exists in the server, so no need to keep its id
-            }
-        }
-
-        // delete all feeds which were not returned by the server, so which do not exist in it
-        if (!accountFeedIds.isEmpty())
-            database.feedDao().deleteByIds(accountFeedIds);
 
         List<Feed> insertedFeeds = new ArrayList<>();
-        if (!newFeeds.isEmpty()) {
+        if (!insertedFeedsIds.isEmpty()) {
-            long[] newFeedIds = database.feedDao().insert(newFeeds);
+            insertedFeeds.addAll(database.feedDao().selectFromIdList(insertedFeedsIds));
-
-            insertedFeeds.addAll(database.feedDao().selectFromIdList(newFeedIds));
             setFaviconUtils(insertedFeeds);
         }
 
-
         return insertedFeeds;
     }
 
     private void insertFolders(List<NextNewsFolder> folders, Account account) {
-        List<Folder> newFolders = new ArrayList<>();
+        database.folderDao().upsert(folders, account);
-
-        List<Long> accountFolderIds = database.folderDao().getFolderRemoteIdsOfAccount(account.getId());
-
-        for (NextNewsFolder nextNewsFolder : folders) {
-
-            if (!database.folderDao().remoteFolderExists(nextNewsFolder.getId(), account.getId())) {
-                Folder folder = new Folder(nextNewsFolder.getName());
-                folder.setRemoteId(nextNewsFolder.getId());
-                folder.setAccountId(account.getId());
-
-                newFolders.add(folder);
-            } else {
-                database.folderDao().updateName(nextNewsFolder.getId(), account.getId(),
-                        nextNewsFolder.getName());
-                accountFolderIds.remove((long) nextNewsFolder.getId());
-            }
-
-        }
-
-        if (!accountFolderIds.isEmpty())
-            database.folderDao().deleteByIds(accountFolderIds);
-
-        if (!newFolders.isEmpty())
-            database.folderDao().insert(newFolders);
     }
 
     private void insertItems(List<NextNewsItem> items, Account account, boolean initialSync) {
         List<Item> newItems = new ArrayList<>();
 
         for (NextNewsItem nextNewsItem : items) {
-            Feed feed = database.feedDao().getFeedByRemoteId(nextNewsItem.getFeedId(), account.getId());
+            int feedId = database.feedDao().getFeedIdByRemoteId(nextNewsItem.getFeedId(), account.getId());
 
-            if (!initialSync && feed != null) {
+            if (!initialSync && feedId > 0) {
-                if (database.itemDao().remoteItemExists(nextNewsItem.getId(), feed.getId()))
+                if (database.itemDao().remoteItemExists(nextNewsItem.getId(), feedId))
                     break;
             }
 
-            Item item = ItemMatcher.nextNewsItemToItem(nextNewsItem, feed);
+            Item item = ItemMatcher.nextNewsItemToItem(nextNewsItem, feedId);
             item.setReadTime(Utils.readTimeFromString(item.getContent()));
 
             newItems.add(item);

app/src/main/java/com/readrops/app/utils/ItemMatcher.java

@@ -1,6 +1,5 @@
 package com.readrops.app.utils;
 
-import com.readrops.app.database.entities.Feed;
 import com.readrops.app.database.entities.Item;
 import com.readrops.readropslibrary.services.nextcloudnews.json.NextNewsItem;
 
@@ -9,7 +8,7 @@ import org.joda.time.LocalDateTime;
 
 public final class ItemMatcher {
 
-    public static Item nextNewsItemToItem(NextNewsItem nextNewsItem, Feed feed) {
+    public static Item nextNewsItemToItem(NextNewsItem nextNewsItem, int feedId) {
         Item item = new Item();
 
         item.setRemoteId(nextNewsItem.getId());
@@ -29,7 +28,7 @@ public final class ItemMatcher {
         item.setGuid(nextNewsItem.getGuid());
         item.setRead(!nextNewsItem.isUnread());
 
-        item.setFeedId(feed.getId());
+        item.setFeedId(feedId);
 
         return item;
     }

readropslibrary/src/main/java/com/readrops/readropslibrary/services/nextcloudnews/NextNewsAPI.java

@@ -148,6 +148,9 @@ public class NextNewsAPI {
     }
 
     private void putModifiedItems(SyncData data, SyncResult syncResult) throws IOException {
+        if (data.getReadItems().size() == 0 && data.getUnreadItems().size() == 0)
+            return;
+
         Response readItemsResponse = api.setArticlesState(StateType.READ.name().toLowerCase(),
                 new NextNewsItemIds(data.getReadItems())).execute();