Improve sync speed

Put feed and folder insertion/update logic in transactions. Get only the feed id for an item insertion instead of getting the full feed
This commit is contained in:
Shinokuni 2019-08-04 17:29:54 +02:00
parent 1ee0f50801
commit 3c523dfe2c
8 changed files with 166 additions and 163 deletions

View File

@ -2,42 +2,30 @@ package com.readrops.app.database.dao;
import androidx.lifecycle.LiveData;
import androidx.room.Dao;
import androidx.room.Delete;
import androidx.room.Insert;
import androidx.room.Query;
import androidx.room.Update;
import com.readrops.app.database.entities.Account;
import java.util.List;
@Dao
public interface AccountDao {
public abstract class AccountDao implements BaseDao<Account> {
@Query("Select * from Account")
LiveData<List<Account>> selectAll();
@Insert
long insert(Account account);
@Update
void update(Account account);
@Delete
void delete(Account account);
public abstract LiveData<List<Account>> selectAll();
@Query("Update Account set last_modified = :lastModified Where id = :accountId")
void updateLastModified(int accountId, long lastModified);
public abstract void updateLastModified(int accountId, long lastModified);
@Query("Update Account set current_account = 0 Where id Not In (:accountId)")
void deselectOldCurrentAccount(int accountId);
public abstract void deselectOldCurrentAccount(int accountId);
@Query("Update Account set current_account = 1 Where id = :accountId")
void setCurrentAccount(int accountId);
public abstract void setCurrentAccount(int accountId);
@Query("Select count(*) From Account Where account_type = :accountType")
Integer getAccountCountByType(int accountType);
public abstract Integer getAccountCountByType(int accountType);
@Query("Select count(*) From Account")
Integer getAccountCount();
public abstract Integer getAccountCount();
}

View File

@ -0,0 +1,28 @@
package com.readrops.app.database.dao;
import androidx.room.Delete;
import androidx.room.Insert;
import androidx.room.Update;
import java.util.List;
public interface BaseDao<T> {
@Insert
long insert(T entity);
@Insert
List<Long> insert(List<T> entities);
@Update
void update(T entity);
@Update
void update(List<T> entities);
@Delete
void delete(T entity);
@Delete
void delete(List<T> entities);
}

View File

@ -3,74 +3,107 @@ package com.readrops.app.database.dao;
import androidx.lifecycle.LiveData;
import androidx.room.Dao;
import androidx.room.Insert;
import androidx.room.Query;
import androidx.room.Transaction;
import com.readrops.app.database.entities.Account;
import com.readrops.app.database.entities.Feed;
import com.readrops.app.database.pojo.FeedWithFolder;
import com.readrops.app.utils.FeedMatcher;
import com.readrops.readropslibrary.services.nextcloudnews.json.NextNewsFeed;
import java.util.ArrayList;
import java.util.List;
@Dao
public interface FeedDao {
public abstract class FeedDao implements BaseDao<Feed> {
@Query("Select * from Feed Where account_id = :accountId order by name ASC")
List<Feed> getAllFeeds(int accountId);
@Insert
long insert(Feed feed);
@Insert
long[] insert(List<Feed> feeds);
public abstract List<Feed> getAllFeeds(int accountId);
@Query("Delete From Feed Where id = :feedId")
void delete(int feedId);
public abstract void delete(int feedId);
@Query("Select case When :feedUrl In (Select url from Feed Where account_id = :accountId) Then 1 else 0 end")
boolean feedExists(String feedUrl, int accountId);
public abstract boolean feedExists(String feedUrl, int accountId);
@Query("Select case When :remoteId In (Select remoteId from Feed Where account_id = :accountId) Then 1 else 0 end")
boolean remoteFeedExists(int remoteId, int accountId);
public abstract boolean remoteFeedExists(int remoteId, int accountId);
@Query("Select count(*) from Feed Where account_id = :accountId")
int getFeedCount(int accountId);
public abstract int getFeedCount(int accountId);
@Query("Select * from Feed Where url = :feedUrl")
Feed getFeedByUrl(String feedUrl);
public abstract Feed getFeedByUrl(String feedUrl);
@Query("Select * from Feed Where remoteId = :remoteId And account_id = :accountId")
Feed getFeedByRemoteId(int remoteId, int accountId);
@Query("Select id from Feed Where remoteId = :remoteId And account_id = :accountId")
public abstract int getFeedIdByRemoteId(int remoteId, int accountId);
@Query("Select * from Feed Where folder_id = :folderId")
List<Feed> getFeedsByFolder(int folderId);
public abstract List<Feed> getFeedsByFolder(int folderId);
@Query("Select * from Feed Where account_id = :accountId And folder_id is null")
List<Feed> getFeedsWithoutFolder(int accountId);
public abstract List<Feed> getFeedsWithoutFolder(int accountId);
@Query("Update Feed set etag = :etag, last_modified = :lastModified Where id = :feedId")
void updateHeaders(String etag, String lastModified, int feedId);
public abstract void updateHeaders(String etag, String lastModified, int feedId);
@Query("Update Feed set name = :feedName, url = :feedUrl, folder_id = :folderId Where id = :feedId")
void updateFeedFields(int feedId, String feedName, String feedUrl, Integer folderId);
public abstract void updateFeedFields(int feedId, String feedName, String feedUrl, Integer folderId);
@Query("Update Feed set name = :name, folder_id = :folderId Where remoteId = :remoteFeedId And account_id = :accountId")
void updateNameAndFolder(int remoteFeedId, int accountId, String name, Integer folderId);
public abstract void updateNameAndFolder(int remoteFeedId, int accountId, String name, Integer folderId);
@Query("Update Feed set text_color = :textColor, background_color = :bgColor Where id = :feedId")
void updateColors(int feedId, int textColor, int bgColor);
public abstract void updateColors(int feedId, int textColor, int bgColor);
@Query("Select Feed.name as feed_name, Feed.id as feed_id, Folder.name as folder_name, Folder.id as folder_id, Folder.remoteId as folder_remoteId," +
"Feed.description as feed_description, Feed.icon_url as feed_icon_url, Feed.url as feed_url, Feed.folder_id as feed_folder_id" +
", Feed.siteUrl as feed_siteUrl, Feed.remoteId as feed_remoteId from Feed Left Join Folder on Feed.folder_id = Folder.id Where Feed.account_id = :accountId Order by Feed.name")
LiveData<List<FeedWithFolder>> getAllFeedsWithFolder(int accountId);
public abstract LiveData<List<FeedWithFolder>> getAllFeedsWithFolder(int accountId);
@Query("Select * From Feed Where id in (:ids)")
List<Feed> selectFromIdList(long[] ids);
public abstract List<Feed> selectFromIdList(List<Long> ids);
@Query("Select remoteId From Feed Where account_id = :accountId")
List<Long> getFeedRemoteIdsOfAccount(int accountId);
public abstract List<Long> getFeedRemoteIdsOfAccount(int accountId);
@Query("Delete from Feed Where id in (:ids)")
void deleteByIds(List<Long> ids);
public abstract void deleteByIds(List<Long> ids);
@Query("Select id From Folder Where remoteId = :remoteId And account_id = :accountId")
abstract int getRemoteFolderLocalId(int remoteId, int accountId);
/**
* Insert, update and delete feeds, by account
* @param nextNewsFeeds feeds to insert or update
* @param account owner of the feeds
* @return the list of the inserted feeds ids
*/
@Transaction
public List<Long> upsert(List<NextNewsFeed> nextNewsFeeds, Account account) {
List<Long> accountFeedIds = getFeedRemoteIdsOfAccount(account.getId());
List<Feed> feedsToInsert = new ArrayList<>();
for (NextNewsFeed nextNewsFeed : nextNewsFeeds) {
Feed feed = FeedMatcher.nextNewsFeedToFeed(nextNewsFeed, account);
Integer folderId = nextNewsFeed.getId() == 0 ? null : getRemoteFolderLocalId(nextNewsFeed.getFolderId(), account.getId());
if (remoteFeedExists(nextNewsFeed.getId(), account.getId())) {
updateNameAndFolder(nextNewsFeed.getId(), account.getId(), nextNewsFeed.getTitle(), folderId);
accountFeedIds.remove((long) nextNewsFeed.getId());
} else {
feed.setFolderId(folderId);
feedsToInsert.add(feed);
}
}
if (!accountFeedIds.isEmpty())
deleteByIds(accountFeedIds);
return insert(feedsToInsert);
}
}

View File

@ -2,51 +2,68 @@ package com.readrops.app.database.dao;
import androidx.lifecycle.LiveData;
import androidx.room.Dao;
import androidx.room.Delete;
import androidx.room.Insert;
import androidx.room.Query;
import androidx.room.Update;
import androidx.room.Transaction;
import com.readrops.app.database.entities.Account;
import com.readrops.app.database.entities.Folder;
import com.readrops.readropslibrary.services.nextcloudnews.json.NextNewsFolder;
import java.util.ArrayList;
import java.util.List;
@Dao
public interface FolderDao {
public abstract class FolderDao implements BaseDao<Folder> {
@Query("Select * from Folder Where account_id = :accountId Order By name ASC")
LiveData<List<Folder>> getAllFolders(int accountId);
public abstract LiveData<List<Folder>> getAllFolders(int accountId);
@Query("Select * from Folder Where account_id = :accountId Order By name ASC")
List<Folder> getFolders(int accountId);
@Insert
long insert(Folder folder);
@Insert
long[] insert(List<Folder> folders);
@Update
void update(Folder folder);
public abstract List<Folder> getFolders(int accountId);
@Query("Update Folder set name = :name Where remoteId = :remoteFolderId And account_id = :accountId")
void updateName(int remoteFolderId, int accountId, String name);
@Delete
void delete(Folder folder);
@Query("Select id From Folder Where remoteId = :remoteId And account_id = :accountId")
int getRemoteFolderLocalId(int remoteId, int accountId);
public abstract void updateName(int remoteFolderId, int accountId, String name);
@Query("Select case When :remoteId In (Select remoteId From Folder Where account_id = :accountId) Then 1 else 0 end")
boolean remoteFolderExists(int remoteId, int accountId);
public abstract boolean remoteFolderExists(int remoteId, int accountId);
@Query("Select * from Folder Where id = :folderId")
Folder select(int folderId);
public abstract Folder select(int folderId);
@Query("Select remoteId From Folder Where account_id = :accountId")
List<Long> getFolderRemoteIdsOfAccount(int accountId);
public abstract List<Long> getFolderRemoteIdsOfAccount(int accountId);
@Query("Delete From Folder Where id in (:ids)")
void deleteByIds(List<Long> ids);
public abstract void deleteByIds(List<Long> ids);
/**
* Insert, update and delete folders
* @param nextNewsFolders folders to insert or update
* @param account owner of the feeds
* @return the list of the inserted feeds ids
*/
@Transaction
public List<Long> upsert(List<NextNewsFolder> nextNewsFolders, Account account) {
List<Long> accountFolderIds = getFolderRemoteIdsOfAccount(account.getId());
List<Folder> foldersToInsert = new ArrayList<>();
for (NextNewsFolder nextNewsFolder : nextNewsFolders) {
if (remoteFolderExists(nextNewsFolder.getId(), account.getId())) {
updateName(nextNewsFolder.getId(), account.getId(), nextNewsFolder.getName());
accountFolderIds.remove((long) nextNewsFolder.getId());
} else {
Folder folder = new Folder(nextNewsFolder.getName());
folder.setRemoteId(nextNewsFolder.getId());
folder.setAccountId(account.getId());
foldersToInsert.add(folder);
}
}
if (!accountFolderIds.isEmpty())
deleteByIds(accountFolderIds);
return insert(foldersToInsert);
}
}

View File

@ -4,7 +4,6 @@ package com.readrops.app.database.dao;
import androidx.lifecycle.LiveData;
import androidx.paging.PageKeyedDataSource;
import androidx.room.Dao;
import androidx.room.Insert;
import androidx.room.Query;
import androidx.room.RawQuery;
import androidx.sqlite.db.SupportSQLiteQuery;
@ -17,22 +16,16 @@ import com.readrops.app.database.pojo.ItemWithFeed;
import java.util.List;
@Dao
public interface ItemDao {
public abstract class ItemDao implements BaseDao<Item> {
@RawQuery(observedEntities = {Item.class, Folder.class, Feed.class})
PageKeyedDataSource.Factory<Integer, ItemWithFeed> selectAll(SupportSQLiteQuery query);
public abstract PageKeyedDataSource.Factory<Integer, ItemWithFeed> selectAll(SupportSQLiteQuery query);
@Query("Select case When :guid In (Select guid From Item Inner Join Feed on Item.feed_id = Feed.id and account_id = :accountId) Then 1 else 0 end")
boolean itemExists(String guid, int accountId);
public abstract boolean itemExists(String guid, int accountId);
@Query("Select case When :remoteId In (Select remoteId from Item) And :feedId In (Select feed_id From Item) Then 1 else 0 end")
boolean remoteItemExists(int remoteId, int feedId);
@Insert
long insert(Item item);
@Insert
long[] insert(List<Item> items);
public abstract boolean remoteItemExists(int remoteId, int feedId);
/**
* Set an item read or unread
@ -42,31 +35,31 @@ public interface ItemDao {
* @param readChanged
*/
@Query("Update Item Set read_changed = :readChanged, read = :readState Where id = :itemId")
void setReadState(int itemId, int readState, int readChanged);
public abstract void setReadState(int itemId, int readState, int readChanged);
@Query("Update Item set read_changed = 1, read = :readState")
void setAllItemsReadState(int readState);
public abstract void setAllItemsReadState(int readState);
@Query("Update Item set read_changed = 1, read = :readState Where feed_id = :feedId")
void setAllItemsReadState(int feedId, int readState);
public abstract void setAllItemsReadState(int feedId, int readState);
@Query("Update Item set read_it_later = 1 Where id = :itemId")
void setReadItLater(int itemId);
public abstract void setReadItLater(int itemId);
@Query("Select count(*) From Item Where feed_id = :feedId And read = 0")
int getUnreadCount(int feedId);
public abstract int getUnreadCount(int feedId);
@Query("Select title, Item.description, content, link, pub_date, image_link, author, read, text_color, " +
"background_color, read_time, Feed.name, Feed.id as feedId, siteUrl, Folder.id as folder_id, " +
"Folder.name as folder_name from Item Inner Join Feed On Item.feed_id = Feed.id Left Join Folder on Folder.id = Feed.folder_id Where Item.id = :id")
LiveData<ItemWithFeed> getItemById(int id);
public abstract LiveData<ItemWithFeed> getItemById(int id);
@Query("Select remoteId From Item Where read_changed = 1 And read = 1")
List<Integer> getReadChanges();
public abstract List<Integer> getReadChanges();
@Query("Select remoteId From Item Where read_changed = 1 And read = 0")
List<Integer> getUnreadChanges();
public abstract List<Integer> getUnreadChanges();
@Query("Update Item set read_changed = 0")
void resetReadChanges();
public abstract void resetReadChanges();
}

View File

@ -9,7 +9,6 @@ import com.readrops.app.database.entities.Feed;
import com.readrops.app.database.entities.Folder;
import com.readrops.app.database.entities.Item;
import com.readrops.app.utils.FeedInsertionResult;
import com.readrops.app.utils.FeedMatcher;
import com.readrops.app.utils.ItemMatcher;
import com.readrops.app.utils.ParsingResult;
import com.readrops.app.utils.Utils;
@ -83,10 +82,12 @@ public class NextNewsRepository extends ARepository {
syncData.setUnreadItems(database.itemDao().getUnreadChanges());
}
TimingLogger timings = new TimingLogger(TAG, "nextcloud news " + syncType.name().toLowerCase());
SyncResult syncResult = newsAPI.sync(account.toCredentials(), syncType, syncData);
timings.addSplit("server queries");
if (!syncResult.isError()) {
TimingLogger timings = new TimingLogger(TAG, "sync");
insertFolders(syncResult.getFolders(), account);
timings.addSplit("insert folders");
@ -262,92 +263,33 @@ public class NextNewsRepository extends ARepository {
}
private List<Feed> insertFeeds(List<NextNewsFeed> feeds, Account account) {
List<Feed> newFeeds = new ArrayList<>();
// get remote feeds ids in local
List<Long> accountFeedIds = database.feedDao().getFeedRemoteIdsOfAccount(account.getId());
for (NextNewsFeed nextNewsFeed : feeds) {
Feed feed = FeedMatcher.nextNewsFeedToFeed(nextNewsFeed, account);
if (!database.feedDao().remoteFeedExists(nextNewsFeed.getId(), account.getId())) {
// if the Nextcloud feed has a folder, it is already inserted, so we have to get its local id
if (nextNewsFeed.getFolderId() != 0) {
int folderId = database.folderDao().getRemoteFolderLocalId(nextNewsFeed.getFolderId(), account.getId());
if (folderId != 0)
feed.setFolderId(folderId);
} else
feed.setFolderId(null);
newFeeds.add(feed);
} else { // update feed
Integer folderId = nextNewsFeed.getId() == 0 ? null :
database.folderDao().getRemoteFolderLocalId(nextNewsFeed.getFolderId(), account.getId());
database.feedDao().updateNameAndFolder(nextNewsFeed.getId(), account.getId(),
nextNewsFeed.getTitle(), folderId);
accountFeedIds.remove((long) nextNewsFeed.getId()); // the feeds exists in the server, so no need to keep its id
}
}
// delete all feeds which were not returned by the server, so which do not exist in it
if (!accountFeedIds.isEmpty())
database.feedDao().deleteByIds(accountFeedIds);
List<Long> insertedFeedsIds = database.feedDao().upsert(feeds, account);
List<Feed> insertedFeeds = new ArrayList<>();
if (!newFeeds.isEmpty()) {
long[] newFeedIds = database.feedDao().insert(newFeeds);
insertedFeeds.addAll(database.feedDao().selectFromIdList(newFeedIds));
if (!insertedFeedsIds.isEmpty()) {
insertedFeeds.addAll(database.feedDao().selectFromIdList(insertedFeedsIds));
setFaviconUtils(insertedFeeds);
}
return insertedFeeds;
}
private void insertFolders(List<NextNewsFolder> folders, Account account) {
List<Folder> newFolders = new ArrayList<>();
List<Long> accountFolderIds = database.folderDao().getFolderRemoteIdsOfAccount(account.getId());
for (NextNewsFolder nextNewsFolder : folders) {
if (!database.folderDao().remoteFolderExists(nextNewsFolder.getId(), account.getId())) {
Folder folder = new Folder(nextNewsFolder.getName());
folder.setRemoteId(nextNewsFolder.getId());
folder.setAccountId(account.getId());
newFolders.add(folder);
} else {
database.folderDao().updateName(nextNewsFolder.getId(), account.getId(),
nextNewsFolder.getName());
accountFolderIds.remove((long) nextNewsFolder.getId());
}
}
if (!accountFolderIds.isEmpty())
database.folderDao().deleteByIds(accountFolderIds);
if (!newFolders.isEmpty())
database.folderDao().insert(newFolders);
database.folderDao().upsert(folders, account);
}
private void insertItems(List<NextNewsItem> items, Account account, boolean initialSync) {
List<Item> newItems = new ArrayList<>();
for (NextNewsItem nextNewsItem : items) {
Feed feed = database.feedDao().getFeedByRemoteId(nextNewsItem.getFeedId(), account.getId());
int feedId = database.feedDao().getFeedIdByRemoteId(nextNewsItem.getFeedId(), account.getId());
if (!initialSync && feed != null) {
if (database.itemDao().remoteItemExists(nextNewsItem.getId(), feed.getId()))
if (!initialSync && feedId > 0) {
if (database.itemDao().remoteItemExists(nextNewsItem.getId(), feedId))
break;
}
Item item = ItemMatcher.nextNewsItemToItem(nextNewsItem, feed);
Item item = ItemMatcher.nextNewsItemToItem(nextNewsItem, feedId);
item.setReadTime(Utils.readTimeFromString(item.getContent()));
newItems.add(item);

View File

@ -1,6 +1,5 @@
package com.readrops.app.utils;
import com.readrops.app.database.entities.Feed;
import com.readrops.app.database.entities.Item;
import com.readrops.readropslibrary.services.nextcloudnews.json.NextNewsItem;
@ -9,7 +8,7 @@ import org.joda.time.LocalDateTime;
public final class ItemMatcher {
public static Item nextNewsItemToItem(NextNewsItem nextNewsItem, Feed feed) {
public static Item nextNewsItemToItem(NextNewsItem nextNewsItem, int feedId) {
Item item = new Item();
item.setRemoteId(nextNewsItem.getId());
@ -29,7 +28,7 @@ public final class ItemMatcher {
item.setGuid(nextNewsItem.getGuid());
item.setRead(!nextNewsItem.isUnread());
item.setFeedId(feed.getId());
item.setFeedId(feedId);
return item;
}

View File

@ -148,6 +148,9 @@ public class NextNewsAPI {
}
private void putModifiedItems(SyncData data, SyncResult syncResult) throws IOException {
if (data.getReadItems().size() == 0 && data.getUnreadItems().size() == 0)
return;
Response readItemsResponse = api.setArticlesState(StateType.READ.name().toLowerCase(),
new NextNewsItemIds(data.getReadItems())).execute();