FreshRSS
/
Readrops
mirror of https://github.com/readrops/Readrops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rework Fever synchronization to be more efficient

This commit is contained in:
Shinokuni 2024-08-07 14:45:44 +02:00
parent 732ae4efa4
commit 89c4dfad1a
4 changed files with 85 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
api/src/main/java/com/readrops/api/services/fever/FeverDataSource.kt
View File

@ -3,8 +3,11 @@ package com.readrops.api.services.fever
import com.readrops.api.services.SyncType
import com.readrops.api.services.fever.adapters.FeverAPIAdapter
import com.readrops.api.utils.ApiUtils
import com.readrops.db.entities.Item
import com.squareup.moshi.Moshi
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import okhttp3.MultipartBody
class FeverDataSource(private val service: FeverService) {
@ -21,54 +24,68 @@ class FeverDataSource(private val service: FeverService) {
}
suspend fun synchronize(
login: String,
password: String,
syncType: SyncType,
syncData: FeverSyncData,
body: MultipartBody
): FeverSyncResult {
syncData: FeverSyncData
): FeverSyncResult = with(CoroutineScope(Dispatchers.IO)) {
val body = getFeverRequestBody(login, password)
if (syncType == SyncType.INITIAL_SYNC) {
val unreadIds = service.getUnreadItemsIds(body)
.reversed()
.subList(0, MAX_ITEMS_IDS)
return FeverSyncResult().apply {
listOf(
async { feverFeeds = service.getFeeds(body) },
async { folders = service.getFolders(body) },
async {
unreadIds = service.getUnreadItemsIds(body)
.reversed()
.subList(0, MAX_ITEMS_IDS)
var lastId = unreadIds.first()
val items = arrayListOf<Item>()
repeat(INITIAL_SYNC_ITEMS_REQUESTS_COUNT) {
val newItems = service.getItems(body, lastId, null)
var lastId = unreadIds.first()
items = buildList {
repeat(INITIAL_SYNC_ITEMS_REQUESTS_COUNT) {
val newItems = service.getItems(body, lastId, null)
lastId = newItems.last().remoteId!!
items += newItems
lastId = newItems.last().remoteId!!
addAll(newItems)
}
}
sinceId = unreadIds.first().toLong()
},
async { starredIds = service.getStarredItemsIds(body) },
async { favicons = listOf() }
)
.awaitAll()
}
return FeverSyncResult(
feverFeeds = service.getFeeds(body),
folders = service.getFolders(body),
items = items,
unreadIds = unreadIds,
starredIds = service.getStarredItemsIds(body),
favicons = listOf(),
sinceId = unreadIds.first().toLong(),
)
} else {
val items = arrayListOf<Item>()
var sinceId = syncData.sinceId
return FeverSyncResult().apply {
listOf(
async { folders = service.getFolders(body) },
async { feverFeeds = service.getFeeds(body) },
async { unreadIds = service.getUnreadItemsIds(body) },
async { starredIds = service.getStarredItemsIds(body) },
async { favicons = listOf() },
async {
items = buildList {
var sinceId = syncData.sinceId
while (true) {
val newItems = service.getItems(body, null, sinceId)
while (true) {
val newItems = service.getItems(body, null, sinceId)
if (newItems.isEmpty()) break
sinceId = newItems.first().remoteId!!
items += newItems
if (newItems.isEmpty()) break
sinceId = newItems.first().remoteId!!
addAll(newItems)
}
}
if (items.isNotEmpty()) items.first().remoteId!!.toLong() else sinceId.toLong()
}
)
.awaitAll()
}
return FeverSyncResult(
feverFeeds = service.getFeeds(body),
folders = service.getFolders(body),
items = items,
unreadIds = service.getUnreadItemsIds(body),
starredIds = service.getStarredItemsIds(body),
favicons = listOf(),
sinceId = if (items.isNotEmpty()) items.first().remoteId!!.toLong() else sinceId.toLong(),
)
}
}

14
api/src/main/java/com/readrops/api/services/fever/FeverSyncResult.kt
View File

@ -6,11 +6,11 @@ import com.readrops.db.entities.Folder
import com.readrops.db.entities.Item
data class FeverSyncResult(
val feverFeeds: FeverFeeds,
val folders: List<Folder>,
val items: List<Item>,
val unreadIds: List<String>,
val starredIds: List<String>,
val favicons: List<Favicon>,
val sinceId: Long = 0,
var feverFeeds: FeverFeeds = FeverFeeds(),
var folders: List<Folder> = listOf(),
var items: List<Item> = listOf(),
var unreadIds: List<String> = listOf(),
var starredIds: List<String> = listOf(),
var favicons: List<Favicon> = listOf(),
var sinceId: Long = 0,
)

9
api/src/main/java/com/readrops/api/services/fever/adapters/FeverFeedsAdapter.kt
View File

@ -5,13 +5,14 @@ import com.readrops.api.utils.exceptions.ParseException
import com.readrops.api.utils.extensions.nextNonEmptyString
import com.readrops.api.utils.extensions.nextNullableString
import com.readrops.api.utils.extensions.skipField
import com.readrops.api.utils.extensions.skipToEnd
import com.readrops.db.entities.Feed
import com.squareup.moshi.*
import com.squareup.moshi.JsonAdapter
import com.squareup.moshi.JsonReader
import com.squareup.moshi.JsonWriter
data class FeverFeeds(
val feeds: List<Feed>,
val feedsGroups: Map<Int, List<Int>>
val feeds: List<Feed> = listOf(),
val feedsGroups: Map<Int, List<Int>> = emptyMap()
)
class FeverFeedsAdapter : JsonAdapter<FeverFeeds>() {

30
app/src/main/java/com/readrops/app/repositories/FeverRepository.kt
View File

@ -41,20 +41,24 @@ class FeverRepository(
}
return feverDataSource.synchronize(
account.login!!,
account.password!!,
syncType,
FeverSyncData(account.lastModified.toString()),
getFeverRequestBody()
FeverSyncData(account.lastModified.toString())
).run {
insertFolders(folders)
insertFeeds(feverFeeds)
val newFeeds = insertFeeds(feverFeeds)
insertItems(items)
val newItems = insertItems(items)
insertItemsIds(unreadIds, starredIds.toMutableList())
// We store the id to use for the next synchronisation even if it's not a timestamp
database.accountDao().updateLastModified(sinceId, account.id)
SyncResult()
SyncResult(
items = newItems,
feeds = newFeeds
)
}
}
@ -144,7 +148,7 @@ class FeverRepository(
database.folderDao().upsertFolders(folders, account)
}
private suspend fun insertFeeds(feverFeeds: FeverFeeds) = with(feverFeeds) {
private suspend fun insertFeeds(feverFeeds: FeverFeeds): List<Feed> = with(feverFeeds) {
for (feed in feeds) {
for ((folderId, feedsIds) in feedsGroups) {
if (feedsIds.contains(feed.remoteId!!.toInt())) {
@ -154,23 +158,23 @@ class FeverRepository(
}
feeds.forEach { it.accountId = account.id }
database.feedDao().upsertFeeds(feeds, account)
return database.feedDao().upsertFeeds(feeds, account)
}
private suspend fun insertItems(items: List<Item>): List<Item> {
val newItems = arrayListOf<Item>()
val itemsFeedsIds = mutableMapOf<String, Int>()
val itemsFeedsIds = mutableMapOf<String?, Int>()
for (item in items) {
var feedId: Int?
val feedId: Int
if (itemsFeedsIds.containsKey(item.feedRemoteId)) {
feedId = itemsFeedsIds[item.feedRemoteId]
feedId = itemsFeedsIds.getValue(item.feedRemoteId)
} else {
//feedId = database.feedDao().getFeedIdByRemoteId(item.feedRemoteId!!, account.id)
// itemsFeedsIds[item.feedRemoteId!!] = feedId
feedId = database.feedDao().selectRemoteFeedLocalId(item.feedRemoteId!!, account.id)
itemsFeedsIds[item.feedRemoteId!!] = feedId
}
//item.feedId = feedId!!
item.feedId = feedId
item.text?.let { item.readTime = Utils.readTimeFromString(it) }
newItems += item