improvement(Sync): Save items in batches #82 #331

This commit is contained in:
Artem Chepurnoy 2024-06-19 08:33:24 +03:00
parent 2ac3591fab
commit 2133829978
No known key found for this signature in database
GPG Key ID: FAC37D0CF674043E
1 changed files with 20 additions and 8 deletions

View File

@ -169,7 +169,7 @@ suspend fun <
) )
localDeleteById(localDeletedCipherIds) localDeleteById(localDeletedCipherIds)
val localPutCipherDecoded = df.localPutCipher val localPutCipherDecodedIos = df.localPutCipher
.map { (localOrNull, remote) -> .map { (localOrNull, remote) ->
ioEffect { ioEffect {
val remoteId = remote.let(remoteLens.getId) val remoteId = remote.let(remoteLens.getId)
@ -191,6 +191,17 @@ suspend fun <
remoteDecodedFallback(remote, localOrNull, e) remoteDecodedFallback(remote, localOrNull, e)
} }
} }
// Save the decoded items in groups. This way if a sync takes
// a lot of time we at least save some intermediate progress and
// later start from it.
localPutCipherDecodedIos
.windowed(
size = 1000,
step = 1000,
partialWindows = true,
)
.forEach { ios ->
val localPutCipherDecoded = ios
.parallel(Dispatchers.Default) .parallel(Dispatchers.Default)
.measure { duration, v -> .measure { duration, v ->
val msg = "[local] Decoding $name took $duration; " + val msg = "[local] Decoding $name took $duration; " +
@ -199,6 +210,7 @@ suspend fun <
} }
.bind() .bind()
localPut(localPutCipherDecoded) localPut(localPutCipherDecoded)
}
// //
// Write changes to remote server. // Write changes to remote server.