From f0dbb92d76dbdb2b52489e3abc0e58cd23351658 Mon Sep 17 00:00:00 2001 From: ganfra Date: Tue, 30 Jun 2020 19:45:17 +0200 Subject: [PATCH] Attempt to clean db [WIP] --- .../database/helper/ChunkEntityHelper.kt | 1 + .../internal/database/model/ChunkEntity.kt | 1 + .../internal/session/DefaultSession.kt | 55 +++++++++++++++++++ .../internal/session/filter/FilterUtil.kt | 3 +- 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/ChunkEntityHelper.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/ChunkEntityHelper.kt index d86151e694..bc681e4eb8 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/ChunkEntityHelper.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/ChunkEntityHelper.kt @@ -115,6 +115,7 @@ internal fun ChunkEntity.addTimelineEvent(roomId: String, true } } + numberOfTimelineEvents++ timelineEvents.add(timelineEventEntity) } diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/ChunkEntity.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/ChunkEntity.kt index 19bf72970c..b1e08fdeea 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/ChunkEntity.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/model/ChunkEntity.kt @@ -27,6 +27,7 @@ internal open class ChunkEntity(@Index var prevToken: String? = null, @Index var nextToken: String? = null, var stateEvents: RealmList = RealmList(), var timelineEvents: RealmList = RealmList(), + var numberOfTimelineEvents: Long= 0, // Only one chunk will have isLastForward == true @Index var isLastForward: Boolean = false, @Index var isLastBackward: Boolean = false diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt index e32ba7e63c..f8ddd3d345 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/DefaultSession.kt @@ -51,9 +51,21 @@ import im.vector.matrix.android.api.session.user.UserService import im.vector.matrix.android.api.session.widgets.WidgetService import im.vector.matrix.android.internal.auth.SessionParamsStore import im.vector.matrix.android.internal.crypto.DefaultCryptoService +import im.vector.matrix.android.internal.database.awaitTransaction +import im.vector.matrix.android.internal.database.helper.nextDisplayIndex +import im.vector.matrix.android.internal.database.model.ChunkEntity +import im.vector.matrix.android.internal.database.model.ChunkEntityFields +import im.vector.matrix.android.internal.database.model.CurrentStateEventEntity +import im.vector.matrix.android.internal.database.model.CurrentStateEventEntityFields +import im.vector.matrix.android.internal.database.model.EventEntity +import im.vector.matrix.android.internal.database.model.RoomEntity +import im.vector.matrix.android.internal.database.model.TimelineEventEntity +import im.vector.matrix.android.internal.database.model.TimelineEventEntityFields +import im.vector.matrix.android.internal.di.SessionDatabase import im.vector.matrix.android.internal.di.SessionId import im.vector.matrix.android.internal.di.WorkManagerProvider import im.vector.matrix.android.internal.session.identity.DefaultIdentityService +import im.vector.matrix.android.internal.session.room.timeline.PaginationDirection import im.vector.matrix.android.internal.session.room.timeline.TimelineEventDecryptor import im.vector.matrix.android.internal.session.sync.SyncTokenStore import im.vector.matrix.android.internal.session.sync.job.SyncThread @@ -61,6 +73,7 @@ import im.vector.matrix.android.internal.session.sync.job.SyncWorker import im.vector.matrix.android.internal.task.TaskExecutor import im.vector.matrix.android.internal.util.MatrixCoroutineDispatchers import im.vector.matrix.android.internal.util.createUIHandler +import io.realm.RealmConfiguration import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.greenrobot.eventbus.EventBus @@ -77,6 +90,7 @@ internal class DefaultSession @Inject constructor( private val eventBus: EventBus, @SessionId override val sessionId: String, + @SessionDatabase private val realmConfiguration: RealmConfiguration, private val lifecycleObservers: Set<@JvmSuppressWildcards SessionLifecycleObserver>, private val sessionListeners: SessionListeners, private val roomService: Lazy, @@ -151,6 +165,47 @@ internal class DefaultSession @Inject constructor( } eventBus.register(this) timelineEventDecryptor.start() + taskExecutor.executorScope.launch(Dispatchers.Default) { + awaitTransaction(realmConfiguration) { realm -> + val allRooms = realm.where(RoomEntity::class.java).findAll() + val numberOfEvents = realm.where(EventEntity::class.java).findAll().size + val numberOfTimelineEvents = realm.where(TimelineEventEntity::class.java).findAll().size + Timber.v("Number of events in db: $numberOfEvents | Number of timeline events in db: $numberOfTimelineEvents") + Timber.v("Number of rooms in db: ${allRooms.size}") + if (numberOfTimelineEvents < 30_000L) { + Timber.v("Db is low enough") + } else { + + val hugeChunks = realm.where(ChunkEntity::class.java).greaterThan(ChunkEntityFields.NUMBER_OF_TIMELINE_EVENTS, 250).findAll() + Timber.v("There are ${hugeChunks.size} chunks to clean") + for (chunk in hugeChunks) { + val maxDisplayIndex = chunk.nextDisplayIndex(PaginationDirection.FORWARDS) + val thresholdDisplayIndex = maxDisplayIndex - 250 + val eventsToRemove = chunk.timelineEvents.where().lessThan(TimelineEventEntityFields.DISPLAY_INDEX, thresholdDisplayIndex).findAll() + Timber.v("There are ${eventsToRemove.size} events to clean in chunk: ${chunk.identifier()} from room ${chunk.room?.first()?.roomId}") + chunk.numberOfTimelineEvents = chunk.numberOfTimelineEvents - eventsToRemove.size + eventsToRemove.forEach { + val canDeleteRoot = it.root?.stateKey == null + if (canDeleteRoot) { + it.root?.deleteFromRealm() + } + it.readReceipts?.readReceipts?.deleteAllFromRealm() + it.readReceipts?.deleteFromRealm() + it.annotations?.apply { + editSummary?.deleteFromRealm() + pollResponseSummary?.deleteFromRealm() + referencesSummaryEntity?.deleteFromRealm() + reactionsSummary.deleteAllFromRealm() + } + it.annotations?.deleteFromRealm() + it.readReceipts?.deleteFromRealm() + it.deleteFromRealm() + } + } + } + + } + } } override fun requireBackgroundSync() { diff --git a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/filter/FilterUtil.kt b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/filter/FilterUtil.kt index 53ede5ad45..d95a53cb07 100644 --- a/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/filter/FilterUtil.kt +++ b/matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/filter/FilterUtil.kt @@ -98,7 +98,8 @@ internal object FilterUtil { state = filter.room.state?.copy(lazyLoadMembers = true) ?: RoomEventFilter(lazyLoadMembers = true) ) - ?: RoomFilter(state = RoomEventFilter(lazyLoadMembers = true)) + ?: RoomFilter(state = RoomEventFilter(lazyLoadMembers = true), + timeline = RoomEventFilter(limit = 1500)) ) } else { val newRoomEventFilter = filter.room?.state?.copy(lazyLoadMembers = null)?.takeIf { it.hasData() }