Timeline/Sync: Fix some issues

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/helper/ChunkEntityHelper.kt

@@ -24,6 +24,8 @@ import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.EventEntityFields
 import im.vector.matrix.android.internal.database.model.ReadReceiptEntity
 import im.vector.matrix.android.internal.database.model.ReadReceiptsSummaryEntity
+import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntity
+import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntityFields
 import im.vector.matrix.android.internal.database.model.TimelineEventEntity
 import im.vector.matrix.android.internal.database.model.TimelineEventEntityFields
 import im.vector.matrix.android.internal.database.query.find
@@ -31,8 +33,10 @@ import im.vector.matrix.android.internal.database.query.getOrCreate
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.extensions.assertIsManaged
 import im.vector.matrix.android.internal.session.room.timeline.PaginationDirection
+import io.realm.Realm
 import io.realm.Sort
 import io.realm.kotlin.createObject
+import kotlinx.coroutines.coroutineScope
 import timber.log.Timber
 
 internal fun ChunkEntity.deleteOnCascade() {
@@ -43,6 +47,7 @@ internal fun ChunkEntity.deleteOnCascade() {
 
 internal fun ChunkEntity.merge(roomId: String, chunkToMerge: ChunkEntity, direction: PaginationDirection) {
     assertIsManaged()
+    val localRealm = this.realm
     val eventsToMerge: List<TimelineEventEntity>
     if (direction == PaginationDirection.FORWARDS) {
         this.nextToken = chunkToMerge.nextToken
@@ -59,15 +64,32 @@ internal fun ChunkEntity.merge(roomId: String, chunkToMerge: ChunkEntity, direct
     return eventsToMerge
             .forEach {
                 if (timelineEvents.find(it.eventId) == null) {
-                    it.displayIndex = nextDisplayIndex(direction)
+                    val eventId = it.eventId
-                    this.timelineEvents.add(it)
+                    if (timelineEvents.find(eventId) != null) {
+                        return
+                    }
+                    val displayIndex = nextDisplayIndex(direction)
+                    val localId = TimelineEventEntity.nextId(realm)
+                    val copied = localRealm.createObject<TimelineEventEntity>().apply {
+                        this.localId = localId
+                        this.root = it.root
+                        this.eventId = it.eventId
+                        this.roomId = it.roomId
+                        this.annotations = it.annotations
+                        this.readReceipts = it.readReceipts
+                        this.displayIndex = displayIndex
+                        this.senderAvatar = it.senderAvatar
+                        this.senderName = it.senderName
+                        this.isUniqueDisplayName = it.isUniqueDisplayName
+                    }
+                    timelineEvents.add(copied)
                 }
             }
 }
 
 internal fun ChunkEntity.addStateEvent(roomId: String, stateEvent: EventEntity, direction: PaginationDirection) {
-    if (direction == PaginationDirection.FORWARDS) {
+    if (direction == PaginationDirection.BACKWARDS) {
-        Timber.v("We don't keep chunk state events when paginating forward")
+        Timber.v("We don't keep chunk state events when paginating backward")
     } else {
         val stateKey = stateEvent.stateKey ?: return
         val type = stateEvent.type
@@ -97,27 +119,8 @@ internal fun ChunkEntity.addTimelineEvent(roomId: String,
     val localId = TimelineEventEntity.nextId(realm)
     val senderId = eventEntity.sender ?: ""
 
-    val readReceiptsSummaryEntity = ReadReceiptsSummaryEntity.where(realm, eventId).findFirst()
-            ?: realm.createObject<ReadReceiptsSummaryEntity>(eventId).apply {
-                this.roomId = roomId
-            }
-
     // Update RR for the sender of a new message with a dummy one
-
+    val readReceiptsSummaryEntity = handleReadReceipts(realm, roomId, eventEntity, senderId)
-    val originServerTs = eventEntity.originServerTs
-    if (originServerTs != null) {
-        val timestampOfEvent = originServerTs.toDouble()
-        val readReceiptOfSender = ReadReceiptEntity.getOrCreate(realm, roomId = roomId, userId = senderId)
-        // If the synced RR is older, update
-        if (timestampOfEvent > readReceiptOfSender.originServerTs) {
-            val previousReceiptsSummary = ReadReceiptsSummaryEntity.where(realm, eventId = readReceiptOfSender.eventId).findFirst()
-            readReceiptOfSender.eventId = eventId
-            readReceiptOfSender.originServerTs = timestampOfEvent
-            previousReceiptsSummary?.readReceipts?.remove(readReceiptOfSender)
-            readReceiptsSummaryEntity.readReceipts.add(readReceiptOfSender)
-        }
-    }
-
     val timelineEventEntity = realm.createObject<TimelineEventEntity>().apply {
         this.localId = localId
         this.root = eventEntity
@@ -126,19 +129,53 @@ internal fun ChunkEntity.addTimelineEvent(roomId: String,
         this.annotations = EventAnnotationsSummaryEntity.where(realm, eventId).findFirst()
         this.readReceipts = readReceiptsSummaryEntity
         this.displayIndex = displayIndex
-
         val roomMemberContent = roomMemberContentsByUser[senderId]
-        val isUnique = roomMemberContentsByUser.values.find {
-            roomMemberContent != it &&
-                    it?.displayName == roomMemberContent?.displayName
-        } == null
         this.senderAvatar = roomMemberContent?.avatarUrl
         this.senderName = roomMemberContent?.displayName
-        this.isUniqueDisplayName = isUnique
+        if (roomMemberContent?.displayName != null) {
+            val isHistoricalUnique = roomMemberContentsByUser.values.find {
+                roomMemberContent != it &&
+                        it?.displayName == roomMemberContent.displayName
+            } == null
+            isUniqueDisplayName = if (isLastForward) {
+                val isLiveUnique = RoomMemberSummaryEntity
+                        .where(realm, roomId)
+                        .equalTo(RoomMemberSummaryEntityFields.DISPLAY_NAME, roomMemberContent.displayName)
+                        .findAll().none {
+                            !roomMemberContentsByUser.containsKey(it.userId)
+                        }
+                isHistoricalUnique && isLiveUnique
+            } else {
+                isHistoricalUnique
+            }
+        } else {
+            isUniqueDisplayName = true
+        }
     }
     timelineEvents.add(timelineEventEntity)
 }
 
+private fun handleReadReceipts(realm: Realm, roomId: String, eventEntity: EventEntity, senderId: String): ReadReceiptsSummaryEntity {
+    val readReceiptsSummaryEntity = ReadReceiptsSummaryEntity.where(realm, eventEntity.eventId).findFirst()
+            ?: realm.createObject<ReadReceiptsSummaryEntity>(eventEntity.eventId).apply {
+                this.roomId = roomId
+            }
+    val originServerTs = eventEntity.originServerTs
+    if (originServerTs != null) {
+        val timestampOfEvent = originServerTs.toDouble()
+        val readReceiptOfSender = ReadReceiptEntity.getOrCreate(realm, roomId = roomId, userId = senderId)
+        // If the synced RR is older, update
+        if (timestampOfEvent > readReceiptOfSender.originServerTs) {
+            val previousReceiptsSummary = ReadReceiptsSummaryEntity.where(realm, eventId = readReceiptOfSender.eventId).findFirst()
+            readReceiptOfSender.eventId = eventEntity.eventId
+            readReceiptOfSender.originServerTs = timestampOfEvent
+            previousReceiptsSummary?.readReceipts?.remove(readReceiptOfSender)
+            readReceiptsSummaryEntity.readReceipts.add(readReceiptOfSender)
+        }
+    }
+    return readReceiptsSummaryEntity
+}
+
 internal fun ChunkEntity.nextDisplayIndex(direction: PaginationDirection): Int {
     return when (direction) {
         PaginationDirection.FORWARDS  -> {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/EventEntityQueries.kt

@@ -44,10 +44,14 @@ internal fun EventEntity.Companion.whereType(realm: Realm,
     return query.equalTo(EventEntityFields.TYPE, type)
 }
 
-internal fun EventEntity.Companion.types(realm: Realm,
+internal fun EventEntity.Companion.whereTypes(realm: Realm,
-                                         typeList: List<String> = emptyList()): RealmQuery<EventEntity> {
+                                              typeList: List<String> = emptyList(),
+                                              roomId: String? = null): RealmQuery<EventEntity> {
     val query = realm.where<EventEntity>()
     query.`in`(EventEntityFields.TYPE, typeList.toTypedArray())
+    if (roomId != null) {
+        query.equalTo(EventEntityFields.ROOM_ID, roomId)
+    }
     return query
 }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/database/query/TimelineEventEntityQueries.kt

@@ -38,17 +38,6 @@ internal fun TimelineEventEntity.Companion.whereRoomId(realm: Realm,
     return realm.where<TimelineEventEntity>().equalTo(TimelineEventEntityFields.ROOM_ID, roomId)
 }
 
-internal fun TimelineEventEntity.Companion.whereType(realm: Realm,
-                                                     type: String,
-                                                     roomId: String? = null): RealmQuery<TimelineEventEntity> {
-    val query = realm.where<TimelineEventEntity>()
-    query.equalTo(TimelineEventEntityFields.ROOT.TYPE, type)
-    if (roomId != null) {
-        query.equalTo(TimelineEventEntityFields.ROOM_ID, roomId)
-    }
-    return query
-}
-
 internal fun TimelineEventEntity.Companion.findWithSenderMembershipEvent(realm: Realm, senderMembershipEventId: String): List<TimelineEventEntity> {
     return realm.where<TimelineEventEntity>()
             .equalTo(TimelineEventEntityFields.SENDER_MEMBERSHIP_EVENT_ID, senderMembershipEventId)

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/EventRelationsAggregationUpdater.kt

@@ -20,7 +20,7 @@ import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.internal.database.RealmLiveEntityObserver
 import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.EventEntity
-import im.vector.matrix.android.internal.database.query.types
+import im.vector.matrix.android.internal.database.query.whereTypes
 import im.vector.matrix.android.internal.di.SessionDatabase
 import im.vector.matrix.android.internal.di.UserId
 import io.realm.OrderedCollectionChangeSet
@@ -43,7 +43,7 @@ internal class EventRelationsAggregationUpdater @Inject constructor(
         RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
 
     override val query = Monarchy.Query<EventEntity> {
-        EventEntity.types(it, listOf(
+        EventEntity.whereTypes(it, listOf(
                 EventType.MESSAGE,
                 EventType.REDACTION,
                 EventType.REACTION,

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/create/RoomCreateEventLiveObserver.kt

@@ -27,7 +27,7 @@ import im.vector.matrix.android.internal.database.awaitTransaction
 import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
-import im.vector.matrix.android.internal.database.query.types
+import im.vector.matrix.android.internal.database.query.whereTypes
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.di.SessionDatabase
 import io.realm.OrderedCollectionChangeSet
@@ -41,7 +41,7 @@ internal class RoomCreateEventLiveObserver @Inject constructor(@SessionDatabase
     : RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
 
     override val query = Monarchy.Query<EventEntity> {
-        EventEntity.types(it, listOf(EventType.STATE_ROOM_CREATE))
+        EventEntity.whereTypes(it, listOf(EventType.STATE_ROOM_CREATE))
     }
 
     override fun onChange(results: RealmResults<EventEntity>, changeSet: OrderedCollectionChangeSet) {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/prune/EventsPruner.kt

@@ -21,7 +21,7 @@ import im.vector.matrix.android.api.session.events.model.EventType
 import im.vector.matrix.android.internal.database.RealmLiveEntityObserver
 import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.EventEntity
-import im.vector.matrix.android.internal.database.query.types
+import im.vector.matrix.android.internal.database.query.whereTypes
 import im.vector.matrix.android.internal.di.SessionDatabase
 import io.realm.OrderedCollectionChangeSet
 import io.realm.RealmConfiguration
@@ -38,7 +38,7 @@ internal class EventsPruner @Inject constructor(@SessionDatabase realmConfigurat
                                                 private val pruneEventTask: PruneEventTask) :
         RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
 
-    override val query = Monarchy.Query<EventEntity> { EventEntity.types(it, listOf(EventType.REDACTION)) }
+    override val query = Monarchy.Query<EventEntity> { EventEntity.whereTypes(it, listOf(EventType.REDACTION)) }
 
     override fun onChange(results: RealmResults<EventEntity>, changeSet: OrderedCollectionChangeSet) {
         Timber.v("Event pruner called with ${changeSet.insertions.size} insertions")

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/prune/PruneEventTask.kt

@@ -95,10 +95,12 @@ internal class DefaultPruneEventTask @Inject constructor(private val monarchy: M
 //                }
             }
         }
-        // TODO : make it work again. Maybe waits for SQL rework...
         if (typeToPrune == EventType.STATE_ROOM_MEMBER && stateKey != null) {
-            TimelineEventEntity.findWithSenderMembershipEvent(realm, eventToPrune.eventId)
+            TimelineEventEntity.findWithSenderMembershipEvent(realm, eventToPrune.eventId).forEach {
-
+                it.senderName = null
+                it.isUniqueDisplayName = false
+                it.senderAvatar = null
+            }
         }
     }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/DefaultGetContextOfEventTask.kt

@@ -38,13 +38,12 @@ internal class DefaultGetContextOfEventTask @Inject constructor(
         private val eventBus: EventBus
 ) : GetContextOfEventTask {
 
-
     override suspend fun execute(params: GetContextOfEventTask.Params): TokenChunkEventPersistor.Result {
         val filter = filterRepository.getRoomFilter()
         val response = executeRequest<EventContextResponse>(eventBus) {
             // We are limiting the response to the event with eventId to be sure we don't have any issue with potential merging process.
             apiCall = roomAPI.getContextOfEvent(params.roomId, params.eventId, 0, filter)
         }
-        return tokenChunkEventPersistor.insertInDb(response, params.roomId, PaginationDirection.BACKWARDS)
+        return tokenChunkEventPersistor.insertInDb(response, params.roomId, PaginationDirection.FORWARDS)
     }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/timeline/TokenChunkEventPersistor.kt

@@ -38,6 +38,8 @@ import im.vector.matrix.android.internal.database.query.create
 import im.vector.matrix.android.internal.database.query.find
 import im.vector.matrix.android.internal.database.query.findAllIncludingEvents
 import im.vector.matrix.android.internal.database.query.findLastLiveChunkFromRoom
+import im.vector.matrix.android.internal.database.query.getOrCreate
+import im.vector.matrix.android.internal.database.query.getOrNull
 import im.vector.matrix.android.internal.database.query.latestEvent
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.session.room.RoomSummaryUpdater
@@ -195,21 +197,12 @@ internal class TokenChunkEventPersistor @Inject constructor(private val monarchy
         val eventList = receivedChunk.events
         val stateEvents = receivedChunk.stateEvents
 
-        realm.where(CurrentStateEventEntity::class.java)
-                .equalTo(CurrentStateEventEntityFields.ROOM_ID, roomId)
-                .equalTo(CurrentStateEventEntityFields.TYPE, EventType.STATE_ROOM_MEMBER)
-                .findAll()
-                .forEach {
-                    val roomMember = ContentMapper.map(it.root?.content).toModel<RoomMemberContent>()
-                    roomMemberContentsByUser[it.stateKey] = roomMember
-                }
-
         for (stateEvent in stateEvents) {
             val stateEventEntity = stateEvent.toEntity(roomId, SendState.SYNCED).let {
                 realm.copyToRealmOrUpdate(it)
             }
             currentChunk.addStateEvent(roomId, stateEventEntity, direction)
-            if (stateEvent.type == EventType.STATE_ROOM_MEMBER && stateEvent.stateKey != null && !stateEvent.isRedacted()) {
+            if (stateEvent.type == EventType.STATE_ROOM_MEMBER && stateEvent.stateKey != null) {
                 roomMemberContentsByUser[stateEvent.stateKey] = stateEvent.content.toModel<RoomMemberContent>()
             }
         }
@@ -222,14 +215,15 @@ internal class TokenChunkEventPersistor @Inject constructor(private val monarchy
             val eventEntity = event.toEntity(roomId, SendState.SYNCED).let {
                 realm.copyToRealmOrUpdate(it)
             }
-            if (event.type == EventType.STATE_ROOM_MEMBER && event.stateKey != null && !event.isRedacted()) {
+            if (event.type == EventType.STATE_ROOM_MEMBER && event.stateKey != null) {
-                val contentToUse = if (direction == PaginationDirection.FORWARDS) {
+                val contentToUse = if (direction == PaginationDirection.BACKWARDS) {
-                    event.content
-                } else {
                     event.prevContent
+                } else {
+                    event.content
                 }
                 roomMemberContentsByUser[event.stateKey] = contentToUse.toModel<RoomMemberContent>()
             }
+
             currentChunk.addTimelineEvent(roomId, eventEntity, direction, roomMemberContentsByUser)
         }
         val chunks = ChunkEntity.findAllIncludingEvents(realm, eventIds)
@@ -240,8 +234,14 @@ internal class TokenChunkEventPersistor @Inject constructor(private val monarchy
                 chunksToDelete.add(it)
             }
         }
+        val shouldUpdateSummary = chunksToDelete.isNotEmpty() && currentChunk.isLastForward && direction == PaginationDirection.FORWARDS
         chunksToDelete.forEach {
-            it.deleteFromRealm()
+            it.deleteOnCascade()
+        }
+        if (shouldUpdateSummary) {
+            val roomSummaryEntity = RoomSummaryEntity.getOrCreate(realm, roomId)
+            val latestPreviewableEvent = TimelineEventEntity.latestEvent(realm, roomId, includesSending = true, filterTypes = RoomSummaryUpdater.PREVIEWABLE_TYPES)
+            roomSummaryEntity.latestPreviewableEvent = latestPreviewableEvent
         }
         RoomEntity.where(realm, roomId).findFirst()?.addOrUpdate(currentChunk)
     }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/tombstone/RoomTombstoneEventLiveObserver.kt

@@ -27,7 +27,7 @@ import im.vector.matrix.android.internal.database.awaitTransaction
 import im.vector.matrix.android.internal.database.mapper.asDomain
 import im.vector.matrix.android.internal.database.model.EventEntity
 import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
-import im.vector.matrix.android.internal.database.query.types
+import im.vector.matrix.android.internal.database.query.whereTypes
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.di.SessionDatabase
 import io.realm.OrderedCollectionChangeSet
@@ -41,7 +41,7 @@ internal class RoomTombstoneEventLiveObserver @Inject constructor(@SessionDataba
     : RealmLiveEntityObserver<EventEntity>(realmConfiguration) {
 
     override val query = Monarchy.Query<EventEntity> {
-        EventEntity.types(it, listOf(EventType.STATE_ROOM_TOMBSTONE))
+        EventEntity.whereTypes(it, listOf(EventType.STATE_ROOM_TOMBSTONE))
     }
 
     override fun onChange(results: RealmResults<EventEntity>, changeSet: OrderedCollectionChangeSet) {

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/sync/RoomSyncHandler.kt

@@ -31,11 +31,11 @@ import im.vector.matrix.android.internal.database.mapper.ContentMapper
 import im.vector.matrix.android.internal.database.mapper.toEntity
 import im.vector.matrix.android.internal.database.model.ChunkEntity
 import im.vector.matrix.android.internal.database.model.CurrentStateEventEntity
-import im.vector.matrix.android.internal.database.model.CurrentStateEventEntityFields
 import im.vector.matrix.android.internal.database.model.RoomEntity
 import im.vector.matrix.android.internal.database.query.find
 import im.vector.matrix.android.internal.database.query.findLastLiveChunkFromRoom
 import im.vector.matrix.android.internal.database.query.getOrCreate
+import im.vector.matrix.android.internal.database.query.getOrNull
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.session.DefaultInitialSyncProgressService
 import im.vector.matrix.android.internal.session.mapWithProgress
@@ -218,15 +218,6 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle
         val eventIds = ArrayList<String>(eventList.size)
         val roomMemberContentsByUser = HashMap<String, RoomMemberContent?>()
 
-        realm.where(CurrentStateEventEntity::class.java)
-                .equalTo(CurrentStateEventEntityFields.ROOM_ID, roomId)
-                .equalTo(CurrentStateEventEntityFields.TYPE, EventType.STATE_ROOM_MEMBER)
-                .findAll()
-                .forEach {
-                    val roomMember = ContentMapper.map(it.root?.content).toModel<RoomMemberContent>()
-                    roomMemberContentsByUser[it.stateKey] = roomMember
-                }
-
         for (event in eventList) {
             if (event.eventId == null || event.senderId == null) {
                 continue
@@ -235,7 +226,7 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle
             val eventEntity = event.toEntity(roomId, SendState.SYNCED).let {
                 realm.copyToRealmOrUpdate(it)
             }
-            if (event.isStateEvent() && event.stateKey != null && !event.isRedacted()) {
+            if (event.isStateEvent() && event.stateKey != null) {
                 CurrentStateEventEntity.getOrCreate(realm, roomId, event.stateKey, event.type).apply {
                     eventId = event.eventId
                     root = eventEntity
@@ -245,6 +236,11 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle
                     roomMemberEventHandler.handle(realm, roomEntity.roomId, event)
                 }
             }
+            roomMemberContentsByUser.getOrPut(event.senderId) {
+                // If we don't have any new state on this user, get it from db
+                val rootStateEvent = CurrentStateEventEntity.getOrNull(realm, roomId, event.senderId, EventType.STATE_ROOM_MEMBER)?.root
+                ContentMapper.map(rootStateEvent?.content).toModel()
+            }
             chunkEntity.addTimelineEvent(roomId, eventEntity, PaginationDirection.FORWARDS, roomMemberContentsByUser)
             // Give info to crypto module
             cryptoService.onLiveEvent(roomEntity.roomId, event)