Merge pull request #1297 from vector-im/feature/xsigning_trust_optimization

Feature/xsigning trust optimization

matrix-sdk-android/src/androidTest/java/im/vector/matrix/android/internal/crypto/CryptoStoreHelper.kt

@@ -20,6 +20,8 @@ import im.vector.matrix.android.api.auth.data.Credentials
 import im.vector.matrix.android.internal.crypto.store.IMXCryptoStore
 import im.vector.matrix.android.internal.crypto.store.db.RealmCryptoStore
 import im.vector.matrix.android.internal.crypto.store.db.RealmCryptoStoreModule
+import im.vector.matrix.android.internal.crypto.store.db.mapper.CrossSigningKeysMapper
+import im.vector.matrix.android.internal.di.MoshiProvider
 import io.realm.RealmConfiguration
 import kotlin.random.Random
 
@@ -31,6 +33,7 @@ internal class CryptoStoreHelper {
                         .name("test.realm")
                         .modules(RealmCryptoStoreModule())
                         .build(),
+                crossSigningKeysMapper = CrossSigningKeysMapper(MoshiProvider.providesMoshi()),
                 credentials = createCredential())
     }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/api/session/room/model/RoomSummary.kt

@@ -49,7 +49,6 @@ data class RoomSummary constructor(
         val inviterId: String? = null,
         val typingRoomMemberIds: List<String> = emptyList(),
         val breadcrumbsIndex: Int = NOT_IN_BREADCRUMBS,
-        // TODO Plug it
         val roomEncryptionTrustLevel: RoomEncryptionTrustLevel? = null
 ) {
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/CryptoModule.kt

@@ -112,6 +112,7 @@ internal abstract class CryptoModule {
         @SessionScope
         fun providesRealmConfiguration(@SessionFilesDirectory directory: File,
                                        @UserMd5 userMd5: String,
+                                       realmCryptoStoreMigration: RealmCryptoStoreMigration,
                                        realmKeysUtils: RealmKeysUtils): RealmConfiguration {
             return RealmConfiguration.Builder()
                     .directory(directory)
@@ -121,7 +122,7 @@ internal abstract class CryptoModule {
                     .name("crypto_store.realm")
                     .modules(RealmCryptoStoreModule())
                     .schemaVersion(RealmCryptoStoreMigration.CRYPTO_STORE_SCHEMA_VERSION)
-                    .migration(RealmCryptoStoreMigration)
+                    .migration(realmCryptoStoreMigration)
                     .build()
         }
 

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/crosssigning/ShieldTrustUpdater.kt

@@ -15,18 +15,20 @@
  */
 package im.vector.matrix.android.internal.crypto.crosssigning
 
+import im.vector.matrix.android.api.extensions.orFalse
 import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntity
 import im.vector.matrix.android.internal.database.model.RoomMemberSummaryEntityFields
+import im.vector.matrix.android.internal.database.model.RoomSummaryEntity
 import im.vector.matrix.android.internal.database.query.where
 import im.vector.matrix.android.internal.di.SessionDatabase
 import im.vector.matrix.android.internal.session.room.RoomSummaryUpdater
+import im.vector.matrix.android.internal.session.room.membership.RoomMemberHelper
 import im.vector.matrix.android.internal.task.TaskExecutor
-import im.vector.matrix.android.internal.util.MatrixCoroutineDispatchers
 import im.vector.matrix.android.internal.util.createBackgroundHandler
 import io.realm.Realm
 import io.realm.RealmConfiguration
+import kotlinx.coroutines.android.asCoroutineDispatcher
 import kotlinx.coroutines.launch
-import kotlinx.coroutines.withContext
 import org.greenrobot.eventbus.EventBus
 import org.greenrobot.eventbus.Subscribe
 import timber.log.Timber
@@ -38,13 +40,13 @@ internal class ShieldTrustUpdater @Inject constructor(
         private val eventBus: EventBus,
         private val computeTrustTask: ComputeTrustTask,
         private val taskExecutor: TaskExecutor,
-        private val coroutineDispatchers: MatrixCoroutineDispatchers,
         @SessionDatabase private val sessionRealmConfiguration: RealmConfiguration,
         private val roomSummaryUpdater: RoomSummaryUpdater
 ) {
 
     companion object {
         private val BACKGROUND_HANDLER = createBackgroundHandler("SHIELD_CRYPTO_DB_THREAD")
+        private val BACKGROUND_HANDLER_DISPATCHER = BACKGROUND_HANDLER.asCoroutineDispatcher()
     }
 
     private val backgroundSessionRealm = AtomicReference<Realm>()
@@ -76,14 +78,11 @@ internal class ShieldTrustUpdater @Inject constructor(
         if (!isStarted.get()) {
             return
         }
-        taskExecutor.executorScope.launch(coroutineDispatchers.crypto) {
+        taskExecutor.executorScope.launch(BACKGROUND_HANDLER_DISPATCHER) {
             val updatedTrust = computeTrustTask.execute(ComputeTrustTask.Params(update.userIds))
             // We need to send that back to session base
-
+            backgroundSessionRealm.get()?.executeTransaction { realm ->
-            BACKGROUND_HANDLER.post {
+                roomSummaryUpdater.updateShieldTrust(realm, update.roomId, updatedTrust)
-                backgroundSessionRealm.get()?.executeTransaction { realm ->
-                    roomSummaryUpdater.updateShieldTrust(realm, update.roomId, updatedTrust)
-                }
             }
         }
     }
@@ -93,45 +92,30 @@ internal class ShieldTrustUpdater @Inject constructor(
         if (!isStarted.get()) {
             return
         }
-
         onCryptoDevicesChange(update.userIds)
     }
 
     private fun onCryptoDevicesChange(users: List<String>) {
-        BACKGROUND_HANDLER.post {
+        taskExecutor.executorScope.launch(BACKGROUND_HANDLER_DISPATCHER) {
-            val impactedRoomsId = backgroundSessionRealm.get()?.where(RoomMemberSummaryEntity::class.java)
+            val realm = backgroundSessionRealm.get() ?: return@launch
-                    ?.`in`(RoomMemberSummaryEntityFields.USER_ID, users.toTypedArray())
+            val distinctRoomIds = realm.where(RoomMemberSummaryEntity::class.java)
-                    ?.findAll()
+                    .`in`(RoomMemberSummaryEntityFields.USER_ID, users.toTypedArray())
-                    ?.map { it.roomId }
+                    .distinct(RoomMemberSummaryEntityFields.ROOM_ID)
-                    ?.distinct()
+                    .findAll()
+                    .map { it.roomId }
 
-            val map = HashMap<String, List<String>>()
+            distinctRoomIds.forEach { roomId ->
-            impactedRoomsId?.forEach { roomId ->
+                val roomSummary = RoomSummaryEntity.where(realm, roomId).findFirst()
-                backgroundSessionRealm.get()?.let { realm ->
+                if (roomSummary?.isEncrypted.orFalse()) {
-                    RoomMemberSummaryEntity.where(realm, roomId)
+                    val allActiveRoomMembers = RoomMemberHelper(realm, roomId).getActiveRoomMemberIds()
-                            .findAll()
+                    try {
-                            .let { results ->
+                        // Can throw if the crypto database has been closed in between, in this case log and ignore?
-                                map[roomId] = results.map { it.userId }
+                        val updatedTrust = computeTrustTask.execute(ComputeTrustTask.Params(allActiveRoomMembers))
-                            }
+                        realm.executeTransaction {
-                }
+                            roomSummaryUpdater.updateShieldTrust(it, roomId, updatedTrust)
-            }
-
-            map.forEach { entry ->
-                val roomId = entry.key
-                val userList = entry.value
-                taskExecutor.executorScope.launch {
-                    withContext(coroutineDispatchers.crypto) {
-                        try {
-                            // Can throw if the crypto database has been closed in between, in this case log and ignore?
-                            val updatedTrust = computeTrustTask.execute(ComputeTrustTask.Params(userList))
-                            BACKGROUND_HANDLER.post {
-                                backgroundSessionRealm.get()?.executeTransaction { realm ->
-                                    roomSummaryUpdater.updateShieldTrust(realm, roomId, updatedTrust)
-                                }
-                            }
-                        } catch (failure: Throwable) {
-                            Timber.e(failure)
                         }
+                    } catch (failure: Throwable) {
+                        Timber.e(failure)
                     }
                 }
             }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/Helper.kt

@@ -62,6 +62,7 @@ fun doRealmTransaction(realmConfiguration: RealmConfiguration, action: (Realm) -
         realm.executeTransaction { action.invoke(it) }
     }
 }
+
 fun doRealmTransactionAsync(realmConfiguration: RealmConfiguration, action: (Realm) -> Unit) {
     Realm.getInstance(realmConfiguration).use { realm ->
         realm.executeTransactionAsync { action.invoke(it) }
@@ -79,31 +80,26 @@ fun serializeForRealm(o: Any?): String? {
     val baos = ByteArrayOutputStream()
     val gzis = CompatUtil.createGzipOutputStream(baos)
     val out = ObjectOutputStream(gzis)
-
+    out.use {
-    out.writeObject(o)
+        it.writeObject(o)
-    out.close()
+    }
-
     return Base64.encodeToString(baos.toByteArray(), Base64.DEFAULT)
 }
 
 /**
  * Do the opposite of serializeForRealm.
  */
+@Suppress("UNCHECKED_CAST")
 fun <T> deserializeFromRealm(string: String?): T? {
     if (string == null) {
         return null
     }
-
     val decodedB64 = Base64.decode(string.toByteArray(), Base64.DEFAULT)
 
     val bais = ByteArrayInputStream(decodedB64)
     val gzis = GZIPInputStream(bais)
     val ois = ObjectInputStream(gzis)
-
+    return ois.use {
-    @Suppress("UNCHECKED_CAST")
+        it.readObject() as T
-    val result = ois.readObject() as T
+    }
-
-    ois.close()
-
-    return result
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/RealmCryptoStore.kt

@@ -36,7 +36,6 @@ import im.vector.matrix.android.internal.crypto.OutgoingGossipingRequestState
 import im.vector.matrix.android.internal.crypto.OutgoingRoomKeyRequest
 import im.vector.matrix.android.internal.crypto.OutgoingSecretRequest
 import im.vector.matrix.android.internal.crypto.algorithms.olm.OlmDecryptionResult
-import im.vector.matrix.android.internal.crypto.crosssigning.DeviceTrustLevel
 import im.vector.matrix.android.internal.crypto.model.CryptoCrossSigningKey
 import im.vector.matrix.android.internal.crypto.model.CryptoDeviceInfo
 import im.vector.matrix.android.internal.crypto.model.OlmInboundGroupSessionWrapper
@@ -46,6 +45,7 @@ import im.vector.matrix.android.internal.crypto.model.toEntity
 import im.vector.matrix.android.internal.crypto.store.IMXCryptoStore
 import im.vector.matrix.android.internal.crypto.store.PrivateKeysInfo
 import im.vector.matrix.android.internal.crypto.store.SavedKeyBackupKeyInfo
+import im.vector.matrix.android.internal.crypto.store.db.mapper.CrossSigningKeysMapper
 import im.vector.matrix.android.internal.crypto.store.db.model.CrossSigningInfoEntity
 import im.vector.matrix.android.internal.crypto.store.db.model.CrossSigningInfoEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.CryptoMapper
@@ -91,6 +91,7 @@ import kotlin.collections.set
 @SessionScope
 internal class RealmCryptoStore @Inject constructor(
         @CryptoDatabase private val realmConfiguration: RealmConfiguration,
+        private val crossSigningKeysMapper: CrossSigningKeysMapper,
         private val credentials: Credentials) : IMXCryptoStore {
 
     /* ==========================================================================================
@@ -200,9 +201,9 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getDeviceId(): String {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
-            it.where<CryptoMetadataEntity>().findFirst()
+            it.where<CryptoMetadataEntity>().findFirst()?.deviceId
-        }?.deviceId ?: ""
+        } ?: ""
     }
 
     override fun saveOlmAccount() {
@@ -256,24 +257,25 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getUserDevice(userId: String, deviceId: String): CryptoDeviceInfo? {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<DeviceInfoEntity>()
                     .equalTo(DeviceInfoEntityFields.PRIMARY_KEY, DeviceInfoEntity.createPrimaryKey(userId, deviceId))
                     .findFirst()
-        }?.let {
+                    ?.let { deviceInfo ->
-            CryptoMapper.mapToModel(it)
+                        CryptoMapper.mapToModel(deviceInfo)
+                    }
         }
     }
 
     override fun deviceWithIdentityKey(identityKey: String): CryptoDeviceInfo? {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<DeviceInfoEntity>()
                     .equalTo(DeviceInfoEntityFields.IDENTITY_KEY, identityKey)
                     .findFirst()
+                    ?.let { deviceInfo ->
+                        CryptoMapper.mapToModel(deviceInfo)
+                    }
         }
-                ?.let {
-                    CryptoMapper.mapToModel(it)
-                }
     }
 
     override fun storeUserDevices(userId: String, devices: Map<String, CryptoDeviceInfo>?) {
@@ -309,36 +311,19 @@ internal class RealmCryptoStore @Inject constructor(
                         } else {
                             CrossSigningInfoEntity.getOrCreate(realm, userId).let { signingInfo ->
                                 // What should we do if we detect a change of the keys?
-
                                 val existingMaster = signingInfo.getMasterKey()
                                 if (existingMaster != null && existingMaster.publicKeyBase64 == masterKey.unpaddedBase64PublicKey) {
-                                    // update signatures?
+                                    crossSigningKeysMapper.update(existingMaster, masterKey)
-                                    existingMaster.putSignatures(masterKey.signatures)
-                                    existingMaster.usages = masterKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                            ?: RealmList()
                                 } else {
-                                    val keyEntity = realm.createObject(KeyInfoEntity::class.java).apply {
+                                    val keyEntity = crossSigningKeysMapper.map(masterKey)
-                                        this.publicKeyBase64 = masterKey.unpaddedBase64PublicKey
-                                        this.usages = masterKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                                ?: RealmList()
-                                        this.putSignatures(masterKey.signatures)
-                                    }
                                     signingInfo.setMasterKey(keyEntity)
                                 }
 
                                 val existingSelfSigned = signingInfo.getSelfSignedKey()
                                 if (existingSelfSigned != null && existingSelfSigned.publicKeyBase64 == selfSigningKey.unpaddedBase64PublicKey) {
-                                    // update signatures?
+                                    crossSigningKeysMapper.update(existingSelfSigned, selfSigningKey)
-                                    existingSelfSigned.putSignatures(selfSigningKey.signatures)
-                                    existingSelfSigned.usages = selfSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                            ?: RealmList()
                                 } else {
-                                    val keyEntity = realm.createObject(KeyInfoEntity::class.java).apply {
+                                    val keyEntity = crossSigningKeysMapper.map(selfSigningKey)
-                                        this.publicKeyBase64 = selfSigningKey.unpaddedBase64PublicKey
-                                        this.usages = selfSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                                ?: RealmList()
-                                        this.putSignatures(selfSigningKey.signatures)
-                                    }
                                     signingInfo.setSelfSignedKey(keyEntity)
                                 }
 
@@ -346,21 +331,12 @@ internal class RealmCryptoStore @Inject constructor(
                                 if (userSigningKey != null) {
                                     val existingUSK = signingInfo.getUserSigningKey()
                                     if (existingUSK != null && existingUSK.publicKeyBase64 == userSigningKey.unpaddedBase64PublicKey) {
-                                        // update signatures?
+                                        crossSigningKeysMapper.update(existingUSK, userSigningKey)
-                                        existingUSK.putSignatures(userSigningKey.signatures)
-                                        existingUSK.usages = userSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                                ?: RealmList()
                                     } else {
-                                        val keyEntity = realm.createObject(KeyInfoEntity::class.java).apply {
+                                        val keyEntity = crossSigningKeysMapper.map(userSigningKey)
-                                            this.publicKeyBase64 = userSigningKey.unpaddedBase64PublicKey
-                                            this.usages = userSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
-                                                    ?: RealmList()
-                                            this.putSignatures(userSigningKey.signatures)
-                                        }
                                         signingInfo.setUserSignedKey(keyEntity)
                                     }
                                 }
-
                                 userEntity.crossSigningInfoEntity = signingInfo
                             }
                         }
@@ -369,14 +345,16 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getCrossSigningPrivateKeys(): PrivateKeysInfo? {
-        return doRealmQueryAndCopy(realmConfiguration) { realm ->
+        return doWithRealm(realmConfiguration) { realm ->
-            realm.where<CryptoMetadataEntity>().findFirst()
+            realm.where<CryptoMetadataEntity>()
-        }?.let {
+                    .findFirst()
-            PrivateKeysInfo(
+                    ?.let {
-                    master = it.xSignMasterPrivateKey,
+                        PrivateKeysInfo(
-                    selfSigned = it.xSignSelfSignedPrivateKey,
+                                master = it.xSignMasterPrivateKey,
-                    user = it.xSignUserPrivateKey
+                                selfSigned = it.xSignSelfSignedPrivateKey,
-            )
+                                user = it.xSignUserPrivateKey
+                        )
+                    }
         }
     }
 
@@ -400,16 +378,18 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getKeyBackupRecoveryKeyInfo(): SavedKeyBackupKeyInfo? {
-        return doRealmQueryAndCopy(realmConfiguration) { realm ->
+        return doWithRealm(realmConfiguration) { realm ->
-            realm.where<CryptoMetadataEntity>().findFirst()
+            realm.where<CryptoMetadataEntity>()
-        }?.let {
+                    .findFirst()
-            val key = it.keyBackupRecoveryKey
+                    ?.let {
-            val version = it.keyBackupRecoveryKeyVersion
+                        val key = it.keyBackupRecoveryKey
-            if (!key.isNullOrBlank() && !version.isNullOrBlank()) {
+                        val version = it.keyBackupRecoveryKeyVersion
-                SavedKeyBackupKeyInfo(recoveryKey = key, version = version)
+                        if (!key.isNullOrBlank() && !version.isNullOrBlank()) {
-            } else {
+                            SavedKeyBackupKeyInfo(recoveryKey = key, version = version)
-                null
+                        } else {
-            }
+                            null
+                        }
+                    }
         }
     }
 
@@ -430,24 +410,30 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getUserDevices(userId: String): Map<String, CryptoDeviceInfo>? {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<UserEntity>()
                     .equalTo(UserEntityFields.USER_ID, userId)
                     .findFirst()
+                    ?.devices
+                    ?.map { deviceInfo ->
+                        CryptoMapper.mapToModel(deviceInfo)
+                    }
+                    ?.associateBy { cryptoDevice ->
+                        cryptoDevice.deviceId
+                    }
         }
-                ?.devices
-                ?.map { CryptoMapper.mapToModel(it) }
-                ?.associateBy { it.deviceId }
     }
 
     override fun getUserDeviceList(userId: String): List<CryptoDeviceInfo>? {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<UserEntity>()
                     .equalTo(UserEntityFields.USER_ID, userId)
                     .findFirst()
+                    ?.devices
+                    ?.map { deviceInfo ->
+                        CryptoMapper.mapToModel(deviceInfo)
+                    }
         }
-                ?.devices
-                ?.map { CryptoMapper.mapToModel(it) }
     }
 
     override fun getLiveDeviceList(userId: String): LiveData<List<CryptoDeviceInfo>> {
@@ -503,17 +489,16 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getRoomAlgorithm(roomId: String): String? {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
-            CryptoRoomEntity.getById(it, roomId)
+            CryptoRoomEntity.getById(it, roomId)?.algorithm
         }
-                ?.algorithm
     }
 
     override fun shouldEncryptForInvitedMembers(roomId: String): Boolean {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
-            CryptoRoomEntity.getById(it, roomId)
+            CryptoRoomEntity.getById(it, roomId)?.shouldEncryptForInvitedMembers
         }
-                ?.shouldEncryptForInvitedMembers ?: false
+                ?: false
     }
 
     override fun setShouldEncryptForInvitedMembers(roomId: String, shouldEncryptForInvitedMembers: Boolean) {
@@ -577,24 +562,24 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getLastUsedSessionId(deviceKey: String): String? {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<OlmSessionEntity>()
                     .equalTo(OlmSessionEntityFields.DEVICE_KEY, deviceKey)
                     .sort(OlmSessionEntityFields.LAST_RECEIVED_MESSAGE_TS, Sort.DESCENDING)
                     .findFirst()
+                    ?.sessionId
         }
-                ?.sessionId
     }
 
     override fun getDeviceSessionIds(deviceKey: String): MutableSet<String> {
-        return doRealmQueryAndCopyList(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<OlmSessionEntity>()
                     .equalTo(OlmSessionEntityFields.DEVICE_KEY, deviceKey)
                     .findAll()
+                    .mapNotNull { sessionEntity ->
+                        sessionEntity.sessionId
+                    }
         }
-                .mapNotNull {
-                    it.sessionId
-                }
                 .toMutableSet()
     }
 
@@ -641,12 +626,12 @@ internal class RealmCryptoStore @Inject constructor(
 
         // If not in cache (or not found), try to read it from realm
         if (inboundGroupSessionToRelease[key] == null) {
-            doRealmQueryAndCopy(realmConfiguration) {
+            doWithRealm(realmConfiguration) {
                 it.where<OlmInboundGroupSessionEntity>()
                         .equalTo(OlmInboundGroupSessionEntityFields.PRIMARY_KEY, key)
                         .findFirst()
+                        ?.getInboundGroupSession()
             }
-                    ?.getInboundGroupSession()
                     ?.let {
                         inboundGroupSessionToRelease[key] = it
                     }
@@ -660,13 +645,13 @@ internal class RealmCryptoStore @Inject constructor(
      * so there is no need to use or update `inboundGroupSessionToRelease` for native memory management
      */
     override fun getInboundGroupSessions(): MutableList<OlmInboundGroupSessionWrapper> {
-        return doRealmQueryAndCopyList(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<OlmInboundGroupSessionEntity>()
                     .findAll()
+                    .mapNotNull { inboundGroupSessionEntity ->
+                        inboundGroupSessionEntity.getInboundGroupSession()
+                    }
         }
-                .mapNotNull {
-                    it.getInboundGroupSession()
-                }
                 .toMutableList()
     }
 
@@ -755,13 +740,14 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun inboundGroupSessionsToBackup(limit: Int): List<OlmInboundGroupSessionWrapper> {
-        return doRealmQueryAndCopyList(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<OlmInboundGroupSessionEntity>()
                     .equalTo(OlmInboundGroupSessionEntityFields.BACKED_UP, false)
                     .limit(limit.toLong())
                     .findAll()
-        }.mapNotNull { inboundGroupSession ->
+                    .mapNotNull { inboundGroupSession ->
-            inboundGroupSession.getInboundGroupSession()
+                        inboundGroupSession.getInboundGroupSession()
+                    }
         }
     }
 
@@ -785,10 +771,9 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getGlobalBlacklistUnverifiedDevices(): Boolean {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
-            it.where<CryptoMetadataEntity>().findFirst()
+            it.where<CryptoMetadataEntity>().findFirst()?.globalBlacklistUnverifiedDevices
-        }?.globalBlacklistUnverifiedDevices
+        } ?: false
-                ?: false
     }
 
     override fun setRoomsListBlacklistUnverifiedDevices(roomIds: List<String>) {
@@ -811,28 +796,28 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getRoomsListBlacklistUnverifiedDevices(): MutableList<String> {
-        return doRealmQueryAndCopyList(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<CryptoRoomEntity>()
                     .equalTo(CryptoRoomEntityFields.BLACKLIST_UNVERIFIED_DEVICES, true)
                     .findAll()
+                    .mapNotNull { cryptoRoom ->
+                        cryptoRoom.roomId
+                    }
         }
-                .mapNotNull {
-                    it.roomId
-                }
                 .toMutableList()
     }
 
     override fun getDeviceTrackingStatuses(): MutableMap<String, Int> {
-        return doRealmQueryAndCopyList(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<UserEntity>()
                     .findAll()
+                    .associateBy { user ->
+                        user.userId!!
+                    }
+                    .mapValues { entry ->
+                        entry.value.deviceTrackingStatus
+                    }
         }
-                .associateBy {
-                    it.userId!!
-                }
-                .mapValues {
-                    it.value.deviceTrackingStatus
-                }
                 .toMutableMap()
     }
 
@@ -847,12 +832,12 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getDeviceTrackingStatus(userId: String, defaultValue: Int): Int {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<UserEntity>()
                     .equalTo(UserEntityFields.USER_ID, userId)
                     .findFirst()
+                    ?.deviceTrackingStatus
         }
-                ?.deviceTrackingStatus
                 ?: defaultValue
     }
 
@@ -1089,63 +1074,65 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getIncomingRoomKeyRequest(userId: String, deviceId: String, requestId: String): IncomingRoomKeyRequest? {
-        return doRealmQueryAndCopyList(realmConfiguration) { realm ->
+        return doWithRealm(realmConfiguration) { realm ->
             realm.where<IncomingGossipingRequestEntity>()
                     .equalTo(IncomingGossipingRequestEntityFields.TYPE_STR, GossipRequestType.KEY.name)
                     .equalTo(IncomingGossipingRequestEntityFields.OTHER_DEVICE_ID, deviceId)
                     .equalTo(IncomingGossipingRequestEntityFields.OTHER_USER_ID, userId)
                     .findAll()
-        }.mapNotNull { entity ->
+                    .mapNotNull { entity ->
-            entity.toIncomingGossipingRequest() as? IncomingRoomKeyRequest
+                        entity.toIncomingGossipingRequest() as? IncomingRoomKeyRequest
-        }.firstOrNull()
+                    }
+                    .firstOrNull()
+        }
     }
 
     override fun getPendingIncomingRoomKeyRequests(): List<IncomingRoomKeyRequest> {
-        return doRealmQueryAndCopyList(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<IncomingGossipingRequestEntity>()
                     .equalTo(IncomingGossipingRequestEntityFields.TYPE_STR, GossipRequestType.KEY.name)
                     .equalTo(IncomingGossipingRequestEntityFields.REQUEST_STATE_STR, GossipingRequestState.PENDING.name)
                     .findAll()
+                    .map { entity ->
+                        IncomingRoomKeyRequest(
+                                userId = entity.otherUserId,
+                                deviceId = entity.otherDeviceId,
+                                requestId = entity.requestId,
+                                requestBody = entity.getRequestedKeyInfo(),
+                                localCreationTimestamp = entity.localCreationTimestamp
+                        )
+                    }
         }
-                .map { entity ->
-                    IncomingRoomKeyRequest(
-                            userId = entity.otherUserId,
-                            deviceId = entity.otherDeviceId,
-                            requestId = entity.requestId,
-                            requestBody = entity.getRequestedKeyInfo(),
-                            localCreationTimestamp = entity.localCreationTimestamp
-                    )
-                }
     }
 
     override fun getPendingIncomingGossipingRequests(): List<IncomingShareRequestCommon> {
-        return doRealmQueryAndCopyList(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
             it.where<IncomingGossipingRequestEntity>()
                     .equalTo(IncomingGossipingRequestEntityFields.REQUEST_STATE_STR, GossipingRequestState.PENDING.name)
                     .findAll()
-        }
+                    .mapNotNull { entity ->
-                .mapNotNull { entity ->
+                        when (entity.type) {
-                    when (entity.type) {
+                            GossipRequestType.KEY    -> {
-                        GossipRequestType.KEY    -> {
+                                IncomingRoomKeyRequest(
-                            IncomingRoomKeyRequest(
+                                        userId = entity.otherUserId,
-                                    userId = entity.otherUserId,
+                                        deviceId = entity.otherDeviceId,
-                                    deviceId = entity.otherDeviceId,
+                                        requestId = entity.requestId,
-                                    requestId = entity.requestId,
+                                        requestBody = entity.getRequestedKeyInfo(),
-                                    requestBody = entity.getRequestedKeyInfo(),
+                                        localCreationTimestamp = entity.localCreationTimestamp
-                                    localCreationTimestamp = entity.localCreationTimestamp
+                                )
-                            )
+                            }
-                        }
+                            GossipRequestType.SECRET -> {
-                        GossipRequestType.SECRET -> {
+                                IncomingSecretShareRequest(
-                            IncomingSecretShareRequest(
+                                        userId = entity.otherUserId,
-                                    userId = entity.otherUserId,
+                                        deviceId = entity.otherDeviceId,
-                                    deviceId = entity.otherDeviceId,
+                                        requestId = entity.requestId,
-                                    requestId = entity.requestId,
+                                        secretName = entity.getRequestedSecretName(),
-                                    secretName = entity.getRequestedSecretName(),
+                                        localCreationTimestamp = entity.localCreationTimestamp
-                                    localCreationTimestamp = entity.localCreationTimestamp
+                                )
-                            )
+                            }
                         }
                     }
-                }
+        }
     }
 
     override fun storeIncomingGossipingRequest(request: IncomingShareRequestCommon, ageLocalTS: Long?) {
@@ -1183,9 +1170,9 @@ internal class RealmCryptoStore @Inject constructor(
      * Cross Signing
      * ========================================================================================== */
     override fun getMyCrossSigningInfo(): MXCrossSigningInfo? {
-        return doRealmQueryAndCopy(realmConfiguration) {
+        return doWithRealm(realmConfiguration) {
-            it.where<CryptoMetadataEntity>().findFirst()
+            it.where<CryptoMetadataEntity>().findFirst()?.userId
-        }?.userId?.let {
+        }?.let {
             getCrossSigningInfo(it)
         }
     }
@@ -1304,33 +1291,24 @@ internal class RealmCryptoStore @Inject constructor(
     }
 
     override fun getCrossSigningInfo(userId: String): MXCrossSigningInfo? {
-        return doRealmQueryAndCopy(realmConfiguration) { realm ->
+        return doWithRealm(realmConfiguration) { realm ->
-            realm.where(CrossSigningInfoEntity::class.java)
+            val crossSigningInfo = realm.where(CrossSigningInfoEntity::class.java)
                     .equalTo(CrossSigningInfoEntityFields.USER_ID, userId)
                     .findFirst()
-        }?.let { xsignInfo ->
+            if (crossSigningInfo == null) {
-            mapCrossSigningInfoEntity(xsignInfo)
+                null
+            } else {
+                mapCrossSigningInfoEntity(crossSigningInfo)
+            }
         }
     }
 
     private fun mapCrossSigningInfoEntity(xsignInfo: CrossSigningInfoEntity): MXCrossSigningInfo {
+        val userId = xsignInfo.userId ?: ""
         return MXCrossSigningInfo(
-                userId = xsignInfo.userId ?: "",
+                userId = userId,
                 crossSigningKeys = xsignInfo.crossSigningKeys.mapNotNull {
-                    val pubKey = it.publicKeyBase64 ?: return@mapNotNull null
+                    crossSigningKeysMapper.map(userId, it)
-                    CryptoCrossSigningKey(
-                            userId = xsignInfo.userId ?: "",
-                            keys = mapOf("ed25519:$pubKey" to pubKey),
-                            usages = it.usages.map { it },
-                            signatures = it.getSignatures(),
-                            trustLevel = it.trustLevelEntity?.let {
-                                DeviceTrustLevel(
-                                        crossSigningVerified = it.crossSignedVerified ?: false,
-                                        locallyVerified = it.locallyVerified ?: false
-                                )
-                            }
-
-                    )
                 }
         )
     }
@@ -1341,26 +1319,7 @@ internal class RealmCryptoStore @Inject constructor(
                     realm.where<CrossSigningInfoEntity>()
                             .equalTo(UserEntityFields.USER_ID, userId)
                 },
-                { entity ->
+                { mapCrossSigningInfoEntity(it) }
-                    MXCrossSigningInfo(
-                            userId = userId,
-                            crossSigningKeys = entity.crossSigningKeys.mapNotNull {
-                                val pubKey = it.publicKeyBase64 ?: return@mapNotNull null
-                                CryptoCrossSigningKey(
-                                        userId = userId,
-                                        keys = mapOf("ed25519:$pubKey" to pubKey),
-                                        usages = it.usages.map { it },
-                                        signatures = it.getSignatures(),
-                                        trustLevel = it.trustLevelEntity?.let {
-                                            DeviceTrustLevel(
-                                                    crossSigningVerified = it.crossSignedVerified ?: false,
-                                                    locallyVerified = it.locallyVerified ?: false
-                                            )
-                                        }
-                                )
-                            }
-                    )
-                }
         )
         return Transformations.map(liveData) {
             it.firstOrNull().toOptional()
@@ -1402,17 +1361,8 @@ internal class RealmCryptoStore @Inject constructor(
             // existing.crossSigningKeys.forEach { it.deleteFromRealm() }
             val xkeys = RealmList<KeyInfoEntity>()
             info.crossSigningKeys.forEach { cryptoCrossSigningKey ->
-                xkeys.add(
+                val keyEntity = crossSigningKeysMapper.map(cryptoCrossSigningKey)
-                        realm.createObject(KeyInfoEntity::class.java).also { keyInfoEntity ->
+                xkeys.add(keyEntity)
-                            keyInfoEntity.publicKeyBase64 = cryptoCrossSigningKey.unpaddedBase64PublicKey
-                            keyInfoEntity.usages = cryptoCrossSigningKey.usages?.let { RealmList(*it.toTypedArray()) }
-                                    ?: RealmList()
-                            keyInfoEntity.putSignatures(cryptoCrossSigningKey.signatures)
-                            // TODO how to handle better, check if same keys?
-                            // reset trust
-                            keyInfoEntity.trustLevelEntity = null
-                        }
-                )
             }
             existing.crossSigningKeys = xkeys
         }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/RealmCryptoStoreMigration.kt

@@ -20,6 +20,7 @@ import com.squareup.moshi.Moshi
 import com.squareup.moshi.Types
 import im.vector.matrix.android.api.util.JsonDict
 import im.vector.matrix.android.internal.crypto.model.MXDeviceInfo
+import im.vector.matrix.android.internal.crypto.store.db.mapper.CrossSigningKeysMapper
 import im.vector.matrix.android.internal.crypto.store.db.model.CrossSigningInfoEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.CryptoMetadataEntityFields
 import im.vector.matrix.android.internal.crypto.store.db.model.DeviceInfoEntityFields
@@ -33,11 +34,14 @@ import im.vector.matrix.android.internal.di.SerializeNulls
 import io.realm.DynamicRealm
 import io.realm.RealmMigration
 import timber.log.Timber
+import javax.inject.Inject
 
-internal object RealmCryptoStoreMigration : RealmMigration {
+internal class RealmCryptoStoreMigration @Inject constructor(private val crossSigningKeysMapper: CrossSigningKeysMapper) : RealmMigration {
 
     // Version 1L added Cross Signing info persistence
-    const val CRYPTO_STORE_SCHEMA_VERSION = 3L
+    companion object {
+        const val CRYPTO_STORE_SCHEMA_VERSION = 4L
+    }
 
     override fun migrate(realm: DynamicRealm, oldVersion: Long, newVersion: Long) {
         Timber.v("Migrating Realm Crypto from $oldVersion to $newVersion")
@@ -45,6 +49,7 @@ internal object RealmCryptoStoreMigration : RealmMigration {
         if (oldVersion <= 0) migrateTo1(realm)
         if (oldVersion <= 1) migrateTo2(realm)
         if (oldVersion <= 2) migrateTo3(realm)
+        if (oldVersion <= 3) migrateTo4(realm)
     }
 
     private fun migrateTo1(realm: DynamicRealm) {
@@ -193,4 +198,18 @@ internal object RealmCryptoStoreMigration : RealmMigration {
                 ?.addField(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY, String::class.java)
                 ?.addField(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY_VERSION, String::class.java)
     }
+
+    private fun migrateTo4(realm: DynamicRealm) {
+        Timber.d("Updating KeyInfoEntity table")
+        val keyInfoEntities = realm.where("KeyInfoEntity").findAll()
+        try {
+            keyInfoEntities.forEach {
+                val stringSignatures = it.getString(KeyInfoEntityFields.SIGNATURES)
+                val objectSignatures: Map<String, Map<String, String>>? = deserializeFromRealm(stringSignatures)
+                val jsonSignatures = crossSigningKeysMapper.serializeSignatures(objectSignatures)
+                it.setString(KeyInfoEntityFields.SIGNATURES, jsonSignatures)
+            }
+        } catch (failure: Throwable) {
+        }
+    }
 }

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/mapper/CrossSigningKeysMapper.kt

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2020 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package im.vector.matrix.android.internal.crypto.store.db.mapper
+
+import com.squareup.moshi.Moshi
+import com.squareup.moshi.Types
+import im.vector.matrix.android.internal.crypto.crosssigning.DeviceTrustLevel
+import im.vector.matrix.android.internal.crypto.model.CryptoCrossSigningKey
+import im.vector.matrix.android.internal.crypto.store.db.model.KeyInfoEntity
+import io.realm.RealmList
+import timber.log.Timber
+import javax.inject.Inject
+
+internal class CrossSigningKeysMapper @Inject constructor(moshi: Moshi) {
+
+    private val signaturesAdapter = moshi.adapter<Map<String, Map<String, String>>>(Types.newParameterizedType(
+            Map::class.java,
+            String::class.java,
+            Any::class.java
+    ))
+
+    fun update(keyInfo: KeyInfoEntity, cryptoCrossSigningKey: CryptoCrossSigningKey) {
+        // update signatures?
+        keyInfo.signatures = serializeSignatures(cryptoCrossSigningKey.signatures)
+        keyInfo.usages = cryptoCrossSigningKey.usages?.toTypedArray()?.let { RealmList(*it) }
+                ?: RealmList()
+    }
+
+    fun map(userId: String?, keyInfo: KeyInfoEntity?): CryptoCrossSigningKey? {
+        val pubKey = keyInfo?.publicKeyBase64 ?: return null
+        return CryptoCrossSigningKey(
+                userId = userId ?: "",
+                keys = mapOf("ed25519:$pubKey" to pubKey),
+                usages = keyInfo.usages.map { it },
+                signatures = deserializeSignatures(keyInfo.signatures),
+                trustLevel = keyInfo.trustLevelEntity?.let {
+                    DeviceTrustLevel(
+                            crossSigningVerified = it.crossSignedVerified ?: false,
+                            locallyVerified = it.locallyVerified ?: false
+                    )
+                }
+        )
+    }
+
+    fun map(keyInfo: CryptoCrossSigningKey): KeyInfoEntity {
+        return KeyInfoEntity().apply {
+            publicKeyBase64 = keyInfo.unpaddedBase64PublicKey
+            usages = keyInfo.usages?.let { RealmList(*it.toTypedArray()) } ?: RealmList()
+            signatures = serializeSignatures(keyInfo.signatures)
+            // TODO how to handle better, check if same keys?
+            // reset trust
+            trustLevelEntity = null
+        }
+    }
+
+    fun serializeSignatures(signatures: Map<String, Map<String, String>>?): String {
+        return signaturesAdapter.toJson(signatures)
+    }
+
+    fun deserializeSignatures(signatures: String?): Map<String, Map<String, String>>? {
+        if (signatures == null) {
+            return null
+        }
+        return try {
+            signaturesAdapter.fromJson(signatures)
+        } catch (failure: Throwable) {
+            Timber.e(failure)
+            null
+        }
+    }
+}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/crypto/store/db/model/KeyInfoEntity.kt

@@ -16,8 +16,6 @@
 
 package im.vector.matrix.android.internal.crypto.store.db.model
 
-import im.vector.matrix.android.internal.crypto.store.db.deserializeFromRealm
-import im.vector.matrix.android.internal.crypto.store.db.serializeForRealm
 import io.realm.RealmList
 import io.realm.RealmObject
 
@@ -31,15 +29,4 @@ internal open class KeyInfoEntity(
          */
         var signatures: String? = null,
         var trustLevelEntity: TrustLevelEntity? = null
-) : RealmObject() {
+) : RealmObject()
-
-    // Deserialize data
-    fun getSignatures(): Map<String, Map<String, String>>? {
-        return deserializeFromRealm(signatures)
-    }
-
-    // Serialize data
-    fun putSignatures(deviceInfo: Map<String, Map<String, String>>?) {
-        signatures = serializeForRealm(deviceInfo)
-    }
-}

matrix-sdk-android/src/main/java/im/vector/matrix/android/internal/session/room/RoomSummaryUpdater.kt

@@ -152,7 +152,7 @@ internal class RoomSummaryUpdater @Inject constructor(
 
         if (updateMembers) {
             val otherRoomMembers = RoomMemberHelper(realm, roomId)
-                    .queryRoomMembersEvent()
+                    .queryActiveRoomMembersEvent()
                     .notEqualTo(RoomMemberSummaryEntityFields.USER_ID, userId)
                     .findAll()
                     .asSequence()