create rust db as a realm migration

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt

@@ -18,14 +18,20 @@ package org.matrix.android.sdk.internal.database
 
 import android.content.Context
 import androidx.test.platform.app.InstrumentationRegistry
+import io.mockk.spyk
 import io.realm.Realm
 import org.junit.After
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
+import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
+import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
+import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
 import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
 import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
 import org.matrix.android.sdk.internal.util.time.Clock
+import java.io.File
+import java.security.KeyStore
 
 class CryptoSanityMigrationTest {
     @get:Rule val configurationFactory = TestRealmConfigurationFactory()
@@ -43,14 +49,28 @@ class CryptoSanityMigrationTest {
         realm?.close()
     }
 
+    private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
+
     @Test
     fun cryptoDatabaseShouldMigrateGracefully() {
         val realmName = "crypto_store_20.realm"
-        val migration = RealmCryptoStoreMigration(object : Clock {
+
+        val migration = RealmCryptoStoreMigration(
+                object : Clock {
                     override fun epochMillis(): Long {
                         return 0L
                     }
-        })
+                },
+                RustEncryptionConfiguration(
+                        "foo",
+                        RealmKeysUtils(
+                                context,
+                                SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
+                        )
+                ),
+                File("test_rust")
+        )
+
         val realmConfiguration = configurationFactory.createConfiguration(
                 realmName,
                 "7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca",

matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt

@@ -19,6 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.migration
 import android.content.Context
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.platform.app.InstrumentationRegistry
+import io.mockk.spyk
 import io.realm.Realm
 import io.realm.kotlin.where
 import org.amshove.kluent.internal.assertEquals
@@ -31,32 +32,33 @@ import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.matrix.android.sdk.InstrumentedTest
+import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
+import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
+import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
 import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
 import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
 import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
+import org.matrix.android.sdk.internal.database.RealmKeysUtils
-import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields
 import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory
-import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation
 import org.matrix.android.sdk.internal.util.time.Clock
 import org.matrix.olm.OlmAccount
 import org.matrix.olm.OlmManager
 import org.matrix.rustcomponents.sdk.crypto.OlmMachine
 import java.io.File
+import java.security.KeyStore
 
 @RunWith(AndroidJUnit4::class)
-class ElementAndroidToElementRMigrationTest : InstrumentedTest {
+class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest {
 
     @get:Rule val configurationFactory = TestRealmConfigurationFactory()
 
-    lateinit var context: Context
+    var context: Context = InstrumentationRegistry.getInstrumentation().context
     var realm: Realm? = null
 
     @Before
     fun setUp() {
         // Ensure Olm is initialized
         OlmManager()
-        context = InstrumentationRegistry.getInstrumentation().context
     }
 
     @After
@@ -64,7 +66,22 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
         realm?.close()
     }
 
+    private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
+
+    private val rustEncryptionConfiguration = RustEncryptionConfiguration(
+            "foo",
+            RealmKeysUtils(
+                    context,
+                    SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
+            )
+    )
+
+    private val fakeClock = object : Clock {
+        override fun epochMillis() = 0L
+    }
+
     @Test
+<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
     fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() {
         testMigrate(false)
     }
@@ -76,10 +93,13 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
     }
 
     private fun testMigrate(migrateGroupSessions: Boolean) {
+=======
+    fun dynamic_migration_to_rust() {
+        val targetFile = File(configurationFactory.root, "rust-sdk")
+
+>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
         val realmName = "crypto_store_migration_16.realm"
-        val migration = RealmCryptoStoreMigration(object : Clock {
+        val migration = RealmCryptoStoreMigration(fakeClock, rustEncryptionConfiguration, targetFile)
-            override fun epochMillis() = 0L
-        })
 
         val realmConfiguration = configurationFactory.createConfiguration(
                 realmName,
@@ -91,12 +111,12 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
         configurationFactory.copyRealmFromAssets(context, realmName, realmName)
 
         realm = Realm.getInstance(realmConfiguration)
-
         val metaData = realm!!.where<CryptoMetadataEntity>().findFirst()!!
         val userId = metaData.userId!!
         val deviceId = metaData.deviceId!!
         val olmAccount = metaData.getOlmAccount()!!
 
+<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
         val extractor = MigrateEAtoEROperation(migrateGroupSessions)
 
         val targetFile = File(configurationFactory.root, "rust-sdk")
@@ -104,6 +124,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
         extractor.execute(realmConfiguration, targetFile, null)
 
         val machine = OlmMachine(userId, deviceId, targetFile.path, null)
+=======
+        val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase())
+>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
 
         assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"])
         assertNotNull(machine.getBackupKeys())
@@ -112,6 +135,7 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
         assertTrue(crossSigningStatus.hasSelfSigning)
         assertTrue(crossSigningStatus.hasUserSigning)
 
+<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
         if (migrateGroupSessions) {
             val inboundGroupSessionEntities = realm!!.where<OlmInboundGroupSessionEntity>().findAll()
             assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt())
@@ -122,19 +146,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
                     .findAll()
             assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt())
         }
+=======
+        // How to check that olm sessions have been migrated?
+        // Can see it from logs
+>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
     }
-
-//    @Test
-//    fun given_an_empty_crypto_store_realm_file_then_migration_should_not_happen() {
-//        val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = false)
-//        Realm.getInstance(realmConfiguration).use {
-//            assertTrue(it.isEmpty)
-//        }
-//        val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null)
-//        assertNull(machine.getBackupKeys())
-//        val crossSigningStatus = machine.crossSigningStatus()
-//        assertFalse(crossSigningStatus.hasMaster)
-//        assertFalse(crossSigningStatus.hasSelfSigning)
-//        assertFalse(crossSigningStatus.hasUserSigning)
-//    }
 }

matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt

@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2021 The Matrix.org Foundation C.I.C.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.matrix.android.sdk.internal.session
-
-import io.realm.RealmConfiguration
-import timber.log.Timber
-import java.io.File
-
-class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) {
-
-    fun execute(cryptoRealm: RealmConfiguration, sessionFilesDir: File, passphrase: String?): File {
-        // to remove unused warning
-        Timber.v("Not used in kotlin crypto $cryptoRealm ${"*".repeat(passphrase?.length ?: 0)} lazy:$migrateGroupSessions")
-        // no op in kotlinCrypto
-        return sessionFilesDir
-    }
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt

@@ -44,7 +44,6 @@ import org.matrix.android.sdk.api.session.permalinks.PermalinkService
 import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService
 import org.matrix.android.sdk.api.session.typing.TypingUsersTracker
 import org.matrix.android.sdk.api.util.md5
-import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
 import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorageService
 import org.matrix.android.sdk.internal.crypto.tasks.DefaultRedactEventTask
 import org.matrix.android.sdk.internal.crypto.tasks.RedactEventTask
@@ -53,7 +52,6 @@ import org.matrix.android.sdk.internal.database.RealmSessionProvider
 import org.matrix.android.sdk.internal.database.SessionRealmConfigurationFactory
 import org.matrix.android.sdk.internal.di.Authenticated
 import org.matrix.android.sdk.internal.di.CacheDirectory
-import org.matrix.android.sdk.internal.di.CryptoDatabase
 import org.matrix.android.sdk.internal.di.DeviceId
 import org.matrix.android.sdk.internal.di.SessionDatabase
 import org.matrix.android.sdk.internal.di.SessionDownloadsDirectory
@@ -100,11 +98,9 @@ import org.matrix.android.sdk.internal.session.typing.DefaultTypingUsersTracker
 import org.matrix.android.sdk.internal.session.user.accountdata.DefaultSessionAccountDataService
 import org.matrix.android.sdk.internal.session.widgets.DefaultWidgetURLFormatter
 import retrofit2.Retrofit
-import timber.log.Timber
 import java.io.File
 import javax.inject.Provider
 import javax.inject.Qualifier
-import kotlin.system.measureTimeMillis
 
 @Qualifier
 @Retention(AnnotationRetention.RUNTIME)
@@ -189,17 +185,8 @@ internal abstract class SessionModule {
         @SessionScope
         fun providesRustCryptoFilesDir(
                 @SessionFilesDirectory parent: File,
-                @CryptoDatabase realmConfiguration: RealmConfiguration,
-                rustEncryptionConfiguration: RustEncryptionConfiguration,
         ): File {
-            val target = File(parent, "rustFlavor")
+            return File(parent, "rustFlavor")
-            val file: File
-            measureTimeMillis {
-                file = MigrateEAtoEROperation().execute(realmConfiguration, target, rustEncryptionConfiguration.getDatabasePassphrase())
-            }.let { duration ->
-                Timber.v("Migrating to ER in $duration ms")
-            }
-            return file
         }
 
         @JvmStatic

matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto.store.db
+
+import io.realm.DynamicRealm
+import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo001Legacy
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo002Legacy
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo003RiotX
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo004
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo005
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo006
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo007
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo008
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo009
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo010
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo011
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo012
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo013
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo014
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo015
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo016
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo017
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo018
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo019
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo020
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo021
+import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo022
+import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory
+import org.matrix.android.sdk.internal.util.database.MatrixRealmMigration
+import org.matrix.android.sdk.internal.util.time.Clock
+import java.io.File
+import javax.inject.Inject
+
+/**
+ * Schema version history:
+ *  0, 1, 2: legacy Riot-Android;
+ *  3: migrate to RiotX schema;
+ *  4, 5, 6, 7, 8, 9: migrations from RiotX (which was previously 1, 2, 3, 4, 5, 6).
+ */
+internal class RealmCryptoStoreMigration @Inject constructor(
+        private val clock: Clock,
+        private val rustEncryptionConfiguration: RustEncryptionConfiguration,
+        @SessionRustFilesDirectory
+        private val rustDirectory: File,
+) : MatrixRealmMigration(
+        dbName = "Crypto",
+        schemaVersion = 22L,
+) {
+    /**
+     * Forces all RealmCryptoStoreMigration instances to be equal.
+     * Avoids Realm throwing when multiple instances of the migration are set.
+     */
+    override fun equals(other: Any?) = other is RealmCryptoStoreMigration
+    override fun hashCode() = 5000
+
+    override fun doMigrate(realm: DynamicRealm, oldVersion: Long) {
+        if (oldVersion < 1) MigrateCryptoTo001Legacy(realm).perform()
+        if (oldVersion < 2) MigrateCryptoTo002Legacy(realm).perform()
+        if (oldVersion < 3) MigrateCryptoTo003RiotX(realm).perform()
+        if (oldVersion < 4) MigrateCryptoTo004(realm).perform()
+        if (oldVersion < 5) MigrateCryptoTo005(realm).perform()
+        if (oldVersion < 6) MigrateCryptoTo006(realm).perform()
+        if (oldVersion < 7) MigrateCryptoTo007(realm).perform()
+        if (oldVersion < 8) MigrateCryptoTo008(realm, clock).perform()
+        if (oldVersion < 9) MigrateCryptoTo009(realm).perform()
+        if (oldVersion < 10) MigrateCryptoTo010(realm).perform()
+        if (oldVersion < 11) MigrateCryptoTo011(realm).perform()
+        if (oldVersion < 12) MigrateCryptoTo012(realm).perform()
+        if (oldVersion < 13) MigrateCryptoTo013(realm).perform()
+        if (oldVersion < 14) MigrateCryptoTo014(realm).perform()
+        if (oldVersion < 15) MigrateCryptoTo015(realm).perform()
+        if (oldVersion < 16) MigrateCryptoTo016(realm).perform()
+        if (oldVersion < 17) MigrateCryptoTo017(realm).perform()
+        if (oldVersion < 18) MigrateCryptoTo018(realm).perform()
+        if (oldVersion < 19) MigrateCryptoTo019(realm).perform()
+        if (oldVersion < 20) MigrateCryptoTo020(realm).perform()
+        if (oldVersion < 21) MigrateCryptoTo021(realm).perform()
+        if (oldVersion < 22) MigrateCryptoTo022(realm, rustDirectory, rustEncryptionConfiguration).perform()
+    }
+}

matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2023 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto.store.db.migration
+
+import io.realm.DynamicRealm
+import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
+import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation
+import org.matrix.android.sdk.internal.util.database.RealmMigrator
+import java.io.File
+
+/**
+ * This migration creates the rust database and migrates from legacy crypto
+ */
+internal class MigrateCryptoTo022(
+        realm: DynamicRealm,
+        private val rustDirectory: File,
+        private val rustEncryptionConfiguration: RustEncryptionConfiguration
+) : RealmMigrator(
+        realm,
+        22
+) {
+    override fun doMigrate(realm: DynamicRealm) {
+        // Migrate to rust!
+        val migrateOperation = MigrateEAtoEROperation()
+        migrateOperation.dynamicExecute(realm, rustDirectory, rustEncryptionConfiguration.getDatabasePassphrase())
+
+        // wa can't delete all for now, but we can do some cleaning
+        realm.schema.get("OlmSessionEntity")?.transform {
+            it.deleteFromRealm()
+        }
+
+        // a future migration will clean the rest
+    }
+}

matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt

@@ -24,12 +24,9 @@ import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
 import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
 import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
 import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
-import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
 import org.matrix.olm.OlmSession
 import org.matrix.olm.OlmUtility
-import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport
 import org.matrix.rustcomponents.sdk.crypto.MigrationData
-import org.matrix.rustcomponents.sdk.crypto.PickledAccount
 import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession
 import org.matrix.rustcomponents.sdk.crypto.PickledSession
 import timber.log.Timber
@@ -40,7 +37,7 @@ private val charset = Charset.forName("UTF-8")
 
 internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = false) {
 
-    fun extractData(realm: Realm, importPartial: ((MigrationData) -> Unit)) {
+    fun extractData(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) {
         return try {
             extract(realm, importPartial)
         } catch (failure: Throwable) {
@@ -57,89 +54,33 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f
         }
     }
 
-    private fun extract(realm: Realm, importPartial: ((MigrationData) -> Unit)) {
+    private fun extract(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) {
-        val metadataEntity = realm.where<CryptoMetadataEntity>().findFirst()
-                ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity")
-
         val pickleKey = OlmUtility.getRandomKey()
 
-        val masterKey = metadataEntity.xSignMasterPrivateKey
+        val baseExtract = realm.getPickledAccount(pickleKey)
-        val userKey = metadataEntity.xSignUserPrivateKey
-        val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey
-
-        val userId = metadataEntity.userId
-                ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
-        val deviceId = metadataEntity.deviceId
-                ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
-
-        val backupVersion = metadataEntity.backupVersion
-        val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey
-
-        val isOlmAccountShared = metadataEntity.deviceKeysSentToServer
-
-        val olmAccount = metadataEntity.getOlmAccount()
-                ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
-        val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
-        olmAccount.oneTimeKeys()
-        val pickledAccount = PickledAccount(
-                userId = userId,
-                deviceId = deviceId,
-                pickle = pickledOlmAccount,
-                shared = isOlmAccountShared,
-                uploadedSignedKeyCount = 50
-        )
-
-        val baseExtract = MigrationData(
-                account = pickledAccount,
-                pickleKey = pickleKey.map { it.toUByte() },
-                crossSigning = CrossSigningKeyExport(
-                        masterKey = masterKey,
-                        selfSigningKey = selfSignedKey,
-                        userSigningKey = userKey
-                ),
-                sessions = emptyList(),
-                backupRecoveryKey = backupRecoveryKey,
-                trackedUsers = emptyList(),
-                inboundGroupSessions = emptyList(),
-                backupVersion = backupVersion,
-                // TODO import room settings from legacy DB
-                roomSettings = emptyMap()
-        )
         // import the account asap
         importPartial(baseExtract)
 
         val chunkSize = 500
-        realm.where<UserEntity>()
+        realm.trackedUsersChunk(500) {
-                .findAll()
-                .chunked(chunkSize) { chunk ->
-                    val trackedUserIds = chunk.mapNotNull { it.userId }
             importPartial(
-                            baseExtract.copy(trackedUsers = trackedUserIds)
+                    baseExtract.copy(trackedUsers = it)
             )
         }
 
         var migratedOlmSessionCount = 0
-        var readTime = 0L
         var writeTime = 0L
         measureTimeMillis {
-            realm.where<OlmSessionEntity>().findAll()
+            realm.pickledOlmSessions(pickleKey, chunkSize) { pickledSessions ->
-                    .chunked(chunkSize) { chunk ->
+                migratedOlmSessionCount += pickledSessions.size
-                        migratedOlmSessionCount += chunk.size
-                        val export: List<PickledSession>
-                        measureTimeMillis {
-                            export = chunk.map { it.toPickledSession(pickleKey) }
-                        }.also {
-                            readTime += it
-                        }
                 measureTimeMillis {
                     importPartial(
-                                    baseExtract.copy(sessions = export)
+                            baseExtract.copy(sessions = pickledSessions)
                     )
                 }.also { writeTime += it }
             }
         }.also {
             Timber.i("Migration: took $it ms to migrate $migratedOlmSessionCount olm sessions")
-            Timber.i("Migration: extract time $readTime")
             Timber.i("Migration: rust import time $writeTime")
         }
 

matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt

@@ -0,0 +1,244 @@
+/*
+ * Copyright (c) 2023 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.crypto.store.db.migration.rust
+
+import io.realm.kotlin.where
+import okhttp3.internal.toImmutableList
+import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
+import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
+import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntityFields
+import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
+import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntityFields
+import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
+import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields
+import org.matrix.olm.OlmAccount
+import org.matrix.olm.OlmSession
+import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport
+import org.matrix.rustcomponents.sdk.crypto.MigrationData
+import org.matrix.rustcomponents.sdk.crypto.PickledAccount
+import org.matrix.rustcomponents.sdk.crypto.PickledSession
+import java.nio.charset.Charset
+
+sealed class RealmToMigrate {
+    data class DynamicRealm(val realm: io.realm.DynamicRealm) : RealmToMigrate()
+    data class ClassicRealm(val realm: io.realm.Realm) : RealmToMigrate()
+}
+
+fun RealmToMigrate.hasExistingData(): Boolean {
+    return when (this) {
+        is RealmToMigrate.ClassicRealm -> {
+            !this.realm.isEmpty &&
+                    // Check if there is a MetaData object
+                    this.realm.where<CryptoMetadataEntity>().count() > 0 &&
+                    this.realm.where<CryptoMetadataEntity>().findFirst()?.olmAccountData != null
+        }
+        is RealmToMigrate.DynamicRealm -> {
+            return true
+        }
+    }
+}
+
+@Throws
+fun RealmToMigrate.getPickledAccount(pickleKey: ByteArray): MigrationData {
+    return when (this) {
+        is RealmToMigrate.ClassicRealm -> {
+            val metadataEntity = realm.where<CryptoMetadataEntity>().findFirst()
+                    ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity")
+
+            val masterKey = metadataEntity.xSignMasterPrivateKey
+            val userKey = metadataEntity.xSignUserPrivateKey
+            val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey
+
+            val userId = metadataEntity.userId
+                    ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
+            val deviceId = metadataEntity.deviceId
+                    ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
+
+            val backupVersion = metadataEntity.backupVersion
+            val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey
+
+            val isOlmAccountShared = metadataEntity.deviceKeysSentToServer
+
+            val olmAccount = metadataEntity.getOlmAccount()
+                    ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
+            val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
+
+            val pickledAccount = PickledAccount(
+                    userId = userId,
+                    deviceId = deviceId,
+                    pickle = pickledOlmAccount,
+                    shared = isOlmAccountShared,
+                    uploadedSignedKeyCount = 50
+            )
+            MigrationData(
+                    account = pickledAccount,
+                    pickleKey = pickleKey.map { it.toUByte() },
+                    crossSigning = CrossSigningKeyExport(
+                            masterKey = masterKey,
+                            selfSigningKey = selfSignedKey,
+                            userSigningKey = userKey
+                    ),
+                    sessions = emptyList(),
+                    backupRecoveryKey = backupRecoveryKey,
+                    trackedUsers = emptyList(),
+                    inboundGroupSessions = emptyList(),
+                    backupVersion = backupVersion,
+                    // TODO import room settings from legacy DB
+                    roomSettings = emptyMap()
+            )
+        }
+        is RealmToMigrate.DynamicRealm -> {
+            val cryptoMetadataEntitySchema = realm.schema.get("CryptoMetadataEntity")
+                    ?: throw java.lang.IllegalStateException("Missing Metadata entity")
+
+            var migrationData: MigrationData? = null
+            cryptoMetadataEntitySchema.transform { dynMetaData ->
+
+                val serializedOlmAccount = dynMetaData.getString(CryptoMetadataEntityFields.OLM_ACCOUNT_DATA)
+
+                val masterKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_MASTER_PRIVATE_KEY)
+                val userKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_USER_PRIVATE_KEY)
+                val selfSignedKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_SELF_SIGNED_PRIVATE_KEY)
+
+                val userId = dynMetaData.getString(CryptoMetadataEntityFields.USER_ID)
+                        ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
+                val deviceId = dynMetaData.getString(CryptoMetadataEntityFields.DEVICE_ID)
+                        ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
+
+                val backupVersion = dynMetaData.getString(CryptoMetadataEntityFields.BACKUP_VERSION)
+                val backupRecoveryKey = dynMetaData.getString(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY)
+
+                val isOlmAccountShared = dynMetaData.getBoolean(CryptoMetadataEntityFields.DEVICE_KEYS_SENT_TO_SERVER)
+
+                val olmAccount = deserializeFromRealm<OlmAccount>(serializedOlmAccount)
+                        ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
+
+                val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
+
+                val pickledAccount = PickledAccount(
+                        userId = userId,
+                        deviceId = deviceId,
+                        pickle = pickledOlmAccount,
+                        shared = isOlmAccountShared,
+                        uploadedSignedKeyCount = 50
+                )
+
+                migrationData = MigrationData(
+                        account = pickledAccount,
+                        pickleKey = pickleKey.map { it.toUByte() },
+                        crossSigning = CrossSigningKeyExport(
+                                masterKey = masterKey,
+                                selfSigningKey = selfSignedKey,
+                                userSigningKey = userKey
+                        ),
+                        sessions = emptyList(),
+                        backupRecoveryKey = backupRecoveryKey,
+                        trackedUsers = emptyList(),
+                        inboundGroupSessions = emptyList(),
+                        backupVersion = backupVersion,
+                        // TODO import room settings from legacy DB
+                        roomSettings = emptyMap()
+                )
+            }
+            migrationData!!
+        }
+    }
+}
+
+fun RealmToMigrate.trackedUsersChunk(chunkSize: Int, onChunk: ((List<String>) -> Unit)) {
+    when (this) {
+        is RealmToMigrate.ClassicRealm -> {
+            realm.where<UserEntity>()
+                    .findAll()
+                    .chunked(chunkSize)
+                    .onEach {
+                        onChunk(it.mapNotNull { it.userId })
+                    }
+        }
+        is RealmToMigrate.DynamicRealm ->  {
+            val userList = mutableListOf<String>()
+            realm.schema.get("UserEntity")?.transform {
+                val userId = it.getString(UserEntityFields.USER_ID)
+                // should we check the tracking status?
+                userList.add(userId)
+                if (userList.size > chunkSize) {
+                    onChunk(userList.toImmutableList())
+                    userList.clear()
+                }
+            }
+            if (userList.isNotEmpty()) {
+                onChunk(userList)
+            }
+        }
+    }
+}
+
+fun RealmToMigrate.pickledOlmSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List<PickledSession>) -> Unit)) {
+    when (this) {
+        is RealmToMigrate.ClassicRealm -> {
+            realm.where<OlmSessionEntity>().findAll()
+                    .chunked(chunkSize) { chunk ->
+                        val export = chunk.map { it.toPickledSession(pickleKey) }
+                        onChunk(export)
+                    }
+        }
+        is RealmToMigrate.DynamicRealm ->  {
+            val pickledSessions = mutableListOf<PickledSession>()
+            realm.schema.get("OlmSessionEntity")?.transform {
+                val sessionData = it.getString(OlmSessionEntityFields.OLM_SESSION_DATA)
+                val deviceKey = it.getString(OlmSessionEntityFields.DEVICE_KEY)
+                val lastReceivedMessageTs = it.getLong(OlmSessionEntityFields.LAST_RECEIVED_MESSAGE_TS)
+                val olmSession = deserializeFromRealm<OlmSession>(sessionData)!!
+                val pickle = olmSession.pickle(pickleKey, StringBuffer()).asString()
+                val pickledSession =  PickledSession(
+                        pickle = pickle,
+                        senderKey = deviceKey,
+                        createdUsingFallbackKey = false,
+                        creationTime = lastReceivedMessageTs.toString(),
+                        lastUseTime = lastReceivedMessageTs.toString()
+                )
+                // should we check the tracking status?
+                pickledSessions.add(pickledSession)
+                if (pickledSessions.size > chunkSize) {
+                    onChunk(pickledSessions.toImmutableList())
+                    pickledSessions.clear()
+                }
+            }
+            if (pickledSessions.isNotEmpty()) {
+                onChunk(pickledSessions)
+            }
+        }
+    }
+}
+
+private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession {
+    val deviceKey = this.deviceKey ?: ""
+    val lastReceivedMessageTs = this.lastReceivedMessageTs
+    val olmSessionStr = this.olmSessionData
+    val olmSession = deserializeFromRealm<OlmSession>(olmSessionStr)!!
+    val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString()
+    return PickledSession(
+            pickle = pickledOlmSession,
+            senderKey = deviceKey,
+            createdUsingFallbackKey = false,
+            creationTime = lastReceivedMessageTs.toString(),
+            lastUseTime = lastReceivedMessageTs.toString()
+    )
+}
+
+private val charset = Charset.forName("UTF-8")
+private fun ByteArray.asString() = String(this, charset)

matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt

@@ -16,9 +16,11 @@
 
 package org.matrix.android.sdk.internal.session
 
+import io.realm.DynamicRealm
 import io.realm.Realm
 import io.realm.RealmConfiguration
 import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.ExtractMigrationDataUseCase
+import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.RealmToMigrate
 import org.matrix.rustcomponents.sdk.crypto.ProgressListener
 import timber.log.Timber
 import java.io.File
@@ -40,9 +42,8 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false)
                         Timber.v("OnProgress: $progress/$total")
                     }
                 }
-
                 Realm.getInstance(cryptoRealm).use { realm ->
-                    extractMigrationData.extractData(realm) {
+                    extractMigrationData.extractData(RealmToMigrate.ClassicRealm(realm)) {
                         org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener)
                     }
                 }
@@ -53,4 +54,25 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false)
         }
         return rustFilesDir
     }
+
+    fun dynamicExecute(dynamicRealm: DynamicRealm, rustFilesDir: File, passphrase: String?) {
+        if (!rustFilesDir.exists()) {
+            rustFilesDir.mkdir()
+        }
+        val extractMigrationData = ExtractMigrationDataUseCase()
+
+        try {
+            val progressListener = object : ProgressListener {
+                override fun onProgress(progress: Int, total: Int) {
+                    Timber.v("OnProgress: $progress/$total")
+                }
+            }
+            extractMigrationData.extractData(RealmToMigrate.DynamicRealm(dynamicRealm)) {
+                org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener)
+            }
+        } catch (failure: Throwable) {
+            Timber.e(failure, "Failure while calling rust migration method")
+            throw failure
+        }
+    }
 }