filippodb
/
SchildiChat-android
mirror of https://github.com/SchildiChat/SchildiChat-android.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

create rust db as a realm migration

This commit is contained in:
valere 2023-05-06 09:48:01 +02:00 committed by Benoit Marty
parent 62ec1eb505
commit fd186c1f32
10 changed files with 491 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt
View File

@ -18,14 +18,20 @@ package org.matrix.android.sdk.internal.database
import android.content.Context
import androidx.test.platform.app.InstrumentationRegistry
import io.mockk.spyk
import io.realm.Realm
import org.junit.After
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
import org.matrix.android.sdk.internal.util.time.Clock
import java.io.File
import java.security.KeyStore
class CryptoSanityMigrationTest {
@get:Rule val configurationFactory = TestRealmConfigurationFactory()
@ -43,14 +49,28 @@ class CryptoSanityMigrationTest {
realm?.close()
}
private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
@Test
fun cryptoDatabaseShouldMigrateGracefully() {
val realmName = "crypto_store_20.realm"
val migration = RealmCryptoStoreMigration(object : Clock {
val migration = RealmCryptoStoreMigration(
object : Clock {
override fun epochMillis(): Long {
return 0L
}
})
},
RustEncryptionConfiguration(
"foo",
RealmKeysUtils(
context,
SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
)
),
File("test_rust")
)
val realmConfiguration = configurationFactory.createConfiguration(
realmName,
"7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca",

62
matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt → matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
View File

@ -19,6 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.migration
import android.content.Context
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import io.mockk.spyk
import io.realm.Realm
import io.realm.kotlin.where
import org.amshove.kluent.internal.assertEquals
@ -31,32 +32,33 @@ import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.matrix.android.sdk.InstrumentedTest
import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields
import org.matrix.android.sdk.internal.database.RealmKeysUtils
import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory
import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation
import org.matrix.android.sdk.internal.util.time.Clock
import org.matrix.olm.OlmAccount
import org.matrix.olm.OlmManager
import org.matrix.rustcomponents.sdk.crypto.OlmMachine
import java.io.File
import java.security.KeyStore
@RunWith(AndroidJUnit4::class)
class ElementAndroidToElementRMigrationTest : InstrumentedTest {
class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest {
@get:Rule val configurationFactory = TestRealmConfigurationFactory()
lateinit var context: Context
var context: Context = InstrumentationRegistry.getInstrumentation().context
var realm: Realm? = null
@Before
fun setUp() {
// Ensure Olm is initialized
OlmManager()
context = InstrumentationRegistry.getInstrumentation().context
}
@After
@ -64,7 +66,22 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
realm?.close()
}
private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
private val rustEncryptionConfiguration = RustEncryptionConfiguration(
"foo",
RealmKeysUtils(
context,
SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
)
)
private val fakeClock = object : Clock {
override fun epochMillis() = 0L
}
@Test
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() {
testMigrate(false)
}
@ -76,10 +93,13 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
}
private fun testMigrate(migrateGroupSessions: Boolean) {
=======
fun dynamic_migration_to_rust() {
val targetFile = File(configurationFactory.root, "rust-sdk")
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
val realmName = "crypto_store_migration_16.realm"
val migration = RealmCryptoStoreMigration(object : Clock {
override fun epochMillis() = 0L
})
val migration = RealmCryptoStoreMigration(fakeClock, rustEncryptionConfiguration, targetFile)
val realmConfiguration = configurationFactory.createConfiguration(
realmName,
@ -91,12 +111,12 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
configurationFactory.copyRealmFromAssets(context, realmName, realmName)
realm = Realm.getInstance(realmConfiguration)
val metaData = realm!!.where<CryptoMetadataEntity>().findFirst()!!
val userId = metaData.userId!!
val deviceId = metaData.deviceId!!
val olmAccount = metaData.getOlmAccount()!!
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
val extractor = MigrateEAtoEROperation(migrateGroupSessions)
val targetFile = File(configurationFactory.root, "rust-sdk")
@ -104,6 +124,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
extractor.execute(realmConfiguration, targetFile, null)
val machine = OlmMachine(userId, deviceId, targetFile.path, null)
=======
val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase())
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"])
assertNotNull(machine.getBackupKeys())
@ -112,6 +135,7 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
assertTrue(crossSigningStatus.hasSelfSigning)
assertTrue(crossSigningStatus.hasUserSigning)
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
if (migrateGroupSessions) {
val inboundGroupSessionEntities = realm!!.where<OlmInboundGroupSessionEntity>().findAll()
assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt())
@ -122,19 +146,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
.findAll()
assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt())
}
=======
// How to check that olm sessions have been migrated?
// Can see it from logs
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
}
// @Test
// fun given_an_empty_crypto_store_realm_file_then_migration_should_not_happen() {
// val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = false)
// Realm.getInstance(realmConfiguration).use {
// assertTrue(it.isEmpty)
// }
// val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null)
// assertNull(machine.getBackupKeys())
// val crossSigningStatus = machine.crossSigningStatus()
// assertFalse(crossSigningStatus.hasMaster)
// assertFalse(crossSigningStatus.hasSelfSigning)
// assertFalse(crossSigningStatus.hasUserSigning)
// }
}

0
matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt → matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt
View File

31
matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt
View File

@ -1,31 +0,0 @@
/*
* Copyright (c) 2021 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.session
import io.realm.RealmConfiguration
import timber.log.Timber
import java.io.File
class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) {
fun execute(cryptoRealm: RealmConfiguration, sessionFilesDir: File, passphrase: String?): File {
// to remove unused warning
Timber.v("Not used in kotlin crypto $cryptoRealm ${"*".repeat(passphrase?.length ?: 0)} lazy:$migrateGroupSessions")
// no op in kotlinCrypto
return sessionFilesDir
}
}

15
matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt
View File

@ -44,7 +44,6 @@ import org.matrix.android.sdk.api.session.permalinks.PermalinkService
import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService
import org.matrix.android.sdk.api.session.typing.TypingUsersTracker
import org.matrix.android.sdk.api.util.md5
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorageService
import org.matrix.android.sdk.internal.crypto.tasks.DefaultRedactEventTask
import org.matrix.android.sdk.internal.crypto.tasks.RedactEventTask
@ -53,7 +52,6 @@ import org.matrix.android.sdk.internal.database.RealmSessionProvider
import org.matrix.android.sdk.internal.database.SessionRealmConfigurationFactory
import org.matrix.android.sdk.internal.di.Authenticated
import org.matrix.android.sdk.internal.di.CacheDirectory
import org.matrix.android.sdk.internal.di.CryptoDatabase
import org.matrix.android.sdk.internal.di.DeviceId
import org.matrix.android.sdk.internal.di.SessionDatabase
import org.matrix.android.sdk.internal.di.SessionDownloadsDirectory
@ -100,11 +98,9 @@ import org.matrix.android.sdk.internal.session.typing.DefaultTypingUsersTracker
import org.matrix.android.sdk.internal.session.user.accountdata.DefaultSessionAccountDataService
import org.matrix.android.sdk.internal.session.widgets.DefaultWidgetURLFormatter
import retrofit2.Retrofit
import timber.log.Timber
import java.io.File
import javax.inject.Provider
import javax.inject.Qualifier
import kotlin.system.measureTimeMillis
@Qualifier
@Retention(AnnotationRetention.RUNTIME)
@ -189,17 +185,8 @@ internal abstract class SessionModule {
@SessionScope
fun providesRustCryptoFilesDir(
@SessionFilesDirectory parent: File,
@CryptoDatabase realmConfiguration: RealmConfiguration,
rustEncryptionConfiguration: RustEncryptionConfiguration,
): File {
val target = File(parent, "rustFlavor")
val file: File
measureTimeMillis {
file = MigrateEAtoEROperation().execute(realmConfiguration, target, rustEncryptionConfiguration.getDatabasePassphrase())
}.let { duration ->
Timber.v("Migrating to ER in $duration ms")
}
return file
return File(parent, "rustFlavor")
}
@JvmStatic

95
matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt Normal file
View File

@ -0,0 +1,95 @@
/*
* Copyright 2020 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.db
import io.realm.DynamicRealm
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo001Legacy
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo002Legacy
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo003RiotX
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo004
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo005
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo006
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo007
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo008
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo009
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo010
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo011
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo012
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo013
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo014
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo015
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo016
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo017
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo018
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo019
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo020
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo021
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo022
import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory
import org.matrix.android.sdk.internal.util.database.MatrixRealmMigration
import org.matrix.android.sdk.internal.util.time.Clock
import java.io.File
import javax.inject.Inject
/**
* Schema version history:
* 0, 1, 2: legacy Riot-Android;
* 3: migrate to RiotX schema;
* 4, 5, 6, 7, 8, 9: migrations from RiotX (which was previously 1, 2, 3, 4, 5, 6).
*/
internal class RealmCryptoStoreMigration @Inject constructor(
private val clock: Clock,
private val rustEncryptionConfiguration: RustEncryptionConfiguration,
@SessionRustFilesDirectory
private val rustDirectory: File,
) : MatrixRealmMigration(
dbName = "Crypto",
schemaVersion = 22L,
) {
/**
* Forces all RealmCryptoStoreMigration instances to be equal.
* Avoids Realm throwing when multiple instances of the migration are set.
*/
override fun equals(other: Any?) = other is RealmCryptoStoreMigration
override fun hashCode() = 5000
override fun doMigrate(realm: DynamicRealm, oldVersion: Long) {
if (oldVersion < 1) MigrateCryptoTo001Legacy(realm).perform()
if (oldVersion < 2) MigrateCryptoTo002Legacy(realm).perform()
if (oldVersion < 3) MigrateCryptoTo003RiotX(realm).perform()
if (oldVersion < 4) MigrateCryptoTo004(realm).perform()
if (oldVersion < 5) MigrateCryptoTo005(realm).perform()
if (oldVersion < 6) MigrateCryptoTo006(realm).perform()
if (oldVersion < 7) MigrateCryptoTo007(realm).perform()
if (oldVersion < 8) MigrateCryptoTo008(realm, clock).perform()
if (oldVersion < 9) MigrateCryptoTo009(realm).perform()
if (oldVersion < 10) MigrateCryptoTo010(realm).perform()
if (oldVersion < 11) MigrateCryptoTo011(realm).perform()
if (oldVersion < 12) MigrateCryptoTo012(realm).perform()
if (oldVersion < 13) MigrateCryptoTo013(realm).perform()
if (oldVersion < 14) MigrateCryptoTo014(realm).perform()
if (oldVersion < 15) MigrateCryptoTo015(realm).perform()
if (oldVersion < 16) MigrateCryptoTo016(realm).perform()
if (oldVersion < 17) MigrateCryptoTo017(realm).perform()
if (oldVersion < 18) MigrateCryptoTo018(realm).perform()
if (oldVersion < 19) MigrateCryptoTo019(realm).perform()
if (oldVersion < 20) MigrateCryptoTo020(realm).perform()
if (oldVersion < 21) MigrateCryptoTo021(realm).perform()
if (oldVersion < 22) MigrateCryptoTo022(realm, rustDirectory, rustEncryptionConfiguration).perform()
}
}

48
matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2023 New Vector Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.db.migration
import io.realm.DynamicRealm
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation
import org.matrix.android.sdk.internal.util.database.RealmMigrator
import java.io.File
/**
* This migration creates the rust database and migrates from legacy crypto
*/
internal class MigrateCryptoTo022(
realm: DynamicRealm,
private val rustDirectory: File,
private val rustEncryptionConfiguration: RustEncryptionConfiguration
) : RealmMigrator(
realm,
22
) {
override fun doMigrate(realm: DynamicRealm) {
// Migrate to rust!
val migrateOperation = MigrateEAtoEROperation()
migrateOperation.dynamicExecute(realm, rustDirectory, rustEncryptionConfiguration.getDatabasePassphrase())
// wa can't delete all for now, but we can do some cleaning
realm.schema.get("OlmSessionEntity")?.transform {
it.deleteFromRealm()
}
// a future migration will clean the rest
}
}

75
matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt
View File

@ -24,12 +24,9 @@ import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
import org.matrix.olm.OlmSession
import org.matrix.olm.OlmUtility
import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport
import org.matrix.rustcomponents.sdk.crypto.MigrationData
import org.matrix.rustcomponents.sdk.crypto.PickledAccount
import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession
import org.matrix.rustcomponents.sdk.crypto.PickledSession
import timber.log.Timber
@ -40,7 +37,7 @@ private val charset = Charset.forName("UTF-8")
internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = false) {
fun extractData(realm: Realm, importPartial: ((MigrationData) -> Unit)) {
fun extractData(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) {
return try {
extract(realm, importPartial)
} catch (failure: Throwable) {
@ -57,89 +54,33 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f
}
}
private fun extract(realm: Realm, importPartial: ((MigrationData) -> Unit)) {
val metadataEntity = realm.where<CryptoMetadataEntity>().findFirst()
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity")
private fun extract(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) {
val pickleKey = OlmUtility.getRandomKey()
val masterKey = metadataEntity.xSignMasterPrivateKey
val userKey = metadataEntity.xSignUserPrivateKey
val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey
val userId = metadataEntity.userId
?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
val deviceId = metadataEntity.deviceId
?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
val backupVersion = metadataEntity.backupVersion
val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey
val isOlmAccountShared = metadataEntity.deviceKeysSentToServer
val olmAccount = metadataEntity.getOlmAccount()
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
olmAccount.oneTimeKeys()
val pickledAccount = PickledAccount(
userId = userId,
deviceId = deviceId,
pickle = pickledOlmAccount,
shared = isOlmAccountShared,
uploadedSignedKeyCount = 50
)
val baseExtract = MigrationData(
account = pickledAccount,
pickleKey = pickleKey.map { it.toUByte() },
crossSigning = CrossSigningKeyExport(
masterKey = masterKey,
selfSigningKey = selfSignedKey,
userSigningKey = userKey
),
sessions = emptyList(),
backupRecoveryKey = backupRecoveryKey,
trackedUsers = emptyList(),
inboundGroupSessions = emptyList(),
backupVersion = backupVersion,
// TODO import room settings from legacy DB
roomSettings = emptyMap()
)
val baseExtract = realm.getPickledAccount(pickleKey)
// import the account asap
importPartial(baseExtract)
val chunkSize = 500
realm.where<UserEntity>()
.findAll()
.chunked(chunkSize) { chunk ->
val trackedUserIds = chunk.mapNotNull { it.userId }
realm.trackedUsersChunk(500) {
importPartial(
baseExtract.copy(trackedUsers = trackedUserIds)
baseExtract.copy(trackedUsers = it)
)
}
var migratedOlmSessionCount = 0
var readTime = 0L
var writeTime = 0L
measureTimeMillis {
realm.where<OlmSessionEntity>().findAll()
.chunked(chunkSize) { chunk ->
migratedOlmSessionCount += chunk.size
val export: List<PickledSession>
measureTimeMillis {
export = chunk.map { it.toPickledSession(pickleKey) }
}.also {
readTime += it
}
realm.pickledOlmSessions(pickleKey, chunkSize) { pickledSessions ->
migratedOlmSessionCount += pickledSessions.size
measureTimeMillis {
importPartial(
baseExtract.copy(sessions = export)
baseExtract.copy(sessions = pickledSessions)
)
}.also { writeTime += it }
}
}.also {
Timber.i("Migration: took $it ms to migrate $migratedOlmSessionCount olm sessions")
Timber.i("Migration: extract time $readTime")
Timber.i("Migration: rust import time $writeTime")
}

244
matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt Normal file
View File

@ -0,0 +1,244 @@
/*
* Copyright (c) 2023 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.db.migration.rust
import io.realm.kotlin.where
import okhttp3.internal.toImmutableList
import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntityFields
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntityFields
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields
import org.matrix.olm.OlmAccount
import org.matrix.olm.OlmSession
import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport
import org.matrix.rustcomponents.sdk.crypto.MigrationData
import org.matrix.rustcomponents.sdk.crypto.PickledAccount
import org.matrix.rustcomponents.sdk.crypto.PickledSession
import java.nio.charset.Charset
sealed class RealmToMigrate {
data class DynamicRealm(val realm: io.realm.DynamicRealm) : RealmToMigrate()
data class ClassicRealm(val realm: io.realm.Realm) : RealmToMigrate()
}
fun RealmToMigrate.hasExistingData(): Boolean {
return when (this) {
is RealmToMigrate.ClassicRealm -> {
!this.realm.isEmpty &&
// Check if there is a MetaData object
this.realm.where<CryptoMetadataEntity>().count() > 0 &&
this.realm.where<CryptoMetadataEntity>().findFirst()?.olmAccountData != null
}
is RealmToMigrate.DynamicRealm -> {
return true
}
}
}
@Throws
fun RealmToMigrate.getPickledAccount(pickleKey: ByteArray): MigrationData {
return when (this) {
is RealmToMigrate.ClassicRealm -> {
val metadataEntity = realm.where<CryptoMetadataEntity>().findFirst()
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity")
val masterKey = metadataEntity.xSignMasterPrivateKey
val userKey = metadataEntity.xSignUserPrivateKey
val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey
val userId = metadataEntity.userId
?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
val deviceId = metadataEntity.deviceId
?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
val backupVersion = metadataEntity.backupVersion
val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey
val isOlmAccountShared = metadataEntity.deviceKeysSentToServer
val olmAccount = metadataEntity.getOlmAccount()
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
val pickledAccount = PickledAccount(
userId = userId,
deviceId = deviceId,
pickle = pickledOlmAccount,
shared = isOlmAccountShared,
uploadedSignedKeyCount = 50
)
MigrationData(
account = pickledAccount,
pickleKey = pickleKey.map { it.toUByte() },
crossSigning = CrossSigningKeyExport(
masterKey = masterKey,
selfSigningKey = selfSignedKey,
userSigningKey = userKey
),
sessions = emptyList(),
backupRecoveryKey = backupRecoveryKey,
trackedUsers = emptyList(),
inboundGroupSessions = emptyList(),
backupVersion = backupVersion,
// TODO import room settings from legacy DB
roomSettings = emptyMap()
)
}
is RealmToMigrate.DynamicRealm -> {
val cryptoMetadataEntitySchema = realm.schema.get("CryptoMetadataEntity")
?: throw java.lang.IllegalStateException("Missing Metadata entity")
var migrationData: MigrationData? = null
cryptoMetadataEntitySchema.transform { dynMetaData ->
val serializedOlmAccount = dynMetaData.getString(CryptoMetadataEntityFields.OLM_ACCOUNT_DATA)
val masterKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_MASTER_PRIVATE_KEY)
val userKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_USER_PRIVATE_KEY)
val selfSignedKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_SELF_SIGNED_PRIVATE_KEY)
val userId = dynMetaData.getString(CryptoMetadataEntityFields.USER_ID)
?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
val deviceId = dynMetaData.getString(CryptoMetadataEntityFields.DEVICE_ID)
?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
val backupVersion = dynMetaData.getString(CryptoMetadataEntityFields.BACKUP_VERSION)
val backupRecoveryKey = dynMetaData.getString(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY)
val isOlmAccountShared = dynMetaData.getBoolean(CryptoMetadataEntityFields.DEVICE_KEYS_SENT_TO_SERVER)
val olmAccount = deserializeFromRealm<OlmAccount>(serializedOlmAccount)
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
val pickledAccount = PickledAccount(
userId = userId,
deviceId = deviceId,
pickle = pickledOlmAccount,
shared = isOlmAccountShared,
uploadedSignedKeyCount = 50
)
migrationData = MigrationData(
account = pickledAccount,
pickleKey = pickleKey.map { it.toUByte() },
crossSigning = CrossSigningKeyExport(
masterKey = masterKey,
selfSigningKey = selfSignedKey,
userSigningKey = userKey
),
sessions = emptyList(),
backupRecoveryKey = backupRecoveryKey,
trackedUsers = emptyList(),
inboundGroupSessions = emptyList(),
backupVersion = backupVersion,
// TODO import room settings from legacy DB
roomSettings = emptyMap()
)
}
migrationData!!
}
}
}
fun RealmToMigrate.trackedUsersChunk(chunkSize: Int, onChunk: ((List<String>) -> Unit)) {
when (this) {
is RealmToMigrate.ClassicRealm -> {
realm.where<UserEntity>()
.findAll()
.chunked(chunkSize)
.onEach {
onChunk(it.mapNotNull { it.userId })
}
}
is RealmToMigrate.DynamicRealm -> {
val userList = mutableListOf<String>()
realm.schema.get("UserEntity")?.transform {
val userId = it.getString(UserEntityFields.USER_ID)
// should we check the tracking status?
userList.add(userId)
if (userList.size > chunkSize) {
onChunk(userList.toImmutableList())
userList.clear()
}
}
if (userList.isNotEmpty()) {
onChunk(userList)
}
}
}
}
fun RealmToMigrate.pickledOlmSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List<PickledSession>) -> Unit)) {
when (this) {
is RealmToMigrate.ClassicRealm -> {
realm.where<OlmSessionEntity>().findAll()
.chunked(chunkSize) { chunk ->
val export = chunk.map { it.toPickledSession(pickleKey) }
onChunk(export)
}
}
is RealmToMigrate.DynamicRealm -> {
val pickledSessions = mutableListOf<PickledSession>()
realm.schema.get("OlmSessionEntity")?.transform {
val sessionData = it.getString(OlmSessionEntityFields.OLM_SESSION_DATA)
val deviceKey = it.getString(OlmSessionEntityFields.DEVICE_KEY)
val lastReceivedMessageTs = it.getLong(OlmSessionEntityFields.LAST_RECEIVED_MESSAGE_TS)
val olmSession = deserializeFromRealm<OlmSession>(sessionData)!!
val pickle = olmSession.pickle(pickleKey, StringBuffer()).asString()
val pickledSession = PickledSession(
pickle = pickle,
senderKey = deviceKey,
createdUsingFallbackKey = false,
creationTime = lastReceivedMessageTs.toString(),
lastUseTime = lastReceivedMessageTs.toString()
)
// should we check the tracking status?
pickledSessions.add(pickledSession)
if (pickledSessions.size > chunkSize) {
onChunk(pickledSessions.toImmutableList())
pickledSessions.clear()
}
}
if (pickledSessions.isNotEmpty()) {
onChunk(pickledSessions)
}
}
}
}
private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession {
val deviceKey = this.deviceKey ?: ""
val lastReceivedMessageTs = this.lastReceivedMessageTs
val olmSessionStr = this.olmSessionData
val olmSession = deserializeFromRealm<OlmSession>(olmSessionStr)!!
val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString()
return PickledSession(
pickle = pickledOlmSession,
senderKey = deviceKey,
createdUsingFallbackKey = false,
creationTime = lastReceivedMessageTs.toString(),
lastUseTime = lastReceivedMessageTs.toString()
)
}
private val charset = Charset.forName("UTF-8")
private fun ByteArray.asString() = String(this, charset)

26
matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt
View File

@ -16,9 +16,11 @@
package org.matrix.android.sdk.internal.session
import io.realm.DynamicRealm
import io.realm.Realm
import io.realm.RealmConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.ExtractMigrationDataUseCase
import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.RealmToMigrate
import org.matrix.rustcomponents.sdk.crypto.ProgressListener
import timber.log.Timber
import java.io.File
@ -40,9 +42,8 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false)
Timber.v("OnProgress: $progress/$total")
}
}
Realm.getInstance(cryptoRealm).use { realm ->
extractMigrationData.extractData(realm) {
extractMigrationData.extractData(RealmToMigrate.ClassicRealm(realm)) {
org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener)
}
}
@ -53,4 +54,25 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false)
}
return rustFilesDir
}
fun dynamicExecute(dynamicRealm: DynamicRealm, rustFilesDir: File, passphrase: String?) {
if (!rustFilesDir.exists()) {
rustFilesDir.mkdir()
}
val extractMigrationData = ExtractMigrationDataUseCase()
try {
val progressListener = object : ProgressListener {
override fun onProgress(progress: Int, total: Int) {
Timber.v("OnProgress: $progress/$total")
}
}
extractMigrationData.extractData(RealmToMigrate.DynamicRealm(dynamicRealm)) {
org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener)
}
} catch (failure: Throwable) {
Timber.e(failure, "Failure while calling rust migration method")
throw failure
}
}
}