post rebase fix

This commit is contained in:
valere 2023-05-09 09:26:47 +02:00 committed by Benoit Marty
parent 49054d8a15
commit 98841bbe3d
9 changed files with 165 additions and 126 deletions

View File

@ -27,20 +27,20 @@ import org.junit.After
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Ignore
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.matrix.android.sdk.InstrumentedTest
import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
import org.matrix.android.sdk.internal.crypto.store.db.RustMigrationInfoProvider
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.database.RealmKeysUtils
import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory
import org.matrix.android.sdk.internal.util.time.Clock
import org.matrix.android.sdk.test.shared.createTimberTestRule
import org.matrix.olm.OlmAccount
import org.matrix.olm.OlmManager
import org.matrix.rustcomponents.sdk.crypto.OlmMachine
@ -48,10 +48,13 @@ import java.io.File
import java.security.KeyStore
@RunWith(AndroidJUnit4::class)
class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest {
class DynamicElementAndroidToElementRMigrationTest {
@get:Rule val configurationFactory = TestRealmConfigurationFactory()
@Rule
fun timberTestRule() = createTimberTestRule()
var context: Context = InstrumentationRegistry.getInstrumentation().context
var realm: Realm? = null
@ -81,25 +84,26 @@ class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest {
}
@Test
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() {
testMigrate(false)
}
@Test
@Ignore("We don't migrate group session for now, and it makes test suite unstable")
fun given_a_valid_crypto_store_realm_file_no_lazy_then_migration_should_be_successful() {
testMigrate(true)
}
private fun testMigrate(migrateGroupSessions: Boolean) {
=======
fun dynamic_migration_to_rust() {
val targetFile = File(configurationFactory.root, "rust-sdk")
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
val realmName = "crypto_store_migration_16.realm"
val migration = RealmCryptoStoreMigration(fakeClock, rustEncryptionConfiguration, targetFile)
val infoProvider = RustMigrationInfoProvider(
targetFile,
rustEncryptionConfiguration
).apply {
migrateMegolmGroupSessions = migrateGroupSessions
}
val migration = RealmCryptoStoreMigration(fakeClock, infoProvider)
val realmConfiguration = configurationFactory.createConfiguration(
realmName,
@ -116,17 +120,7 @@ class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest {
val deviceId = metaData.deviceId!!
val olmAccount = metaData.getOlmAccount()!!
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
val extractor = MigrateEAtoEROperation(migrateGroupSessions)
val targetFile = File(configurationFactory.root, "rust-sdk")
extractor.execute(realmConfiguration, targetFile, null)
val machine = OlmMachine(userId, deviceId, targetFile.path, null)
=======
val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase())
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"])
assertNotNull(machine.getBackupKeys())
@ -135,20 +129,12 @@ class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest {
assertTrue(crossSigningStatus.hasSelfSigning)
assertTrue(crossSigningStatus.hasUserSigning)
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
if (migrateGroupSessions) {
val inboundGroupSessionEntities = realm!!.where<OlmInboundGroupSessionEntity>().findAll()
assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt())
val backedUpInboundGroupSessionEntities = realm!!
.where<OlmInboundGroupSessionEntity>()
.equalTo(OlmInboundGroupSessionEntityFields.BACKED_UP, true)
.findAll()
assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt())
assertTrue("Some outbound sessions should be migrated", machine.roomKeyCounts().total.toInt() > 0)
assertTrue("There are some backed-up sessions", machine.roomKeyCounts().backedUp.toInt() > 0)
} else {
assertTrue(machine.roomKeyCounts().total.toInt() == 0)
assertTrue(machine.roomKeyCounts().backedUp.toInt() == 0)
}
=======
// How to check that olm sessions have been migrated?
// Can see it from logs
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
}
}

View File

@ -29,6 +29,7 @@ import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
import org.matrix.android.sdk.internal.crypto.store.db.RustMigrationInfoProvider
import org.matrix.android.sdk.internal.util.time.Clock
import org.matrix.olm.OlmManager
import java.io.File
@ -58,12 +59,8 @@ class CryptoSanityMigrationTest {
fun cryptoDatabaseShouldMigrateGracefully() {
val realmName = "crypto_store_20.realm"
val migration = RealmCryptoStoreMigration(
object : Clock {
override fun epochMillis(): Long {
return 0L
}
},
val rustMigrationInfo = RustMigrationInfoProvider(
File(configurationFactory.root, "test_rust"),
RustEncryptionConfiguration(
"foo",
RealmKeysUtils(
@ -71,7 +68,14 @@ class CryptoSanityMigrationTest {
SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
)
),
File(configurationFactory.root, "rust-sdk")
)
val migration = RealmCryptoStoreMigration(
object : Clock {
override fun epochMillis(): Long {
return 0L
}
},
rustMigrationInfo
)
val realmConfiguration = configurationFactory.createConfiguration(

View File

@ -17,7 +17,6 @@
package org.matrix.android.sdk.internal.crypto.store.db
import io.realm.DynamicRealm
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo001Legacy
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo002Legacy
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo003RiotX
@ -40,10 +39,8 @@ import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo020
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo021
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo022
import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory
import org.matrix.android.sdk.internal.util.database.MatrixRealmMigration
import org.matrix.android.sdk.internal.util.time.Clock
import java.io.File
import javax.inject.Inject
/**
@ -54,9 +51,7 @@ import javax.inject.Inject
*/
internal class RealmCryptoStoreMigration @Inject constructor(
private val clock: Clock,
private val rustEncryptionConfiguration: RustEncryptionConfiguration,
@SessionRustFilesDirectory
private val rustDirectory: File,
private val rustMigrationInfoProvider: RustMigrationInfoProvider,
) : MatrixRealmMigration(
dbName = "Crypto",
schemaVersion = 22L,
@ -90,6 +85,11 @@ internal class RealmCryptoStoreMigration @Inject constructor(
if (oldVersion < 19) MigrateCryptoTo019(realm).perform()
if (oldVersion < 20) MigrateCryptoTo020(realm).perform()
if (oldVersion < 21) MigrateCryptoTo021(realm).perform()
if (oldVersion < 22) MigrateCryptoTo022(realm, rustDirectory, rustEncryptionConfiguration).perform()
if (oldVersion < 22) MigrateCryptoTo022(
realm,
rustMigrationInfoProvider.rustDirectory,
rustMigrationInfoProvider.rustEncryptionConfiguration,
rustMigrationInfoProvider.migrateMegolmGroupSessions
).perform()
}
}

View File

@ -0,0 +1,31 @@
/*
* Copyright 2023 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.matrix.android.sdk.internal.crypto.store.db
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory
import java.io.File
import javax.inject.Inject
internal class RustMigrationInfoProvider @Inject constructor(
@SessionRustFilesDirectory
val rustDirectory: File,
val rustEncryptionConfiguration: RustEncryptionConfiguration
) {
var migrateMegolmGroupSessions: Boolean = false
}

View File

@ -28,14 +28,15 @@ import java.io.File
internal class MigrateCryptoTo022(
realm: DynamicRealm,
private val rustDirectory: File,
private val rustEncryptionConfiguration: RustEncryptionConfiguration
private val rustEncryptionConfiguration: RustEncryptionConfiguration,
private val migrateMegolmGroupSessions: Boolean = false
) : RealmMigrator(
realm,
22
) {
override fun doMigrate(realm: DynamicRealm) {
// Migrate to rust!
val migrateOperation = MigrateEAtoEROperation()
val migrateOperation = MigrateEAtoEROperation(migrateMegolmGroupSessions)
migrateOperation.dynamicExecute(realm, rustDirectory, rustEncryptionConfiguration.getDatabasePassphrase())
// wa can't delete all for now, but we can do some cleaning

View File

@ -16,4 +16,5 @@
package org.matrix.android.sdk.internal.crypto.store.db.migration.rust
object ExtractMigrationDataFailure : java.lang.RuntimeException("Can't proceed with migration, crypto store is empty or some necessary data is missing.")
data class ExtractMigrationDataFailure(override val cause: Throwable) :
java.lang.RuntimeException("Can't proceed with migration, crypto store is empty or some necessary data is missing.", cause)

View File

@ -19,29 +19,19 @@ package org.matrix.android.sdk.internal.crypto.store.db.migration.rust
import io.realm.Realm
import io.realm.RealmConfiguration
import io.realm.kotlin.where
import org.matrix.android.sdk.api.extensions.orFalse
import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
import org.matrix.olm.OlmSession
import org.matrix.olm.OlmUtility
import org.matrix.rustcomponents.sdk.crypto.MigrationData
import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession
import org.matrix.rustcomponents.sdk.crypto.PickledSession
import timber.log.Timber
import java.nio.charset.Charset
import kotlin.system.measureTimeMillis
private val charset = Charset.forName("UTF-8")
internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = false) {
internal class ExtractMigrationDataUseCase(private val migrateGroupSessions: Boolean = false) {
fun extractData(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) {
return try {
extract(realm, importPartial)
} catch (failure: Throwable) {
throw ExtractMigrationDataFailure
throw ExtractMigrationDataFailure(failure)
}
}
@ -88,75 +78,19 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f
// We are going to do it lazyly when decryption fails
if (migrateGroupSessions) {
var migratedInboundGroupSessionCount = 0
readTime = 0
writeTime = 0
measureTimeMillis {
realm.where<OlmInboundGroupSessionEntity>()
.findAll()
.chunked(chunkSize) { chunk ->
val export: List<PickledInboundGroupSession>
measureTimeMillis {
export = chunk.mapNotNull { it.toPickledInboundGroupSession(pickleKey) }
}.also {
readTime += it
}
migratedInboundGroupSessionCount += export.size
realm.pickledOlmGroupSessions(pickleKey, chunkSize) { pickledSessions ->
migratedInboundGroupSessionCount += pickledSessions.size
measureTimeMillis {
importPartial(
baseExtract.copy(inboundGroupSessions = export)
baseExtract.copy(inboundGroupSessions = pickledSessions)
)
}.also {
writeTime += it
}
}.also { writeTime += it }
}
}.also {
Timber.i("Migration: took $it ms to migrate $migratedInboundGroupSessionCount group sessions")
Timber.i("Migration: extract time $readTime")
Timber.i("Migration: rust import time $writeTime")
}
}
// return baseExtract
}
private fun OlmInboundGroupSessionEntity.toPickledInboundGroupSession(pickleKey: ByteArray): PickledInboundGroupSession? {
val senderKey = this.senderKey ?: return null
val backedUp = this.backedUp
val olmInboundGroupSession = this.getOlmGroupSession() ?: return null.also {
Timber.w("Rust db migration: Failed to migrated group session $sessionId")
}
val data = this.getData() ?: return null.also {
Timber.w("Rust db migration: Failed to migrated group session $sessionId, no meta data")
}
val roomId = data.roomId ?: return null.also {
Timber.w("Rust db migration: Failed to migrated group session $sessionId, no roomId")
}
val pickledInboundGroupSession = olmInboundGroupSession.pickle(pickleKey, StringBuffer()).asString()
return PickledInboundGroupSession(
pickle = pickledInboundGroupSession,
senderKey = senderKey,
signingKey = data.keysClaimed.orEmpty(),
roomId = roomId,
forwardingChains = data.forwardingCurve25519KeyChain.orEmpty(),
imported = data.trusted.orFalse().not(),
backedUp = backedUp
)
}
private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession {
val deviceKey = this.deviceKey ?: ""
val lastReceivedMessageTs = this.lastReceivedMessageTs
val olmSessionStr = this.olmSessionData
val olmSession = deserializeFromRealm<OlmSession>(olmSessionStr)!!
val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString()
return PickledSession(
pickle = pickledOlmSession,
senderKey = deviceKey,
createdUsingFallbackKey = false,
creationTime = lastReceivedMessageTs.toString(),
lastUseTime = lastReceivedMessageTs.toString()
)
}
private fun ByteArray.asString() = String(this, charset)
}

View File

@ -18,19 +18,27 @@ package org.matrix.android.sdk.internal.crypto.store.db.migration.rust
import io.realm.kotlin.where
import okhttp3.internal.toImmutableList
import org.matrix.android.sdk.api.extensions.orFalse
import org.matrix.android.sdk.internal.crypto.model.InboundGroupSessionData
import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntityFields
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntityFields
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields
import org.matrix.android.sdk.internal.di.MoshiProvider
import org.matrix.olm.OlmAccount
import org.matrix.olm.OlmInboundGroupSession
import org.matrix.olm.OlmSession
import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport
import org.matrix.rustcomponents.sdk.crypto.MigrationData
import org.matrix.rustcomponents.sdk.crypto.PickledAccount
import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession
import org.matrix.rustcomponents.sdk.crypto.PickledSession
import timber.log.Timber
import java.nio.charset.Charset
sealed class RealmToMigrate {
@ -225,6 +233,80 @@ fun RealmToMigrate.pickledOlmSessions(pickleKey: ByteArray, chunkSize: Int, onCh
}
}
private val sessionDataAdapter = MoshiProvider.providesMoshi()
.adapter(InboundGroupSessionData::class.java)
fun RealmToMigrate.pickledOlmGroupSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List<PickledInboundGroupSession>) -> Unit)) {
when (this) {
is RealmToMigrate.ClassicRealm -> {
realm.where<OlmInboundGroupSessionEntity>()
.findAll()
.chunked(chunkSize) { chunk ->
val export = chunk.mapNotNull { it.toPickledInboundGroupSession(pickleKey) }
onChunk(export)
}
}
is RealmToMigrate.DynamicRealm -> {
val pickledSessions = mutableListOf<PickledInboundGroupSession>()
realm.schema.get("OlmInboundGroupSessionEntity")?.transform {
val senderKey = it.getString(OlmInboundGroupSessionEntityFields.SENDER_KEY)
val roomId = it.getString(OlmInboundGroupSessionEntityFields.ROOM_ID)
val backedUp = it.getBoolean(OlmInboundGroupSessionEntityFields.BACKED_UP)
val serializedOlmInboundGroupSession = it.getString(OlmInboundGroupSessionEntityFields.SERIALIZED_OLM_INBOUND_GROUP_SESSION)
val inboundSession = deserializeFromRealm<OlmInboundGroupSession>(serializedOlmInboundGroupSession) ?: return@transform Unit.also {
Timber.w("Rust db migration: Failed to migrated group session, no meta data")
}
val sessionData = it.getString(OlmInboundGroupSessionEntityFields.INBOUND_GROUP_SESSION_DATA_JSON).let { json ->
sessionDataAdapter.fromJson(json)
} ?: return@transform Unit.also {
Timber.w("Rust db migration: Failed to migrated group session, no meta data")
}
val pickle = inboundSession.pickle(pickleKey, StringBuffer()).asString()
val pickledSession = PickledInboundGroupSession(
pickle = pickle,
senderKey = senderKey,
signingKey = sessionData.keysClaimed.orEmpty(),
roomId = roomId,
forwardingChains = sessionData.forwardingCurve25519KeyChain.orEmpty(),
imported = sessionData.trusted.orFalse().not(),
backedUp = backedUp
)
// should we check the tracking status?
pickledSessions.add(pickledSession)
if (pickledSessions.size > chunkSize) {
onChunk(pickledSessions.toImmutableList())
pickledSessions.clear()
}
}
if (pickledSessions.isNotEmpty()) {
onChunk(pickledSessions)
}
}
}
}
private fun OlmInboundGroupSessionEntity.toPickledInboundGroupSession(pickleKey: ByteArray): PickledInboundGroupSession? {
val senderKey = this.senderKey ?: return null
val backedUp = this.backedUp
val olmInboundGroupSession = this.getOlmGroupSession() ?: return null.also {
Timber.w("Rust db migration: Failed to migrated group session $sessionId")
}
val data = this.getData() ?: return null.also {
Timber.w("Rust db migration: Failed to migrated group session $sessionId, no meta data")
}
val roomId = data.roomId ?: return null.also {
Timber.w("Rust db migration: Failed to migrated group session $sessionId, no roomId")
}
val pickledInboundGroupSession = olmInboundGroupSession.pickle(pickleKey, StringBuffer()).asString()
return PickledInboundGroupSession(
pickle = pickledInboundGroupSession,
senderKey = senderKey,
signingKey = data.keysClaimed.orEmpty(),
roomId = roomId,
forwardingChains = data.forwardingCurve25519KeyChain.orEmpty(),
imported = data.trusted.orFalse().not(),
backedUp = backedUp
)
}
private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession {
val deviceKey = this.deviceKey ?: ""
val lastReceivedMessageTs = this.lastReceivedMessageTs

View File

@ -59,7 +59,7 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false)
if (!rustFilesDir.exists()) {
rustFilesDir.mkdir()
}
val extractMigrationData = ExtractMigrationDataUseCase()
val extractMigrationData = ExtractMigrationDataUseCase(migrateGroupSessions)
try {
val progressListener = object : ProgressListener {