diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 985da7d83a..b135aa5c9e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: with: cache-read-only: ${{ github.ref != 'refs/heads/develop' }} - name: Assemble ${{ matrix.target }} debug apk - run: ./gradlew assemble${{ matrix.target }}KotlinCryptoDebug $CI_GRADLE_ARG_PROPERTIES + run: ./gradlew assemble${{ matrix.target }}RustCryptoDebug $CI_GRADLE_ARG_PROPERTIES - name: Upload ${{ matrix.target }} debug APKs uses: actions/upload-artifact@v3 with: @@ -57,7 +57,7 @@ jobs: with: cache-read-only: ${{ github.ref != 'refs/heads/develop' }} - name: Assemble GPlay unsigned apk - run: ./gradlew clean assembleGplayKotlinCryptoRelease $CI_GRADLE_ARG_PROPERTIES + run: ./gradlew clean assembleGplayRustCryptoRelease $CI_GRADLE_ARG_PROPERTIES - name: Upload Gplay unsigned APKs uses: actions/upload-artifact@v3 with: @@ -79,7 +79,7 @@ jobs: - name: Execute exodus-standalone uses: docker://exodusprivacy/exodus-standalone:latest with: - args: /github/workspace/gplayKotlinCrypto/release/vector-gplay-kotlinCrypto-universal-release-unsigned.apk -j -o /github/workspace/exodus.json + args: /github/workspace/gplayRustCrypto/release/vector-gplay-rustCrypto-universal-release-unsigned.apk -j -o /github/workspace/exodus.json - name: Upload exodus json report uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 62e169e49f..7c509218b0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -34,7 +34,7 @@ jobs: yes n | towncrier build --version nightly - name: Build and upload Gplay Nightly APK run: | - ./gradlew assembleGplayKotlinCryptoNightly appDistributionUploadGplayKotlinCryptoNightly $CI_GRADLE_ARG_PROPERTIES + ./gradlew assembleGplayRustCryptoNightly appDistributionUploadGplayRustCryptoNightly $CI_GRADLE_ARG_PROPERTIES env: ELEMENT_ANDROID_NIGHTLY_KEYID: ${{ secrets.ELEMENT_ANDROID_NIGHTLY_KEYID }} ELEMENT_ANDROID_NIGHTLY_KEYPASSWORD: ${{ secrets.ELEMENT_ANDROID_NIGHTLY_KEYPASSWORD }} diff --git a/build.gradle b/build.gradle index 7d7f132e95..3060747977 100644 --- a/build.gradle +++ b/build.gradle @@ -323,7 +323,7 @@ tasks.register("recordScreenshots", GradleBuild) { tasks.register("verifyScreenshots", GradleBuild) { startParameter.projectProperties.screenshot = "" - tasks = [':vector:verifyPaparazziKotlinCryptoDebug'] + tasks = [':vector:verifyPaparazziRustCryptoDebug'] } ext.initScreenshotTests = { project -> diff --git a/changelog.d/8390.feature b/changelog.d/8390.feature new file mode 100644 index 0000000000..cc430a5a42 --- /dev/null +++ b/changelog.d/8390.feature @@ -0,0 +1 @@ +Element Android is now using the Crypto Rust SDK. Migration of user's data should be done at first launch after application upgrade. diff --git a/changelog.d/8405.sdk b/changelog.d/8405.sdk new file mode 100644 index 0000000000..c8d165fcdc --- /dev/null +++ b/changelog.d/8405.sdk @@ -0,0 +1 @@ +Add crypto database migration 22, that extract account and olm session to the new rust DB format diff --git a/docs/nightly_build.md b/docs/nightly_build.md index 223fe14824..ea515e90eb 100644 --- a/docs/nightly_build.md +++ b/docs/nightly_build.md @@ -48,7 +48,7 @@ mv towncrier.toml towncrier.toml.bak sed 's/CHANGES\.md/CHANGES_NIGHTLY\.md/' towncrier.toml.bak > towncrier.toml rm towncrier.toml.bak yes n | towncrier build --version nightly -./gradlew assembleGplayKotlinCryptoNightly appDistributionUploadGplayKotlinCryptoNightly $CI_GRADLE_ARG_PROPERTIES +./gradlew assembleGplayRustCryptoNightly appDistributionUploadRustKotlinCryptoNightly $CI_GRADLE_ARG_PROPERTIES ``` Then you can reset the change on the codebase. diff --git a/flavor.gradle b/flavor.gradle index 688eade4fc..946040e4ed 100644 --- a/flavor.gradle +++ b/flavor.gradle @@ -5,13 +5,13 @@ android { productFlavors { kotlinCrypto { dimension "crypto" - isDefault = true // versionName "${versionMajor}.${versionMinor}.${versionPatch}${getFdroidVersionSuffix()}" // buildConfigField "String", "SHORT_FLAVOR_DESCRIPTION", "\"JC\"" // buildConfigField "String", "FLAVOR_DESCRIPTION", "\"KotlinCrypto\"" } rustCrypto { dimension "crypto" + isDefault = true // // versionName "${versionMajor}.${versionMinor}.${versionPatch}${getFdroidVersionSuffix()}" // buildConfigField "String", "SHORT_FLAVOR_DESCRIPTION", "\"RC\"" // buildConfigField "String", "FLAVOR_DESCRIPTION", "\"RustCrypto\"" diff --git a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt b/matrix-sdk-android/src/androidTestKotlinCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt similarity index 90% rename from matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt rename to matrix-sdk-android/src/androidTestKotlinCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt index 2643bf643a..b4f07eff5a 100644 --- a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt +++ b/matrix-sdk-android/src/androidTestKotlinCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt @@ -46,11 +46,15 @@ class CryptoSanityMigrationTest { @Test fun cryptoDatabaseShouldMigrateGracefully() { val realmName = "crypto_store_20.realm" - val migration = RealmCryptoStoreMigration(object : Clock { - override fun epochMillis(): Long { - return 0L - } - }) + + val migration = RealmCryptoStoreMigration( + object : Clock { + override fun epochMillis(): Long { + return 0L + } + } + ) + val realmConfiguration = configurationFactory.createConfiguration( realmName, "7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca", diff --git a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt similarity index 62% rename from matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt rename to matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt index a0847a7ad3..52a75d0653 100644 --- a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt +++ b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt @@ -19,6 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.migration import android.content.Context import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry +import io.mockk.spyk import io.realm.Realm import io.realm.kotlin.where import org.amshove.kluent.internal.assertEquals @@ -30,33 +31,39 @@ import org.junit.Ignore import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith -import org.matrix.android.sdk.InstrumentedTest +import org.matrix.android.sdk.TestBuildVersionSdkIntProvider +import org.matrix.android.sdk.api.securestorage.SecretStoringUtils +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule +import org.matrix.android.sdk.internal.crypto.store.db.RustMigrationInfoProvider import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity +import org.matrix.android.sdk.internal.database.RealmKeysUtils import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory -import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation import org.matrix.android.sdk.internal.util.time.Clock +import org.matrix.android.sdk.test.shared.createTimberTestRule import org.matrix.olm.OlmAccount import org.matrix.olm.OlmManager import org.matrix.rustcomponents.sdk.crypto.OlmMachine import java.io.File +import java.security.KeyStore @RunWith(AndroidJUnit4::class) -class ElementAndroidToElementRMigrationTest : InstrumentedTest { +class DynamicElementAndroidToElementRMigrationTest { @get:Rule val configurationFactory = TestRealmConfigurationFactory() - lateinit var context: Context + @Rule + fun timberTestRule() = createTimberTestRule() + + var context: Context = InstrumentationRegistry.getInstrumentation().context var realm: Realm? = null @Before fun setUp() { // Ensure Olm is initialized OlmManager() - context = InstrumentationRegistry.getInstrumentation().context } @After @@ -64,22 +71,42 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { realm?.close() } + private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) } + + private val rustEncryptionConfiguration = RustEncryptionConfiguration( + "foo", + RealmKeysUtils( + context, + SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false) + ) + ) + + private val fakeClock = object : Clock { + override fun epochMillis() = 0L + } + @Test fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() { testMigrate(false) } @Test - @Ignore("We don't migrate group session for now, and it makes test suite unstable") + @Ignore("We don't migrate group sessions for now, and it's making this test suite unstable") fun given_a_valid_crypto_store_realm_file_no_lazy_then_migration_should_be_successful() { testMigrate(true) } private fun testMigrate(migrateGroupSessions: Boolean) { + val targetFile = File(configurationFactory.root, "rust-sdk") + val realmName = "crypto_store_migration_16.realm" - val migration = RealmCryptoStoreMigration(object : Clock { - override fun epochMillis() = 0L - }) + val infoProvider = RustMigrationInfoProvider( + targetFile, + rustEncryptionConfiguration + ).apply { + migrateMegolmGroupSessions = migrateGroupSessions + } + val migration = RealmCryptoStoreMigration(fakeClock, infoProvider) val realmConfiguration = configurationFactory.createConfiguration( realmName, @@ -91,19 +118,12 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { configurationFactory.copyRealmFromAssets(context, realmName, realmName) realm = Realm.getInstance(realmConfiguration) - val metaData = realm!!.where().findFirst()!! val userId = metaData.userId!! val deviceId = metaData.deviceId!! val olmAccount = metaData.getOlmAccount()!! - val extractor = MigrateEAtoEROperation(migrateGroupSessions) - - val targetFile = File(configurationFactory.root, "rust-sdk") - - extractor.execute(realmConfiguration, targetFile, null) - - val machine = OlmMachine(userId, deviceId, targetFile.path, null) + val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase()) assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"]) assertNotNull(machine.getBackupKeys()) @@ -113,28 +133,15 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { assertTrue(crossSigningStatus.hasUserSigning) if (migrateGroupSessions) { - val inboundGroupSessionEntities = realm!!.where().findAll() - assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt()) - - val backedUpInboundGroupSessionEntities = realm!! - .where() - .equalTo(OlmInboundGroupSessionEntityFields.BACKED_UP, true) - .findAll() - assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt()) + assertTrue("Some outbound sessions should be migrated", machine.roomKeyCounts().total.toInt() > 0) + assertTrue("There are some backed-up sessions", machine.roomKeyCounts().backedUp.toInt() > 0) + } else { + assertTrue(machine.roomKeyCounts().total.toInt() == 0) + assertTrue(machine.roomKeyCounts().backedUp.toInt() == 0) } - } -// @Test -// fun given_an_empty_crypto_store_realm_file_then_migration_should_not_happen() { -// val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = false) -// Realm.getInstance(realmConfiguration).use { -// assertTrue(it.isEmpty) -// } -// val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null) -// assertNull(machine.getBackupKeys()) -// val crossSigningStatus = machine.crossSigningStatus() -// assertFalse(crossSigningStatus.hasMaster) -// assertFalse(crossSigningStatus.hasSelfSigning) -// assertFalse(crossSigningStatus.hasUserSigning) -// } + // legacy olm sessions should have been deleted + val remainingOlmSessions = realm!!.where().findAll().size + assertEquals("legacy olm sessions should have been removed from store", 0, remainingOlmSessions) + } } diff --git a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt new file mode 100644 index 0000000000..828c0f51d4 --- /dev/null +++ b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2023 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.database + +import android.content.Context +import androidx.test.platform.app.InstrumentationRegistry +import io.mockk.spyk +import io.realm.Realm +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.matrix.android.sdk.TestBuildVersionSdkIntProvider +import org.matrix.android.sdk.api.securestorage.SecretStoringUtils +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration +import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration +import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule +import org.matrix.android.sdk.internal.crypto.store.db.RustMigrationInfoProvider +import org.matrix.android.sdk.internal.util.time.Clock +import org.matrix.olm.OlmManager +import java.io.File +import java.security.KeyStore + +class CryptoSanityMigrationTest { + @get:Rule val configurationFactory = TestRealmConfigurationFactory() + + lateinit var context: Context + var realm: Realm? = null + + @Before + fun setUp() { + // Ensure Olm is initialized + OlmManager() + context = InstrumentationRegistry.getInstrumentation().context + } + + @After + fun tearDown() { + realm?.close() + } + + private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) } + + @Test + fun cryptoDatabaseShouldMigrateGracefully() { + val realmName = "crypto_store_20.realm" + + val rustMigrationInfo = RustMigrationInfoProvider( + File(configurationFactory.root, "test_rust"), + RustEncryptionConfiguration( + "foo", + RealmKeysUtils( + context, + SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false) + ) + ), + ) + val migration = RealmCryptoStoreMigration( + object : Clock { + override fun epochMillis(): Long { + return 0L + } + }, + rustMigrationInfo + ) + + val realmConfiguration = configurationFactory.createConfiguration( + realmName, + "7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca", + RealmCryptoStoreModule(), + migration.schemaVersion, + migration + ) + configurationFactory.copyRealmFromAssets(context, realmName, realmName) + + realm = Realm.getInstance(realmConfiguration) + } +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt b/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt similarity index 100% rename from matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt rename to matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt index f0aaf8e59e..4e778e04cf 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt @@ -44,7 +44,6 @@ import org.matrix.android.sdk.api.session.permalinks.PermalinkService import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService import org.matrix.android.sdk.api.session.typing.TypingUsersTracker import org.matrix.android.sdk.api.util.md5 -import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorageService import org.matrix.android.sdk.internal.crypto.tasks.DefaultRedactEventTask import org.matrix.android.sdk.internal.crypto.tasks.RedactEventTask @@ -53,7 +52,6 @@ import org.matrix.android.sdk.internal.database.RealmSessionProvider import org.matrix.android.sdk.internal.database.SessionRealmConfigurationFactory import org.matrix.android.sdk.internal.di.Authenticated import org.matrix.android.sdk.internal.di.CacheDirectory -import org.matrix.android.sdk.internal.di.CryptoDatabase import org.matrix.android.sdk.internal.di.DeviceId import org.matrix.android.sdk.internal.di.SessionDatabase import org.matrix.android.sdk.internal.di.SessionDownloadsDirectory @@ -100,11 +98,9 @@ import org.matrix.android.sdk.internal.session.typing.DefaultTypingUsersTracker import org.matrix.android.sdk.internal.session.user.accountdata.DefaultSessionAccountDataService import org.matrix.android.sdk.internal.session.widgets.DefaultWidgetURLFormatter import retrofit2.Retrofit -import timber.log.Timber import java.io.File import javax.inject.Provider import javax.inject.Qualifier -import kotlin.system.measureTimeMillis @Qualifier @Retention(AnnotationRetention.RUNTIME) @@ -189,17 +185,8 @@ internal abstract class SessionModule { @SessionScope fun providesRustCryptoFilesDir( @SessionFilesDirectory parent: File, - @CryptoDatabase realmConfiguration: RealmConfiguration, - rustEncryptionConfiguration: RustEncryptionConfiguration, ): File { - val target = File(parent, "rustFlavor") - val file: File - measureTimeMillis { - file = MigrateEAtoEROperation().execute(realmConfiguration, target, rustEncryptionConfiguration.getDatabasePassphrase()) - }.let { duration -> - Timber.v("Migrating to ER in $duration ms") - } - return file + return File(parent, "rustFlavor") } @JvmStatic diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/RustCrossSigningService.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/RustCrossSigningService.kt index 85c55f31c3..5a200a59ff 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/RustCrossSigningService.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/RustCrossSigningService.kt @@ -44,7 +44,7 @@ internal class RustCrossSigningService @Inject constructor( override suspend fun isCrossSigningVerified(): Boolean { return when (val identity = olmMachine.getIdentity(olmMachine.userId())) { is OwnUserIdentity -> identity.verified() - else -> false + else -> false } } @@ -177,7 +177,7 @@ internal class RustCrossSigningService @Inject constructor( throw IllegalArgumentException("This device [$deviceId] is not known, or not yours") } } else { - throw IllegalArgumentException("This device [$deviceId] is not known") + throw IllegalArgumentException("This device [$deviceId] is not known") } } @@ -238,6 +238,6 @@ internal class RustCrossSigningService @Inject constructor( override fun checkOtherMSKTrusted(myCrossSigningInfo: MXCrossSigningInfo?, otherInfo: MXCrossSigningInfo?): UserTrustResult { // is this needed in rust? should be moved to internal API? - TODO() + return UserTrustResult.Failure("Not used in rust") } } diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/RustCryptoService.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/RustCryptoService.kt index c66fb59a0b..4769d57dc9 100755 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/RustCryptoService.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/RustCryptoService.kt @@ -266,6 +266,15 @@ internal class RustCryptoService @Inject constructor( Timber.tag(loggerTag.value).v("Failed create an Olm machine: $throwable") } + // After the initial rust migration the current keys & signature might not be there + // The session is then in an invalid state and can fire unexpected verify popups + // this will only do network request once. + cryptoCoroutineScope.launch(coroutineDispatchers.io) { + tryOrNull { + downloadKeysIfNeeded(listOf(myUserId), false) + } + } + // We try to enable key backups, if the backup version on the server is trusted, // we're gonna continue backing up. cryptoCoroutineScope.launch { diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt new file mode 100644 index 0000000000..99734f654f --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt @@ -0,0 +1,95 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db + +import io.realm.DynamicRealm +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo001Legacy +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo002Legacy +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo003RiotX +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo004 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo005 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo006 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo007 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo008 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo009 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo010 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo011 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo012 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo013 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo014 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo015 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo016 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo017 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo018 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo019 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo020 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo021 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo022 +import org.matrix.android.sdk.internal.util.database.MatrixRealmMigration +import org.matrix.android.sdk.internal.util.time.Clock +import javax.inject.Inject + +/** + * Schema version history: + * 0, 1, 2: legacy Riot-Android; + * 3: migrate to RiotX schema; + * 4, 5, 6, 7, 8, 9: migrations from RiotX (which was previously 1, 2, 3, 4, 5, 6). + */ +internal class RealmCryptoStoreMigration @Inject constructor( + private val clock: Clock, + private val rustMigrationInfoProvider: RustMigrationInfoProvider, +) : MatrixRealmMigration( + dbName = "Crypto", + schemaVersion = 22L, +) { + /** + * Forces all RealmCryptoStoreMigration instances to be equal. + * Avoids Realm throwing when multiple instances of the migration are set. + */ + override fun equals(other: Any?) = other is RealmCryptoStoreMigration + override fun hashCode() = 5000 + + override fun doMigrate(realm: DynamicRealm, oldVersion: Long) { + if (oldVersion < 1) MigrateCryptoTo001Legacy(realm).perform() + if (oldVersion < 2) MigrateCryptoTo002Legacy(realm).perform() + if (oldVersion < 3) MigrateCryptoTo003RiotX(realm).perform() + if (oldVersion < 4) MigrateCryptoTo004(realm).perform() + if (oldVersion < 5) MigrateCryptoTo005(realm).perform() + if (oldVersion < 6) MigrateCryptoTo006(realm).perform() + if (oldVersion < 7) MigrateCryptoTo007(realm).perform() + if (oldVersion < 8) MigrateCryptoTo008(realm, clock).perform() + if (oldVersion < 9) MigrateCryptoTo009(realm).perform() + if (oldVersion < 10) MigrateCryptoTo010(realm).perform() + if (oldVersion < 11) MigrateCryptoTo011(realm).perform() + if (oldVersion < 12) MigrateCryptoTo012(realm).perform() + if (oldVersion < 13) MigrateCryptoTo013(realm).perform() + if (oldVersion < 14) MigrateCryptoTo014(realm).perform() + if (oldVersion < 15) MigrateCryptoTo015(realm).perform() + if (oldVersion < 16) MigrateCryptoTo016(realm).perform() + if (oldVersion < 17) MigrateCryptoTo017(realm).perform() + if (oldVersion < 18) MigrateCryptoTo018(realm).perform() + if (oldVersion < 19) MigrateCryptoTo019(realm).perform() + if (oldVersion < 20) MigrateCryptoTo020(realm).perform() + if (oldVersion < 21) MigrateCryptoTo021(realm).perform() + if (oldVersion < 22) MigrateCryptoTo022( + realm, + rustMigrationInfoProvider.rustDirectory, + rustMigrationInfoProvider.rustEncryptionConfiguration, + rustMigrationInfoProvider.migrateMegolmGroupSessions + ).perform() + } +} diff --git a/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RustMigrationInfoProvider.kt similarity index 50% rename from matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt rename to matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RustMigrationInfoProvider.kt index 3fd6d1ecf1..667990468c 100644 --- a/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RustMigrationInfoProvider.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 The Matrix.org Foundation C.I.C. + * Copyright 2023 The Matrix.org Foundation C.I.C. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,18 @@ * limitations under the License. */ -package org.matrix.android.sdk.internal.session +package org.matrix.android.sdk.internal.crypto.store.db -import io.realm.RealmConfiguration -import timber.log.Timber +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration +import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory import java.io.File +import javax.inject.Inject -class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) { +internal class RustMigrationInfoProvider @Inject constructor( + @SessionRustFilesDirectory + val rustDirectory: File, + val rustEncryptionConfiguration: RustEncryptionConfiguration +) { - fun execute(cryptoRealm: RealmConfiguration, sessionFilesDir: File, passphrase: String?): File { - // to remove unused warning - Timber.v("Not used in kotlin crypto $cryptoRealm ${"*".repeat(passphrase?.length ?: 0)} lazy:$migrateGroupSessions") - // no op in kotlinCrypto - return sessionFilesDir - } + var migrateMegolmGroupSessions: Boolean = false } diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt new file mode 100644 index 0000000000..d0f612aa87 --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2023 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db.migration + +import io.realm.DynamicRealm +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration +import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation +import org.matrix.android.sdk.internal.util.database.RealmMigrator +import java.io.File + +/** + * This migration creates the rust database and migrates from legacy crypto + */ +internal class MigrateCryptoTo022( + realm: DynamicRealm, + private val rustDirectory: File, + private val rustEncryptionConfiguration: RustEncryptionConfiguration, + private val migrateMegolmGroupSessions: Boolean = false +) : RealmMigrator( + realm, + 22 +) { + override fun doMigrate(realm: DynamicRealm) { + // Migrate to rust! + val migrateOperation = MigrateEAtoEROperation(migrateMegolmGroupSessions) + migrateOperation.dynamicExecute(realm, rustDirectory, rustEncryptionConfiguration.getDatabasePassphrase()) + + // wa can't delete all for now, but we can do some cleaning + realm.schema.get("OlmSessionEntity")?.transform { + it.deleteFromRealm() + } + + // a future migration will clean the rest + } +} diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataFailure.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataFailure.kt index 8e86402916..fb4bd1c8fe 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataFailure.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataFailure.kt @@ -16,4 +16,5 @@ package org.matrix.android.sdk.internal.crypto.store.db.migration.rust -object ExtractMigrationDataFailure : java.lang.RuntimeException("Can't proceed with migration, crypto store is empty or some necessary data is missing.") +data class ExtractMigrationDataFailure(override val cause: Throwable) : + java.lang.RuntimeException("Can't proceed with migration, crypto store is empty or some necessary data is missing.", cause) diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt index da125b8e16..3dae9a6b13 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt @@ -19,32 +19,19 @@ package org.matrix.android.sdk.internal.crypto.store.db.migration.rust import io.realm.Realm import io.realm.RealmConfiguration import io.realm.kotlin.where -import org.matrix.android.sdk.api.extensions.orFalse -import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity -import org.matrix.olm.OlmSession import org.matrix.olm.OlmUtility -import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport import org.matrix.rustcomponents.sdk.crypto.MigrationData -import org.matrix.rustcomponents.sdk.crypto.PickledAccount -import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession -import org.matrix.rustcomponents.sdk.crypto.PickledSession import timber.log.Timber -import java.nio.charset.Charset import kotlin.system.measureTimeMillis -private val charset = Charset.forName("UTF-8") +internal class ExtractMigrationDataUseCase(private val migrateGroupSessions: Boolean = false) { -internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = false) { - - fun extractData(realm: Realm, importPartial: ((MigrationData) -> Unit)) { + fun extractData(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) { return try { extract(realm, importPartial) } catch (failure: Throwable) { - throw ExtractMigrationDataFailure + throw ExtractMigrationDataFailure(failure) } } @@ -57,89 +44,33 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f } } - private fun extract(realm: Realm, importPartial: ((MigrationData) -> Unit)) { - val metadataEntity = realm.where().findFirst() - ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity") - + private fun extract(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) { val pickleKey = OlmUtility.getRandomKey() - val masterKey = metadataEntity.xSignMasterPrivateKey - val userKey = metadataEntity.xSignUserPrivateKey - val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey - - val userId = metadataEntity.userId - ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null") - val deviceId = metadataEntity.deviceId - ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null") - - val backupVersion = metadataEntity.backupVersion - val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey - - val isOlmAccountShared = metadataEntity.deviceKeysSentToServer - - val olmAccount = metadataEntity.getOlmAccount() - ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account") - val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString() - olmAccount.oneTimeKeys() - val pickledAccount = PickledAccount( - userId = userId, - deviceId = deviceId, - pickle = pickledOlmAccount, - shared = isOlmAccountShared, - uploadedSignedKeyCount = 50 - ) - - val baseExtract = MigrationData( - account = pickledAccount, - pickleKey = pickleKey.map { it.toUByte() }, - crossSigning = CrossSigningKeyExport( - masterKey = masterKey, - selfSigningKey = selfSignedKey, - userSigningKey = userKey - ), - sessions = emptyList(), - backupRecoveryKey = backupRecoveryKey, - trackedUsers = emptyList(), - inboundGroupSessions = emptyList(), - backupVersion = backupVersion, - // TODO import room settings from legacy DB - roomSettings = emptyMap() - ) + val baseExtract = realm.getPickledAccount(pickleKey) // import the account asap importPartial(baseExtract) val chunkSize = 500 - realm.where() - .findAll() - .chunked(chunkSize) { chunk -> - val trackedUserIds = chunk.mapNotNull { it.userId } - importPartial( - baseExtract.copy(trackedUsers = trackedUserIds) - ) - } + realm.trackedUsersChunk(500) { + importPartial( + baseExtract.copy(trackedUsers = it) + ) + } var migratedOlmSessionCount = 0 - var readTime = 0L var writeTime = 0L measureTimeMillis { - realm.where().findAll() - .chunked(chunkSize) { chunk -> - migratedOlmSessionCount += chunk.size - val export: List - measureTimeMillis { - export = chunk.map { it.toPickledSession(pickleKey) } - }.also { - readTime += it - } - measureTimeMillis { - importPartial( - baseExtract.copy(sessions = export) - ) - }.also { writeTime += it } - } + realm.pickledOlmSessions(pickleKey, chunkSize) { pickledSessions -> + migratedOlmSessionCount += pickledSessions.size + measureTimeMillis { + importPartial( + baseExtract.copy(sessions = pickledSessions) + ) + }.also { writeTime += it } + } }.also { Timber.i("Migration: took $it ms to migrate $migratedOlmSessionCount olm sessions") - Timber.i("Migration: extract time $readTime") Timber.i("Migration: rust import time $writeTime") } @@ -147,75 +78,19 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f // We are going to do it lazyly when decryption fails if (migrateGroupSessions) { var migratedInboundGroupSessionCount = 0 - readTime = 0 - writeTime = 0 measureTimeMillis { - realm.where() - .findAll() - .chunked(chunkSize) { chunk -> - val export: List - measureTimeMillis { - export = chunk.mapNotNull { it.toPickledInboundGroupSession(pickleKey) } - }.also { - readTime += it - } - migratedInboundGroupSessionCount += export.size - measureTimeMillis { - importPartial( - baseExtract.copy(inboundGroupSessions = export) - ) - }.also { - writeTime += it - } - } + realm.pickledOlmGroupSessions(pickleKey, chunkSize) { pickledSessions -> + migratedInboundGroupSessionCount += pickledSessions.size + measureTimeMillis { + importPartial( + baseExtract.copy(inboundGroupSessions = pickledSessions) + ) + }.also { writeTime += it } + } }.also { Timber.i("Migration: took $it ms to migrate $migratedInboundGroupSessionCount group sessions") - Timber.i("Migration: extract time $readTime") Timber.i("Migration: rust import time $writeTime") } } - -// return baseExtract } - - private fun OlmInboundGroupSessionEntity.toPickledInboundGroupSession(pickleKey: ByteArray): PickledInboundGroupSession? { - val senderKey = this.senderKey ?: return null - val backedUp = this.backedUp - val olmInboundGroupSession = this.getOlmGroupSession() ?: return null.also { - Timber.w("Rust db migration: Failed to migrated group session $sessionId") - } - val data = this.getData() ?: return null.also { - Timber.w("Rust db migration: Failed to migrated group session $sessionId, no meta data") - } - val roomId = data.roomId ?: return null.also { - Timber.w("Rust db migration: Failed to migrated group session $sessionId, no roomId") - } - val pickledInboundGroupSession = olmInboundGroupSession.pickle(pickleKey, StringBuffer()).asString() - return PickledInboundGroupSession( - pickle = pickledInboundGroupSession, - senderKey = senderKey, - signingKey = data.keysClaimed.orEmpty(), - roomId = roomId, - forwardingChains = data.forwardingCurve25519KeyChain.orEmpty(), - imported = data.trusted.orFalse().not(), - backedUp = backedUp - ) - } - - private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession { - val deviceKey = this.deviceKey ?: "" - val lastReceivedMessageTs = this.lastReceivedMessageTs - val olmSessionStr = this.olmSessionData - val olmSession = deserializeFromRealm(olmSessionStr)!! - val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString() - return PickledSession( - pickle = pickledOlmSession, - senderKey = deviceKey, - createdUsingFallbackKey = false, - creationTime = lastReceivedMessageTs.toString(), - lastUseTime = lastReceivedMessageTs.toString() - ) - } - - private fun ByteArray.asString() = String(this, charset) } diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt new file mode 100644 index 0000000000..608f68fc3d --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2023 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db.migration.rust + +import io.realm.kotlin.where +import okhttp3.internal.toImmutableList +import org.matrix.android.sdk.api.extensions.orFalse +import org.matrix.android.sdk.internal.crypto.model.InboundGroupSessionData +import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm +import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields +import org.matrix.android.sdk.internal.di.MoshiProvider +import org.matrix.olm.OlmAccount +import org.matrix.olm.OlmInboundGroupSession +import org.matrix.olm.OlmSession +import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport +import org.matrix.rustcomponents.sdk.crypto.MigrationData +import org.matrix.rustcomponents.sdk.crypto.PickledAccount +import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession +import org.matrix.rustcomponents.sdk.crypto.PickledSession +import timber.log.Timber +import java.nio.charset.Charset + +sealed class RealmToMigrate { + data class DynamicRealm(val realm: io.realm.DynamicRealm) : RealmToMigrate() + data class ClassicRealm(val realm: io.realm.Realm) : RealmToMigrate() +} + +fun RealmToMigrate.hasExistingData(): Boolean { + return when (this) { + is RealmToMigrate.ClassicRealm -> { + !this.realm.isEmpty && + // Check if there is a MetaData object + this.realm.where().count() > 0 && + this.realm.where().findFirst()?.olmAccountData != null + } + is RealmToMigrate.DynamicRealm -> { + return true + } + } +} + +@Throws +fun RealmToMigrate.getPickledAccount(pickleKey: ByteArray): MigrationData { + return when (this) { + is RealmToMigrate.ClassicRealm -> { + val metadataEntity = realm.where().findFirst() + ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity") + + val masterKey = metadataEntity.xSignMasterPrivateKey + val userKey = metadataEntity.xSignUserPrivateKey + val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey + + val userId = metadataEntity.userId + ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null") + val deviceId = metadataEntity.deviceId + ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null") + + val backupVersion = metadataEntity.backupVersion + val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey + + val isOlmAccountShared = metadataEntity.deviceKeysSentToServer + + val olmAccount = metadataEntity.getOlmAccount() + ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account") + val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString() + + val pickledAccount = PickledAccount( + userId = userId, + deviceId = deviceId, + pickle = pickledOlmAccount, + shared = isOlmAccountShared, + uploadedSignedKeyCount = 50 + ) + MigrationData( + account = pickledAccount, + pickleKey = pickleKey.map { it.toUByte() }, + crossSigning = CrossSigningKeyExport( + masterKey = masterKey, + selfSigningKey = selfSignedKey, + userSigningKey = userKey + ), + sessions = emptyList(), + backupRecoveryKey = backupRecoveryKey, + trackedUsers = emptyList(), + inboundGroupSessions = emptyList(), + backupVersion = backupVersion, + // TODO import room settings from legacy DB + roomSettings = emptyMap() + ) + } + is RealmToMigrate.DynamicRealm -> { + val cryptoMetadataEntitySchema = realm.schema.get("CryptoMetadataEntity") + ?: throw java.lang.IllegalStateException("Missing Metadata entity") + + var migrationData: MigrationData? = null + cryptoMetadataEntitySchema.transform { dynMetaData -> + + val serializedOlmAccount = dynMetaData.getString(CryptoMetadataEntityFields.OLM_ACCOUNT_DATA) + + val masterKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_MASTER_PRIVATE_KEY) + val userKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_USER_PRIVATE_KEY) + val selfSignedKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_SELF_SIGNED_PRIVATE_KEY) + + val userId = dynMetaData.getString(CryptoMetadataEntityFields.USER_ID) + ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null") + val deviceId = dynMetaData.getString(CryptoMetadataEntityFields.DEVICE_ID) + ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null") + + val backupVersion = dynMetaData.getString(CryptoMetadataEntityFields.BACKUP_VERSION) + val backupRecoveryKey = dynMetaData.getString(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY) + + val isOlmAccountShared = dynMetaData.getBoolean(CryptoMetadataEntityFields.DEVICE_KEYS_SENT_TO_SERVER) + + val olmAccount = deserializeFromRealm(serializedOlmAccount) + ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account") + + val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString() + + val pickledAccount = PickledAccount( + userId = userId, + deviceId = deviceId, + pickle = pickledOlmAccount, + shared = isOlmAccountShared, + uploadedSignedKeyCount = 50 + ) + + migrationData = MigrationData( + account = pickledAccount, + pickleKey = pickleKey.map { it.toUByte() }, + crossSigning = CrossSigningKeyExport( + masterKey = masterKey, + selfSigningKey = selfSignedKey, + userSigningKey = userKey + ), + sessions = emptyList(), + backupRecoveryKey = backupRecoveryKey, + trackedUsers = emptyList(), + inboundGroupSessions = emptyList(), + backupVersion = backupVersion, + // TODO import room settings from legacy DB + roomSettings = emptyMap() + ) + } + migrationData!! + } + } +} + +fun RealmToMigrate.trackedUsersChunk(chunkSize: Int, onChunk: ((List) -> Unit)) { + when (this) { + is RealmToMigrate.ClassicRealm -> { + realm.where() + .findAll() + .chunked(chunkSize) + .onEach { + onChunk(it.mapNotNull { it.userId }) + } + } + is RealmToMigrate.DynamicRealm -> { + val userList = mutableListOf() + realm.schema.get("UserEntity")?.transform { + val userId = it.getString(UserEntityFields.USER_ID) + // should we check the tracking status? + userList.add(userId) + if (userList.size > chunkSize) { + onChunk(userList.toImmutableList()) + userList.clear() + } + } + if (userList.isNotEmpty()) { + onChunk(userList) + } + } + } +} + +fun RealmToMigrate.pickledOlmSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List) -> Unit)) { + when (this) { + is RealmToMigrate.ClassicRealm -> { + realm.where().findAll() + .chunked(chunkSize) { chunk -> + val export = chunk.map { it.toPickledSession(pickleKey) } + onChunk(export) + } + } + is RealmToMigrate.DynamicRealm -> { + val pickledSessions = mutableListOf() + realm.schema.get("OlmSessionEntity")?.transform { + val sessionData = it.getString(OlmSessionEntityFields.OLM_SESSION_DATA) + val deviceKey = it.getString(OlmSessionEntityFields.DEVICE_KEY) + val lastReceivedMessageTs = it.getLong(OlmSessionEntityFields.LAST_RECEIVED_MESSAGE_TS) + val olmSession = deserializeFromRealm(sessionData)!! + val pickle = olmSession.pickle(pickleKey, StringBuffer()).asString() + val pickledSession = PickledSession( + pickle = pickle, + senderKey = deviceKey, + createdUsingFallbackKey = false, + creationTime = lastReceivedMessageTs.toString(), + lastUseTime = lastReceivedMessageTs.toString() + ) + // should we check the tracking status? + pickledSessions.add(pickledSession) + if (pickledSessions.size > chunkSize) { + onChunk(pickledSessions.toImmutableList()) + pickledSessions.clear() + } + } + if (pickledSessions.isNotEmpty()) { + onChunk(pickledSessions) + } + } + } +} + +private val sessionDataAdapter = MoshiProvider.providesMoshi() + .adapter(InboundGroupSessionData::class.java) +fun RealmToMigrate.pickledOlmGroupSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List) -> Unit)) { + when (this) { + is RealmToMigrate.ClassicRealm -> { + realm.where() + .findAll() + .chunked(chunkSize) { chunk -> + val export = chunk.mapNotNull { it.toPickledInboundGroupSession(pickleKey) } + onChunk(export) + } + } + is RealmToMigrate.DynamicRealm -> { + val pickledSessions = mutableListOf() + realm.schema.get("OlmInboundGroupSessionEntity")?.transform { + val senderKey = it.getString(OlmInboundGroupSessionEntityFields.SENDER_KEY) + val roomId = it.getString(OlmInboundGroupSessionEntityFields.ROOM_ID) + val backedUp = it.getBoolean(OlmInboundGroupSessionEntityFields.BACKED_UP) + val serializedOlmInboundGroupSession = it.getString(OlmInboundGroupSessionEntityFields.SERIALIZED_OLM_INBOUND_GROUP_SESSION) + val inboundSession = deserializeFromRealm(serializedOlmInboundGroupSession) ?: return@transform Unit.also { + Timber.w("Rust db migration: Failed to migrated group session, no meta data") + } + val sessionData = it.getString(OlmInboundGroupSessionEntityFields.INBOUND_GROUP_SESSION_DATA_JSON).let { json -> + sessionDataAdapter.fromJson(json) + } ?: return@transform Unit.also { + Timber.w("Rust db migration: Failed to migrated group session, no meta data") + } + val pickle = inboundSession.pickle(pickleKey, StringBuffer()).asString() + val pickledSession = PickledInboundGroupSession( + pickle = pickle, + senderKey = senderKey, + signingKey = sessionData.keysClaimed.orEmpty(), + roomId = roomId, + forwardingChains = sessionData.forwardingCurve25519KeyChain.orEmpty(), + imported = sessionData.trusted.orFalse().not(), + backedUp = backedUp + ) + // should we check the tracking status? + pickledSessions.add(pickledSession) + if (pickledSessions.size > chunkSize) { + onChunk(pickledSessions.toImmutableList()) + pickledSessions.clear() + } + } + if (pickledSessions.isNotEmpty()) { + onChunk(pickledSessions) + } + } + } +} + +private fun OlmInboundGroupSessionEntity.toPickledInboundGroupSession(pickleKey: ByteArray): PickledInboundGroupSession? { + val senderKey = this.senderKey ?: return null + val backedUp = this.backedUp + val olmInboundGroupSession = this.getOlmGroupSession() ?: return null.also { + Timber.w("Rust db migration: Failed to migrated group session $sessionId") + } + val data = this.getData() ?: return null.also { + Timber.w("Rust db migration: Failed to migrated group session $sessionId, no meta data") + } + val roomId = data.roomId ?: return null.also { + Timber.w("Rust db migration: Failed to migrated group session $sessionId, no roomId") + } + val pickledInboundGroupSession = olmInboundGroupSession.pickle(pickleKey, StringBuffer()).asString() + return PickledInboundGroupSession( + pickle = pickledInboundGroupSession, + senderKey = senderKey, + signingKey = data.keysClaimed.orEmpty(), + roomId = roomId, + forwardingChains = data.forwardingCurve25519KeyChain.orEmpty(), + imported = data.trusted.orFalse().not(), + backedUp = backedUp + ) +} +private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession { + val deviceKey = this.deviceKey ?: "" + val lastReceivedMessageTs = this.lastReceivedMessageTs + val olmSessionStr = this.olmSessionData + val olmSession = deserializeFromRealm(olmSessionStr)!! + val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString() + return PickledSession( + pickle = pickledOlmSession, + senderKey = deviceKey, + createdUsingFallbackKey = false, + creationTime = lastReceivedMessageTs.toString(), + lastUseTime = lastReceivedMessageTs.toString() + ) +} + +private val charset = Charset.forName("UTF-8") +private fun ByteArray.asString() = String(this, charset) diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt index 741b5a4c8f..b4944edbb9 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt @@ -16,9 +16,11 @@ package org.matrix.android.sdk.internal.session +import io.realm.DynamicRealm import io.realm.Realm import io.realm.RealmConfiguration import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.ExtractMigrationDataUseCase +import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.RealmToMigrate import org.matrix.rustcomponents.sdk.crypto.ProgressListener import timber.log.Timber import java.io.File @@ -40,9 +42,8 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) Timber.v("OnProgress: $progress/$total") } } - Realm.getInstance(cryptoRealm).use { realm -> - extractMigrationData.extractData(realm) { + extractMigrationData.extractData(RealmToMigrate.ClassicRealm(realm)) { org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener) } } @@ -53,4 +54,25 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) } return rustFilesDir } + + fun dynamicExecute(dynamicRealm: DynamicRealm, rustFilesDir: File, passphrase: String?) { + if (!rustFilesDir.exists()) { + rustFilesDir.mkdir() + } + val extractMigrationData = ExtractMigrationDataUseCase(migrateGroupSessions) + + try { + val progressListener = object : ProgressListener { + override fun onProgress(progress: Int, total: Int) { + Timber.v("OnProgress: $progress/$total") + } + } + extractMigrationData.extractData(RealmToMigrate.DynamicRealm(dynamicRealm)) { + org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener) + } + } catch (failure: Throwable) { + Timber.e(failure, "Failure while calling rust migration method") + throw failure + } + } } diff --git a/vector-app/build.gradle b/vector-app/build.gradle index f76fb466ad..e3ab104b46 100644 --- a/vector-app/build.gradle +++ b/vector-app/build.gradle @@ -334,16 +334,16 @@ android { kotlinCrypto { dimension "crypto" - isDefault = true // versionName "${versionMajor}.${versionMinor}.${versionPatch}${getFdroidVersionSuffix()}" buildConfigField "String", "CRYPTO_FLAVOR_DESCRIPTION", "\"olm-crypto\"" // buildConfigField "String", "FLAVOR_DESCRIPTION", "\"KotlinCrypto\"" } rustCrypto { dimension "crypto" - applicationIdSuffix ".corroded" - versionNameSuffix "-R" - resValue "string", "app_name", "ER" + isDefault = true + // applicationIdSuffix ".corroded" + // versionNameSuffix "-R" + // resValue "string", "app_name", "ER" // // versionName "${versionMajor}.${versionMinor}.${versionPatch}${getFdroidVersionSuffix()}" buildConfigField "String", "CRYPTO_FLAVOR_DESCRIPTION", "\"rust-crypto\"" diff --git a/vector-app/src/gplayRustCrypto/debug/google-services.json b/vector-app/src/gplayRustCrypto/debug/google-services.json deleted file mode 100644 index 465df9d167..0000000000 --- a/vector-app/src/gplayRustCrypto/debug/google-services.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "project_info": { - "project_number": "912726360885", - "firebase_url": "https://vector-alpha.firebaseio.com", - "project_id": "vector-alpha", - "storage_bucket": "vector-alpha.appspot.com" - }, - "client": [ - { - "client_info": { - "mobilesdk_app_id": "1:912726360885:android:ee00f33109f786a800427c", - "android_client_info": { - "package_name": "im.vector.app.corroded.debug" - } - }, - "oauth_client": [ - { - "client_id": "912726360885-e87n3jva9uoj4vbidvijq78ebg02asv2.apps.googleusercontent.com", - "client_type": 3 - } - ], - "api_key": [ - { - "current_key": "AIzaSyAFZX8IhIfgzdOZvxDP_ISO5WYoU7jmQ5c" - } - ], - "services": { - "appinvite_service": { - "other_platform_oauth_client": [ - { - "client_id": "912726360885-e87n3jva9uoj4vbidvijq78ebg02asv2.apps.googleusercontent.com", - "client_type": 3 - } - ] - } - } - } - ], - "configuration_version": "1" -} diff --git a/vector-app/src/gplayRustCrypto/nightly/google-services.json b/vector-app/src/gplayRustCrypto/nightly/google-services.json deleted file mode 100644 index 9eb7172825..0000000000 --- a/vector-app/src/gplayRustCrypto/nightly/google-services.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "project_info": { - "project_number": "912726360885", - "firebase_url": "https://vector-alpha.firebaseio.com", - "project_id": "vector-alpha", - "storage_bucket": "vector-alpha.appspot.com" - }, - "client": [ - { - "client_info": { - "mobilesdk_app_id": "1:912726360885:android:94fb99347eaa36d100427c", - "android_client_info": { - "package_name": "im.vector.app.corroded.nightly" - } - }, - "oauth_client": [ - { - "client_id": "912726360885-e87n3jva9uoj4vbidvijq78ebg02asv2.apps.googleusercontent.com", - "client_type": 3 - } - ], - "api_key": [ - { - "current_key": "AIzaSyAFZX8IhIfgzdOZvxDP_ISO5WYoU7jmQ5c" - } - ], - "services": { - "appinvite_service": { - "other_platform_oauth_client": [ - { - "client_id": "912726360885-e87n3jva9uoj4vbidvijq78ebg02asv2.apps.googleusercontent.com", - "client_type": 3 - } - ] - } - } - } - ], - "configuration_version": "1" -} diff --git a/vector-app/src/gplayRustCrypto/release/google-services.json b/vector-app/src/gplayRustCrypto/release/google-services.json deleted file mode 100644 index 3647bcfd18..0000000000 --- a/vector-app/src/gplayRustCrypto/release/google-services.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "project_info": { - "project_number": "912726360885", - "firebase_url": "https://vector-alpha.firebaseio.com", - "project_id": "vector-alpha", - "storage_bucket": "vector-alpha.appspot.com" - }, - "client": [ - { - "client_info": { - "mobilesdk_app_id": "1:912726360885:android:94fb99347eaa36d100427c", - "android_client_info": { - "package_name": "im.vector.app.corroded" - } - }, - "oauth_client": [ - { - "client_id": "912726360885-e87n3jva9uoj4vbidvijq78ebg02asv2.apps.googleusercontent.com", - "client_type": 3 - } - ], - "api_key": [ - { - "current_key": "AIzaSyAFZX8IhIfgzdOZvxDP_ISO5WYoU7jmQ5c" - } - ], - "services": { - "appinvite_service": { - "other_platform_oauth_client": [ - { - "client_id": "912726360885-e87n3jva9uoj4vbidvijq78ebg02asv2.apps.googleusercontent.com", - "client_type": 3 - } - ] - } - } - } - ], - "configuration_version": "1" -} diff --git a/vector-app/src/gplayRustCryptoNightly/res/values/colors.xml b/vector-app/src/gplayRustCryptoNightly/res/values/colors.xml deleted file mode 100644 index 2ec78e4096..0000000000 --- a/vector-app/src/gplayRustCryptoNightly/res/values/colors.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - #FF5964 - diff --git a/vector/src/main/java/im/vector/app/features/analytics/impl/DefaultVectorAnalytics.kt b/vector/src/main/java/im/vector/app/features/analytics/impl/DefaultVectorAnalytics.kt index 2a7d0ac975..d70dba773a 100644 --- a/vector/src/main/java/im/vector/app/features/analytics/impl/DefaultVectorAnalytics.kt +++ b/vector/src/main/java/im/vector/app/features/analytics/impl/DefaultVectorAnalytics.kt @@ -19,6 +19,7 @@ package im.vector.app.features.analytics.impl import com.posthog.android.Options import com.posthog.android.PostHog import com.posthog.android.Properties +import im.vector.app.BuildConfig import im.vector.app.core.di.NamedGlobalScope import im.vector.app.features.analytics.AnalyticsConfig import im.vector.app.features.analytics.VectorAnalytics @@ -214,6 +215,9 @@ class DefaultVectorAnalytics @Inject constructor( private fun Map.toPostHogUserProperties(): Properties { return Properties().apply { putAll(this@toPostHogUserProperties.filter { it.value != null }) + if (BuildConfig.FLAVOR == "rustCrypto") { + put("crypto", "rust") + } } } diff --git a/vector/src/main/java/im/vector/app/features/navigation/DefaultNavigator.kt b/vector/src/main/java/im/vector/app/features/navigation/DefaultNavigator.kt index 3f1638a5e2..d2628fcf0f 100644 --- a/vector/src/main/java/im/vector/app/features/navigation/DefaultNavigator.kt +++ b/vector/src/main/java/im/vector/app/features/navigation/DefaultNavigator.kt @@ -241,12 +241,8 @@ class DefaultNavigator @Inject constructor( } if (context is AppCompatActivity) { - TODO() -// VerificationBottomSheet.withArgs( -// roomId = null, -// otherUserId = otherUserId, -// transactionId = sasTransactionId -// ).show(context.supportFragmentManager, "REQPOP") + SelfVerificationBottomSheet.forTransaction(tx.transactionId) + .show(context.supportFragmentManager, "VERIF") } } } @@ -254,16 +250,14 @@ class DefaultNavigator @Inject constructor( override fun requestSessionVerification(context: Context, otherSessionId: String) { coroutineScope.launch { val session = sessionHolder.getSafeActiveSession() ?: return@launch -// val pr = - session.cryptoService().verificationService().requestSelfKeyVerification( - supportedVerificationMethodsProvider.provide() + val request = session.cryptoService().verificationService().requestDeviceVerification( + supportedVerificationMethodsProvider.provide(), + session.myUserId, + otherSessionId ) if (context is AppCompatActivity) { - TODO() -// VerificationBottomSheet.withArgs( -// otherUserId = session.myUserId, -// transactionId = pr.requestId() -// ).show(context.supportFragmentManager) + SelfVerificationBottomSheet.forTransaction(request.transactionId) + .show(context.supportFragmentManager, "VERIF") } } }