Migration test and cleaning

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/RealmSessionStoreMigration43Test.kt

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.database
+
+import android.content.Context
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import io.realm.Realm
+import org.amshove.kluent.fail
+import org.amshove.kluent.shouldBe
+import org.amshove.kluent.shouldBeEqualTo
+import org.amshove.kluent.shouldNotBe
+import org.junit.After
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.session.room.model.message.MessageContent
+import org.matrix.android.sdk.internal.database.mapper.EventMapper
+import org.matrix.android.sdk.internal.database.model.EventAnnotationsSummaryEntity
+import org.matrix.android.sdk.internal.database.model.SessionRealmModule
+import org.matrix.android.sdk.internal.database.query.where
+import org.matrix.android.sdk.internal.util.Normalizer
+
+@RunWith(AndroidJUnit4::class)
+class RealmSessionStoreMigration43Test {
+
+    @get:Rule val configurationFactory = TestRealmConfigurationFactory()
+
+    lateinit var context: Context
+    var realm: Realm? = null
+
+    @Before
+    fun setUp() {
+        context = InstrumentationRegistry.getInstrumentation().context
+    }
+
+    @After
+    fun tearDown() {
+        realm?.close()
+    }
+
+    @Test
+    fun migrationShouldBeNeeed() {
+        val realmName = "session_42.realm"
+        val realmConfiguration = configurationFactory.createConfiguration(
+                realmName,
+                "efa9ab2c77ae06b0e767ffdb1c45b12be3c77d48d94f1ac41a7cd1d637fc59ac41f869a250453074e21ce13cfe7ed535593e7d150c08ce2bad7a2ab8c7b841f0",
+                SessionRealmModule(),
+                43,
+                null
+        )
+        configurationFactory.copyRealmFromAssets(context, realmName, realmName)
+
+        try {
+            realm = Realm.getInstance(realmConfiguration)
+            fail("Should need a migration")
+        } catch (failure: Throwable) {
+            // nop
+        }
+    }
+
+    //  Database key for alias `session_db_e00482619b2597069b1f192b86de7da9`: efa9ab2c77ae06b0e767ffdb1c45b12be3c77d48d94f1ac41a7cd1d637fc59ac41f869a250453074e21ce13cfe7ed535593e7d150c08ce2bad7a2ab8c7b841f0
+    // $WEJ8U6Zsx3TDZx3qmHIOKh-mXe5kqL_MnPcIkStEwwI
+    // $11EtAQ8RYcudJVtw7e6B5Vm4ufCqKTOWKblY2U_wrpo
+    @Test
+    fun testMigration43() {
+        val realmName = "session_42.realm"
+        val migration = RealmSessionStoreMigration(Normalizer())
+        val realmConfiguration = configurationFactory.createConfiguration(
+                realmName,
+                "efa9ab2c77ae06b0e767ffdb1c45b12be3c77d48d94f1ac41a7cd1d637fc59ac41f869a250453074e21ce13cfe7ed535593e7d150c08ce2bad7a2ab8c7b841f0",
+                SessionRealmModule(),
+                43,
+                migration
+        )
+        configurationFactory.copyRealmFromAssets(context, realmName, realmName)
+
+        realm = Realm.getInstance(realmConfiguration)
+
+        // assert that the edit from 42 are migrated
+        val editions = EventAnnotationsSummaryEntity
+                .where(realm!!, "\$WEJ8U6Zsx3TDZx3qmHIOKh-mXe5kqL_MnPcIkStEwwI")
+                .findFirst()
+                ?.editSummary
+                ?.editions
+
+        editions shouldNotBe null
+        editions!!.size shouldBe 1
+        val firstEdition = editions.first()
+        firstEdition?.eventId shouldBeEqualTo "\$DvOyA8vJxwGfTaJG3OEJVcL4isShyaVDnprihy38W28"
+        firstEdition?.isLocalEcho shouldBeEqualTo false
+
+        val editEvent = EventMapper.map(firstEdition!!.event!!)
+        val body = editEvent.content.toModel<MessageContent>()?.body
+        body shouldBeEqualTo "* Message 2 with edit"
+
+        // assert that the edit from 42 are migrated
+        val editionsOfE2E = EventAnnotationsSummaryEntity
+                .where(realm!!, "\$11EtAQ8RYcudJVtw7e6B5Vm4ufCqKTOWKblY2U_wrpo")
+                .findFirst()
+                ?.editSummary
+                ?.editions
+
+        editionsOfE2E shouldNotBe null
+        editionsOfE2E!!.size shouldBe 1
+        val firstEditionE2E = editionsOfE2E.first()
+        firstEditionE2E?.eventId shouldBeEqualTo "\$HUwJOQRCJwfPv7XSKvBPcvncjM0oR3q2tGIIIdv9Zts"
+        firstEditionE2E?.isLocalEcho shouldBeEqualTo false
+
+        val editEventE2E = EventMapper.map(firstEditionE2E!!.event!!)
+        val body2 = editEventE2E.getClearContent().toModel<MessageContent>()?.body
+        body2 shouldBeEqualTo "* Message 2, e2e edit"
+    }
+}

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/SessionSanityMigrationTest.kt

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2020 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.internal.database
+
+import android.content.Context
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import io.realm.Realm
+import org.junit.After
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.matrix.android.sdk.internal.database.model.SessionRealmModule
+import org.matrix.android.sdk.internal.util.Normalizer
+
+@RunWith(AndroidJUnit4::class)
+class SessionSanityMigrationTest {
+
+    @get:Rule val configurationFactory = TestRealmConfigurationFactory()
+
+    lateinit var context: Context
+    var realm: Realm? = null
+
+    @Before
+    fun setUp() {
+        context = InstrumentationRegistry.getInstrumentation().context
+    }
+
+    @After
+    fun tearDown() {
+        realm?.close()
+    }
+
+    @Test
+    fun sessionDatabaseShouldMigrateGracefully() {
+        val realmName = "session_42.realm"
+        val migration = RealmSessionStoreMigration(Normalizer())
+        val realmConfiguration = configurationFactory.createConfiguration(
+                realmName,
+                "efa9ab2c77ae06b0e767ffdb1c45b12be3c77d48d94f1ac41a7cd1d637fc59ac41f869a250453074e21ce13cfe7ed535593e7d150c08ce2bad7a2ab8c7b841f0",
+                SessionRealmModule(),
+                migration.schemaVersion,
+                migration
+        )
+        configurationFactory.copyRealmFromAssets(context, realmName, realmName)
+
+        realm = Realm.getInstance(realmConfiguration)
+    }
+}

matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/TestRealmConfigurationFactory.kt

@@ -0,0 +1,196 @@
+/*
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.matrix.android.sdk.internal.database
+
+import android.content.Context
+import androidx.test.platform.app.InstrumentationRegistry
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import io.realm.RealmMigration
+import org.junit.rules.TemporaryFolder
+import org.junit.runner.Description
+import org.junit.runners.model.Statement
+import java.io.File
+import java.io.FileOutputStream
+import java.io.IOException
+import java.io.InputStream
+import java.lang.IllegalStateException
+import java.util.Collections
+import java.util.Locale
+import java.util.concurrent.ConcurrentHashMap
+import kotlin.Throws
+
+/**
+ * Based on https://github.com/realm/realm-java/blob/master/realm/realm-library/src/testUtils/java/io/realm/TestRealmConfigurationFactory.java
+ */
+class TestRealmConfigurationFactory : TemporaryFolder() {
+    private val map: Map<RealmConfiguration, Boolean> = ConcurrentHashMap()
+    private val configurations = Collections.newSetFromMap(map)
+    @get:Synchronized private var isUnitTestFailed = false
+    private var testName = ""
+    private var tempFolder: File? = null
+
+    override fun apply(base: Statement, description: Description): Statement {
+        return object : Statement() {
+            @Throws(Throwable::class)
+            override fun evaluate() {
+                setTestName(description)
+                before()
+                try {
+                    base.evaluate()
+                } catch (throwable: Throwable) {
+                    setUnitTestFailed()
+                    throw throwable
+                } finally {
+                    after()
+                }
+            }
+        }
+    }
+
+    @Throws(Throwable::class)
+    override fun before() {
+        Realm.init(InstrumentationRegistry.getInstrumentation().targetContext)
+        super.before()
+    }
+
+    override fun after() {
+        try {
+            for (configuration in configurations) {
+                Realm.deleteRealm(configuration)
+            }
+        } catch (e: IllegalStateException) {
+            // Only throws the exception caused by deleting the opened Realm if the test case itself doesn't throw.
+            if (!isUnitTestFailed) {
+                throw e
+            }
+        } finally {
+            // This will delete the temp directory.
+            super.after()
+        }
+    }
+
+    @Throws(IOException::class)
+    override fun create() {
+        super.create()
+        tempFolder = File(super.getRoot(), testName)
+        check(!(tempFolder!!.exists() && !tempFolder!!.delete())) { "Could not delete folder: " + tempFolder!!.absolutePath }
+        check(tempFolder!!.mkdir()) { "Could not create folder: " + tempFolder!!.absolutePath }
+    }
+
+    override fun getRoot(): File {
+        checkNotNull(tempFolder) { "the temporary folder has not yet been created" }
+        return tempFolder!!
+    }
+
+    /**
+     * To be called in the [.apply].
+     */
+    protected fun setTestName(description: Description) {
+        testName = description.displayName
+    }
+
+    @Synchronized
+    fun setUnitTestFailed() {
+        isUnitTestFailed = true
+    }
+
+    // This builder creates a configuration that is *NOT* managed.
+    // You have to delete it yourself.
+    private fun createConfigurationBuilder(): RealmConfiguration.Builder {
+        return RealmConfiguration.Builder().directory(root)
+    }
+
+    fun String.decodeHex(): ByteArray {
+        check(length % 2 == 0) { "Must have an even length" }
+        return chunked(2)
+                .map { it.toInt(16).toByte() }
+                .toByteArray()
+    }
+
+    fun createConfiguration(
+            name: String,
+            key: String?,
+            module: Any,
+            schemaVersion: Long,
+            migration: RealmMigration?
+    ): RealmConfiguration {
+        val builder = createConfigurationBuilder()
+        builder
+                .directory(root)
+                .name(name)
+                .apply {
+                    if (key != null) {
+                        encryptionKey(key.decodeHex())
+                    }
+                }
+                .modules(module)
+                // Allow writes on UI
+                .allowWritesOnUiThread(true)
+                .schemaVersion(schemaVersion)
+                .apply {
+                    migration?.let { migration(it) }
+                }
+        val configuration = builder.build()
+        configurations.add(configuration)
+        return configuration
+    }
+
+    // Copies a Realm file from assets to temp dir
+    @Throws(IOException::class)
+    fun copyRealmFromAssets(context: Context, realmPath: String, newName: String) {
+        val config = RealmConfiguration.Builder()
+                .directory(root)
+                .name(newName)
+                .build()
+        copyRealmFromAssets(context, realmPath, config)
+    }
+
+    @Throws(IOException::class)
+    fun copyRealmFromAssets(context: Context, realmPath: String, config: RealmConfiguration) {
+        check(!File(config.path).exists()) { String.format(Locale.ENGLISH, "%s exists!", config.path) }
+        val outFile = File(config.realmDirectory, config.realmFileName)
+        copyFileFromAssets(context, realmPath, outFile)
+    }
+
+    @Throws(IOException::class)
+    fun copyFileFromAssets(context: Context, assetPath: String?, outFile: File?) {
+        var stream: InputStream? = null
+        var os: FileOutputStream? = null
+        try {
+            stream = context.assets.open(assetPath!!)
+            os = FileOutputStream(outFile)
+            val buf = ByteArray(1024)
+            var bytesRead: Int
+            while (stream.read(buf).also { bytesRead = it } > -1) {
+                os.write(buf, 0, bytesRead)
+            }
+        } finally {
+            if (stream != null) {
+                try {
+                    stream.close()
+                } catch (ignore: IOException) {
+                }
+            }
+            if (os != null) {
+                try {
+                    os.close()
+                } catch (ignore: IOException) {
+                }
+            }
+        }
+    }
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/EventExt.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.matrix.android.sdk.api.session.events.model
+
+fun Event.toValidDecryptedEvent(): ValidDecryptedEvent? {
+    if (!this.isEncrypted()) return null
+    val decryptedContent = this.getDecryptedContent() ?: return null
+    val eventId = this.eventId ?: return null
+    val roomId = this.roomId ?: return null
+    val type = this.getDecryptedType() ?: return null
+    val senderKey = this.getSenderKey() ?: return null
+    val algorithm = this.content?.get("algorithm") as? String ?: return null
+
+    // copy the relation as it's in clear in the encrypted content
+    val updatedContent = this.content.get("m.relates_to")?.let {
+        decryptedContent.toMutableMap().apply {
+            put("m.relates_to", it)
+        }
+    } ?: decryptedContent
+    return ValidDecryptedEvent(
+            type = type,
+            eventId = eventId,
+            clearContent = updatedContent,
+            prevContent = this.prevContent,
+            originServerTs = this.originServerTs ?: 0,
+            cryptoSenderKey = senderKey,
+            roomId = roomId,
+            unsignedData = this.unsignedData,
+            redacts = this.redacts,
+            algorithm = algorithm
+    )
+}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/migration/MigrateSessionTo043.kt

@@ -18,8 +18,8 @@ package org.matrix.android.sdk.internal.database.migration
 
 import io.realm.DynamicRealm
 import org.matrix.android.sdk.internal.database.model.EditionOfEventFields
-import org.matrix.android.sdk.internal.database.model.EventEntity
 import org.matrix.android.sdk.internal.database.model.EventEntityFields
+import org.matrix.android.sdk.internal.database.query.where
 import org.matrix.android.sdk.internal.util.database.RealmMigrator
 
 internal class MigrateSessionTo043(realm: DynamicRealm) : RealmMigrator(realm, 43) {
@@ -27,11 +27,9 @@ internal class MigrateSessionTo043(realm: DynamicRealm) : RealmMigrator(realm, 4
     override fun doMigrate(realm: DynamicRealm) {
         // content(string) & senderId(string) have been removed and replaced by a link to the actual event
         realm.schema.get("EditionOfEvent")
-                ?.removeField("senderId")
-                ?.removeField("content")
                 ?.addRealmObjectField(EditionOfEventFields.EVENT.`$`, realm.schema.get("EventEntity")!!)
                 ?.transform { dynamicObject ->
-                    realm.where(EventEntity::javaClass.name)
+                    realm.where("EventEntity")
                             .equalTo(EventEntityFields.EVENT_ID, dynamicObject.getString(EditionOfEventFields.EVENT_ID))
                             .equalTo(EventEntityFields.SENDER, dynamicObject.getString("senderId"))
                             .findFirst()
@@ -39,5 +37,7 @@ internal class MigrateSessionTo043(realm: DynamicRealm) : RealmMigrator(realm, 4
                                 dynamicObject.setObject(EditionOfEventFields.EVENT.`$`, it)
                             }
                 }
+                ?.removeField("senderId")
+                ?.removeField("content")
     }
 }

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/events/EventExt.kt

@@ -17,7 +17,6 @@
 package org.matrix.android.sdk.internal.session.events
 
 import org.matrix.android.sdk.api.session.events.model.Event
-import org.matrix.android.sdk.api.session.events.model.ValidDecryptedEvent
 import org.matrix.android.sdk.api.session.events.model.toModel
 import org.matrix.android.sdk.api.session.room.model.RoomMemberContent
 
@@ -34,32 +33,3 @@ internal fun Event.getFixedRoomMemberContent(): RoomMemberContent? {
         content
     }
 }
-
-fun Event.toValidDecryptedEvent(): ValidDecryptedEvent? {
-    if (!this.isEncrypted()) return null
-    val decryptedContent = this.getDecryptedContent() ?: return null
-    val eventId = this.eventId ?: return null
-    val roomId = this.roomId ?: return null
-    val type = this.getDecryptedType() ?: return null
-    val senderKey = this.getSenderKey() ?: return null
-    val algorithm = this.content?.get("algorithm") as? String ?: return null
-
-    // copy the relation as it's in clear in the encrypted content
-    val updatedContent = this.content.get("m.relates_to")?.let {
-        decryptedContent.toMutableMap().apply {
-            put("m.relates_to", it)
-        }
-    } ?: decryptedContent
-    return ValidDecryptedEvent(
-            type = type,
-            eventId = eventId,
-            clearContent = updatedContent,
-            prevContent = this.prevContent,
-            originServerTs = this.originServerTs ?: 0,
-            cryptoSenderKey = senderKey,
-            roomId = roomId,
-            unsignedData = this.unsignedData,
-            redacts = this.redacts,
-            algorithm = algorithm
-    )
-}

matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/room/EventEditValidator.kt

@@ -21,9 +21,9 @@ import org.matrix.android.sdk.api.session.events.model.LocalEcho
 import org.matrix.android.sdk.api.session.events.model.RelationType
 import org.matrix.android.sdk.api.session.events.model.getRelationContent
 import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.session.events.model.toValidDecryptedEvent
 import org.matrix.android.sdk.api.session.room.model.message.MessageContent
 import org.matrix.android.sdk.internal.crypto.store.IMXCryptoStore
-import org.matrix.android.sdk.internal.session.events.toValidDecryptedEvent
 import timber.log.Timber
 import javax.inject.Inject
 

matrix-sdk-android/src/test/java/org/matrix/android/sdk/internal/database/mapper/EditAggregationSummaryMapperTest.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 New Vector Ltd
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

matrix-sdk-android/src/test/java/org/matrix/android/sdk/internal/session/event/ValidDecryptedEventTest.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 New Vector Ltd
+ * Copyright 2022 The Matrix.org Foundation C.I.C.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,8 +26,8 @@ import org.matrix.android.sdk.api.session.events.model.EventType
 import org.matrix.android.sdk.api.session.events.model.content.EncryptedEventContent
 import org.matrix.android.sdk.api.session.events.model.toContent
 import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.session.events.model.toValidDecryptedEvent
 import org.matrix.android.sdk.api.session.room.model.message.MessageTextContent
-import org.matrix.android.sdk.internal.session.events.toValidDecryptedEvent
 
 class ValidDecryptedEventTest {
 

vector/src/main/java/im/vector/app/features/home/room/detail/timeline/helper/MessageInformationDataFactory.kt

@@ -37,6 +37,7 @@ import org.matrix.android.sdk.api.session.events.model.getMsgType
 import org.matrix.android.sdk.api.session.events.model.isAttachmentMessage
 import org.matrix.android.sdk.api.session.events.model.isSticker
 import org.matrix.android.sdk.api.session.events.model.toModel
+import org.matrix.android.sdk.api.session.events.model.toValidDecryptedEvent
 import org.matrix.android.sdk.api.session.room.model.ReferencesAggregatedContent
 import org.matrix.android.sdk.api.session.room.model.RoomSummary
 import org.matrix.android.sdk.api.session.room.model.message.MessageType
@@ -44,7 +45,6 @@ import org.matrix.android.sdk.api.session.room.model.message.MessageVerification
 import org.matrix.android.sdk.api.session.room.send.SendState
 import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent
 import org.matrix.android.sdk.api.session.room.timeline.hasBeenEdited
-import org.matrix.android.sdk.internal.session.events.toValidDecryptedEvent
 import javax.inject.Inject
 
 /**