Remove Item.guid field, use Item.remoteId instead for all account types

This commit is contained in:
Shinokuni 2024-08-04 15:34:27 +02:00
parent 44b2858cb0
commit a00ef31cf7
18 changed files with 96 additions and 71 deletions

View File

@ -4,7 +4,7 @@ import com.gitlab.mvysny.konsumexml.Konsumer
import com.gitlab.mvysny.konsumexml.Names
import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore
import com.readrops.api.localfeed.XmlAdapter
import com.readrops.api.utils.*
import com.readrops.api.utils.DateUtils
import com.readrops.api.utils.exceptions.ParseException
import com.readrops.api.utils.extensions.nonNullText
import com.readrops.api.utils.extensions.nullableText
@ -22,7 +22,7 @@ class ATOMItemAdapter : XmlAdapter<Item> {
konsumer.allChildrenAutoIgnore(names) {
when (tagName) {
"title" -> title = nonNullText()
"id" -> guid = nullableText()
"id" -> remoteId = nullableText()
"updated" -> pubDate = DateUtils.parse(nullableText())
"link" -> parseLink(this, this@apply)
"author" -> allChildrenAutoIgnore("name") { author = nullableText() }
@ -35,7 +35,7 @@ class ATOMItemAdapter : XmlAdapter<Item> {
validateItem(item)
if (item.pubDate == null) item.pubDate = LocalDateTime.now()
if (item.guid == null) item.guid = item.link
if (item.remoteId == null) item.remoteId = item.link
item
} catch (e: Exception) {

View File

@ -33,7 +33,7 @@ class JSONItemsAdapter : JsonAdapter<List<Item>>() {
while (hasNext()) {
with(item) {
when (selectName(names)) {
0 -> guid = nextNonEmptyString()
0 -> remoteId = nextNonEmptyString()
1 -> link = nextNonEmptyString()
2 -> title = nextNonEmptyString()
3 -> contentHtml = nextNullableString()

View File

@ -5,7 +5,7 @@ import com.gitlab.mvysny.konsumexml.Names
import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore
import com.readrops.api.localfeed.XmlAdapter
import com.readrops.api.localfeed.XmlAdapter.Companion.AUTHORS_MAX
import com.readrops.api.utils.*
import com.readrops.api.utils.DateUtils
import com.readrops.api.utils.exceptions.ParseException
import com.readrops.api.utils.extensions.nonNullText
import com.readrops.api.utils.extensions.nullableText
@ -40,7 +40,7 @@ class RSS1ItemAdapter : XmlAdapter<Item> {
if (item.pubDate == null) item.pubDate = LocalDateTime.now()
if (item.link == null) item.link = about
?: throw ParseException("RSS1 link or about element is required")
item.guid = item.link
item.remoteId = item.link
if (authors.filterNotNull().isNotEmpty()) item.author = authors.filterNotNull()
.joinToString(limit = AUTHORS_MAX)

View File

@ -1,9 +1,13 @@
package com.readrops.api.localfeed.rss2
import com.gitlab.mvysny.konsumexml.*
import com.gitlab.mvysny.konsumexml.Konsumer
import com.gitlab.mvysny.konsumexml.KonsumerException
import com.gitlab.mvysny.konsumexml.Names
import com.gitlab.mvysny.konsumexml.allChildrenAutoIgnore
import com.readrops.api.localfeed.XmlAdapter
import com.readrops.api.localfeed.XmlAdapter.Companion.AUTHORS_MAX
import com.readrops.api.utils.*
import com.readrops.api.utils.ApiUtils
import com.readrops.api.utils.DateUtils
import com.readrops.api.utils.exceptions.ParseException
import com.readrops.api.utils.extensions.nonNullText
import com.readrops.api.utils.extensions.nullableText
@ -29,7 +33,7 @@ class RSS2ItemAdapter : XmlAdapter<Item> {
"dc:creator" -> creators += nullableText()
"pubDate" -> pubDate = DateUtils.parse(nullableText())
"dc:date" -> pubDate = DateUtils.parse(nullableText())
"guid" -> guid = nullableText()
"guid" -> remoteId = nullableText()
"description" -> description = nullableTextRecursively()
"content:encoded" -> content = nullableTextRecursively()
"enclosure" -> parseEnclosure(this, item = this@apply)
@ -81,7 +85,7 @@ class RSS2ItemAdapter : XmlAdapter<Item> {
validateItem(this)
if (pubDate == null) pubDate = LocalDateTime.now()
if (guid == null) guid = link
if (remoteId == null) remoteId = link
if (author == null && creators.filterNotNull().isNotEmpty())
author = creators.filterNotNull().joinToString(limit = AUTHORS_MAX)
}

View File

@ -1,11 +1,11 @@
package com.readrops.api.services.nextcloudnews.adapters
import android.annotation.SuppressLint
import com.readrops.db.entities.Item
import com.readrops.api.utils.ApiUtils
import com.readrops.api.utils.exceptions.ParseException
import com.readrops.api.utils.extensions.nextNonEmptyString
import com.readrops.api.utils.extensions.nextNullableString
import com.readrops.db.entities.Item
import com.squareup.moshi.JsonAdapter
import com.squareup.moshi.JsonReader
import com.squareup.moshi.JsonWriter
@ -49,7 +49,6 @@ class NextcloudNewsItemsAdapter : JsonAdapter<List<Item>>() {
8 -> feedRemoteId = reader.nextInt().toString()
9 -> isRead = !reader.nextBoolean() // the negation is important here
10 -> isStarred = reader.nextBoolean()
11 -> guid = reader.nextNullableString()
else -> reader.skipValue()
}
}
@ -73,6 +72,6 @@ class NextcloudNewsItemsAdapter : JsonAdapter<List<Item>>() {
companion object {
val NAMES: JsonReader.Options = JsonReader.Options.of("id", "url", "title", "author",
"pubDate", "body", "enclosureMime", "enclosureLink", "feedId", "unread", "starred", "guidHash")
"pubDate", "body", "enclosureMime", "enclosureLink", "feedId", "unread", "starred")
}
}

View File

@ -9,7 +9,6 @@ import junit.framework.TestCase.assertEquals
import org.junit.Assert.assertThrows
import org.junit.Assert.assertTrue
import org.junit.Test
import java.lang.Exception
class ATOMAdapterTest {
@ -37,7 +36,7 @@ class ATOMAdapterTest {
assertEquals(pubDate, DateUtils.parse("2020-09-06T21:09:59Z"))
assertEquals(author, "Shinokuni")
assertEquals(description, "Summary")
assertEquals(guid, "tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac")
assertEquals(remoteId, "tag:github.com,2008:Grit::Commit/c15f093a1bc4211e85f8d1817c9073e307afe5ac")
TestCase.assertNotNull(content)
}
}

View File

@ -9,7 +9,6 @@ import com.squareup.moshi.Moshi
import com.squareup.moshi.Types
import junit.framework.TestCase
import junit.framework.TestCase.assertEquals
import junit.framework.TestCase.assertTrue
import okio.Buffer
import org.junit.Assert.assertThrows
import org.junit.Test
@ -40,7 +39,7 @@ class JSONFeedAdapterTest {
with(items[0]) {
assertEquals(items.size, 10)
assertEquals(guid, "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html")
assertEquals(remoteId, "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html")
assertEquals(title, "Acorn and 10.13")
assertEquals(link, "http://flyingmeat.com/blog/archives/2017/9/acorn_and_10.13.html")
assertEquals(pubDate, DateUtils.parse("2017-09-25T14:27:27-07:00"))

View File

@ -35,7 +35,7 @@ class RSS1AdapterTest {
assertEquals(title, "Google Expands its Flutter Development Kit To Windows Apps")
assertEquals(link!!.trim(), "https://developers.slashdot.org/story/20/09/23/1616231/google-expands-" +
"its-flutter-development-kit-to-windows-apps?utm_source=rss1.0mainlinkanon&utm_medium=feed")
assertEquals(guid!!.trim(), "https://developers.slashdot.org/story/20/09/23/1616231/google-expands-" +
assertEquals(remoteId!!.trim(), "https://developers.slashdot.org/story/20/09/23/1616231/google-expands-" +
"its-flutter-development-kit-to-windows-apps?utm_source=rss1.0mainlinkanon&utm_medium=feed")
assertEquals(pubDate, DateUtils.parse("2020-09-23T16:15:00+00:00"))
assertEquals(author, "msmash")

View File

@ -36,7 +36,7 @@ class RSS2AdapterTest {
assertEquals(pubDate, DateUtils.parse("Tue, 25 Aug 2020 17:15:49 +0000"))
assertEquals(author, "Author 1")
assertEquals(description, "<a href=\"https://news.ycombinator.com/item?id=24273602\">Comments</a>")
assertEquals(guid, "https://www.bbc.com/news/world-africa-53887947")
assertEquals(remoteId, "https://www.bbc.com/news/world-africa-53887947")
}
}
@ -55,7 +55,7 @@ class RSS2AdapterTest {
val stream = TestUtils.loadResource("localfeed/rss2/rss_items_other_namespaces.xml")
val item = adapter.fromXml(stream.konsumeXml()).second[0]
assertEquals(item.guid, "guid")
assertEquals(item.remoteId, "guid")
assertEquals(item.author, "creator 1, creator 2, creator 3, creator 4")
assertEquals(item.pubDate, DateUtils.parse("2020-08-05T14:03:48Z"))
assertEquals(item.content, "content:encoded")

View File

@ -25,7 +25,6 @@ class NextcloudNewsItemsAdapterTest {
with(item) {
assertEquals(remoteId, "3443")
assertEquals(guid, "3059047a572cd9cd5d0bf645faffd077")
assertEquals(link, "http://grulja.wordpress.com/2013/04/29/plasma-nm-after-the-solid-sprint/")
assertEquals(title, "Plasma-nm after the solid sprint")
assertEquals(author, "Jan Grulich (grulja)")

View File

@ -90,7 +90,7 @@ class LocalRSSRepositoryTest : KoinTest {
assertTrue { result.first.items.isNotEmpty() }
assertTrue {
database.itemDao().itemExists(result.first.items.first().guid!!, account.id)
database.itemDao().itemExists(result.first.items.first().remoteId!!, account.id)
}
}
@ -110,7 +110,7 @@ class LocalRSSRepositoryTest : KoinTest {
assertTrue { result.first.items.isNotEmpty() }
assertTrue {
database.itemDao().itemExists(result.first.items.first().guid!!, account.id)
database.itemDao().itemExists(result.first.items.first().remoteId!!, account.id)
}
}
}

View File

@ -87,7 +87,7 @@ class LocalRSSRepository(
val newItems = mutableListOf<Item>()
for (item in items) {
if (!database.itemDao().itemExists(item.guid!!, feed.accountId)) {
if (!database.itemDao().itemExists(item.remoteId!!, feed.accountId)) {
if (item.description != null) {
item.cleanDescription = Jsoup.parse(item.description).text()
}

View File

@ -2,7 +2,7 @@
"formatVersion": 1,
"database": {
"version": 4,
"identityHash": "5043c0aff2ed15f8cbe250e35bb9129e",
"identityHash": "7059fa306ee4013c51c8a521e04e3e31",
"entities": [
{
"tableName": "Feed",
@ -153,7 +153,7 @@
},
{
"tableName": "Item",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `description` TEXT, `clean_description` TEXT, `link` TEXT, `image_link` TEXT, `author` TEXT, `pub_date` INTEGER, `content` TEXT, `feed_id` INTEGER NOT NULL, `guid` TEXT, `read_time` REAL NOT NULL, `read` INTEGER NOT NULL, `starred` INTEGER NOT NULL, `read_it_later` INTEGER NOT NULL, `remoteId` TEXT, FOREIGN KEY(`feed_id`) REFERENCES `Feed`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `description` TEXT, `clean_description` TEXT, `link` TEXT, `image_link` TEXT, `author` TEXT, `pub_date` INTEGER, `content` TEXT, `feed_id` INTEGER NOT NULL, `read_time` REAL NOT NULL, `read` INTEGER NOT NULL, `starred` INTEGER NOT NULL, `read_it_later` INTEGER NOT NULL, `remoteId` TEXT, FOREIGN KEY(`feed_id`) REFERENCES `Feed`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "id",
@ -215,12 +215,6 @@
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "guid",
"columnName": "guid",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "readTime",
"columnName": "read_time",
@ -267,15 +261,6 @@
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_Item_feed_id` ON `${TABLE_NAME}` (`feed_id`)"
},
{
"name": "index_Item_guid",
"unique": false,
"columnNames": [
"guid"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_Item_guid` ON `${TABLE_NAME}` (`guid`)"
}
],
"foreignKeys": [
@ -547,7 +532,7 @@
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '5043c0aff2ed15f8cbe250e35bb9129e')"
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '7059fa306ee4013c51c8a521e04e3e31')"
]
}
}

View File

@ -3,6 +3,7 @@ package com.readrops.db
import androidx.room.testing.MigrationTestHelper
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry
import junit.framework.Assert.assertEquals
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
@ -39,4 +40,18 @@ class MigrationsTest {
close()
}
}
@Test
fun migrate3to4() {
helper.createDatabase(dbName, 3).apply {
execSQL("Insert Into Account(account_type, last_modified, current_account, notifications_enabled) Values(0, 0, 0, 0)")
execSQL("Insert Into Feed(text_color, background_color, account_id, notification_enabled) Values(0, 0, 3, 0)")
execSQL("Insert Into Item(title, feed_id, read_time, read, starred, read_it_later, guid) values(\"test\", 12, 0, 0, 0, 0, \"guid\")")
}
helper.runMigrationsAndValidate(dbName, 4, true, MigrationFrom3To4).apply {
val remoteId = compileStatement("Select remoteId From Item").simpleQueryForString()
assertEquals("guid", remoteId)
}
}
}

View File

@ -1,6 +1,5 @@
package com.readrops.db
import androidx.room.AutoMigration
import androidx.room.Database
import androidx.room.RoomDatabase
import androidx.room.TypeConverters
@ -21,10 +20,8 @@ import com.readrops.db.entities.account.Account
@Database(
entities = [Feed::class, Item::class, Folder::class, Account::class,
ItemStateChange::class, ItemState::class], version = 4,
autoMigrations = [
AutoMigration(3, 4)
]
ItemStateChange::class, ItemState::class],
version = 4
)
@TypeConverters(Converters::class)
abstract class Database : RoomDatabase() {
@ -57,7 +54,7 @@ object MigrationFrom2To3 : Migration(2, 3) {
db.execSQL("""CREATE TABLE IF NOT EXISTS `ItemStateChange` (`id` INTEGER NOT NULL, `read_change` INTEGER NOT NULL, `star_change` INTEGER NOT NULL, `account_id` INTEGER NOT NULL, PRIMARY KEY(`id`), FOREIGN KEY(`account_id`) REFERENCES `Account`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )""")
db.execSQL("""CREATE TABLE IF NOT EXISTS `ItemState` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `read` INTEGER NOT NULL, `starred` INTEGER NOT NULL, `remote_id` TEXT NOT NULL, `account_id` INTEGER NOT NULL, FOREIGN KEY(`account_id`) REFERENCES `Account`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )""")
// removing read_changed and adding starred fields. Table is recreated to keep field order
// removing read_changed and adding starred fields
db.execSQL("""CREATE TABLE IF NOT EXISTS `Item_MERGE_TABLE` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `description` TEXT, `clean_description` TEXT, `link` TEXT, `image_link` TEXT, `author` TEXT, `pub_date` INTEGER, `content` TEXT, `feed_id` INTEGER NOT NULL, `guid` TEXT, `read_time` REAL NOT NULL, `read` INTEGER NOT NULL, `starred` INTEGER NOT NULL, `read_it_later` INTEGER NOT NULL, `remoteId` TEXT, FOREIGN KEY(`feed_id`) REFERENCES `Feed`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )""")
db.execSQL("""INSERT INTO `Item_MERGE_TABLE` (`id`,`title`,`description`,`clean_description`,`link`,`image_link`,`author`,`pub_date`,`content`,`feed_id`,`guid`,`read_time`,`read`,`read_it_later`,`remoteId`,`starred`) SELECT `Item`.`id`,`Item`.`title`,`Item`.`description`,`Item`.`clean_description`,`Item`.`link`,`Item`.`image_link`,`Item`.`author`,`Item`.`pub_date`,`Item`.`content`,`Item`.`feed_id`,`Item`.`guid`,`Item`.`read_time`,`Item`.`read`,`Item`.`read_it_later`,`Item`.`remoteId`,0 FROM `Item`""")
db.execSQL("""DROP TABLE IF EXISTS `Item`""")
@ -66,3 +63,24 @@ object MigrationFrom2To3 : Migration(2, 3) {
db.execSQL("""CREATE INDEX IF NOT EXISTS `index_Item_guid` ON `Item` (`guid`)""")
}
}
object MigrationFrom3To4 : Migration(3, 4) {
override fun migrate(db: SupportSQLiteDatabase) {
// add unique index to ItemState.(account_id, remote_id)
db.execSQL("CREATE TABLE IF NOT EXISTS `_new_ItemState` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `read` INTEGER NOT NULL, `starred` INTEGER NOT NULL, `remote_id` TEXT NOT NULL, `account_id` INTEGER NOT NULL, FOREIGN KEY(`account_id`) REFERENCES `Account`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )")
db.execSQL("INSERT INTO `_new_ItemState` (`id`,`read`,`starred`,`remote_id`,`account_id`) SELECT `id`,`read`,`starred`,`remote_id`,`account_id` FROM `ItemState`")
db.execSQL("DROP TABLE `ItemState`")
db.execSQL("ALTER TABLE `_new_ItemState` RENAME TO `ItemState`")
db.execSQL("CREATE UNIQUE INDEX IF NOT EXISTS `index_ItemState_remote_id_account_id` ON `ItemState` (`remote_id`, `account_id`)")
// remove guid, use remoteId local accounts
db.execSQL("Update Item set remoteId = guid Where remoteId is NULL")
db.execSQL("CREATE TABLE IF NOT EXISTS `_new_Item` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `description` TEXT, `clean_description` TEXT, `link` TEXT, `image_link` TEXT, `author` TEXT, `pub_date` INTEGER, `content` TEXT, `feed_id` INTEGER NOT NULL, `read_time` REAL NOT NULL, `read` INTEGER NOT NULL, `starred` INTEGER NOT NULL, `read_it_later` INTEGER NOT NULL, `remoteId` TEXT, FOREIGN KEY(`feed_id`) REFERENCES `Feed`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )")
db.execSQL("INSERT INTO `_new_Item`(`id`, `title`, `description`, `clean_description`, `link`, `image_link`, `author`, `pub_date`, `content`, `feed_id`, `read_time`, `read`, `starred`, `read_it_later`, `remoteId`) SELECT `id`, `title`, `description`, `clean_description`, `link`, `image_link`, `author`, `pub_date`, `content`, `feed_id`, `read_time`, `read`, `starred`, `read_it_later`, `remoteId` FROM `Item`")
db.execSQL("DROP TABLE IF EXISTS `Item`")
db.execSQL("ALTER TABLE `_new_Item` RENAME TO `Item`")
db.execSQL("CREATE INDEX IF NOT EXISTS `index_Item_feed_id` ON `Item` (`feed_id`)")
}
}

View File

@ -7,7 +7,7 @@ val dbModule = module {
single(createdAtStart = true) {
Room.databaseBuilder(get(), Database::class.java, "readrops-db")
.addMigrations(MigrationFrom1To2, MigrationFrom2To3)
.addMigrations(MigrationFrom1To2, MigrationFrom2To3, MigrationFrom3To4)
.build()
}
}

View File

@ -64,6 +64,6 @@ abstract class ItemDao : BaseDao<Item> {
abstract fun selectFeedUnreadItemsCount(query: SupportSQLiteQuery):
Flow<Map<@MapColumn(columnName = "feed_id") Int, @MapColumn(columnName = "item_count") Int>>
@Query("Select case When :guid In (Select guid From Item Inner Join Feed on Item.feed_id = Feed.id and account_id = :accountId) Then 1 else 0 end")
abstract suspend fun itemExists(guid: String, accountId: Int): Boolean
@Query("Select case When :remoteId In (Select Item.remoteId From Item Inner Join Feed on Item.feed_id = Feed.id and account_id = :accountId) Then 1 else 0 end")
abstract suspend fun itemExists(remoteId: String, accountId: Int): Boolean
}

View File

@ -7,8 +7,16 @@ import androidx.room.Ignore
import androidx.room.PrimaryKey
import org.joda.time.LocalDateTime
@Entity(foreignKeys = [ForeignKey(entity = Feed::class, parentColumns = ["id"],
childColumns = ["feed_id"], onDelete = ForeignKey.CASCADE)])
@Entity(
foreignKeys = [
ForeignKey(
entity = Feed::class,
parentColumns = ["id"],
childColumns = ["feed_id"],
onDelete = ForeignKey.CASCADE
)
]
)
data class Item(
@PrimaryKey(autoGenerate = true) var id: Int = 0,
var title: String? = null,
@ -20,7 +28,6 @@ data class Item(
@ColumnInfo(name = "pub_date") var pubDate: LocalDateTime? = null,
var content: String? = null,
@ColumnInfo(name = "feed_id", index = true) var feedId: Int = 0,
@ColumnInfo(index = true) var guid: String? = null,
@ColumnInfo(name = "read_time") var readTime: Double = 0.0,
@ColumnInfo(name = "read") var isRead: Boolean = false,
@ColumnInfo(name = "starred") var isStarred: Boolean = false,