From 4a32b00c606a953993af03ed89df9da026026992 Mon Sep 17 00:00:00 2001 From: Ash Date: Wed, 15 Mar 2023 22:47:09 +0800 Subject: [PATCH] Optimize Fever API integration (#363) --- .../5.json | 377 ++++++++++++++++++ .../java/me/ash/reader/data/dao/FeedDao.kt | 14 + .../java/me/ash/reader/data/dao/GroupDao.kt | 11 + .../ash/reader/data/model/account/Account.kt | 2 + .../data/repository/AbstractRssRepository.kt | 18 +- .../data/repository/FeverRssRepository.kt | 167 ++++---- .../me/ash/reader/data/source/RYDatabase.kt | 15 +- 7 files changed, 517 insertions(+), 87 deletions(-) create mode 100644 app/schemas/me.ash.reader.data.source.RYDatabase/5.json diff --git a/app/schemas/me.ash.reader.data.source.RYDatabase/5.json b/app/schemas/me.ash.reader.data.source.RYDatabase/5.json new file mode 100644 index 00000000..1f1a26eb --- /dev/null +++ b/app/schemas/me.ash.reader.data.source.RYDatabase/5.json @@ -0,0 +1,377 @@ +{ + "formatVersion": 1, + "database": { + "version": 5, + "identityHash": "2b86f20200ed2c56f5ae8d0565cf0f26", + "entities": [ + { + "tableName": "account", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` TEXT NOT NULL, `type` INTEGER NOT NULL, `updateAt` INTEGER, `lastArticleId` TEXT, `syncInterval` INTEGER NOT NULL DEFAULT 30, `syncOnStart` INTEGER NOT NULL DEFAULT 0, `syncOnlyOnWiFi` INTEGER NOT NULL DEFAULT 0, `syncOnlyWhenCharging` INTEGER NOT NULL DEFAULT 0, `keepArchived` INTEGER NOT NULL DEFAULT 2592000000, `syncBlockList` TEXT NOT NULL DEFAULT '', `securityKey` TEXT DEFAULT 'CvJ1PKM8EW8=')", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "type", + "columnName": "type", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "updateAt", + "columnName": "updateAt", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "lastArticleId", + "columnName": "lastArticleId", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "syncInterval", + "columnName": "syncInterval", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "30" + }, + { + "fieldPath": "syncOnStart", + "columnName": "syncOnStart", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "syncOnlyOnWiFi", + "columnName": "syncOnlyOnWiFi", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "syncOnlyWhenCharging", + "columnName": "syncOnlyWhenCharging", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "keepArchived", + "columnName": "keepArchived", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "2592000000" + }, + { + "fieldPath": "syncBlockList", + "columnName": "syncBlockList", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "securityKey", + "columnName": "securityKey", + "affinity": "TEXT", + "notNull": false, + "defaultValue": "'CvJ1PKM8EW8='" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "feed", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `icon` TEXT, `url` TEXT NOT NULL, `groupId` TEXT NOT NULL, `accountId` INTEGER NOT NULL, `isNotification` INTEGER NOT NULL, `isFullContent` INTEGER NOT NULL, PRIMARY KEY(`id`), FOREIGN KEY(`groupId`) REFERENCES `group`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "icon", + "columnName": "icon", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "groupId", + "columnName": "groupId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isNotification", + "columnName": "isNotification", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isFullContent", + "columnName": "isFullContent", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_feed_groupId", + "unique": false, + "columnNames": [ + "groupId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_feed_groupId` ON `${TABLE_NAME}` (`groupId`)" + }, + { + "name": "index_feed_accountId", + "unique": false, + "columnNames": [ + "accountId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_feed_accountId` ON `${TABLE_NAME}` (`accountId`)" + } + ], + "foreignKeys": [ + { + "table": "group", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "groupId" + ], + "referencedColumns": [ + "id" + ] + } + ] + }, + { + "tableName": "article", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `date` INTEGER NOT NULL, `title` TEXT NOT NULL, `author` TEXT, `rawDescription` TEXT NOT NULL, `shortDescription` TEXT NOT NULL, `fullContent` TEXT, `img` TEXT, `link` TEXT NOT NULL, `feedId` TEXT NOT NULL, `accountId` INTEGER NOT NULL, `isUnread` INTEGER NOT NULL, `isStarred` INTEGER NOT NULL, `isReadLater` INTEGER NOT NULL, `updateAt` INTEGER, PRIMARY KEY(`id`), FOREIGN KEY(`feedId`) REFERENCES `feed`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "date", + "columnName": "date", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "author", + "columnName": "author", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "rawDescription", + "columnName": "rawDescription", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "shortDescription", + "columnName": "shortDescription", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fullContent", + "columnName": "fullContent", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "img", + "columnName": "img", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "link", + "columnName": "link", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "feedId", + "columnName": "feedId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isUnread", + "columnName": "isUnread", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isStarred", + "columnName": "isStarred", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isReadLater", + "columnName": "isReadLater", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "updateAt", + "columnName": "updateAt", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_article_feedId", + "unique": false, + "columnNames": [ + "feedId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_article_feedId` ON `${TABLE_NAME}` (`feedId`)" + }, + { + "name": "index_article_accountId", + "unique": false, + "columnNames": [ + "accountId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_article_accountId` ON `${TABLE_NAME}` (`accountId`)" + } + ], + "foreignKeys": [ + { + "table": "feed", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "feedId" + ], + "referencedColumns": [ + "id" + ] + } + ] + }, + { + "tableName": "group", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `accountId` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_group_accountId", + "unique": false, + "columnNames": [ + "accountId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_group_accountId` ON `${TABLE_NAME}` (`accountId`)" + } + ], + "foreignKeys": [] + } + ], + "views": [], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '2b86f20200ed2c56f5ae8d0565cf0f26')" + ] + } +} \ No newline at end of file diff --git a/app/src/main/java/me/ash/reader/data/dao/FeedDao.kt b/app/src/main/java/me/ash/reader/data/dao/FeedDao.kt index 2e978b01..b189bb1c 100644 --- a/app/src/main/java/me/ash/reader/data/dao/FeedDao.kt +++ b/app/src/main/java/me/ash/reader/data/dao/FeedDao.kt @@ -98,4 +98,18 @@ interface FeedDao { @Delete suspend fun delete(vararg feed: Feed) + + suspend fun insertOrUpdate(feeds: List) { + feeds.forEach { + val feed = queryById(it.id) + if (feed == null) { + insert(it) + } else { + // TODO: Consider migrating the fields to be nullable. + it.isNotification = feed.isNotification + it.isFullContent = feed.isFullContent + update(it) + } + } + } } diff --git a/app/src/main/java/me/ash/reader/data/dao/GroupDao.kt b/app/src/main/java/me/ash/reader/data/dao/GroupDao.kt index fe478de1..dcc2c7b1 100644 --- a/app/src/main/java/me/ash/reader/data/dao/GroupDao.kt +++ b/app/src/main/java/me/ash/reader/data/dao/GroupDao.kt @@ -66,4 +66,15 @@ interface GroupDao { @Delete suspend fun delete(vararg group: Group) + + suspend fun insertOrUpdate(groups: List) { + groups.forEach { + val group = queryById(it.id) + if (group == null) { + insert(it) + } else { + update(it) + } + } + } } diff --git a/app/src/main/java/me/ash/reader/data/model/account/Account.kt b/app/src/main/java/me/ash/reader/data/model/account/Account.kt index 766fafed..3d8d703e 100644 --- a/app/src/main/java/me/ash/reader/data/model/account/Account.kt +++ b/app/src/main/java/me/ash/reader/data/model/account/Account.kt @@ -21,6 +21,8 @@ data class Account( var type: AccountType, @ColumnInfo var updateAt: Date? = null, + @ColumnInfo + var lastArticleId: String? = null, @ColumnInfo(defaultValue = "30") var syncInterval: SyncIntervalPreference = SyncIntervalPreference.default, @ColumnInfo(defaultValue = "0") diff --git a/app/src/main/java/me/ash/reader/data/repository/AbstractRssRepository.kt b/app/src/main/java/me/ash/reader/data/repository/AbstractRssRepository.kt index e6375319..65399d01 100644 --- a/app/src/main/java/me/ash/reader/data/repository/AbstractRssRepository.kt +++ b/app/src/main/java/me/ash/reader/data/repository/AbstractRssRepository.kt @@ -181,24 +181,24 @@ abstract class AbstractRssRepository constructor( if (it.syncOnStart.value) { SyncWorker.enqueueOneTimeWork(workManager) } - if (it.syncInterval != SyncIntervalPreference.Manually) { + if (it.syncInterval.value != SyncIntervalPreference.Manually.value) { SyncWorker.enqueuePeriodicWork( workManager = workManager, syncInterval = it.syncInterval, syncOnlyWhenCharging = it.syncOnlyWhenCharging, syncOnlyOnWiFi = it.syncOnlyOnWiFi, ) - } else { - } } else { SyncWorker.enqueueOneTimeWork(workManager) - SyncWorker.enqueuePeriodicWork( - workManager = workManager, - syncInterval = it.syncInterval, - syncOnlyWhenCharging = it.syncOnlyWhenCharging, - syncOnlyOnWiFi = it.syncOnlyOnWiFi, - ) + if (it.syncInterval.value != SyncIntervalPreference.Manually.value) { + SyncWorker.enqueuePeriodicWork( + workManager = workManager, + syncInterval = it.syncInterval, + syncOnlyWhenCharging = it.syncOnlyWhenCharging, + syncOnlyOnWiFi = it.syncOnlyOnWiFi, + ) + } } } } diff --git a/app/src/main/java/me/ash/reader/data/repository/FeverRssRepository.kt b/app/src/main/java/me/ash/reader/data/repository/FeverRssRepository.kt index 0be46e7d..a1d6bb8a 100644 --- a/app/src/main/java/me/ash/reader/data/repository/FeverRssRepository.kt +++ b/app/src/main/java/me/ash/reader/data/repository/FeverRssRepository.kt @@ -80,102 +80,115 @@ class FeverRssRepository @Inject constructor( } /** - * Sync handling for the Fever API. + * Fever API synchronous processing with object's ID to ensure idempotence + * and handle foreign key relationships such as read status, starred status, etc. + * + * When synchronizing articles, 50 articles will be pulled in each round. + * The ID of the 50th article in this round will be recorded and + * used as the starting mark for the next pull until the number of articles + * obtained is 0 or their quantity exceeds 250, at which point the pulling process stops. * * 1. Fetch the Fever groups * 2. Fetch the Fever feeds * 3. Fetch the Fever articles * 4. Fetch the Fever favicons */ - override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result = - supervisorScope { - coroutineWorker.setProgress(SyncWorker.setIsSyncing(true)) + override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result = supervisorScope { + coroutineWorker.setProgress(SyncWorker.setIsSyncing(true)) - try { - val preTime = System.currentTimeMillis() - val accountId = context.currentAccountId - val feverAPI = getFeverAPI() + try { + val preTime = System.currentTimeMillis() + val accountId = context.currentAccountId + val account = accountDao.queryById(accountId)!! + val feverAPI = getFeverAPI() - // 1. Fetch the Fever groups - groupDao.insert( - *feverAPI.getGroups().groups?.map { - Group( - id = accountId.spacerDollar(it.id!!), - name = it.title ?: context.getString(R.string.empty), - accountId = accountId, - ) - }?.toTypedArray() ?: emptyArray() - ) + // 1. Fetch the Fever groups + groupDao.insertOrUpdate( + feverAPI.getGroups().groups?.map { + Group( + id = accountId.spacerDollar(it.id!!), + name = it.title ?: context.getString(R.string.empty), + accountId = accountId, + ) + } ?: emptyList() + ) - // 2. Fetch the Fever feeds - val feedsBody = feverAPI.getFeeds() - val feedsGroupsMap = mutableMapOf() - feedsBody.feeds_groups?.forEach { feedsGroups -> - feedsGroups.group_id?.toString()?.let { groupId -> - feedsGroups.feed_ids?.split(",")?.forEach { feedId -> - feedsGroupsMap[feedId] = groupId - } + // 2. Fetch the Fever feeds + val feedsBody = feverAPI.getFeeds() + val feedsGroupsMap = mutableMapOf() + feedsBody.feeds_groups?.forEach { feedsGroups -> + feedsGroups.group_id?.toString()?.let { groupId -> + feedsGroups.feed_ids?.split(",")?.forEach { feedId -> + feedsGroupsMap[feedId] = groupId } } - feedDao.insert( - *feedsBody.feeds?.map { - Feed( + } + feedDao.insertOrUpdate( + feedsBody.feeds?.map { + Feed( + id = accountId.spacerDollar(it.id!!), + name = it.title ?: context.getString(R.string.empty), + url = it.url!!, + groupId = accountId.spacerDollar(feedsGroupsMap[it.id.toString()]!!), + accountId = accountId, + ) + } ?: emptyList() + ) + + // 3. Fetch the Fever articles (up to unlimited counts) + var sinceId = account.lastArticleId?.dollarLast() ?: "" + var itemsBody = feverAPI.getItemsSince(sinceId) + while (itemsBody.items?.isNotEmpty() == true) { + articleDao.insert( + *itemsBody.items?.map { + Article( id = accountId.spacerDollar(it.id!!), - name = it.title ?: context.getString(R.string.empty), - url = it.url!!, - groupId = accountId.spacerDollar(feedsGroupsMap[it.id.toString()]!!), + date = it.created_on_time?.run { Date(this * 1000) } ?: Date(), + title = Html.fromHtml(it.title ?: context.getString(R.string.empty)).toString(), + author = it.author, + rawDescription = it.html ?: "", + shortDescription = (Readability4JExtended("", it.html ?: "") + .parse().textContent ?: "") + .take(110) + .trim(), + fullContent = it.html, + img = rssHelper.findImg(it.html ?: ""), + link = it.url ?: "", + feedId = accountId.spacerDollar(it.feed_id!!), accountId = accountId, - ) + isUnread = (it.is_read ?: 0) <= 0, + isStarred = (it.is_saved ?: 0) > 0, + updateAt = Date(), + ).also { + sinceId = it.id.dollarLast() + } }?.toTypedArray() ?: emptyArray() ) - - // 3. Fetch the Fever articles (up to unlimited counts) - var sinceId = "" - var itemsBody = feverAPI.getItemsSince(sinceId) - while (itemsBody.items?.isEmpty() == false) { - articleDao.insert( - *itemsBody.items?.map { - Article( - id = accountId.spacerDollar(it.id!!), - date = it.created_on_time?.run { Date(this * 1000) } ?: Date(), - title = Html.fromHtml(it.title ?: context.getString(R.string.empty)).toString(), - author = it.author, - rawDescription = it.html ?: "", - shortDescription = (Readability4JExtended("", it.html ?: "") - .parse().textContent ?: "") - .take(110) - .trim(), - fullContent = it.html, - img = rssHelper.findImg(it.html ?: ""), - link = it.url ?: "", - feedId = accountId.spacerDollar(it.feed_id!!), - accountId = accountId, - isUnread = (it.is_read ?: 0) <= 0, - isStarred = (it.is_saved ?: 0) > 0, - updateAt = Date(), - ).also { - sinceId = it.id.dollarLast() - } - }?.toTypedArray() ?: emptyArray() - ) + if (itemsBody.items?.size!! >= 50) { itemsBody = feverAPI.getItemsSince(sinceId) + } else { + break } - - // TODO: 4. Fetch the Fever favicons - - Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}") - accountDao.queryById(accountId)?.let { account -> - accountDao.update(account.apply { updateAt = Date() }) - } - ListenableWorker.Result.success(SyncWorker.setIsSyncing(false)) - } catch (e: Exception) { - Log.e("RLog", "On sync exception: ${e.message}", e) - withContext(mainDispatcher) { - context.showToast(e.message) - } - ListenableWorker.Result.failure(SyncWorker.setIsSyncing(false)) } + + // TODO: 4. Fetch the Fever favicons + + Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}") + accountDao.update(account.apply { + updateAt = Date() + if (sinceId.isNotEmpty()) { + lastArticleId = accountId.spacerDollar(sinceId) + } + }) + ListenableWorker.Result.success(SyncWorker.setIsSyncing(false)) + } catch (e: Exception) { + Log.e("RLog", "On sync exception: ${e.message}", e) + withContext(mainDispatcher) { + context.showToast(e.message) + } + ListenableWorker.Result.failure(SyncWorker.setIsSyncing(false)) } + } override suspend fun markAsRead( groupId: String?, diff --git a/app/src/main/java/me/ash/reader/data/source/RYDatabase.kt b/app/src/main/java/me/ash/reader/data/source/RYDatabase.kt index e76626b2..8653700a 100644 --- a/app/src/main/java/me/ash/reader/data/source/RYDatabase.kt +++ b/app/src/main/java/me/ash/reader/data/source/RYDatabase.kt @@ -19,7 +19,7 @@ import java.util.* @Database( entities = [Account::class, Feed::class, Article::class, Group::class], - version = 4 + version = 5 ) @TypeConverters( RYDatabase.DateConverters::class, @@ -73,6 +73,7 @@ val allMigrations = arrayOf( MIGRATION_1_2, MIGRATION_2_3, MIGRATION_3_4, + MIGRATION_4_5, ) @Suppress("ClassName") @@ -140,3 +141,15 @@ object MIGRATION_3_4 : Migration(3, 4) { ) } } + +@Suppress("ClassName") +object MIGRATION_4_5 : Migration(4, 5) { + + override fun migrate(database: SupportSQLiteDatabase) { + database.execSQL( + """ + ALTER TABLE account ADD COLUMN lastArticleId TEXT DEFAULT NULL + """.trimIndent() + ) + } +}