FreshRSS/ReadYou
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize Fever API integration (#363)

Browse Source
This commit is contained in:
Ash 2023-03-15 22:47:09 +08:00 committed by GitHub
parent d9b707db80
commit 4a32b00c60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 517 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

377
app/schemas/me.ash.reader.data.source.RYDatabase/5.json Normal file
View File

@ -0,0 +1,377 @@
{
"formatVersion": 1,
"database": {
"version": 5,
"identityHash": "2b86f20200ed2c56f5ae8d0565cf0f26",
"entities": [
{
"tableName": "account",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` TEXT NOT NULL, `type` INTEGER NOT NULL, `updateAt` INTEGER, `lastArticleId` TEXT, `syncInterval` INTEGER NOT NULL DEFAULT 30, `syncOnStart` INTEGER NOT NULL DEFAULT 0, `syncOnlyOnWiFi` INTEGER NOT NULL DEFAULT 0, `syncOnlyWhenCharging` INTEGER NOT NULL DEFAULT 0, `keepArchived` INTEGER NOT NULL DEFAULT 2592000000, `syncBlockList` TEXT NOT NULL DEFAULT '', `securityKey` TEXT DEFAULT 'CvJ1PKM8EW8=')",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "updateAt",
"columnName": "updateAt",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "lastArticleId",
"columnName": "lastArticleId",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "syncInterval",
"columnName": "syncInterval",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "30"
},
{
"fieldPath": "syncOnStart",
"columnName": "syncOnStart",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "syncOnlyOnWiFi",
"columnName": "syncOnlyOnWiFi",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "syncOnlyWhenCharging",
"columnName": "syncOnlyWhenCharging",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "keepArchived",
"columnName": "keepArchived",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "2592000000"
},
{
"fieldPath": "syncBlockList",
"columnName": "syncBlockList",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "securityKey",
"columnName": "securityKey",
"affinity": "TEXT",
"notNull": false,
"defaultValue": "'CvJ1PKM8EW8='"
}
],
"primaryKey": {
"autoGenerate": true,
"columnNames": [
"id"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "feed",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `icon` TEXT, `url` TEXT NOT NULL, `groupId` TEXT NOT NULL, `accountId` INTEGER NOT NULL, `isNotification` INTEGER NOT NULL, `isFullContent` INTEGER NOT NULL, PRIMARY KEY(`id`), FOREIGN KEY(`groupId`) REFERENCES `group`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "icon",
"columnName": "icon",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "url",
"columnName": "url",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "groupId",
"columnName": "groupId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isNotification",
"columnName": "isNotification",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isFullContent",
"columnName": "isFullContent",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
},
"indices": [
{
"name": "index_feed_groupId",
"unique": false,
"columnNames": [
"groupId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_feed_groupId` ON `${TABLE_NAME}` (`groupId`)"
},
{
"name": "index_feed_accountId",
"unique": false,
"columnNames": [
"accountId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_feed_accountId` ON `${TABLE_NAME}` (`accountId`)"
}
],
"foreignKeys": [
{
"table": "group",
"onDelete": "CASCADE",
"onUpdate": "CASCADE",
"columns": [
"groupId"
],
"referencedColumns": [
"id"
]
}
]
},
{
"tableName": "article",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `date` INTEGER NOT NULL, `title` TEXT NOT NULL, `author` TEXT, `rawDescription` TEXT NOT NULL, `shortDescription` TEXT NOT NULL, `fullContent` TEXT, `img` TEXT, `link` TEXT NOT NULL, `feedId` TEXT NOT NULL, `accountId` INTEGER NOT NULL, `isUnread` INTEGER NOT NULL, `isStarred` INTEGER NOT NULL, `isReadLater` INTEGER NOT NULL, `updateAt` INTEGER, PRIMARY KEY(`id`), FOREIGN KEY(`feedId`) REFERENCES `feed`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "date",
"columnName": "date",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "title",
"columnName": "title",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "author",
"columnName": "author",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "rawDescription",
"columnName": "rawDescription",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "shortDescription",
"columnName": "shortDescription",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "fullContent",
"columnName": "fullContent",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "img",
"columnName": "img",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "link",
"columnName": "link",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "feedId",
"columnName": "feedId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isUnread",
"columnName": "isUnread",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isStarred",
"columnName": "isStarred",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isReadLater",
"columnName": "isReadLater",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "updateAt",
"columnName": "updateAt",
"affinity": "INTEGER",
"notNull": false
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
},
"indices": [
{
"name": "index_article_feedId",
"unique": false,
"columnNames": [
"feedId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_article_feedId` ON `${TABLE_NAME}` (`feedId`)"
},
{
"name": "index_article_accountId",
"unique": false,
"columnNames": [
"accountId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_article_accountId` ON `${TABLE_NAME}` (`accountId`)"
}
],
"foreignKeys": [
{
"table": "feed",
"onDelete": "CASCADE",
"onUpdate": "CASCADE",
"columns": [
"feedId"
],
"referencedColumns": [
"id"
]
}
]
},
{
"tableName": "group",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `accountId` INTEGER NOT NULL, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
},
"indices": [
{
"name": "index_group_accountId",
"unique": false,
"columnNames": [
"accountId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_group_accountId` ON `${TABLE_NAME}` (`accountId`)"
}
],
"foreignKeys": []
}
],
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '2b86f20200ed2c56f5ae8d0565cf0f26')"
]
}
}

14
app/src/main/java/me/ash/reader/data/dao/FeedDao.kt
View File

@ -98,4 +98,18 @@ interface FeedDao {
@Delete @Delete
suspend fun delete(vararg feed: Feed) suspend fun delete(vararg feed: Feed)
suspend fun insertOrUpdate(feeds: List<Feed>) {
feeds.forEach {
val feed = queryById(it.id)
if (feed == null) {
insert(it)
} else {
// TODO: Consider migrating the fields to be nullable.
it.isNotification = feed.isNotification
it.isFullContent = feed.isFullContent
update(it)
}
}
}
} }

11
app/src/main/java/me/ash/reader/data/dao/GroupDao.kt
View File

@ -66,4 +66,15 @@ interface GroupDao {
@Delete @Delete
suspend fun delete(vararg group: Group) suspend fun delete(vararg group: Group)
suspend fun insertOrUpdate(groups: List<Group>) {
groups.forEach {
val group = queryById(it.id)
if (group == null) {
insert(it)
} else {
update(it)
}
}
}
} }

2
app/src/main/java/me/ash/reader/data/model/account/Account.kt
View File

@ -21,6 +21,8 @@ data class Account(
var type: AccountType, var type: AccountType,
@ColumnInfo @ColumnInfo
var updateAt: Date? = null, var updateAt: Date? = null,
@ColumnInfo
var lastArticleId: String? = null,
@ColumnInfo(defaultValue = "30") @ColumnInfo(defaultValue = "30")
var syncInterval: SyncIntervalPreference = SyncIntervalPreference.default, var syncInterval: SyncIntervalPreference = SyncIntervalPreference.default,
@ColumnInfo(defaultValue = "0") @ColumnInfo(defaultValue = "0")

6
app/src/main/java/me/ash/reader/data/repository/AbstractRssRepository.kt
View File

@ -181,18 +181,17 @@ abstract class AbstractRssRepository constructor(
if (it.syncOnStart.value) { if (it.syncOnStart.value) {
SyncWorker.enqueueOneTimeWork(workManager) SyncWorker.enqueueOneTimeWork(workManager)
} }
if (it.syncInterval != SyncIntervalPreference.Manually) { if (it.syncInterval.value != SyncIntervalPreference.Manually.value) {
SyncWorker.enqueuePeriodicWork( SyncWorker.enqueuePeriodicWork(
workManager = workManager, workManager = workManager,
syncInterval = it.syncInterval, syncInterval = it.syncInterval,
syncOnlyWhenCharging = it.syncOnlyWhenCharging, syncOnlyWhenCharging = it.syncOnlyWhenCharging,
syncOnlyOnWiFi = it.syncOnlyOnWiFi, syncOnlyOnWiFi = it.syncOnlyOnWiFi,
) )
} else {
} }
} else { } else {
SyncWorker.enqueueOneTimeWork(workManager) SyncWorker.enqueueOneTimeWork(workManager)
if (it.syncInterval.value != SyncIntervalPreference.Manually.value) {
SyncWorker.enqueuePeriodicWork( SyncWorker.enqueuePeriodicWork(
workManager = workManager, workManager = workManager,
syncInterval = it.syncInterval, syncInterval = it.syncInterval,
@ -202,6 +201,7 @@ abstract class AbstractRssRepository constructor(
} }
} }
} }
}
fun pullGroups(): Flow<MutableList<Group>> = fun pullGroups(): Flow<MutableList<Group>> =
groupDao.queryAllGroup(context.currentAccountId).flowOn(dispatcherIO) groupDao.queryAllGroup(context.currentAccountId).flowOn(dispatcherIO)

39
app/src/main/java/me/ash/reader/data/repository/FeverRssRepository.kt
View File

@ -80,31 +80,37 @@ class FeverRssRepository @Inject constructor(
} }
/** /**
* Sync handling for the Fever API. * Fever API synchronous processing with object's ID to ensure idempotence
* and handle foreign key relationships such as read status, starred status, etc.
*
* When synchronizing articles, 50 articles will be pulled in each round.
* The ID of the 50th article in this round will be recorded and
* used as the starting mark for the next pull until the number of articles
* obtained is 0 or their quantity exceeds 250, at which point the pulling process stops.
* *
* 1. Fetch the Fever groups * 1. Fetch the Fever groups
* 2. Fetch the Fever feeds * 2. Fetch the Fever feeds
* 3. Fetch the Fever articles * 3. Fetch the Fever articles
* 4. Fetch the Fever favicons * 4. Fetch the Fever favicons
*/ */
override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result = override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result = supervisorScope {
supervisorScope {
coroutineWorker.setProgress(SyncWorker.setIsSyncing(true)) coroutineWorker.setProgress(SyncWorker.setIsSyncing(true))
try { try {
val preTime = System.currentTimeMillis() val preTime = System.currentTimeMillis()
val accountId = context.currentAccountId val accountId = context.currentAccountId
val account = accountDao.queryById(accountId)!!
val feverAPI = getFeverAPI() val feverAPI = getFeverAPI()
// 1. Fetch the Fever groups // 1. Fetch the Fever groups
groupDao.insert( groupDao.insertOrUpdate(
*feverAPI.getGroups().groups?.map { feverAPI.getGroups().groups?.map {
Group( Group(
id = accountId.spacerDollar(it.id!!), id = accountId.spacerDollar(it.id!!),
name = it.title ?: context.getString(R.string.empty), name = it.title ?: context.getString(R.string.empty),
accountId = accountId, accountId = accountId,
) )
}?.toTypedArray() ?: emptyArray() } ?: emptyList()
) )
// 2. Fetch the Fever feeds // 2. Fetch the Fever feeds
@ -117,8 +123,8 @@ class FeverRssRepository @Inject constructor(
} }
} }
} }
feedDao.insert( feedDao.insertOrUpdate(
*feedsBody.feeds?.map { feedsBody.feeds?.map {
Feed( Feed(
id = accountId.spacerDollar(it.id!!), id = accountId.spacerDollar(it.id!!),
name = it.title ?: context.getString(R.string.empty), name = it.title ?: context.getString(R.string.empty),
@ -126,13 +132,13 @@ class FeverRssRepository @Inject constructor(
groupId = accountId.spacerDollar(feedsGroupsMap[it.id.toString()]!!), groupId = accountId.spacerDollar(feedsGroupsMap[it.id.toString()]!!),
accountId = accountId, accountId = accountId,
) )
}?.toTypedArray() ?: emptyArray() } ?: emptyList()
) )
// 3. Fetch the Fever articles (up to unlimited counts) // 3. Fetch the Fever articles (up to unlimited counts)
var sinceId = "" var sinceId = account.lastArticleId?.dollarLast() ?: ""
var itemsBody = feverAPI.getItemsSince(sinceId) var itemsBody = feverAPI.getItemsSince(sinceId)
while (itemsBody.items?.isEmpty() == false) { while (itemsBody.items?.isNotEmpty() == true) {
articleDao.insert( articleDao.insert(
*itemsBody.items?.map { *itemsBody.items?.map {
Article( Article(
@ -158,15 +164,22 @@ class FeverRssRepository @Inject constructor(
} }
}?.toTypedArray() ?: emptyArray() }?.toTypedArray() ?: emptyArray()
) )
if (itemsBody.items?.size!! >= 50) {
itemsBody = feverAPI.getItemsSince(sinceId) itemsBody = feverAPI.getItemsSince(sinceId)
} else {
break
}
} }
// TODO: 4. Fetch the Fever favicons // TODO: 4. Fetch the Fever favicons
Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}") Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}")
accountDao.queryById(accountId)?.let { account -> accountDao.update(account.apply {
accountDao.update(account.apply { updateAt = Date() }) updateAt = Date()
if (sinceId.isNotEmpty()) {
lastArticleId = accountId.spacerDollar(sinceId)
} }
})
ListenableWorker.Result.success(SyncWorker.setIsSyncing(false)) ListenableWorker.Result.success(SyncWorker.setIsSyncing(false))
} catch (e: Exception) { } catch (e: Exception) {
Log.e("RLog", "On sync exception: ${e.message}", e) Log.e("RLog", "On sync exception: ${e.message}", e)

15
app/src/main/java/me/ash/reader/data/source/RYDatabase.kt
View File

@ -19,7 +19,7 @@ import java.util.*
@Database( @Database(
entities = [Account::class, Feed::class, Article::class, Group::class], entities = [Account::class, Feed::class, Article::class, Group::class],
version = 4 version = 5
) )
@TypeConverters( @TypeConverters(
RYDatabase.DateConverters::class, RYDatabase.DateConverters::class,
@ -73,6 +73,7 @@ val allMigrations = arrayOf(
MIGRATION_1_2, MIGRATION_1_2,
MIGRATION_2_3, MIGRATION_2_3,
MIGRATION_3_4, MIGRATION_3_4,
MIGRATION_4_5,
) )
@Suppress("ClassName") @Suppress("ClassName")
@ -140,3 +141,15 @@ object MIGRATION_3_4 : Migration(3, 4) {
) )
} }
} }
@Suppress("ClassName")
object MIGRATION_4_5 : Migration(4, 5) {
override fun migrate(database: SupportSQLiteDatabase) {
database.execSQL(
"""
ALTER TABLE account ADD COLUMN lastArticleId TEXT DEFAULT NULL
""".trimIndent()
)
}
}