FreshRSS/ReadYou
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Optimize Fever API integration (#363)

Browse Source
This commit is contained in:
Ash 2023-03-15 22:47:09 +08:00 committed by GitHub
parent d9b707db80
commit 4a32b00c60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 517 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

377
app/schemas/me.ash.reader.data.source.RYDatabase/5.json Normal file
View File

@ -0,0 +1,377 @@
{
"formatVersion": 1,
"database": {
"version": 5,
"identityHash": "2b86f20200ed2c56f5ae8d0565cf0f26",
"entities": [
{
"tableName": "account",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` TEXT NOT NULL, `type` INTEGER NOT NULL, `updateAt` INTEGER, `lastArticleId` TEXT, `syncInterval` INTEGER NOT NULL DEFAULT 30, `syncOnStart` INTEGER NOT NULL DEFAULT 0, `syncOnlyOnWiFi` INTEGER NOT NULL DEFAULT 0, `syncOnlyWhenCharging` INTEGER NOT NULL DEFAULT 0, `keepArchived` INTEGER NOT NULL DEFAULT 2592000000, `syncBlockList` TEXT NOT NULL DEFAULT '', `securityKey` TEXT DEFAULT 'CvJ1PKM8EW8=')",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "updateAt",
"columnName": "updateAt",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "lastArticleId",
"columnName": "lastArticleId",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "syncInterval",
"columnName": "syncInterval",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "30"
},
{
"fieldPath": "syncOnStart",
"columnName": "syncOnStart",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "syncOnlyOnWiFi",
"columnName": "syncOnlyOnWiFi",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "syncOnlyWhenCharging",
"columnName": "syncOnlyWhenCharging",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "keepArchived",
"columnName": "keepArchived",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "2592000000"
},
{
"fieldPath": "syncBlockList",
"columnName": "syncBlockList",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "securityKey",
"columnName": "securityKey",
"affinity": "TEXT",
"notNull": false,
"defaultValue": "'CvJ1PKM8EW8='"
}
],
"primaryKey": {
"autoGenerate": true,
"columnNames": [
"id"
]
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "feed",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `icon` TEXT, `url` TEXT NOT NULL, `groupId` TEXT NOT NULL, `accountId` INTEGER NOT NULL, `isNotification` INTEGER NOT NULL, `isFullContent` INTEGER NOT NULL, PRIMARY KEY(`id`), FOREIGN KEY(`groupId`) REFERENCES `group`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "icon",
"columnName": "icon",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "url",
"columnName": "url",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "groupId",
"columnName": "groupId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isNotification",
"columnName": "isNotification",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isFullContent",
"columnName": "isFullContent",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
},
"indices": [
{
"name": "index_feed_groupId",
"unique": false,
"columnNames": [
"groupId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_feed_groupId` ON `${TABLE_NAME}` (`groupId`)"
},
{
"name": "index_feed_accountId",
"unique": false,
"columnNames": [
"accountId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_feed_accountId` ON `${TABLE_NAME}` (`accountId`)"
}
],
"foreignKeys": [
{
"table": "group",
"onDelete": "CASCADE",
"onUpdate": "CASCADE",
"columns": [
"groupId"
],
"referencedColumns": [
"id"
]
}
]
},
{
"tableName": "article",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `date` INTEGER NOT NULL, `title` TEXT NOT NULL, `author` TEXT, `rawDescription` TEXT NOT NULL, `shortDescription` TEXT NOT NULL, `fullContent` TEXT, `img` TEXT, `link` TEXT NOT NULL, `feedId` TEXT NOT NULL, `accountId` INTEGER NOT NULL, `isUnread` INTEGER NOT NULL, `isStarred` INTEGER NOT NULL, `isReadLater` INTEGER NOT NULL, `updateAt` INTEGER, PRIMARY KEY(`id`), FOREIGN KEY(`feedId`) REFERENCES `feed`(`id`) ON UPDATE CASCADE ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "date",
"columnName": "date",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "title",
"columnName": "title",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "author",
"columnName": "author",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "rawDescription",
"columnName": "rawDescription",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "shortDescription",
"columnName": "shortDescription",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "fullContent",
"columnName": "fullContent",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "img",
"columnName": "img",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "link",
"columnName": "link",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "feedId",
"columnName": "feedId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isUnread",
"columnName": "isUnread",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isStarred",
"columnName": "isStarred",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isReadLater",
"columnName": "isReadLater",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "updateAt",
"columnName": "updateAt",
"affinity": "INTEGER",
"notNull": false
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
},
"indices": [
{
"name": "index_article_feedId",
"unique": false,
"columnNames": [
"feedId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_article_feedId` ON `${TABLE_NAME}` (`feedId`)"
},
{
"name": "index_article_accountId",
"unique": false,
"columnNames": [
"accountId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_article_accountId` ON `${TABLE_NAME}` (`accountId`)"
}
],
"foreignKeys": [
{
"table": "feed",
"onDelete": "CASCADE",
"onUpdate": "CASCADE",
"columns": [
"feedId"
],
"referencedColumns": [
"id"
]
}
]
},
{
"tableName": "group",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `name` TEXT NOT NULL, `accountId` INTEGER NOT NULL, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
},
"indices": [
{
"name": "index_group_accountId",
"unique": false,
"columnNames": [
"accountId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_group_accountId` ON `${TABLE_NAME}` (`accountId`)"
}
],
"foreignKeys": []
}
],
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '2b86f20200ed2c56f5ae8d0565cf0f26')"
]
}
}

14
app/src/main/java/me/ash/reader/data/dao/FeedDao.kt
View File

@ -98,4 +98,18 @@ interface FeedDao {
@Delete @Delete
suspend fun delete(vararg feed: Feed) suspend fun delete(vararg feed: Feed)
suspend fun insertOrUpdate(feeds: List<Feed>) {
feeds.forEach {
val feed = queryById(it.id)
if (feed == null) {
insert(it)
} else {
// TODO: Consider migrating the fields to be nullable.
it.isNotification = feed.isNotification
it.isFullContent = feed.isFullContent
update(it)
}
}
}
} }

11
app/src/main/java/me/ash/reader/data/dao/GroupDao.kt
View File

@ -66,4 +66,15 @@ interface GroupDao {
@Delete @Delete
suspend fun delete(vararg group: Group) suspend fun delete(vararg group: Group)
suspend fun insertOrUpdate(groups: List<Group>) {
groups.forEach {
val group = queryById(it.id)
if (group == null) {
insert(it)
} else {
update(it)
}
}
}
} }

2
app/src/main/java/me/ash/reader/data/model/account/Account.kt
View File

@ -21,6 +21,8 @@ data class Account(
var type: AccountType, var type: AccountType,
@ColumnInfo @ColumnInfo
var updateAt: Date? = null, var updateAt: Date? = null,
@ColumnInfo
var lastArticleId: String? = null,
@ColumnInfo(defaultValue = "30") @ColumnInfo(defaultValue = "30")
var syncInterval: SyncIntervalPreference = SyncIntervalPreference.default, var syncInterval: SyncIntervalPreference = SyncIntervalPreference.default,
@ColumnInfo(defaultValue = "0") @ColumnInfo(defaultValue = "0")

18
app/src/main/java/me/ash/reader/data/repository/AbstractRssRepository.kt
View File

@ -181,24 +181,24 @@ abstract class AbstractRssRepository constructor(
if (it.syncOnStart.value) { if (it.syncOnStart.value) {
SyncWorker.enqueueOneTimeWork(workManager) SyncWorker.enqueueOneTimeWork(workManager)
} }
if (it.syncInterval != SyncIntervalPreference.Manually) { if (it.syncInterval.value != SyncIntervalPreference.Manually.value) {
SyncWorker.enqueuePeriodicWork( SyncWorker.enqueuePeriodicWork(
workManager = workManager, workManager = workManager,
syncInterval = it.syncInterval, syncInterval = it.syncInterval,
syncOnlyWhenCharging = it.syncOnlyWhenCharging, syncOnlyWhenCharging = it.syncOnlyWhenCharging,
syncOnlyOnWiFi = it.syncOnlyOnWiFi, syncOnlyOnWiFi = it.syncOnlyOnWiFi,
) )
} else {
} }
} else { } else {
SyncWorker.enqueueOneTimeWork(workManager) SyncWorker.enqueueOneTimeWork(workManager)
SyncWorker.enqueuePeriodicWork( if (it.syncInterval.value != SyncIntervalPreference.Manually.value) {
workManager = workManager, SyncWorker.enqueuePeriodicWork(
syncInterval = it.syncInterval, workManager = workManager,
syncOnlyWhenCharging = it.syncOnlyWhenCharging, syncInterval = it.syncInterval,
syncOnlyOnWiFi = it.syncOnlyOnWiFi, syncOnlyWhenCharging = it.syncOnlyWhenCharging,
) syncOnlyOnWiFi = it.syncOnlyOnWiFi,
)
}
} }
} }
} }

167
app/src/main/java/me/ash/reader/data/repository/FeverRssRepository.kt
View File

@ -80,102 +80,115 @@ class FeverRssRepository @Inject constructor(
} }
/** /**
* Sync handling for the Fever API. * Fever API synchronous processing with object's ID to ensure idempotence
* and handle foreign key relationships such as read status, starred status, etc.
*
* When synchronizing articles, 50 articles will be pulled in each round.
* The ID of the 50th article in this round will be recorded and
* used as the starting mark for the next pull until the number of articles
* obtained is 0 or their quantity exceeds 250, at which point the pulling process stops.
* *
* 1. Fetch the Fever groups * 1. Fetch the Fever groups
* 2. Fetch the Fever feeds * 2. Fetch the Fever feeds
* 3. Fetch the Fever articles * 3. Fetch the Fever articles
* 4. Fetch the Fever favicons * 4. Fetch the Fever favicons
*/ */
override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result = override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result = supervisorScope {
supervisorScope { coroutineWorker.setProgress(SyncWorker.setIsSyncing(true))
coroutineWorker.setProgress(SyncWorker.setIsSyncing(true))
try { try {
val preTime = System.currentTimeMillis() val preTime = System.currentTimeMillis()
val accountId = context.currentAccountId val accountId = context.currentAccountId
val feverAPI = getFeverAPI() val account = accountDao.queryById(accountId)!!
val feverAPI = getFeverAPI()
// 1. Fetch the Fever groups // 1. Fetch the Fever groups
groupDao.insert( groupDao.insertOrUpdate(
*feverAPI.getGroups().groups?.map { feverAPI.getGroups().groups?.map {
Group( Group(
id = accountId.spacerDollar(it.id!!), id = accountId.spacerDollar(it.id!!),
name = it.title ?: context.getString(R.string.empty), name = it.title ?: context.getString(R.string.empty),
accountId = accountId, accountId = accountId,
) )
}?.toTypedArray() ?: emptyArray() } ?: emptyList()
) )
// 2. Fetch the Fever feeds // 2. Fetch the Fever feeds
val feedsBody = feverAPI.getFeeds() val feedsBody = feverAPI.getFeeds()
val feedsGroupsMap = mutableMapOf<String, String>() val feedsGroupsMap = mutableMapOf<String, String>()
feedsBody.feeds_groups?.forEach { feedsGroups -> feedsBody.feeds_groups?.forEach { feedsGroups ->
feedsGroups.group_id?.toString()?.let { groupId -> feedsGroups.group_id?.toString()?.let { groupId ->
feedsGroups.feed_ids?.split(",")?.forEach { feedId -> feedsGroups.feed_ids?.split(",")?.forEach { feedId ->
feedsGroupsMap[feedId] = groupId feedsGroupsMap[feedId] = groupId
}
} }
} }
feedDao.insert( }
*feedsBody.feeds?.map { feedDao.insertOrUpdate(
Feed( feedsBody.feeds?.map {
Feed(
id = accountId.spacerDollar(it.id!!),
name = it.title ?: context.getString(R.string.empty),
url = it.url!!,
groupId = accountId.spacerDollar(feedsGroupsMap[it.id.toString()]!!),
accountId = accountId,
)
} ?: emptyList()
)
// 3. Fetch the Fever articles (up to unlimited counts)
var sinceId = account.lastArticleId?.dollarLast() ?: ""
var itemsBody = feverAPI.getItemsSince(sinceId)
while (itemsBody.items?.isNotEmpty() == true) {
articleDao.insert(
*itemsBody.items?.map {
Article(
id = accountId.spacerDollar(it.id!!), id = accountId.spacerDollar(it.id!!),
name = it.title ?: context.getString(R.string.empty), date = it.created_on_time?.run { Date(this * 1000) } ?: Date(),
url = it.url!!, title = Html.fromHtml(it.title ?: context.getString(R.string.empty)).toString(),
groupId = accountId.spacerDollar(feedsGroupsMap[it.id.toString()]!!), author = it.author,
rawDescription = it.html ?: "",
shortDescription = (Readability4JExtended("", it.html ?: "")
.parse().textContent ?: "")
.take(110)
.trim(),
fullContent = it.html,
img = rssHelper.findImg(it.html ?: ""),
link = it.url ?: "",
feedId = accountId.spacerDollar(it.feed_id!!),
accountId = accountId, accountId = accountId,
) isUnread = (it.is_read ?: 0) <= 0,
isStarred = (it.is_saved ?: 0) > 0,
updateAt = Date(),
).also {
sinceId = it.id.dollarLast()
}
}?.toTypedArray() ?: emptyArray() }?.toTypedArray() ?: emptyArray()
) )
if (itemsBody.items?.size!! >= 50) {
// 3. Fetch the Fever articles (up to unlimited counts)
var sinceId = ""
var itemsBody = feverAPI.getItemsSince(sinceId)
while (itemsBody.items?.isEmpty() == false) {
articleDao.insert(
*itemsBody.items?.map {
Article(
id = accountId.spacerDollar(it.id!!),
date = it.created_on_time?.run { Date(this * 1000) } ?: Date(),
title = Html.fromHtml(it.title ?: context.getString(R.string.empty)).toString(),
author = it.author,
rawDescription = it.html ?: "",
shortDescription = (Readability4JExtended("", it.html ?: "")
.parse().textContent ?: "")
.take(110)
.trim(),
fullContent = it.html,
img = rssHelper.findImg(it.html ?: ""),
link = it.url ?: "",
feedId = accountId.spacerDollar(it.feed_id!!),
accountId = accountId,
isUnread = (it.is_read ?: 0) <= 0,
isStarred = (it.is_saved ?: 0) > 0,
updateAt = Date(),
).also {
sinceId = it.id.dollarLast()
}
}?.toTypedArray() ?: emptyArray()
)
itemsBody = feverAPI.getItemsSince(sinceId) itemsBody = feverAPI.getItemsSince(sinceId)
} else {
break
} }
// TODO: 4. Fetch the Fever favicons
Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}")
accountDao.queryById(accountId)?.let { account ->
accountDao.update(account.apply { updateAt = Date() })
}
ListenableWorker.Result.success(SyncWorker.setIsSyncing(false))
} catch (e: Exception) {
Log.e("RLog", "On sync exception: ${e.message}", e)
withContext(mainDispatcher) {
context.showToast(e.message)
}
ListenableWorker.Result.failure(SyncWorker.setIsSyncing(false))
} }
// TODO: 4. Fetch the Fever favicons
Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}")
accountDao.update(account.apply {
updateAt = Date()
if (sinceId.isNotEmpty()) {
lastArticleId = accountId.spacerDollar(sinceId)
}
})
ListenableWorker.Result.success(SyncWorker.setIsSyncing(false))
} catch (e: Exception) {
Log.e("RLog", "On sync exception: ${e.message}", e)
withContext(mainDispatcher) {
context.showToast(e.message)
}
ListenableWorker.Result.failure(SyncWorker.setIsSyncing(false))
} }
}
override suspend fun markAsRead( override suspend fun markAsRead(
groupId: String?, groupId: String?,

15
app/src/main/java/me/ash/reader/data/source/RYDatabase.kt
View File

@ -19,7 +19,7 @@ import java.util.*
@Database( @Database(
entities = [Account::class, Feed::class, Article::class, Group::class], entities = [Account::class, Feed::class, Article::class, Group::class],
version = 4 version = 5
) )
@TypeConverters( @TypeConverters(
RYDatabase.DateConverters::class, RYDatabase.DateConverters::class,
@ -73,6 +73,7 @@ val allMigrations = arrayOf(
MIGRATION_1_2, MIGRATION_1_2,
MIGRATION_2_3, MIGRATION_2_3,
MIGRATION_3_4, MIGRATION_3_4,
MIGRATION_4_5,
) )
@Suppress("ClassName") @Suppress("ClassName")
@ -140,3 +141,15 @@ object MIGRATION_3_4 : Migration(3, 4) {
) )
} }
} }
@Suppress("ClassName")
object MIGRATION_4_5 : Migration(4, 5) {
override fun migrate(database: SupportSQLiteDatabase) {
database.execSQL(
"""
ALTER TABLE account ADD COLUMN lastArticleId TEXT DEFAULT NULL
""".trimIndent()
)
}
}