mirror of https://github.com/Ashinch/ReadYou.git
refactor(greader): incrementally fetch the unread items by last sync time (#569)
This commit is contained in:
Ash
GitHub
parent
573ee427db
commit
ca9b27a472
|
|
@ -1,17 +1,59 @@
|
|||
package me.ash.reader.domain.repository
|
||||
|
||||
import androidx.paging.PagingSource
|
||||
import androidx.room.*
|
||||
import androidx.room.Dao
|
||||
import androidx.room.Insert
|
||||
import androidx.room.OnConflictStrategy
|
||||
import androidx.room.Query
|
||||
import androidx.room.RewriteQueriesToDropUnusedColumns
|
||||
import androidx.room.Transaction
|
||||
import androidx.room.Update
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import me.ash.reader.domain.model.article.Article
|
||||
import me.ash.reader.domain.model.article.ArticleMeta
|
||||
import me.ash.reader.domain.model.article.ArticleWithFeed
|
||||
import me.ash.reader.domain.model.feed.ImportantNum
|
||||
import java.util.*
|
||||
import java.util.Date
|
||||
|
||||
@Dao
|
||||
interface ArticleDao {
|
||||
|
||||
@Transaction
|
||||
@RewriteQueriesToDropUnusedColumns
|
||||
@Query(
|
||||
"""
|
||||
SELECT count(1)
|
||||
FROM article
|
||||
WHERE feedId = :feedId
|
||||
AND isStarred = :isStarred
|
||||
AND accountId = :accountId
|
||||
"""
|
||||
)
|
||||
fun countByFeedIdWhenIsStarred(
|
||||
accountId: Int,
|
||||
feedId: String,
|
||||
isStarred: Boolean,
|
||||
): Int
|
||||
|
||||
@Transaction
|
||||
@RewriteQueriesToDropUnusedColumns
|
||||
@Query(
|
||||
"""
|
||||
SELECT count(1)
|
||||
FROM article AS a
|
||||
LEFT JOIN feed AS b ON b.id = a.feedId
|
||||
LEFT JOIN `group` AS c ON c.id = b.groupId
|
||||
WHERE c.id = :groupId
|
||||
AND a.isStarred = :isStarred
|
||||
AND a.accountId = :accountId
|
||||
"""
|
||||
)
|
||||
fun countByGroupIdWhenIsStarred(
|
||||
accountId: Int,
|
||||
groupId: String,
|
||||
isStarred: Boolean,
|
||||
): Int
|
||||
|
||||
@Transaction
|
||||
@Query(
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -31,7 +31,8 @@ import me.ash.reader.infrastructure.rss.RssHelper
|
|||
import me.ash.reader.ui.ext.currentAccountId
|
||||
import me.ash.reader.ui.ext.decodeHTML
|
||||
import me.ash.reader.ui.ext.spacerDollar
|
||||
import java.util.*
|
||||
import java.util.Date
|
||||
import java.util.UUID
|
||||
|
||||
abstract class AbstractRssRepository(
|
||||
private val context: Context,
|
||||
|
|
@ -311,13 +312,24 @@ abstract class AbstractRssRepository(
|
|||
feedDao.update(feed)
|
||||
}
|
||||
|
||||
open suspend fun deleteGroup(group: Group) {
|
||||
open suspend fun deleteGroup(group: Group, onlyDeleteNoStarred: Boolean? = false) {
|
||||
val accountId = context.currentAccountId
|
||||
if (onlyDeleteNoStarred == true
|
||||
&& articleDao.countByGroupIdWhenIsStarred(accountId, group.id, true) > 0
|
||||
) {
|
||||
return
|
||||
}
|
||||
deleteArticles(group = group)
|
||||
feedDao.deleteByGroupId(context.currentAccountId, group.id)
|
||||
feedDao.deleteByGroupId(accountId, group.id)
|
||||
groupDao.delete(group)
|
||||
}
|
||||
|
||||
open suspend fun deleteFeed(feed: Feed) {
|
||||
open suspend fun deleteFeed(feed: Feed, onlyDeleteNoStarred: Boolean? = false) {
|
||||
if (onlyDeleteNoStarred == true
|
||||
&& articleDao.countByFeedIdWhenIsStarred(context.currentAccountId, feed.id, true) > 0
|
||||
) {
|
||||
return
|
||||
}
|
||||
deleteArticles(feed = feed)
|
||||
feedDao.delete(feed)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,12 @@ import me.ash.reader.domain.repository.AccountDao
|
|||
import me.ash.reader.domain.repository.ArticleDao
|
||||
import me.ash.reader.domain.repository.FeedDao
|
||||
import me.ash.reader.domain.repository.GroupDao
|
||||
import me.ash.reader.ui.ext.*
|
||||
import me.ash.reader.ui.ext.DataStoreKeys
|
||||
import me.ash.reader.ui.ext.currentAccountId
|
||||
import me.ash.reader.ui.ext.dataStore
|
||||
import me.ash.reader.ui.ext.getDefaultGroupId
|
||||
import me.ash.reader.ui.ext.put
|
||||
import me.ash.reader.ui.ext.showToast
|
||||
import javax.inject.Inject
|
||||
|
||||
class AccountService @Inject constructor(
|
||||
|
|
@ -72,6 +77,7 @@ class AccountService @Inject constructor(
|
|||
Looper.loop()
|
||||
return
|
||||
}
|
||||
rssService.get().cancelSync()
|
||||
accountDao.queryById(accountId)?.let {
|
||||
articleDao.deleteByAccountId(accountId)
|
||||
feedDao.deleteByAccountId(accountId)
|
||||
|
|
|
|||
|
|
@ -25,13 +25,18 @@ import me.ash.reader.infrastructure.android.NotificationHelper
|
|||
import me.ash.reader.infrastructure.di.DefaultDispatcher
|
||||
import me.ash.reader.infrastructure.di.IODispatcher
|
||||
import me.ash.reader.infrastructure.di.MainDispatcher
|
||||
import me.ash.reader.infrastructure.html.Readability
|
||||
import me.ash.reader.infrastructure.rss.RssHelper
|
||||
import me.ash.reader.infrastructure.rss.provider.fever.FeverAPI
|
||||
import me.ash.reader.infrastructure.rss.provider.fever.FeverDTO
|
||||
import me.ash.reader.ui.ext.*
|
||||
import net.dankito.readability4j.extended.Readability4JExtended
|
||||
import java.util.*
|
||||
import me.ash.reader.ui.ext.currentAccountId
|
||||
import me.ash.reader.ui.ext.decodeHTML
|
||||
import me.ash.reader.ui.ext.dollarLast
|
||||
import me.ash.reader.ui.ext.showToast
|
||||
import me.ash.reader.ui.ext.spacerDollar
|
||||
import java.util.Date
|
||||
import javax.inject.Inject
|
||||
import kotlin.collections.set
|
||||
|
||||
class FeverRssService @Inject constructor(
|
||||
@ApplicationContext
|
||||
|
|
@ -96,11 +101,11 @@ class FeverRssService @Inject constructor(
|
|||
throw Exception("Unsupported")
|
||||
}
|
||||
|
||||
override suspend fun deleteGroup(group: Group) {
|
||||
override suspend fun deleteGroup(group: Group, onlyDeleteNoStarred: Boolean?) {
|
||||
throw Exception("Unsupported")
|
||||
}
|
||||
|
||||
override suspend fun deleteFeed(feed: Feed) {
|
||||
override suspend fun deleteFeed(feed: Feed, onlyDeleteNoStarred: Boolean?) {
|
||||
throw Exception("Unsupported")
|
||||
}
|
||||
|
||||
|
|
@ -144,12 +149,6 @@ class FeverRssService @Inject constructor(
|
|||
)
|
||||
} ?: emptyList()
|
||||
groupDao.insertOrUpdate(groups)
|
||||
val groupIds = groups.map { it.id }
|
||||
groupDao.queryAll(accountId).forEach {
|
||||
if (!groupIds.contains(it.id)) {
|
||||
super.deleteGroup(it)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Fetch the Fever feeds
|
||||
val feedsBody = feverAPI.getFeeds()
|
||||
|
|
@ -161,11 +160,6 @@ class FeverRssService @Inject constructor(
|
|||
}
|
||||
}
|
||||
}
|
||||
feedDao.queryAll(accountId).forEach {
|
||||
if (!feedsGroupsMap.contains(it.id.dollarLast())) {
|
||||
super.deleteFeed(it)
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch the Fever favicons
|
||||
val faviconsById = feverAPI.getFavicons().favicons?.associateBy { it.id } ?: emptyMap()
|
||||
|
|
@ -201,10 +195,7 @@ class FeverRssService @Inject constructor(
|
|||
title = it.title.decodeHTML() ?: context.getString(R.string.empty),
|
||||
author = it.author,
|
||||
rawDescription = it.html ?: "",
|
||||
shortDescription = (Readability4JExtended("", it.html ?: "")
|
||||
.parse().textContent ?: "")
|
||||
.take(110)
|
||||
.trim(),
|
||||
shortDescription = Readability.parseToText(it.html, it.url).take(110),
|
||||
fullContent = it.html,
|
||||
img = rssHelper.findImg(it.html ?: ""),
|
||||
link = it.url ?: "",
|
||||
|
|
@ -241,6 +232,20 @@ class FeverRssService @Inject constructor(
|
|||
}
|
||||
}
|
||||
|
||||
// Remove orphaned groups and feeds, after synchronizing the starred/un-starred
|
||||
val groupIds = groups.map { it.id }
|
||||
groupDao.queryAll(accountId).forEach {
|
||||
if (!groupIds.contains(it.id)) {
|
||||
super.deleteGroup(it, true)
|
||||
}
|
||||
}
|
||||
|
||||
feedDao.queryAll(accountId).forEach {
|
||||
if (!feedsGroupsMap.contains(it.id.dollarLast())) {
|
||||
super.deleteFeed(it, true)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}")
|
||||
accountDao.update(account.apply {
|
||||
|
|
|
|||
|
|
@ -25,15 +25,20 @@ import me.ash.reader.infrastructure.android.NotificationHelper
|
|||
import me.ash.reader.infrastructure.di.DefaultDispatcher
|
||||
import me.ash.reader.infrastructure.di.IODispatcher
|
||||
import me.ash.reader.infrastructure.di.MainDispatcher
|
||||
import me.ash.reader.infrastructure.html.Readability
|
||||
import me.ash.reader.infrastructure.rss.RssHelper
|
||||
import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderAPI
|
||||
import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderAPI.Companion.ofCategoryStreamIdToId
|
||||
import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderAPI.Companion.ofFeedStreamIdToId
|
||||
import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderAPI.Companion.ofItemStreamIdToId
|
||||
import me.ash.reader.infrastructure.rss.provider.greader.GoogleReaderDTO
|
||||
import me.ash.reader.ui.ext.*
|
||||
import net.dankito.readability4j.extended.Readability4JExtended
|
||||
import java.util.*
|
||||
import me.ash.reader.ui.ext.currentAccountId
|
||||
import me.ash.reader.ui.ext.decodeHTML
|
||||
import me.ash.reader.ui.ext.dollarLast
|
||||
import me.ash.reader.ui.ext.showToast
|
||||
import me.ash.reader.ui.ext.spacerDollar
|
||||
import java.util.Calendar
|
||||
import java.util.Date
|
||||
import javax.inject.Inject
|
||||
|
||||
class GoogleReaderRssService @Inject constructor(
|
||||
|
|
@ -164,168 +169,226 @@ class GoogleReaderRssService @Inject constructor(
|
|||
super.renameFeed(feed)
|
||||
}
|
||||
|
||||
override suspend fun deleteGroup(group: Group) {
|
||||
override suspend fun deleteGroup(group: Group, onlyDeleteNoStarred: Boolean?) {
|
||||
feedDao.queryByGroupId(context.currentAccountId, group.id)
|
||||
.forEach { deleteFeed(it) }
|
||||
getGoogleReaderAPI().disableTag(group.id.dollarLast())
|
||||
super.deleteGroup(group)
|
||||
super.deleteGroup(group, false)
|
||||
}
|
||||
|
||||
override suspend fun deleteFeed(feed: Feed) {
|
||||
override suspend fun deleteFeed(feed: Feed, onlyDeleteNoStarred: Boolean?) {
|
||||
getGoogleReaderAPI().subscriptionEdit(
|
||||
action = "unsubscribe",
|
||||
destFeedId = feed.id.dollarLast()
|
||||
)
|
||||
super.deleteFeed(feed)
|
||||
super.deleteFeed(feed, false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Google Reader API synchronous processing with object's ID to ensure idempotence
|
||||
* and handle foreign key relationships such as read status, starred status, etc.
|
||||
*
|
||||
* 1. Fetch list of feeds and folders.
|
||||
* 2. Fetch list of tags (it contains folders too, so you need to remove folders found in previous call to get
|
||||
* tags).
|
||||
* 3. Fetch ids of unread items (user can easily have 1000000 unread items so, please, add a limit on how many
|
||||
* articles you sync, 25000 could be a good default, customizable limit is even better).
|
||||
* 4. Fetch ids of starred items (100k starred items are possible, so, please, limit them too, 10-25k limit is a
|
||||
* good default).
|
||||
* 5. Fetch tagged item ids by passing s=user/-/label/TagName parameter.
|
||||
* 6. Remove items that are no longer in unread/starred/tagged ids lists from your local database.
|
||||
* 7. Fetch contents of items missing in database.
|
||||
* 8. Mark/unmark items read/starred/tagged in you app comparing local state and ids you've got from the Google Reader API.
|
||||
* Use edit-tag to sync read/starred/tagged status from your app to Google Reader API.
|
||||
* 1. /reader/api/0/tag/list
|
||||
* - Full list of categories/folders and tags/labels - and for InnoReader compatibility,
|
||||
* including the number of unread items in each tags/labels.
|
||||
*
|
||||
* 2. /reader/api/0/subscription/list
|
||||
* - Full list of subscriptions/feeds, including their category/folder.
|
||||
* - This is where you get a distinction between categories/folders and tags/labels.
|
||||
*
|
||||
* 3. /reader/api/0/stream/contents/user/-/state/com.google/reading-list
|
||||
* (with some filters in parameter to exclude read items with xt,
|
||||
* and get only the new ones with ot, cf. log below)
|
||||
* - List of new unread items and their content
|
||||
* - The response contains among other things the read/unread state,
|
||||
* the starred/not-starred state, and the tags/labels for each entry.
|
||||
* - Since this request is very expensive for the client, the network, and the server,
|
||||
* it is important to use the filters appropriately.
|
||||
* - If there is no new item since the last synchronisation, the response should be empty,
|
||||
* and therefore efficient.
|
||||
*
|
||||
* 4. /reader/api/0/stream/items/ids
|
||||
* (with a filter in parameter to exclude read items with xt)
|
||||
* - Longer list of unread items IDs
|
||||
* - This allows updating the read/unread status of the local cache of articles - assuming
|
||||
* the ones not in the list are read.
|
||||
*
|
||||
* 5. /reader/api/0/stream/contents/user/-/state/com.google/starred
|
||||
* (with some filters in parameter to exclude read items with xt,
|
||||
* and get only the new ones with ot)
|
||||
* - List of new unread starred items and their content
|
||||
* - If there is no new unread starred item since the last synchronisation,
|
||||
* the response should be empty, and therefore efficient
|
||||
* - This is a bit redundant with request 3 and 6,
|
||||
* but with the advantage of being able to retrieve a larger amount of unread starred items.
|
||||
*
|
||||
* 6. /reader/api/0/stream/contents/user/-/state/com.google/starred
|
||||
* (with some other filters, which includes read starred items)
|
||||
* - List of starred items (also read ones) and their content.
|
||||
*
|
||||
* 7. /reader/api/0/stream/items/ids
|
||||
* (with a filter to get only starred ones)
|
||||
* - Longer list of starred items IDs
|
||||
* - This allows updating the starred/non-starred status of
|
||||
* the local cache of articles - assuming the ones not in the list are not starred
|
||||
* - Similar than request 4 but for the starred status.
|
||||
*
|
||||
* @link https://github.com/FreshRSS/FreshRSS/issues/2566#issuecomment-541317776
|
||||
* @link https://github.com/bazqux/bazqux-api?tab=readme-ov-file
|
||||
* @link https://github.com/theoldreader/api
|
||||
*/
|
||||
override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result = supervisorScope {
|
||||
coroutineWorker.setProgress(SyncWorker.setIsSyncing(true))
|
||||
override suspend fun sync(coroutineWorker: CoroutineWorker): ListenableWorker.Result =
|
||||
supervisorScope {
|
||||
coroutineWorker.setProgress(SyncWorker.setIsSyncing(true))
|
||||
|
||||
try {
|
||||
val preTime = System.currentTimeMillis()
|
||||
val accountId = context.currentAccountId
|
||||
val account = accountDao.queryById(accountId)!!
|
||||
val googleReaderAPI = getGoogleReaderAPI()
|
||||
val groupIds = mutableSetOf<String>()
|
||||
val feedIds = mutableSetOf<String>()
|
||||
val lastUpdateAt = Calendar.getInstance().apply {
|
||||
// if (account.updateAt != null) {
|
||||
// time = account.updateAt!!
|
||||
// add(Calendar.HOUR, -1)
|
||||
// } else {
|
||||
time = Date()
|
||||
add(Calendar.MONTH, -1)
|
||||
// }
|
||||
}.time.time / 1000
|
||||
|
||||
// 1. Fetch list of feeds and folders
|
||||
googleReaderAPI.getSubscriptionList()
|
||||
.subscriptions.groupBy { it.categories?.first() }
|
||||
.forEach { (category, feeds) ->
|
||||
val groupId = accountId.spacerDollar(category?.id?.ofCategoryStreamIdToId()!!)
|
||||
|
||||
// Handle folders
|
||||
groupDao.insert(
|
||||
Group(
|
||||
id = groupId,
|
||||
name = category.label!!,
|
||||
accountId = accountId,
|
||||
)
|
||||
)
|
||||
groupIds.add(groupId)
|
||||
|
||||
// Handle feeds
|
||||
feedDao.insertOrUpdate(
|
||||
feeds.map {
|
||||
val feedId = accountId.spacerDollar(it.id?.ofFeedStreamIdToId()!!)
|
||||
Feed(
|
||||
id = feedId,
|
||||
name = it.title.decodeHTML() ?: context.getString(R.string.empty),
|
||||
url = it.url!!,
|
||||
groupId = groupId,
|
||||
accountId = accountId,
|
||||
icon = it.iconUrl
|
||||
).also {
|
||||
feedIds.add(feedId)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// Handle empty icon for feeds
|
||||
val noIconFeeds = feedDao.queryNoIcon(accountId)
|
||||
Log.i("RLog", "sync: $noIconFeeds")
|
||||
noIconFeeds.forEach {
|
||||
it.icon = rssHelper.queryRssIconLink(it.url)
|
||||
try {
|
||||
val preTime = System.currentTimeMillis()
|
||||
val accountId = context.currentAccountId
|
||||
val account = accountDao.queryById(accountId)!!
|
||||
val googleReaderAPI = getGoogleReaderAPI()
|
||||
val groupIds = mutableSetOf<String>()
|
||||
val feedIds = mutableSetOf<String>()
|
||||
val lastUpdateAt = Calendar.getInstance().apply {
|
||||
if (account.updateAt != null) {
|
||||
time = account.updateAt!!
|
||||
add(Calendar.HOUR, -1)
|
||||
} else {
|
||||
time = Date(preTime)
|
||||
add(Calendar.MONTH, -1)
|
||||
}
|
||||
feedDao.update(*noIconFeeds.toTypedArray())
|
||||
}.time.time / 1000
|
||||
|
||||
// 1. Fetch tags (not supported yet)
|
||||
|
||||
// 2. Fetch folder and subscription list
|
||||
googleReaderAPI.getSubscriptionList()
|
||||
.subscriptions.groupBy { it.categories?.first() }
|
||||
.forEach { (category, feeds) ->
|
||||
val groupId =
|
||||
accountId.spacerDollar(category?.id?.ofCategoryStreamIdToId()!!)
|
||||
|
||||
// Handle folders
|
||||
groupDao.insertOrUpdate(
|
||||
listOf(Group(
|
||||
id = groupId,
|
||||
name = category.label!!,
|
||||
accountId = accountId,
|
||||
))
|
||||
)
|
||||
groupIds.add(groupId)
|
||||
|
||||
// Handle feeds
|
||||
feedDao.insertOrUpdate(
|
||||
feeds.map {
|
||||
val feedId = accountId.spacerDollar(it.id?.ofFeedStreamIdToId()!!)
|
||||
Feed(
|
||||
id = feedId,
|
||||
name = it.title.decodeHTML()
|
||||
?: context.getString(R.string.empty),
|
||||
url = it.url!!,
|
||||
groupId = groupId,
|
||||
accountId = accountId,
|
||||
icon = it.iconUrl
|
||||
).also {
|
||||
feedIds.add(feedId)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
// Handle empty icon for feeds
|
||||
feedDao.queryNoIcon(accountId).let {
|
||||
it.forEach { feed ->
|
||||
feed.icon = rssHelper.queryRssIconLink(feed.url)
|
||||
}
|
||||
feedDao.update(*it.toTypedArray())
|
||||
}
|
||||
|
||||
// Remove orphaned groups and feeds
|
||||
groupDao.queryAll(accountId)
|
||||
.filter { it.id !in groupIds }
|
||||
.forEach { super.deleteGroup(it) }
|
||||
feedDao.queryAll(accountId)
|
||||
.filter { it.id !in feedIds }
|
||||
.forEach { super.deleteFeed(it) }
|
||||
|
||||
// 3. Fetch ids of unread items
|
||||
val unreadItems = googleReaderAPI.getUnreadItemIds().itemRefs
|
||||
val unreadIds = unreadItems ?.map { it.id }
|
||||
fetchItemsContents(unreadItems, googleReaderAPI, accountId, feedIds, unreadIds, listOf())
|
||||
|
||||
// 4. Fetch ids of starred items
|
||||
val starredItems = googleReaderAPI.getStarredItemIds().itemRefs
|
||||
val starredIds = starredItems?.map { it.id }
|
||||
fetchItemsContents(starredItems, googleReaderAPI, accountId, feedIds, unreadIds, starredIds)
|
||||
|
||||
// 5. Fetch ids of read items since last month
|
||||
val readItems = googleReaderAPI.getReadItemIds(lastUpdateAt).itemRefs
|
||||
|
||||
// 6. Fetch items contents for ids
|
||||
fetchItemsContents(readItems, googleReaderAPI, accountId, feedIds, unreadIds, starredIds)
|
||||
|
||||
// 7. Mark/unmark items read/starred/tagged in you app comparing
|
||||
// local state and ids you've got from the GoogleReader
|
||||
val articlesMeta = articleDao.queryMetadataAll(accountId)
|
||||
for (meta: ArticleMeta in articlesMeta) {
|
||||
val articleId = meta.id.dollarLast()
|
||||
val shouldBeUnread = unreadIds?.contains(articleId)
|
||||
val shouldBeStarred = starredIds?.contains(articleId)
|
||||
if (meta.isUnread != shouldBeUnread) {
|
||||
articleDao.markAsReadByArticleId(accountId, meta.id, shouldBeUnread ?: true)
|
||||
// 2. Fetch latest unread item contents since last sync
|
||||
var unreadIds = fetchItemIdsAndContinue {
|
||||
googleReaderAPI.getUnreadItemIds(since = lastUpdateAt, continuationId = it)
|
||||
}
|
||||
if (meta.isStarred != shouldBeStarred) {
|
||||
articleDao.markAsStarredByArticleId(accountId, meta.id, shouldBeStarred ?: false)
|
||||
fetchItemsContents(
|
||||
itemIds = unreadIds,
|
||||
googleReaderAPI = googleReaderAPI,
|
||||
accountId = accountId,
|
||||
feedIds = feedIds,
|
||||
unreadIds = unreadIds,
|
||||
starredIds = listOf())
|
||||
|
||||
// 3. Fetch all starred item contents
|
||||
val starredIds = fetchItemIdsAndContinue {
|
||||
googleReaderAPI.getStarredItemIds(continuationId = it)
|
||||
}
|
||||
fetchItemsContents(
|
||||
itemIds = starredIds,
|
||||
googleReaderAPI = googleReaderAPI,
|
||||
accountId = accountId,
|
||||
feedIds = feedIds,
|
||||
unreadIds = unreadIds,
|
||||
starredIds = starredIds
|
||||
)
|
||||
|
||||
// 4. Mark/unmarked items read/starred (/tagged)
|
||||
// Fetch all unread item id list
|
||||
unreadIds = fetchItemIdsAndContinue {
|
||||
googleReaderAPI.getUnreadItemIds(continuationId = it)
|
||||
}
|
||||
val articlesMeta = articleDao.queryMetadataAll(accountId)
|
||||
for (meta: ArticleMeta in articlesMeta) {
|
||||
val articleId = meta.id.dollarLast()
|
||||
val shouldBeRead = !unreadIds.contains(articleId)
|
||||
val shouldBeUnStarred = !starredIds.contains(articleId)
|
||||
if (meta.isUnread && shouldBeRead) {
|
||||
articleDao.markAsReadByArticleId(accountId, meta.id, true)
|
||||
}
|
||||
if (meta.isStarred && shouldBeUnStarred) {
|
||||
articleDao.markAsStarredByArticleId(accountId, meta.id, false)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Remove orphaned groups and feeds, after synchronizing the starred/un-starred
|
||||
groupDao.queryAll(accountId)
|
||||
.filter { it.id !in groupIds }
|
||||
.forEach { super.deleteGroup(it, true) }
|
||||
feedDao.queryAll(accountId)
|
||||
.filter { it.id !in feedIds }
|
||||
.forEach { super.deleteFeed(it, true) }
|
||||
|
||||
// 6. Record the time of this synchronization
|
||||
Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}")
|
||||
accountDao.update(account.apply {
|
||||
updateAt = Date(preTime)
|
||||
})
|
||||
ListenableWorker.Result.success(SyncWorker.setIsSyncing(false))
|
||||
} catch (e: Exception) {
|
||||
Log.e("RLog", "On sync exception: ${e.message}", e)
|
||||
withContext(mainDispatcher) {
|
||||
context.showToast(e.message)
|
||||
}
|
||||
ListenableWorker.Result.failure(SyncWorker.setIsSyncing(false))
|
||||
}
|
||||
|
||||
Log.i("RLog", "onCompletion: ${System.currentTimeMillis() - preTime}")
|
||||
accountDao.update(account.apply {
|
||||
updateAt = Date()
|
||||
readItems?.takeIf { it.isNotEmpty() }?.first()?.id?.let {
|
||||
lastArticleId = accountId.spacerDollar(it)
|
||||
}
|
||||
})
|
||||
ListenableWorker.Result.success(SyncWorker.setIsSyncing(false))
|
||||
} catch (e: Exception) {
|
||||
Log.e("RLog", "On sync exception: ${e.message}", e)
|
||||
withContext(mainDispatcher) {
|
||||
context.showToast(e.message)
|
||||
}
|
||||
ListenableWorker.Result.failure(SyncWorker.setIsSyncing(false))
|
||||
}
|
||||
|
||||
private suspend fun fetchItemIdsAndContinue(getItemIdsFunc: suspend (continuationId: String?) -> GoogleReaderDTO.ItemIds): MutableList<String> {
|
||||
var result = getItemIdsFunc(null)
|
||||
val ids = result.itemRefs?.mapNotNull { it.id }?.toMutableList() ?: return mutableListOf()
|
||||
while (result.continuation != null) {
|
||||
result = getItemIdsFunc(result.continuation)
|
||||
result.itemRefs?.mapNotNull { it.id }?.let { ids.addAll(it) }
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
private suspend fun fetchItemsContents(
|
||||
readIds: List<GoogleReaderDTO.Item>?,
|
||||
itemIds: List<String>?,
|
||||
googleReaderAPI: GoogleReaderAPI,
|
||||
accountId: Int,
|
||||
feedIds: MutableSet<String>,
|
||||
unreadIds: List<String?>?,
|
||||
unreadIds: List<String>?,
|
||||
starredIds: List<String?>?,
|
||||
) {
|
||||
readIds?.map { it.id!! }?.chunked(100)?.forEach { chunkedIds ->
|
||||
itemIds?.chunked(100)?.forEach { chunkedIds ->
|
||||
articleDao.insert(
|
||||
*googleReaderAPI.getItemsContents(chunkedIds).items?.map {
|
||||
val articleId = it.id!!.ofItemStreamIdToId()
|
||||
|
|
@ -335,15 +398,11 @@ class GoogleReaderRssService @Inject constructor(
|
|||
title = it.title.decodeHTML() ?: context.getString(R.string.empty),
|
||||
author = it.author,
|
||||
rawDescription = it.summary?.content ?: "",
|
||||
shortDescription = (Readability4JExtended("", it.summary?.content ?: "")
|
||||
.parse().textContent ?: "")
|
||||
.take(110)
|
||||
.trim(),
|
||||
shortDescription = Readability
|
||||
.parseToText(it.summary?.content, findArticleURL(it)).take(110),
|
||||
fullContent = it.summary?.content ?: "",
|
||||
img = rssHelper.findImg(it.summary?.content ?: ""),
|
||||
link = it.canonical?.first()?.href
|
||||
?: it.alternate?.first()?.href
|
||||
?: it.origin?.htmlUrl ?: "",
|
||||
link = findArticleURL(it),
|
||||
feedId = accountId.spacerDollar(it.origin?.streamId?.ofFeedStreamIdToId()
|
||||
?: feedIds.first()),
|
||||
accountId = accountId,
|
||||
|
|
@ -356,6 +415,10 @@ class GoogleReaderRssService @Inject constructor(
|
|||
}
|
||||
}
|
||||
|
||||
private fun findArticleURL(it: GoogleReaderDTO.Item) = it.canonical?.first()?.href
|
||||
?: it.alternate?.first()?.href
|
||||
?: it.origin?.htmlUrl ?: ""
|
||||
|
||||
override suspend fun markAsRead(
|
||||
groupId: String?,
|
||||
feedId: String?,
|
||||
|
|
@ -397,7 +460,7 @@ class GoogleReaderRssService @Inject constructor(
|
|||
}
|
||||
if (markList.isNotEmpty()) googleReaderAPI.editTag(
|
||||
itemIds = markList,
|
||||
mark = if (!isUnread) GoogleReaderAPI.Stream.READ.tag else null,
|
||||
mark = if (!isUnread) GoogleReaderAPI.Stream.READ.tag else null,
|
||||
unmark = if (isUnread) GoogleReaderAPI.Stream.READ.tag else null,
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
package me.ash.reader.infrastructure.html
|
||||
|
||||
import android.util.Log
|
||||
import net.dankito.readability4j.extended.Readability4JExtended
|
||||
import org.jsoup.nodes.Element
|
||||
|
||||
object Readability {
|
||||
|
||||
fun parseToText(htmlContent: String?, uri: String?): String {
|
||||
htmlContent ?: return ""
|
||||
return try {
|
||||
Readability4JExtended(uri ?: "", htmlContent).parse().textContent?.trim() ?: ""
|
||||
} catch (e: Exception) {
|
||||
Log.e("RLog", "Readability.parseToText '$uri' is error: ", e)
|
||||
""
|
||||
}
|
||||
}
|
||||
|
||||
fun parseToElement(htmlContent: String?, uri: String?): Element? {
|
||||
htmlContent ?: return null
|
||||
return try {
|
||||
Readability4JExtended(uri ?: "", htmlContent).parse().articleContent
|
||||
} catch (e: Exception) {
|
||||
Log.e("RLog", "Readability.parseToElement '$uri' is error: ", e)
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -15,10 +15,10 @@ import me.ash.reader.domain.model.article.Article
|
|||
import me.ash.reader.domain.model.feed.Feed
|
||||
import me.ash.reader.domain.repository.FeedDao
|
||||
import me.ash.reader.infrastructure.di.IODispatcher
|
||||
import me.ash.reader.infrastructure.html.Readability
|
||||
import me.ash.reader.ui.ext.currentAccountId
|
||||
import me.ash.reader.ui.ext.decodeHTML
|
||||
import me.ash.reader.ui.ext.spacerDollar
|
||||
import net.dankito.readability4j.extended.Readability4JExtended
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.executeAsync
|
||||
|
|
@ -53,17 +53,14 @@ class RssHelper @Inject constructor(
|
|||
return withContext(ioDispatcher) {
|
||||
val response = response(okHttpClient, link)
|
||||
val content = response.body.string()
|
||||
val readability4J = Readability4JExtended(link, content)
|
||||
val articleContent = readability4J.parse().articleContent
|
||||
if (articleContent == null) {
|
||||
""
|
||||
} else {
|
||||
val articleContent = Readability.parseToElement(content, link)
|
||||
articleContent?.run {
|
||||
val h1Element = articleContent.selectFirst("h1")
|
||||
if (h1Element != null && h1Element.hasText() && h1Element.text() == title) {
|
||||
h1Element.remove()
|
||||
}
|
||||
articleContent.toString()
|
||||
}
|
||||
} ?: ""
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -115,10 +112,7 @@ class RssHelper @Inject constructor(
|
|||
title = syndEntry.title.decodeHTML() ?: feed.name,
|
||||
author = syndEntry.author,
|
||||
rawDescription = (content ?: desc) ?: "",
|
||||
shortDescription = (Readability4JExtended("", desc ?: content ?: "")
|
||||
.parse().textContent ?: "")
|
||||
.take(110)
|
||||
.trim(),
|
||||
shortDescription = Readability.parseToText(desc ?: content, syndEntry.link).take(110),
|
||||
fullContent = content,
|
||||
img = findImg((content ?: desc) ?: ""),
|
||||
link = syndEntry.link ?: "",
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package me.ash.reader.infrastructure.rss.provider.greader
|
|||
import me.ash.reader.infrastructure.di.USER_AGENT_STRING
|
||||
import me.ash.reader.infrastructure.rss.provider.ProviderAPI
|
||||
import okhttp3.FormBody
|
||||
import okhttp3.Headers.Companion.toHeaders
|
||||
import okhttp3.Request
|
||||
import okhttp3.executeAsync
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
|
@ -42,11 +41,12 @@ class GoogleReaderAPI private constructor(
|
|||
val clResponse = client.newCall(
|
||||
Request.Builder()
|
||||
.url("${serverUrl}accounts/ClientLogin")
|
||||
.header("User-Agent", USER_AGENT_STRING)
|
||||
.post(FormBody.Builder()
|
||||
.add("output", "json")
|
||||
.add("Email", username)
|
||||
.add("Passwd", password)
|
||||
.add("client", USER_AGENT_STRING)
|
||||
.add("client", "ReadYou")
|
||||
.add("accountType", "HOSTED_OR_GOOGLE")
|
||||
.add("service", "reader")
|
||||
.build())
|
||||
|
|
@ -127,8 +127,9 @@ class GoogleReaderAPI private constructor(
|
|||
|
||||
val response = client.newCall(
|
||||
Request.Builder()
|
||||
.url("${serverUrl}${query}?output=json${params?.joinToString(separator = "") { "&${it.first}=${it.second}" } ?: ""}")
|
||||
.header("Authorization", "GoogleLogin auth=${authData.clientLoginToken}")
|
||||
.url("$serverUrl$query?output=json${params?.joinToString(separator = "") { "&${it.first}=${it.second}" } ?: ""}")
|
||||
.addHeader("Authorization", "GoogleLogin auth=${authData.clientLoginToken}")
|
||||
.addHeader("User-Agent", USER_AGENT_STRING)
|
||||
.get()
|
||||
.build())
|
||||
.executeAsync()
|
||||
|
|
@ -160,11 +161,10 @@ class GoogleReaderAPI private constructor(
|
|||
}
|
||||
val response = client.newCall(
|
||||
Request.Builder()
|
||||
.url("${serverUrl}${query}?output=json${params?.joinToString(separator = "") { "&${it.first}=${it.second}" } ?: ""}")
|
||||
.headers(mapOf(
|
||||
"Authorization" to "GoogleLogin auth=${authData.clientLoginToken}",
|
||||
"Content-Type" to "application/x-www-form-urlencoded",
|
||||
).toHeaders())
|
||||
.url("$serverUrl$query?output=json${params?.joinToString(separator = "") { "&${it.first}=${it.second}" } ?: ""}")
|
||||
.addHeader("Authorization", "GoogleLogin auth=${authData.clientLoginToken}")
|
||||
.addHeader("Content-Type", "application/x-www-form-urlencoded")
|
||||
.addHeader("User-Agent", USER_AGENT_STRING)
|
||||
.post(FormBody.Builder()
|
||||
.apply {
|
||||
form?.forEach { add(it.first, it.second) }
|
||||
|
|
@ -191,33 +191,49 @@ class GoogleReaderAPI private constructor(
|
|||
suspend fun getSubscriptionList(): GoogleReaderDTO.SubscriptionList =
|
||||
retryableGetRequest<GoogleReaderDTO.SubscriptionList>("reader/api/0/subscription/list")
|
||||
|
||||
suspend fun getReadItemIds(since: Long): GoogleReaderDTO.ItemIds =
|
||||
suspend fun getReadItemIds(
|
||||
since: Long,
|
||||
limit: String? = MAXIMUM_ITEMS_LIMIT,
|
||||
continuationId: String? = null,
|
||||
): GoogleReaderDTO.ItemIds =
|
||||
retryableGetRequest<GoogleReaderDTO.ItemIds>(
|
||||
query = "reader/api/0/stream/items/ids",
|
||||
params = listOf(
|
||||
Pair("s", Stream.READ.tag),
|
||||
Pair("ot", since.toString()),
|
||||
Pair("n", MAXIMUM_ITEMS_LIMIT),
|
||||
))
|
||||
params = mutableListOf<Pair<String, String>>().apply {
|
||||
add(Pair("s", Stream.READ.tag))
|
||||
add(Pair("ot", since.toString()))
|
||||
limit?.let { add(Pair("n", limit)) }
|
||||
continuationId?.let { add(Pair("c", continuationId)) }
|
||||
}
|
||||
)
|
||||
|
||||
suspend fun getUnreadItemIds(since: Long? = null): GoogleReaderDTO.ItemIds =
|
||||
suspend fun getUnreadItemIds(
|
||||
since: Long? = null,
|
||||
limit: String? = MAXIMUM_ITEMS_LIMIT,
|
||||
continuationId: String? = null,
|
||||
): GoogleReaderDTO.ItemIds =
|
||||
retryableGetRequest<GoogleReaderDTO.ItemIds>(
|
||||
query = "reader/api/0/stream/items/ids",
|
||||
params = mutableListOf<Pair<String, String>>().apply {
|
||||
add(Pair("s", Stream.ALL_ITEMS.tag))
|
||||
add(Pair("xt", Stream.READ.tag))
|
||||
add(Pair("n", MAXIMUM_ITEMS_LIMIT))
|
||||
limit?.let { add(Pair("n", limit)) }
|
||||
since?.let { add(Pair("ot", since.toString())) }
|
||||
continuationId?.let { add(Pair("c", continuationId)) }
|
||||
}
|
||||
)
|
||||
|
||||
suspend fun getStarredItemIds(since: Long? = null): GoogleReaderDTO.ItemIds =
|
||||
suspend fun getStarredItemIds(
|
||||
since: Long? = null,
|
||||
limit: String? = MAXIMUM_ITEMS_LIMIT,
|
||||
continuationId: String? = null,
|
||||
): GoogleReaderDTO.ItemIds =
|
||||
retryableGetRequest<GoogleReaderDTO.ItemIds>(
|
||||
query = "reader/api/0/stream/items/ids",
|
||||
params = mutableListOf<Pair<String, String>>().apply {
|
||||
add(Pair("s", Stream.STARRED.tag))
|
||||
add(Pair("n", MAXIMUM_ITEMS_LIMIT))
|
||||
limit?.let { add(Pair("n", limit)) }
|
||||
since?.let { add(Pair("ot", since.toString())) }
|
||||
continuationId?.let { add(Pair("c", continuationId)) }
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ object GoogleReaderDTO {
|
|||
*/
|
||||
data class ItemIds(
|
||||
val itemRefs: List<Item>?,
|
||||
val continuation: String?,
|
||||
)
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Reference in New Issue