Store article content in CloudKit

This commit is contained in:
Maurice Parker 2020-04-25 20:20:56 -05:00
parent 31b04f626a
commit 1d7cc4d828
8 changed files with 153 additions and 230 deletions

View File

@ -35,7 +35,6 @@
510BD113232C3E9D002692E4 /* WebFeedMetadataFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 510BD112232C3E9D002692E4 /* WebFeedMetadataFile.swift */; };
510E3317244E0CED00E7A6AF /* TwitterMedia.swift in Sources */ = {isa = PBXBuildFile; fileRef = 510E3316244E0CED00E7A6AF /* TwitterMedia.swift */; };
511B9804237CD4270028BCAA /* FeedIdentifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 511B9803237CD4270028BCAA /* FeedIdentifier.swift */; };
5124A1612454C91B00C1245B /* CloudKitFeedRefresher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5124A1602454C91B00C1245B /* CloudKitFeedRefresher.swift */; };
512DD4CB2431000600C17B1F /* CKRecord+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 512DD4CA2431000600C17B1F /* CKRecord+Extensions.swift */; };
512DD4CD2431098700C17B1F /* CloudKitAccountZoneDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 512DD4CC2431098700C17B1F /* CloudKitAccountZoneDelegate.swift */; };
5132AAC42448BAD90077840A /* FeedProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5132AAC12448BAD90077840A /* FeedProvider.swift */; };
@ -286,7 +285,6 @@
511076A3243BD33100D97C8C /* .framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; path = .framework; sourceTree = BUILT_PRODUCTS_DIR; };
511076F4243BD96D00D97C8C /* FeedProvider.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; path = FeedProvider.framework; sourceTree = BUILT_PRODUCTS_DIR; };
511B9803237CD4270028BCAA /* FeedIdentifier.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeedIdentifier.swift; sourceTree = "<group>"; };
5124A1602454C91B00C1245B /* CloudKitFeedRefresher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudKitFeedRefresher.swift; sourceTree = "<group>"; };
512DD4CA2431000600C17B1F /* CKRecord+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CKRecord+Extensions.swift"; sourceTree = "<group>"; };
512DD4CC2431098700C17B1F /* CloudKitAccountZoneDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudKitAccountZoneDelegate.swift; sourceTree = "<group>"; };
5132AAC12448BAD90077840A /* FeedProvider.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FeedProvider.swift; sourceTree = "<group>"; };
@ -558,7 +556,6 @@
519E84A72434C5EF00D238B0 /* CloudKitArticlesZone.swift */,
519E84AB2435019100D238B0 /* CloudKitArticlesZoneDelegate.swift */,
5150FFFD243823B800C1A442 /* CloudKitError.swift */,
5124A1602454C91B00C1245B /* CloudKitFeedRefresher.swift */,
51E4DB2D242633ED0091EB5B /* CloudKitZone.swift */,
51C034DE242D65D20014DC71 /* CloudKitZoneResult.swift */,
);
@ -1175,7 +1172,6 @@
519E84A62433D49000D238B0 /* OPMLNormalizer.swift in Sources */,
9EEEF71F23545CB4009E9D80 /* FeedlySendArticleStatusesOperation.swift in Sources */,
9EBD49C223C67784005AD5CD /* FeedlyEntryIdentifierProviding.swift in Sources */,
5124A1612454C91B00C1245B /* CloudKitFeedRefresher.swift in Sources */,
846E77541F6F00E300A165E2 /* AccountManager.swift in Sources */,
51E490362288C37100C791F0 /* FeedbinDate.swift in Sources */,
9EEAE06E235D002D00E3FEE4 /* FeedlyGetCollectionsService.swift in Sources */,

View File

@ -43,10 +43,6 @@ final class CloudKitAccountDelegate: AccountDelegate {
return refresher
}()
private lazy var cloudKitFeedRefresher: CloudKitFeedRefresher = {
return CloudKitFeedRefresher(refreshProgress: refreshProgress, refresher: refresher, articlesZone: articlesZone)
}()
weak var account: Account?
let behaviors: AccountBehaviors = []
@ -207,8 +203,6 @@ final class CloudKitAccountDelegate: AccountDelegate {
let normalizedItems = OPMLNormalizer.normalize(opmlItems)
// TODO: remove duplicates created by import
self.accountZone.importOPML(rootExternalID: rootExternalID, items: normalizedItems) { _ in
self.initialRefreshAll(for: account, completion: completion)
}
@ -424,11 +418,8 @@ final class CloudKitAccountDelegate: AccountDelegate {
func accountDidInitialize(_ account: Account) {
self.account = account
accountZone.delegate = CloudKitAcountZoneDelegate(account: account, refreshProgress: refreshProgress)
articlesZone.delegate = CloudKitArticlesZoneDelegate(account: account,
database: database,
articlesZone: articlesZone,
refreshProgress: refreshProgress)
accountZone.delegate = CloudKitAcountZoneDelegate(account: account, refreshProgress: refreshProgress, articlesZone: articlesZone)
articlesZone.delegate = CloudKitArticlesZoneDelegate(account: account, database: database, articlesZone: articlesZone)
// Check to see if this is a new account and initialize anything we need
if account.externalID == nil {
@ -546,7 +537,7 @@ private extension CloudKitAccountDelegate {
self.refreshProgress.completeTask()
self.cloudKitFeedRefresher.refresh(account, webFeeds) {
self.combinedRefresh(account, webFeeds) {
self.refreshProgress.clear()
account.metadata.lastArticleFetchEndTime = Date()
}
@ -569,6 +560,73 @@ private extension CloudKitAccountDelegate {
}
func combinedRefresh(_ account: Account, _ webFeeds: Set<WebFeed>, completion: @escaping () -> Void) {
var newArticles = Set<Article>()
var deletedArticles = Set<Article>()
var refresherWebFeeds = Set<WebFeed>()
let group = DispatchGroup()
refreshProgress.addToNumberOfTasksAndRemaining(2)
for webFeed in webFeeds {
if let components = URLComponents(string: webFeed.url), let feedProvider = FeedProviderManager.shared.best(for: components) {
group.enter()
feedProvider.refresh(webFeed) { result in
switch result {
case .success(let parsedItems):
account.update(webFeed.webFeedID, with: parsedItems) { result in
switch result {
case .success(let articleChanges):
newArticles.formUnion(articleChanges.newArticles ?? Set<Article>())
deletedArticles.formUnion(articleChanges.deletedArticles ?? Set<Article>())
self.refreshProgress.completeTask()
group.leave()
case .failure(let error):
os_log(.error, log: self.log, "CloudKit Feed refresh update error: %@.", error.localizedDescription)
self.refreshProgress.completeTask()
group.leave()
}
}
case .failure(let error):
os_log(.error, log: self.log, "CloudKit Feed refresh error: %@.", error.localizedDescription)
self.refreshProgress.completeTask()
group.leave()
}
}
} else {
refresherWebFeeds.insert(webFeed)
}
}
group.enter()
refresher.refreshFeeds(refresherWebFeeds) { refresherNewArticles, refresherDeletedArticles in
newArticles.formUnion(refresherNewArticles)
deletedArticles.formUnion(refresherDeletedArticles)
group.leave()
}
group.notify(queue: DispatchQueue.main) {
self.articlesZone.deleteArticles(deletedArticles) { _ in
self.refreshProgress.completeTask()
self.articlesZone.sendNewArticles(newArticles) { _ in
self.refreshProgress.completeTask()
completion()
}
}
}
}
func createProviderWebFeed(for account: Account, urlComponents: URLComponents, editedName: String?, container: Container, feedProvider: FeedProvider, completion: @escaping (Result<WebFeed, Error>) -> Void) {
refreshProgress.addToNumberOfTasksAndRemaining(5)
@ -736,14 +794,6 @@ private extension CloudKitAccountDelegate {
extension CloudKitAccountDelegate: LocalAccountRefresherDelegate {
func localAccountRefresher(_ refresher: LocalAccountRefresher, didProcess articleChanges: ArticleChanges, completion: @escaping () -> Void) {
let newArticles = articleChanges.newArticles ?? Set<Article>()
let deletedArticles = articleChanges.deletedArticles ?? Set<Article>()
articlesZone.deleteArticles(deletedArticles) { _ in
self.articlesZone.sendNewArticles(newArticles) { _ in
completion()
}
}
}
func localAccountRefresher(_ refresher: LocalAccountRefresher, requestCompletedFor: WebFeed) {

View File

@ -10,6 +10,8 @@ import Foundation
import os.log
import RSWeb
import CloudKit
import RSCore
import Articles
class CloudKitAcountZoneDelegate: CloudKitZoneDelegate {
@ -20,10 +22,12 @@ class CloudKitAcountZoneDelegate: CloudKitZoneDelegate {
weak var account: Account?
weak var refreshProgress: DownloadProgress?
init(account: Account, refreshProgress: DownloadProgress) {
weak var articlesZone: CloudKitArticlesZone?
init(account: Account, refreshProgress: DownloadProgress, articlesZone: CloudKitArticlesZone) {
self.account = account
self.refreshProgress = refreshProgress
self.articlesZone = articlesZone
}
func cloudKitDidModify(changed: [CKRecord], deleted: [CloudKitRecordKey], completion: @escaping (Result<Void, Error>) -> Void) {
@ -193,7 +197,7 @@ private extension CloudKitAcountZoneDelegate {
if let feedProvider = FeedProviderManager.shared.best(for: urlComponents) {
refreshProgress?.addToNumberOfTasksAndRemaining(2)
refreshProgress?.addToNumberOfTasksAndRemaining(4)
feedProvider.assignName(urlComponents) { result in
self.refreshProgress?.completeTask()
switch result {
@ -206,8 +210,21 @@ private extension CloudKitAcountZoneDelegate {
self.refreshProgress?.completeTask()
switch result {
case .success(let parsedItems):
account.update(url.absoluteString, with: parsedItems) { _ in
completion(webFeed)
account.update(url.absoluteString, with: parsedItems) { result in
switch result {
case .success(let articleChanges):
self.articlesZone?.deleteArticles(articleChanges.deletedArticles ?? Set<Article>()) { _ in
self.refreshProgress?.completeTask()
self.articlesZone?.sendNewArticles(articleChanges.newArticles ?? Set<Article>()) { _ in
self.refreshProgress?.completeTask()
completion(webFeed)
}
}
case .failure:
completion(webFeed)
}
}
case .failure:
completion(webFeed)
@ -221,17 +238,36 @@ private extension CloudKitAcountZoneDelegate {
} else {
refreshProgress?.addToNumberOfTasksAndRemaining(1)
refreshProgress?.addToNumberOfTasksAndRemaining(3)
BatchUpdate.shared.start()
InitialFeedDownloader.download(url) { parsedFeed in
self.refreshProgress?.completeTask()
if let parsedFeed = parsedFeed {
account.update(webFeed, with: parsedFeed, { _ in
container.addWebFeed(webFeed)
completion(webFeed)
container.addWebFeed(webFeed)
account.update(webFeed, with: parsedFeed, { result in
BatchUpdate.shared.end()
switch result {
case .success(let articleChanges):
self.articlesZone?.deleteArticles(articleChanges.deletedArticles ?? Set<Article>()) { _ in
self.refreshProgress?.completeTask()
self.articlesZone?.sendNewArticles(articleChanges.newArticles ?? Set<Article>()) { _ in
self.refreshProgress?.completeTask()
completion(webFeed)
}
}
case .failure:
completion(webFeed)
}
})
} else {
BatchUpdate.shared.end()
completion(webFeed)
}
}
}

View File

@ -87,7 +87,11 @@ final class CloudKitArticlesZone: CloudKitZone {
return
}
let records = makeNewStatusRecords(articles)
var records = makeNewStatusRecords(articles)
for article in articles {
records.append(contentsOf: makeArticleRecords(article))
}
saveIfNew(records, completion: completion)
}

View File

@ -22,23 +22,11 @@ class CloudKitArticlesZoneDelegate: CloudKitZoneDelegate {
weak var account: Account?
var database: SyncDatabase
weak var articlesZone: CloudKitArticlesZone?
weak var refreshProgress: DownloadProgress?
private lazy var refresher: LocalAccountRefresher = {
let refresher = LocalAccountRefresher()
refresher.delegate = self
return refresher
}()
private lazy var cloudKitFeedRefresher: CloudKitFeedRefresher = {
return CloudKitFeedRefresher(refreshProgress: refreshProgress, refresher: refresher, articlesZone: articlesZone)
}()
init(account: Account, database: SyncDatabase, articlesZone: CloudKitArticlesZone, refreshProgress: DownloadProgress?) {
init(account: Account, database: SyncDatabase, articlesZone: CloudKitArticlesZone) {
self.account = account
self.database = database
self.articlesZone = articlesZone
self.refreshProgress = refreshProgress
}
func cloudKitDidModify(changed: [CKRecord], deleted: [CloudKitRecordKey], completion: @escaping (Result<Void, Error>) -> Void) {
@ -79,7 +67,7 @@ private extension CloudKitArticlesZoneDelegate {
let receivedUnstarredArticleIDs = Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.starred] == "0" }).map({ $0.externalID }))
let receivedStarredArticleIDs = Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.starred] == "1" }).map({ $0.externalID }))
let receivedStarredArticles = records.filter({ $0.recordType == CloudKitArticlesZone.CloudKitArticle.recordType })
let receivedArticles = records.filter({ $0.recordType == CloudKitArticlesZone.CloudKitArticle.recordType })
let updateableUnreadArticleIDs = receivedUnreadArticleIDs.subtracting(pendingReadStatusArticleIDs)
let updateableReadArticleIDs = receivedReadArticleIDs.subtracting(pendingReadStatusArticleIDs)
@ -90,48 +78,7 @@ private extension CloudKitArticlesZoneDelegate {
group.enter()
account?.markAsUnread(updateableUnreadArticleIDs) { result in
switch result {
case .success(let newArticleStatusIDs):
if newArticleStatusIDs.isEmpty {
group.leave()
} else {
var webFeedExternalIDDict = [String: String]()
for record in records {
if let webFeedExternalID = record[CloudKitArticlesZone.CloudKitArticleStatus.Fields.webFeedExternalID] as? String {
webFeedExternalIDDict[record.externalID] = webFeedExternalID
}
}
var webFeeds = Set<WebFeed>()
for newArticleStatusID in newArticleStatusIDs {
if let webFeedExternalID = webFeedExternalIDDict[newArticleStatusID],
let webFeed = self.account?.existingWebFeed(withExternalID: webFeedExternalID) {
webFeeds.insert(webFeed)
}
}
webFeeds.forEach { $0.dropConditionalGetInfo() }
self.refreshProgress?.addToNumberOfTasksAndRemaining(webFeeds.count)
if webFeeds.isEmpty {
group.leave()
} else {
if let account = self.account {
self.cloudKitFeedRefresher.refresh(account, webFeeds) {
group.leave()
}
} else {
group.leave()
}
}
}
case .failure:
group.leave()
}
group.leave()
}
group.enter()
@ -149,22 +96,21 @@ private extension CloudKitArticlesZoneDelegate {
group.leave()
}
for receivedStarredArticle in receivedStarredArticles {
if let parsedItem = makeParsedItem(receivedStarredArticle) {
group.enter()
self.account?.update(parsedItem.feedURL, with: Set([parsedItem])) { result in
group.leave()
if case .failure(let databaseError) = result {
os_log(.error, log: self.log, "Error occurred while storing starred items: %@", databaseError.localizedDescription)
}
let parsedItems = receivedArticles.compactMap { makeParsedItem($0) }
let webFeedIDsAndItems = Dictionary(grouping: parsedItems, by: { item in item.feedURL } ).mapValues { Set($0) }
for (webFeedID, parsedItems) in webFeedIDsAndItems {
group.enter()
self.account?.update(webFeedID, with: parsedItems) { result in
group.leave()
if case .failure(let databaseError) = result {
os_log(.error, log: self.log, "Error occurred while storing articles: %@", databaseError.localizedDescription)
}
}
}
group.notify(queue: DispatchQueue.main) {
completion(.success(()))
}
}
func makeParsedItem(_ articleRecord: CKRecord) -> ParsedItem? {
@ -207,18 +153,3 @@ private extension CloudKitArticlesZoneDelegate {
}
}
extension CloudKitArticlesZoneDelegate: LocalAccountRefresherDelegate {
func localAccountRefresher(_ refresher: LocalAccountRefresher, didProcess articleChanges: ArticleChanges, completion: @escaping () -> Void) {
completion()
}
func localAccountRefresher(_ refresher: LocalAccountRefresher, requestCompletedFor: WebFeed) {
refreshProgress?.completeTask()
}
func localAccountRefresherDidFinish(_ refresher: LocalAccountRefresher) {
}
}

View File

@ -1,94 +0,0 @@
//
// CloudKitFeedRefresher.swift
// Account
//
// Created by Maurice Parker on 4/25/20.
// Copyright © 2020 Ranchero Software, LLC. All rights reserved.
//
import Foundation
import os.log
import RSWeb
import Articles
final class CloudKitFeedRefresher {
private var log = OSLog(subsystem: Bundle.main.bundleIdentifier!, category: "CloudKit")
weak var refreshProgress: DownloadProgress?
weak var refresher: LocalAccountRefresher?
weak var articlesZone: CloudKitArticlesZone?
init(refreshProgress: DownloadProgress?, refresher: LocalAccountRefresher?, articlesZone: CloudKitArticlesZone?) {
self.refreshProgress = refreshProgress
self.refresher = refresher
self.articlesZone = articlesZone
}
func refresh(_ account: Account, _ webFeeds: Set<WebFeed>, completion: @escaping () -> Void) {
guard let refreshProgress = refreshProgress, let refresher = refresher, let articlesZone = articlesZone else { return }
var newArticles = Set<Article>()
var deletedArticles = Set<Article>()
var refresherWebFeeds = Set<WebFeed>()
let group = DispatchGroup()
refreshProgress.addToNumberOfTasksAndRemaining(2)
for webFeed in webFeeds {
if let components = URLComponents(string: webFeed.url), let feedProvider = FeedProviderManager.shared.best(for: components) {
group.enter()
feedProvider.refresh(webFeed) { result in
switch result {
case .success(let parsedItems):
account.update(webFeed.webFeedID, with: parsedItems) { result in
switch result {
case .success(let articleChanges):
newArticles.formUnion(articleChanges.newArticles ?? Set<Article>())
deletedArticles.formUnion(articleChanges.deletedArticles ?? Set<Article>())
refreshProgress.completeTask()
group.leave()
case .failure(let error):
os_log(.error, log: self.log, "CloudKit Feed refresh update error: %@.", error.localizedDescription)
refreshProgress.completeTask()
group.leave()
}
}
case .failure(let error):
os_log(.error, log: self.log, "CloudKit Feed refresh error: %@.", error.localizedDescription)
refreshProgress.completeTask()
group.leave()
}
}
} else {
refresherWebFeeds.insert(webFeed)
}
}
group.enter()
refresher.refreshFeeds(refresherWebFeeds) {
group.leave()
}
group.notify(queue: DispatchQueue.main) {
articlesZone.deleteArticles(deletedArticles) { _ in
refreshProgress.completeTask()
articlesZone.sendNewArticles(newArticles) { _ in
refreshProgress.completeTask()
completion()
}
}
}
}
}

View File

@ -79,7 +79,7 @@ final class LocalAccountDelegate: AccountDelegate {
refreshProgress.addToNumberOfTasksAndRemaining(refresherWebFeeds.count)
group.enter()
refresher?.refreshFeeds(refresherWebFeeds) {
refresher?.refreshFeeds(refresherWebFeeds) { _, _ in
group.leave()
}
@ -235,10 +235,6 @@ final class LocalAccountDelegate: AccountDelegate {
extension LocalAccountDelegate: LocalAccountRefresherDelegate {
func localAccountRefresher(_ refresher: LocalAccountRefresher, didProcess articleChanges: ArticleChanges, completion: @escaping () -> Void) {
completion()
}
func localAccountRefresher(_ refresher: LocalAccountRefresher, requestCompletedFor: WebFeed) {
refreshProgress.completeTask()
}

View File

@ -14,14 +14,15 @@ import Articles
import ArticlesDatabase
protocol LocalAccountRefresherDelegate {
func localAccountRefresher(_ refresher: LocalAccountRefresher, didProcess: ArticleChanges, completion: @escaping () -> Void)
func localAccountRefresher(_ refresher: LocalAccountRefresher, requestCompletedFor: WebFeed)
func localAccountRefresherDidFinish(_ refresher: LocalAccountRefresher)
}
final class LocalAccountRefresher {
private var completions = [() -> Void]()
var newArticles = Set<Article>()
var deletedArticles = Set<Article>()
private var completion: ((Set<Article>, Set<Article>) -> Void)?
private var isSuspended = false
var delegate: LocalAccountRefresherDelegate?
@ -29,14 +30,12 @@ final class LocalAccountRefresher {
return DownloadSession(delegate: self)
}()
public func refreshFeeds(_ feeds: Set<WebFeed>, completion: (() -> Void)? = nil) {
public func refreshFeeds(_ feeds: Set<WebFeed>, completion: ((Set<Article>, Set<Article>) -> Void)? = nil) {
guard !feeds.isEmpty else {
completion?()
completion?(Set<Article>(), Set<Article>())
return
}
if let completion = completion {
completions.append(completion)
}
self.completion = completion
downloadSession.downloadObjects(feeds as NSSet)
}
@ -105,14 +104,17 @@ extension LocalAccountRefresher: DownloadSessionDelegate {
account.update(feed, with: parsedFeed) { result in
if case .success(let articleChanges) = result {
self.delegate?.localAccountRefresher(self, didProcess: articleChanges) {
if let httpResponse = response as? HTTPURLResponse {
feed.conditionalGetInfo = HTTPConditionalGetInfo(urlResponse: httpResponse)
}
feed.contentHash = dataHash
completion()
self.delegate?.localAccountRefresher(self, requestCompletedFor: feed)
self.newArticles.formUnion(articleChanges.newArticles ?? Set<Article>())
self.deletedArticles.formUnion(articleChanges.deletedArticles ?? Set<Article>())
if let httpResponse = response as? HTTPURLResponse {
feed.conditionalGetInfo = HTTPConditionalGetInfo(urlResponse: httpResponse)
}
feed.contentHash = dataHash
completion()
self.delegate?.localAccountRefresher(self, requestCompletedFor: feed)
} else {
completion()
self.delegate?.localAccountRefresher(self, requestCompletedFor: feed)
@ -167,8 +169,10 @@ extension LocalAccountRefresher: DownloadSessionDelegate {
}
func downloadSessionDidCompleteDownloadObjects(_ downloadSession: DownloadSession) {
completions.forEach({ $0() })
completions = [() -> Void]()
completion?(newArticles, deletedArticles)
completion = nil
newArticles = Set<Article>()
deletedArticles = Set<Article>()
delegate?.localAccountRefresherDidFinish(self)
}