Enhance SyncStatus so that it can communicate new, updated, and deleted

Frameworks/Account/CloudKit/CloudKitAccountDelegate.swift

@@ -112,7 +112,12 @@ final class CloudKitAccountDelegate: AccountDelegate {
 					
 					func processWithArticles(_ articles: Set<Article>) {
 						
-						self.articlesZone.modifyArticles(articles) { result in
+						let articlesDict = articles.reduce(into: [String: Article]()) { result, article in
+							result[article.articleID] = article
+						}
+						let statusedArticles = syncStatuses.map { ($0, articlesDict[$0.articleID]) }
+						
+						self.articlesZone.modifyArticles(statusedArticles) { result in
 							switch result {
 							case .success:
 								self.database.deleteSelectedForProcessing(syncStatuses.map({ $0.articleID }) )
@@ -428,7 +433,7 @@ final class CloudKitAccountDelegate: AccountDelegate {
 
 	func markArticles(for account: Account, articles: Set<Article>, statusKey: ArticleStatus.Key, flag: Bool) -> Set<Article>? {
 		let syncStatuses = articles.map { article in
-			return SyncStatus(articleID: article.articleID, key: statusKey, flag: flag)
+			return SyncStatus(articleID: article.articleID, key: SyncStatus.Key(statusKey), flag: flag)
 		}
 		database.insertStatuses(syncStatuses)
 
@@ -586,10 +591,6 @@ private extension CloudKitAccountDelegate {
 
 	func combinedRefresh(_ account: Account, _ webFeeds: Set<WebFeed>, completion: @escaping () -> Void) {
 		
-		var newArticles = Set<Article>()
-		var updatedArticles = Set<Article>()
-		var deletedArticles = Set<Article>()
-
 		var refresherWebFeeds = Set<WebFeed>()
 		let group = DispatchGroup()
 
@@ -605,14 +606,10 @@ private extension CloudKitAccountDelegate {
 						account.update(webFeed.webFeedID, with: parsedItems) { result in
 							switch result {
 							case .success(let articleChanges):
-								
+								self.storeArticleChanges(new: articleChanges.newArticles, updated: articleChanges.updatedArticles, deleted: articleChanges.deletedArticles) {
-								newArticles.formUnion(articleChanges.newArticles ?? Set<Article>())
-								updatedArticles.formUnion(articleChanges.updatedArticles ?? Set<Article>())
-								deletedArticles.formUnion(articleChanges.deletedArticles ?? Set<Article>())
-
 									self.refreshProgress.completeTask()
 									group.leave()
-								
+								}
 							case .failure(let error):
 								os_log(.error, log: self.log, "CloudKit Feed refresh update error: %@.", error.localizedDescription)
 								self.refreshProgress.completeTask()
@@ -634,14 +631,13 @@ private extension CloudKitAccountDelegate {
 		
 		group.enter()
 		refresher.refreshFeeds(refresherWebFeeds) { refresherNewArticles, refresherUpdatedArticles, refresherDeletedArticles in
-			newArticles.formUnion(refresherNewArticles)
+			self.storeArticleChanges(new: refresherNewArticles, updated: refresherUpdatedArticles, deleted: refresherDeletedArticles) {
-			updatedArticles.formUnion(refresherUpdatedArticles)
-			deletedArticles.formUnion(refresherDeletedArticles)
 				group.leave()
 			}
+		}
 		
 		group.notify(queue: DispatchQueue.main) {
-			self.processRecords(new: newArticles, updated: updatedArticles, deleted: deletedArticles) {
+			self.refreshArticleStatus(for: account) { _ in
 				self.articlesZone.fetchChangesInZone() { _ in
 					self.refreshProgress.completeTask()
 					completion()
@@ -651,53 +647,6 @@ private extension CloudKitAccountDelegate {
 
 	}
 
-	func processRecords(new: Set<Article>, updated: Set<Article>, deleted: Set<Article>, completion: @escaping () -> Void) {
-		
-		self.articlesZone.deleteArticles(deleted) { result in
-			self.refreshProgress.completeTask()
-			switch result {
-			case .success:
-				self.articlesZone.modifyArticles(updated) { result in
-					self.refreshProgress.completeTask()
-					switch result {
-					case .success:
-						self.saveNewArticles(new) {
-							completion()
-						}
-					case .failure(let error):
-						os_log(.error, log: self.log, "CloudKit modify articles error: %@.", error.localizedDescription)
-						completion()
-					}
-				}
-			case .failure(let error):
-				os_log(.error, log: self.log, "CloudKit delete articles error: %@.", error.localizedDescription)
-				completion()
-			}
-		}
-	}
-	
-	func saveNewArticles(_ articles: Set<Article>, completion: @escaping () -> Void) {
-		let group = DispatchGroup()
-		
-		let articleGroups = Array(articles).chunked(into: 300).map { Set($0) }
-		refreshProgress.addToNumberOfTasksAndRemaining(articleGroups.count)
-		
-		for articleGroup in articleGroups {
-			group.enter()
-			self.articlesZone.saveNewArticles(articleGroup) { result in
-				self.refreshProgress.completeTask()
-				group.leave()
-				if case .failure(let error) = result {
-					os_log(.error, log: self.log, "CloudKit new articles error: %@.", error.localizedDescription)
-				}
-			}
-		}
-		
-		group.notify(queue: DispatchQueue.main) {
-			completion()
-		}
-	}
-
 	func createProviderWebFeed(for account: Account, urlComponents: URLComponents, editedName: String?, container: Container, feedProvider: FeedProvider, completion: @escaping (Result<WebFeed, Error>) -> Void) {
 		refreshProgress.addToNumberOfTasksAndRemaining(6)
 		
@@ -731,21 +680,7 @@ private extension CloudKitAccountDelegate {
 								account.update(urlString, with: parsedItems) { result in
 									switch result {
 									case .success:
-
+										self.sendNewArticlesToTheCloud(account, feed, completion: completion)
-										account.fetchArticlesAsync(.webFeed(feed)) { result in
-											switch result {
-											case .success(let articles):
-												self.processRecords(new: articles, updated: Set<Article>(), deleted: Set<Article>()) {
-													self.articlesZone.fetchChangesInZone() { _ in
-														self.refreshProgress.clear()
-														completion(.success(feed))
-													}
-												}
-											case .failure(let error):
-												completion(.failure(error))
-											}
-										}
-										
 									case .failure(let error):
 										self.refreshProgress.clear()
 										completion(.failure(error))
@@ -812,23 +747,8 @@ private extension CloudKitAccountDelegate {
 									self.refreshProgress.completeTask()
 									switch result {
 									case .success(let externalID):
-										
 										feed.externalID = externalID
-										
+										self.sendNewArticlesToTheCloud(account, feed, completion: completion)
-										account.fetchArticlesAsync(.webFeed(feed)) { result in
-											switch result {
-											case .success(let articles):
-												self.processRecords(new: articles, updated: Set<Article>(), deleted: Set<Article>()) {
-													self.articlesZone.fetchChangesInZone() { _ in
-														self.refreshProgress.clear()
-														completion(.success(feed))
-													}
-												}
-											case .failure(let error):
-												completion(.failure(error))
-											}
-										}
-										
 									case .failure(let error):
 										self.refreshProgress.clear()
 										completion(.failure(error))
@@ -859,6 +779,31 @@ private extension CloudKitAccountDelegate {
 		}
 	}
 
+	func sendNewArticlesToTheCloud(_ account: Account, _ feed: WebFeed, completion: @escaping (Result<WebFeed, Error>) -> Void) {
+		account.fetchArticlesAsync(.webFeed(feed)) { result in
+			switch result {
+			case .success(let articles):
+				self.storeArticleChanges(new: articles, updated: Set<Article>(), deleted: Set<Article>()) {
+					self.refreshArticleStatus(for: account) { result in
+						switch result {
+						case .success:
+							self.articlesZone.fetchChangesInZone() { _ in
+								self.refreshProgress.clear()
+								completion(.success(feed))
+							}
+						case .failure(let error):
+							self.refreshProgress.clear()
+							completion(.failure(error))
+						}
+					}
+				}
+			case .failure(let error):
+				self.refreshProgress.clear()
+				completion(.failure(error))
+			}
+		}
+	}
+	
 	func processAccountError(_ account: Account, _ error: Error) {
 		if case CloudKitZoneError.userDeletedZone = error {
 			account.removeFeeds(account.topLevelWebFeeds)
@@ -868,6 +813,42 @@ private extension CloudKitAccountDelegate {
 		}
 	}
 	
+	func storeArticleChanges(new: Set<Article>?, updated: Set<Article>?, deleted: Set<Article>?, completion: @escaping () -> Void) {
+		let group = DispatchGroup()
+		
+		group.enter()
+		insertSyncStatuses(articles: new, statusKey: .new, flag: true) {
+			group.leave()
+		}
+	
+		group.enter()
+		insertSyncStatuses(articles: updated, statusKey: .new, flag: false) {
+			group.leave()
+		}
+		
+		group.enter()
+		insertSyncStatuses(articles: deleted, statusKey: .deleted, flag: true) {
+			group.leave()
+		}
+		
+		group.notify(queue: DispatchQueue.main) {
+			completion()
+		}
+	}
+	
+	func insertSyncStatuses(articles: Set<Article>?, statusKey: SyncStatus.Key, flag: Bool, completion: @escaping () -> Void) {
+		guard let articles = articles else {
+			completion()
+			return
+		}
+		let syncStatuses = articles.map { article in
+			return SyncStatus(articleID: article.articleID, key: statusKey, flag: flag)
+		}
+		database.insertStatuses(syncStatuses) { _ in
+			completion()
+		}
+	}
+
 }
 
 extension CloudKitAccountDelegate: LocalAccountRefresherDelegate {

Frameworks/Account/CloudKit/CloudKitArticlesZone.swift

@@ -12,6 +12,7 @@ import RSParser
 import RSWeb
 import CloudKit
 import Articles
+import SyncDatabase
 
 final class CloudKitArticlesZone: CloudKitZone {
 	
@@ -95,40 +96,50 @@ final class CloudKitArticlesZone: CloudKitZone {
 		delete(ckQuery: ckQuery, completion: completion)
 	}
 	
-	func deleteArticles(_ articles: Set<Article>, completion: @escaping ((Result<Void, Error>) -> Void)) {
+	func modifyArticles(_ statusArticles: [(status: SyncStatus, article: Article?)], completion: @escaping ((Result<Void, Error>) -> Void)) {
-		guard !articles.isEmpty else {
+		guard !statusArticles.isEmpty else {
 			completion(.success(()))
 			return
 		}
 		
-		let recordIDs = articles.map { CKRecord.ID(recordName: $0.articleID, zoneID: Self.zoneID) }
+		var newRecords = [CKRecord]()
-		delete(recordIDs: recordIDs, completion: completion)
+		var modifyRecords = [CKRecord]()
+		var deleteRecordIDs = [CKRecord.ID]()
+		
+		for statusArticle in statusArticles {
+			switch (statusArticle.status.key, statusArticle.status.flag) {
+			case (.new, true):
+				// create status
+				if let article = statusArticle.article {
+					newRecords.append(contentsOf: makeArticleRecords(article))
+				}
+			case (.starred, true), (.read, false):
+				// create status
+				if let article = statusArticle.article {
+					modifyRecords.append(contentsOf: makeArticleRecords(article))
+				}
+			case (.deleted, true):
+				deleteRecordIDs.append(CKRecord.ID(recordName: statusArticle.status.articleID, zoneID: Self.zoneID))
+			default:
+				print()
+				// create status
+				// delete article record
+			}
 		}
 		
-	func modifyArticles(_ articles: Set<Article>, completion: @escaping ((Result<Void, Error>) -> Void)) {
+		saveIfNew(newRecords) { result in
-		guard !articles.isEmpty else {
+			switch result {
-			completion(.success(()))
+			case .success:
-			return
+				self.modify(recordsToSave: modifyRecords, recordIDsToDelete: deleteRecordIDs) { result in
-		}
-		
-		var records = [CKRecord]()
-		
-		let saveArticles = articles.filter { $0.status.read == false || $0.status.starred == true }
-		for saveArticle in saveArticles {
-			records.append(contentsOf: makeArticleRecords(saveArticle))
-		}
-
-		let hollowArticles = articles.subtracting(saveArticles)
-		for hollowArticle in hollowArticles {
-			records.append(contentsOf: makeHollowArticleRecords(hollowArticle))
-		}
-		
-		self.modify(recordsToSave: records, recordIDsToDelete: []) { result in
 					switch result {
 					case .success:
 						completion(.success(()))
 					case .failure(let error):
-				self.handleSendArticleStatusError(error, articles: articles, completion: completion)
+						self.handleSendArticleStatusError(error, statusArticles: statusArticles, completion: completion)
+					}
+				}
+			case .failure(let error):
+				self.handleSendArticleStatusError(error, statusArticles: statusArticles, completion: completion)
 			}
 		}
 	}
@@ -137,12 +148,12 @@ final class CloudKitArticlesZone: CloudKitZone {
 
 private extension CloudKitArticlesZone {
 
-	func handleSendArticleStatusError(_ error: Error, articles: Set<Article>, completion: @escaping ((Result<Void, Error>) -> Void)) {
+	func handleSendArticleStatusError(_ error: Error, statusArticles: [(status: SyncStatus, article: Article?)], completion: @escaping ((Result<Void, Error>) -> Void)) {
 		if case CloudKitZoneError.userDeletedZone = error {
 			self.createZoneRecord() { result in
 				switch result {
 				case .success:
-					self.modifyArticles(articles, completion: completion)
+					self.modifyArticles(statusArticles, completion: completion)
 				case .failure(let error):
 					completion(.failure(error))
 				}

Frameworks/Account/FeedWrangler/FeedWranglerAPICaller.swift

@@ -261,9 +261,12 @@ final class FeedWranglerAPICaller: NSObject {
 			switch status.key {
 			case .read:
 				return URLQueryItem(name: "read", value: status.flag.description)
-				
 			case .starred:
 				return URLQueryItem(name: "starred", value: status.flag.description)
+			case .deleted:
+				return nil
+			case .new:
+				return nil
 			}
 		}
 		queryItems.append(URLQueryItem(name: "feed_item_id", value: articleID))

Frameworks/Account/FeedWrangler/FeedWranglerAccountDelegate.swift

@@ -436,7 +436,7 @@ final class FeedWranglerAccountDelegate: AccountDelegate {
 	}
 	
 	func markArticles(for account: Account, articles: Set<Article>, statusKey: ArticleStatus.Key, flag: Bool) -> Set<Article>? {
-		let syncStatuses = articles.map { SyncStatus(articleID: $0.articleID, key: statusKey, flag: flag)}
+		let syncStatuses = articles.map { SyncStatus(articleID: $0.articleID, key: SyncStatus.Key(statusKey), flag: flag)}
 		database.insertStatuses(syncStatuses)
 		
 		database.selectPendingCount { result in

Frameworks/Account/Feedbin/FeedbinAccountDelegate.swift

@@ -119,10 +119,10 @@ final class FeedbinAccountDelegate: AccountDelegate {
 		database.selectForProcessing { result in
 
 			func processStatuses(_ syncStatuses: [SyncStatus]) {
-				let createUnreadStatuses = syncStatuses.filter { $0.key == ArticleStatus.Key.read && $0.flag == false }
+				let createUnreadStatuses = syncStatuses.filter { $0.key == SyncStatus.Key.read && $0.flag == false }
-				let deleteUnreadStatuses = syncStatuses.filter { $0.key == ArticleStatus.Key.read && $0.flag == true }
+				let deleteUnreadStatuses = syncStatuses.filter { $0.key == SyncStatus.Key.read && $0.flag == true }
-				let createStarredStatuses = syncStatuses.filter { $0.key == ArticleStatus.Key.starred && $0.flag == true }
+				let createStarredStatuses = syncStatuses.filter { $0.key == SyncStatus.Key.starred && $0.flag == true }
-				let deleteStarredStatuses = syncStatuses.filter { $0.key == ArticleStatus.Key.starred && $0.flag == false }
+				let deleteStarredStatuses = syncStatuses.filter { $0.key == SyncStatus.Key.starred && $0.flag == false }
 
 				let group = DispatchGroup()
 				var errorOccurred = false
@@ -537,7 +537,7 @@ final class FeedbinAccountDelegate: AccountDelegate {
 	func markArticles(for account: Account, articles: Set<Article>, statusKey: ArticleStatus.Key, flag: Bool) -> Set<Article>? {
 
 		let syncStatuses = articles.map { article in
-			return SyncStatus(articleID: article.articleID, key: statusKey, flag: flag)
+			return SyncStatus(articleID: article.articleID, key: SyncStatus.Key(statusKey), flag: flag)
 		}
 		database.insertStatuses(syncStatuses)
 

Frameworks/Account/Feedly/FeedlyAccountDelegate.swift

@@ -489,7 +489,7 @@ final class FeedlyAccountDelegate: AccountDelegate {
 	func markArticles(for account: Account, articles: Set<Article>, statusKey: ArticleStatus.Key, flag: Bool) -> Set<Article>? {
 		
 		let syncStatuses = articles.map { article in
-			return SyncStatus(articleID: article.articleID, key: statusKey, flag: flag)
+			return SyncStatus(articleID: article.articleID, key: SyncStatus.Key(statusKey), flag: flag)
 		}
 		
 		database.insertStatuses(syncStatuses)

Frameworks/Account/Feedly/Operations/FeedlySendArticleStatusesOperation.swift

@@ -47,7 +47,7 @@ final class FeedlySendArticleStatusesOperation: FeedlyOperation {
 private extension FeedlySendArticleStatusesOperation {
 
 	func processStatuses(_ pending: [SyncStatus]) {
-		let statuses: [(status: ArticleStatus.Key, flag: Bool, action: FeedlyMarkAction)] = [
+		let statuses: [(status: SyncStatus.Key, flag: Bool, action: FeedlyMarkAction)] = [
 			(.read, false, .unread),
 			(.read, true, .read),
 			(.starred, true, .saved),

Frameworks/Account/NewsBlur/NewsBlurAccountDelegate.swift

@@ -131,16 +131,16 @@ final class NewsBlurAccountDelegate: AccountDelegate {
 
 			func processStatuses(_ syncStatuses: [SyncStatus]) {
 				let createUnreadStatuses = syncStatuses.filter {
-					$0.key == ArticleStatus.Key.read && $0.flag == false
+					$0.key == SyncStatus.Key.read && $0.flag == false
 				}
 				let deleteUnreadStatuses = syncStatuses.filter {
-					$0.key == ArticleStatus.Key.read && $0.flag == true
+					$0.key == SyncStatus.Key.read && $0.flag == true
 				}
 				let createStarredStatuses = syncStatuses.filter {
-					$0.key == ArticleStatus.Key.starred && $0.flag == true
+					$0.key == SyncStatus.Key.starred && $0.flag == true
 				}
 				let deleteStarredStatuses = syncStatuses.filter {
-					$0.key == ArticleStatus.Key.starred && $0.flag == false
+					$0.key == SyncStatus.Key.starred && $0.flag == false
 				}
 
 				let group = DispatchGroup()
@@ -575,7 +575,7 @@ final class NewsBlurAccountDelegate: AccountDelegate {
 
 	func markArticles(for account: Account, articles: Set<Article>, statusKey: ArticleStatus.Key, flag: Bool) -> Set<Article>? {
 		let syncStatuses = articles.map { article in
-			return SyncStatus(articleID: article.articleID, key: statusKey, flag: flag)
+			return SyncStatus(articleID: article.articleID, key: SyncStatus.Key(statusKey), flag: flag)
 		}
 		database.insertStatuses(syncStatuses)
 

Frameworks/Account/ReaderAPI/ReaderAPIAccountDelegate.swift

@@ -123,10 +123,10 @@ final class ReaderAPIAccountDelegate: AccountDelegate {
 		database.selectForProcessing { result in
 
 			func processStatuses(_ syncStatuses: [SyncStatus]) {
-				let createUnreadStatuses = syncStatuses.filter { $0.key == ArticleStatus.Key.read && $0.flag == false }
+				let createUnreadStatuses = syncStatuses.filter { $0.key == SyncStatus.Key.read && $0.flag == false }
-				let deleteUnreadStatuses = syncStatuses.filter { $0.key == ArticleStatus.Key.read && $0.flag == true }
+				let deleteUnreadStatuses = syncStatuses.filter { $0.key == SyncStatus.Key.read && $0.flag == true }
-				let createStarredStatuses = syncStatuses.filter { $0.key == ArticleStatus.Key.starred && $0.flag == true }
+				let createStarredStatuses = syncStatuses.filter { $0.key == SyncStatus.Key.starred && $0.flag == true }
-				let deleteStarredStatuses = syncStatuses.filter { $0.key == ArticleStatus.Key.starred && $0.flag == false }
+				let deleteStarredStatuses = syncStatuses.filter { $0.key == SyncStatus.Key.starred && $0.flag == false }
 
 				let group = DispatchGroup()
 
@@ -412,7 +412,7 @@ final class ReaderAPIAccountDelegate: AccountDelegate {
 	func markArticles(for account: Account, articles: Set<Article>, statusKey: ArticleStatus.Key, flag: Bool) -> Set<Article>? {
 		
 		let syncStatuses = articles.map { article in
-			return SyncStatus(articleID: article.articleID, key: statusKey, flag: flag)
+			return SyncStatus(articleID: article.articleID, key: SyncStatus.Key(statusKey), flag: flag)
 		}
 		database.insertStatuses(syncStatuses)
 		

Frameworks/SyncDatabase/SyncStatus.swift

@@ -12,12 +12,29 @@ import RSDatabase
 
 public struct SyncStatus: Hashable, Equatable {
 	
+	public enum Key: String {
+		case read = "read"
+		case starred = "starred"
+		case deleted = "deleted"
+		case new = "new"
+		
+		public init(_ articleStatusKey: ArticleStatus.Key) {
+			switch articleStatusKey {
+			case .read:
+				self = Self.read
+			case .starred:
+				self = Self.starred
+			}
+		}
+		
+	}
+
 	public let articleID: String
-	public let key: ArticleStatus.Key
+	public let key: SyncStatus.Key
 	public let flag: Bool
 	public let selected: Bool
 	
-	public init(articleID: String, key: ArticleStatus.Key, flag: Bool, selected: Bool = false) {
+	public init(articleID: String, key: SyncStatus.Key, flag: Bool, selected: Bool = false) {
 		self.articleID = articleID
 		self.key = key
 		self.flag = flag

Frameworks/SyncDatabase/SyncStatusTable.swift

@@ -155,7 +155,7 @@ private extension SyncStatusTable {
 	func statusWithRow(_ row: FMResultSet) -> SyncStatus? {
 		guard let articleID = row.string(forColumn: DatabaseKey.articleID),
 			let rawKey = row.string(forColumn: DatabaseKey.key),
-			let key = ArticleStatus.Key(rawValue: rawKey) else {
+			let key = SyncStatus.Key(rawValue: rawKey) else {
 				return nil
 		}