diff --git a/Frameworks/Data/DatabaseID.swift b/Frameworks/Data/DatabaseID.swift index 83249e74d..a7558e184 100644 --- a/Frameworks/Data/DatabaseID.swift +++ b/Frameworks/Data/DatabaseID.swift @@ -12,7 +12,15 @@ import Foundation // * It’s fast // * Collisions aren’t going to happen with feed data +private var databaseIDCache = [String: String]() + public func databaseIDWithString(_ s: String) -> String { - return (s as NSString).rs_md5Hash() + if let identifier = databaseIDCache[s] { + return identifier + } + + let identifier = (s as NSString).rs_md5Hash() + databaseIDCache[s] = identifier + return identifier } diff --git a/Frameworks/Database/ArticlesTable.swift b/Frameworks/Database/ArticlesTable.swift index e9a2a950d..83afb289c 100644 --- a/Frameworks/Database/ArticlesTable.swift +++ b/Frameworks/Database/ArticlesTable.swift @@ -17,7 +17,7 @@ final class ArticlesTable: DatabaseTable { let name: String private weak var account: Account? private let queue: RSDatabaseQueue - private let statusesTable: StatusesTable + private let statusesTable = StatusesTable() private let authorsLookupTable: DatabaseLookupTable private let attachmentsLookupTable: DatabaseLookupTable private let tagsLookupTable: DatabaseLookupTable @@ -32,7 +32,6 @@ final class ArticlesTable: DatabaseTable { self.account = account self.queue = queue - self.statusesTable = StatusesTable(name: DatabaseTableName.statuses) let authorsTable = AuthorsTable(name: DatabaseTableName.authors) self.authorsLookupTable = DatabaseLookupTable(name: DatabaseTableName.authorsLookup, objectIDKey: DatabaseKey.articleID, relatedObjectIDKey: DatabaseKey.authorID, relatedTable: authorsTable, relationshipName: RelationshipName.authors) @@ -80,8 +79,26 @@ final class ArticlesTable: DatabaseTable { // MARK: Updating func update(_ feed: Feed, _ parsedFeed: ParsedFeed, _ completion: @escaping RSVoidCompletionBlock) { - - // TODO + + if parsedFeed.items.isEmpty { + + // Once upon a time in an early version of NetNewsWire there was a bug with this issue. + // The design, at the time, was that NetNewsWire would always show only what’s currently in a feed — + // no more and no less. + // This meant that if a feed had zero items, then all the articles for that feed would be deleted. + // There were two problems with that: + // 1. People didn’t expect articles to just disappear like that. + // 2. Technorati (R.I.P.) had a bug where some of its feeds, at seemingly random times, would have zero items. + // So this hit people. THEY WERE NOT HAPPY. + // These days we just ignore empty feeds. Who cares if they’re empty. It just means less work the app has to do. + + completion() + return + } + + fetchArticlesAsync(feed) { (articles) in + self.updateArticles(articles.dictionary(), parsedFeed.itemsDictionary(with: feed), feed, completion) + } } // MARK: Unread Counts @@ -133,7 +150,7 @@ final class ArticlesTable: DatabaseTable { } } -// MARK: - +// MARK: - Private private extension ArticlesTable { @@ -232,6 +249,14 @@ private extension ArticlesTable { } return articlesWithResultSet(resultSet, database) } + + // MARK: Saving/Updating + + func updateArticles(_ articlesDictionary: [String: Article], _ parsedItemsDictionary: [String: ParsedItem], _ feed: Feed, _ completion: @escaping RSVoidCompletionBlock) { + + + } + } // MARK: - @@ -276,3 +301,6 @@ private struct ArticleCache { } } + + + diff --git a/Frameworks/Database/Database.xcodeproj/project.pbxproj b/Frameworks/Database/Database.xcodeproj/project.pbxproj index 0b191d7e0..500b63d64 100644 --- a/Frameworks/Database/Database.xcodeproj/project.pbxproj +++ b/Frameworks/Database/Database.xcodeproj/project.pbxproj @@ -11,6 +11,7 @@ 843CB9961F34174100EE6581 /* Author+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84F20F901F1810DD00D8E682 /* Author+Database.swift */; }; 844BEE411F0AB3AB004AB7CD /* Database.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 844BEE371F0AB3AA004AB7CD /* Database.framework */; }; 844BEE461F0AB3AB004AB7CD /* DatabaseTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 844BEE451F0AB3AB004AB7CD /* DatabaseTests.swift */; }; + 844ECFC91F5B4F0E005E405A /* ParsedItem+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 844ECFC81F5B4F0E005E405A /* ParsedItem+Database.swift */; }; 845580671F0AEBCD003CCFA1 /* Constants.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845580661F0AEBCD003CCFA1 /* Constants.swift */; }; 845580721F0AEE49003CCFA1 /* AccountInfo.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845580711F0AEE49003CCFA1 /* AccountInfo.swift */; }; 845580761F0AF670003CCFA1 /* Article+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845580751F0AF670003CCFA1 /* Article+Database.swift */; }; @@ -118,6 +119,7 @@ 844BEE401F0AB3AB004AB7CD /* DatabaseTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = DatabaseTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 844BEE451F0AB3AB004AB7CD /* DatabaseTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DatabaseTests.swift; sourceTree = ""; }; 844BEE471F0AB3AB004AB7CD /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 844ECFC81F5B4F0E005E405A /* ParsedItem+Database.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = "ParsedItem+Database.swift"; path = "Extensions/ParsedItem+Database.swift"; sourceTree = ""; }; 845580661F0AEBCD003CCFA1 /* Constants.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Constants.swift; sourceTree = ""; }; 845580711F0AEE49003CCFA1 /* AccountInfo.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountInfo.swift; sourceTree = ""; }; 845580751F0AF670003CCFA1 /* Article+Database.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = "Article+Database.swift"; path = "Extensions/Article+Database.swift"; sourceTree = ""; }; @@ -221,6 +223,7 @@ 84F20F901F1810DD00D8E682 /* Author+Database.swift */, 8455807B1F0C0DBD003CCFA1 /* Attachment+Database.swift */, 84D0DEA01F4A429800073503 /* String+Database.swift */, + 844ECFC81F5B4F0E005E405A /* ParsedItem+Database.swift */, 845580711F0AEE49003CCFA1 /* AccountInfo.swift */, ); name = Extensions; @@ -484,6 +487,7 @@ 84F20F8F1F180D8700D8E682 /* AuthorsTable.swift in Sources */, 84E156EC1F0AB80E00F8CC05 /* ArticlesTable.swift in Sources */, 84E156EE1F0AB81400F8CC05 /* StatusesTable.swift in Sources */, + 844ECFC91F5B4F0E005E405A /* ParsedItem+Database.swift in Sources */, 845580721F0AEE49003CCFA1 /* AccountInfo.swift in Sources */, 84E156EA1F0AB80500F8CC05 /* Database.swift in Sources */, ); diff --git a/Frameworks/Database/Extensions/Article+Database.swift b/Frameworks/Database/Extensions/Article+Database.swift index 109ced656..355638fee 100644 --- a/Frameworks/Database/Extensions/Article+Database.swift +++ b/Frameworks/Database/Extensions/Article+Database.swift @@ -91,4 +91,13 @@ extension Set where Element == Article { return Set(self.flatMap { $0.status }) } + + func dictionary() -> [String: Article] { + + var d = [String: Article]() + for article in self { + d[article.articleID] = article + } + return d + } } diff --git a/Frameworks/Database/Extensions/ParsedItem+Database.swift b/Frameworks/Database/Extensions/ParsedItem+Database.swift new file mode 100644 index 000000000..45dec0d1b --- /dev/null +++ b/Frameworks/Database/Extensions/ParsedItem+Database.swift @@ -0,0 +1,40 @@ +// +// ParsedItem+Database.swift +// Database +// +// Created by Brent Simmons on 9/2/17. +// Copyright © 2017 Ranchero Software. All rights reserved. +// + +import Foundation +import RSParser + +extension ParsedItem { + + private func databaseIdentifierWithFeed(_ feed: Feed) -> String { + + if let identifier = syncServiceID { + return identifier + } + + // Must be, and is, the same calculation as in Article.init. + return databaseIDWithString("\(feed.feedID) \(uniqueID)") + } +} + +extension ParsedFeed { + + func itemsDictionary(with feed: Feed) -> [String: ParsedItem] { + + var d = [String: ParsedItem]() + + for parsedItem in parsedItems { + let identifier = identifierForParsedItem(parsedItem, feed) + d[identifier] = parsedItem + } + + return d + } +} + + diff --git a/Frameworks/Database/StatusesTable.swift b/Frameworks/Database/StatusesTable.swift index 8737bbc81..08ce8d902 100644 --- a/Frameworks/Database/StatusesTable.swift +++ b/Frameworks/Database/StatusesTable.swift @@ -17,30 +17,9 @@ import Data final class StatusesTable: DatabaseTable { - let name: String - let databaseIDKey = DatabaseKey.articleID + let name = DatabaseTableName.statuses private let cache = DatabaseObjectCache() - init(name: String) { - - self.name = name - } - - // Mark: DatabaseTable Methods - - func objectWithRow(_ row: FMResultSet) -> DatabaseObject? { - - if let status = statusWithRow(row) { - return status as DatabaseObject - } - return nil - } - - func save(_ objects: [DatabaseObject], in database: FMDatabase) { - - // TODO - } - // MARK: Fetching func statusWithRow(_ row: FMResultSet) -> ArticleStatus? { @@ -82,68 +61,6 @@ final class StatusesTable: DatabaseTable { updateRowsWithValue(NSNumber(value: flag), valueKey: statusKey, whereKey: DatabaseKey.articleID, matches: Array(articleIDs), database: database) } - - // MARK: Updating - - - -// func attachStatuses(_ articles: Set
, _ database: FMDatabase) { -// -// // Look in cache first. -// attachCachedStatuses(articles) -// let articlesNeedingStatuses = articlesMissingStatuses(articles) -// if articlesNeedingStatuses.isEmpty { -// return -// } -// -// // Fetch from database. -// fetchAndCacheStatusesForArticles(articlesNeedingStatuses, database) -// attachCachedStatuses(articlesNeedingStatuses) -// -// // Create new statuses, and cache and save them in the database. -// // It shouldn’t happen that an Article in the database has no corresponding ArticleStatus, -// // but the case should be handled anyway. -// -// let articlesNeedingStatusesCreated = articlesMissingStatuses(articlesNeedingStatuses) -// if articlesNeedingStatusesCreated.isEmpty { -// return -// } -// createAndSaveStatusesForArticles(articlesNeedingStatusesCreated, database) -// -// assertNoMissingStatuses(articles) -// } - - -// func ensureStatusesForParsedArticles(_ parsedArticles: [ParsedItem], _ callback: @escaping RSVoidCompletionBlock) { -// -// // 1. Check cache for statuses -// // 2. Fetch statuses not found in cache -// // 3. Create, save, and cache statuses not found in database -// -// var articleIDs = Set(parsedArticles.map { $0.articleID }) -// articleIDs = articleIDsMissingStatuses(articleIDs) -// if articleIDs.isEmpty { -// callback() -// return -// } -// -// queue.fetch { (database: FMDatabase!) -> Void in -// -// let statuses = self.fetchStatusesForArticleIDs(articleIDs, database: database) -// -// DispatchQueue.main.async { -// -// self.cache.addObjectsNotCached(Array(statuses)) -// -// let newArticleIDs = self.articleIDsMissingStatuses(articleIDs) -// if !newArticleIDs.isEmpty { -// self.createAndSaveStatusesForArticleIDs(newArticleIDs) -// } -// -// callback() -// } -// } -// } } private extension StatusesTable { @@ -157,46 +74,6 @@ private extension StatusesTable { } } - // MARK: Fetching - -// func fetchAndCacheStatusesForArticles(_ articles: Set
, _ database: FMDatabase) { -// -// fetchAndCacheStatusesForArticleIDs(articles.articleIDs(), database) -// } -// -// func fetchAndCacheStatusesForArticleIDs(_ articleIDs: Set, _ database: FMDatabase) { -// -// if let statuses = fetchStatusesForArticleIDs(articleIDs, database) { -// cache.addObjectsNotCached(Array(statuses)) -// } -// } -// -// func fetchStatusesForArticleIDs(_ articleIDs: Set, _ database: FMDatabase) -> Set? { -// -// guard let resultSet = selectRowsWhere(key: DatabaseKey.articleID, inValues: Array(articleIDs), in: database) else { -// return nil -// } -// return articleStatusesWithResultSet(resultSet) -// } -// -// func articleStatusesWithResultSet(_ resultSet: FMResultSet) -> Set { -// -// return resultSet.mapToSet(articleStatusWithRow) -// } -// -// func articleStatusWithRow(_ row: FMResultSet) -> ArticleStatus? { -// -// guard let articleID = row.string(forColumn: DatabaseKey.articleID) else { -// return nil -// } -// if let cachedStatus = cache[articleID] { -// return cachedStatus -// } -// let status = ArticleStatus(articleID: articleID, row: row) -// cache[articleID] = status -// return status -// } - // MARK: Creating func saveStatuses(_ statuses: Set, _ database: FMDatabase) { @@ -205,6 +82,12 @@ private extension StatusesTable { insertRows(statusArray, insertType: .orIgnore, in: database) } + func cacheStatuses(_ statuses: [ArticleStatus]) { + + let databaseObjects = statuses.map { $0 as DatabaseObject } + cache.addObjectsNotCached(databaseObjects) + } + func createAndSaveStatusesForArticles(_ articles: Set
, _ database: FMDatabase) { let articleIDs = Set(articles.map { $0.articleID }) @@ -215,31 +98,9 @@ private extension StatusesTable { let now = Date() let statuses = articleIDs.map { ArticleStatus(articleID: $0, dateArrived: now) } - let databaseObjects = statuses.map { $0 as DatabaseObject } - cache.addObjectsNotCached(databaseObjects) + cacheStatuses(statuses) saveStatuses(Set(statuses), database) } - // MARK: Utilities - -// func articleIDsMissingCachedStatuses(_ articleIDs: Set) -> Set { -// -// return Set(articleIDs.filter { !cache.objectWithIDIsCached($0) }) -// } -// -// func articlesMissingStatuses(_ articles: Set
) -> Set
{ -// -// return articles.withNilProperty(\Article.status) -// } } - -//extension ParsedItem { -// -// var articleID: String { -// get { -// return "\(feedURL) \(uniqueID)" //Must be same as Article.articleID -// } -// } -//} - diff --git a/Frameworks/RSParser/Feeds/ParsedItem.swift b/Frameworks/RSParser/Feeds/ParsedItem.swift index 8af508ac9..a96ca6539 100644 --- a/Frameworks/RSParser/Feeds/ParsedItem.swift +++ b/Frameworks/RSParser/Feeds/ParsedItem.swift @@ -10,7 +10,8 @@ import Foundation public struct ParsedItem: Hashable { - public let uniqueID: String + public let syncServiceID: String? //Nil when not syncing + public let uniqueID: String //RSS guid, for instance; may be calculated public let feedURL: String public let url: String? public let externalURL: String? @@ -27,8 +28,9 @@ public struct ParsedItem: Hashable { public let attachments: [ParsedAttachment]? public let hashValue: Int - init(uniqueID: String, feedURL: String, url: String?, externalURL: String?, title: String?, contentHTML: String?, contentText: String?, summary: String?, imageURL: String?, bannerImageURL: String?, datePublished: Date?, dateModified: Date?, authors: [ParsedAuthor]?, tags: [String]?, attachments: [ParsedAttachment]?) { + init(syncServiceID: String?, uniqueID: String, feedURL: String, url: String?, externalURL: String?, title: String?, contentHTML: String?, contentText: String?, summary: String?, imageURL: String?, bannerImageURL: String?, datePublished: Date?, dateModified: Date?, authors: [ParsedAuthor]?, tags: [String]?, attachments: [ParsedAttachment]?) { + self.syncServiceID = syncServiceID self.uniqueID = uniqueID self.feedURL = feedURL self.url = url @@ -44,7 +46,7 @@ public struct ParsedItem: Hashable { self.authors = authors self.tags = tags self.attachments = attachments - self.hashValue = uniqueID.hashValue + self.hashValue = articleID.hashValue } public static func ==(lhs: ParsedItem, rhs: ParsedItem) -> Bool { diff --git a/ToDo.ooutline b/ToDo.ooutline index 7ecc05bb7..cdd681035 100644 Binary files a/ToDo.ooutline and b/ToDo.ooutline differ