FreshRSS/NetNewsWire
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make progress toward saving/updating articles.

Browse Source
This commit is contained in:
Brent Simmons 2017-09-02 14:19:42 -07:00
parent d017536d58
commit d33d8a0330
8 changed files with 108 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Frameworks/Data/DatabaseID.swift
View File

@ -12,7 +12,15 @@ import Foundation
// * It’s fast
// * Collisions aren’t going to happen with feed data
private var databaseIDCache = [String: String]()
public func databaseIDWithString(_ s: String) -> String {
return (s as NSString).rs_md5Hash()
if let identifier = databaseIDCache[s] {
return identifier
}
let identifier = (s as NSString).rs_md5Hash()
databaseIDCache[s] = identifier
return identifier
}

38
Frameworks/Database/ArticlesTable.swift
View File

@ -17,7 +17,7 @@ final class ArticlesTable: DatabaseTable {
let name: String
private weak var account: Account?
private let queue: RSDatabaseQueue
private let statusesTable: StatusesTable
private let statusesTable = StatusesTable()
private let authorsLookupTable: DatabaseLookupTable
private let attachmentsLookupTable: DatabaseLookupTable
private let tagsLookupTable: DatabaseLookupTable
@ -32,7 +32,6 @@ final class ArticlesTable: DatabaseTable {
self.account = account
self.queue = queue
self.statusesTable = StatusesTable(name: DatabaseTableName.statuses)
let authorsTable = AuthorsTable(name: DatabaseTableName.authors)
self.authorsLookupTable = DatabaseLookupTable(name: DatabaseTableName.authorsLookup, objectIDKey: DatabaseKey.articleID, relatedObjectIDKey: DatabaseKey.authorID, relatedTable: authorsTable, relationshipName: RelationshipName.authors)
@ -80,8 +79,26 @@ final class ArticlesTable: DatabaseTable {
// MARK: Updating
func update(_ feed: Feed, _ parsedFeed: ParsedFeed, _ completion: @escaping RSVoidCompletionBlock) {
// TODO
if parsedFeed.items.isEmpty {
// Once upon a time in an early version of NetNewsWire there was a bug with this issue.
// The design, at the time, was that NetNewsWire would always show only what’s currently in a feed —
// no more and no less.
// This meant that if a feed had zero items, then all the articles for that feed would be deleted.
// There were two problems with that:
// 1. People didn’t expect articles to just disappear like that.
// 2. Technorati (R.I.P.) had a bug where some of its feeds, at seemingly random times, would have zero items.
// So this hit people. THEY WERE NOT HAPPY.
// These days we just ignore empty feeds. Who cares if they’re empty. It just means less work the app has to do.
completion()
return
}
fetchArticlesAsync(feed) { (articles) in
self.updateArticles(articles.dictionary(), parsedFeed.itemsDictionary(with: feed), feed, completion)
}
}
// MARK: Unread Counts
@ -133,7 +150,7 @@ final class ArticlesTable: DatabaseTable {
}
}
// MARK: -
// MARK: - Private
private extension ArticlesTable {
@ -232,6 +249,14 @@ private extension ArticlesTable {
}
return articlesWithResultSet(resultSet, database)
}
// MARK: Saving/Updating
func updateArticles(_ articlesDictionary: [String: Article], _ parsedItemsDictionary: [String: ParsedItem], _ feed: Feed, _ completion: @escaping RSVoidCompletionBlock) {
}
}
// MARK: -
@ -276,3 +301,6 @@ private struct ArticleCache {
}
}

4
Frameworks/Database/Database.xcodeproj/project.pbxproj
View File

@ -11,6 +11,7 @@
843CB9961F34174100EE6581 /* Author+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84F20F901F1810DD00D8E682 /* Author+Database.swift */; };
844BEE411F0AB3AB004AB7CD /* Database.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 844BEE371F0AB3AA004AB7CD /* Database.framework */; };
844BEE461F0AB3AB004AB7CD /* DatabaseTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 844BEE451F0AB3AB004AB7CD /* DatabaseTests.swift */; };
844ECFC91F5B4F0E005E405A /* ParsedItem+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 844ECFC81F5B4F0E005E405A /* ParsedItem+Database.swift */; };
845580671F0AEBCD003CCFA1 /* Constants.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845580661F0AEBCD003CCFA1 /* Constants.swift */; };
845580721F0AEE49003CCFA1 /* AccountInfo.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845580711F0AEE49003CCFA1 /* AccountInfo.swift */; };
845580761F0AF670003CCFA1 /* Article+Database.swift in Sources */ = {isa = PBXBuildFile; fileRef = 845580751F0AF670003CCFA1 /* Article+Database.swift */; };
@ -118,6 +119,7 @@
844BEE401F0AB3AB004AB7CD /* DatabaseTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = DatabaseTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
844BEE451F0AB3AB004AB7CD /* DatabaseTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DatabaseTests.swift; sourceTree = "<group>"; };
844BEE471F0AB3AB004AB7CD /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
844ECFC81F5B4F0E005E405A /* ParsedItem+Database.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = "ParsedItem+Database.swift"; path = "Extensions/ParsedItem+Database.swift"; sourceTree = "<group>"; };
845580661F0AEBCD003CCFA1 /* Constants.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Constants.swift; sourceTree = "<group>"; };
845580711F0AEE49003CCFA1 /* AccountInfo.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AccountInfo.swift; sourceTree = "<group>"; };
845580751F0AF670003CCFA1 /* Article+Database.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = "Article+Database.swift"; path = "Extensions/Article+Database.swift"; sourceTree = "<group>"; };
@ -221,6 +223,7 @@
84F20F901F1810DD00D8E682 /* Author+Database.swift */,
8455807B1F0C0DBD003CCFA1 /* Attachment+Database.swift */,
84D0DEA01F4A429800073503 /* String+Database.swift */,
844ECFC81F5B4F0E005E405A /* ParsedItem+Database.swift */,
845580711F0AEE49003CCFA1 /* AccountInfo.swift */,
);
name = Extensions;
@ -484,6 +487,7 @@
84F20F8F1F180D8700D8E682 /* AuthorsTable.swift in Sources */,
84E156EC1F0AB80E00F8CC05 /* ArticlesTable.swift in Sources */,
84E156EE1F0AB81400F8CC05 /* StatusesTable.swift in Sources */,
844ECFC91F5B4F0E005E405A /* ParsedItem+Database.swift in Sources */,
845580721F0AEE49003CCFA1 /* AccountInfo.swift in Sources */,
84E156EA1F0AB80500F8CC05 /* Database.swift in Sources */,
);

9
Frameworks/Database/Extensions/Article+Database.swift
View File

@ -91,4 +91,13 @@ extension Set where Element == Article {
return Set<ArticleStatus>(self.flatMap { $0.status })
}
func dictionary() -> [String: Article] {
var d = [String: Article]()
for article in self {
d[article.articleID] = article
}
return d
}
}

40
Frameworks/Database/Extensions/ParsedItem+Database.swift Normal file
View File

@ -0,0 +1,40 @@
//
// ParsedItem+Database.swift
// Database
//
// Created by Brent Simmons on 9/2/17.
// Copyright © 2017 Ranchero Software. All rights reserved.
//
import Foundation
import RSParser
extension ParsedItem {
private func databaseIdentifierWithFeed(_ feed: Feed) -> String {
if let identifier = syncServiceID {
return identifier
}
// Must be, and is, the same calculation as in Article.init.
return databaseIDWithString("\(feed.feedID) \(uniqueID)")
}
}
extension ParsedFeed {
func itemsDictionary(with feed: Feed) -> [String: ParsedItem] {
var d = [String: ParsedItem]()
for parsedItem in parsedItems {
let identifier = identifierForParsedItem(parsedItem, feed)
d[identifier] = parsedItem
}
return d
}
}

155
Frameworks/Database/StatusesTable.swift
View File

@ -17,30 +17,9 @@ import Data
final class StatusesTable: DatabaseTable {
let name: String
let databaseIDKey = DatabaseKey.articleID
let name = DatabaseTableName.statuses
private let cache = DatabaseObjectCache()
init(name: String) {
self.name = name
}
// Mark: DatabaseTable Methods
func objectWithRow(_ row: FMResultSet) -> DatabaseObject? {
if let status = statusWithRow(row) {
return status as DatabaseObject
}
return nil
}
func save(_ objects: [DatabaseObject], in database: FMDatabase) {
// TODO
}
// MARK: Fetching
func statusWithRow(_ row: FMResultSet) -> ArticleStatus? {
@ -82,68 +61,6 @@ final class StatusesTable: DatabaseTable {
updateRowsWithValue(NSNumber(value: flag), valueKey: statusKey, whereKey: DatabaseKey.articleID, matches: Array(articleIDs), database: database)
}
// MARK: Updating
// func attachStatuses(_ articles: Set<Article>, _ database: FMDatabase) {
//
// // Look in cache first.
// attachCachedStatuses(articles)
// let articlesNeedingStatuses = articlesMissingStatuses(articles)
// if articlesNeedingStatuses.isEmpty {
// return
// }
//
// // Fetch from database.
// fetchAndCacheStatusesForArticles(articlesNeedingStatuses, database)
// attachCachedStatuses(articlesNeedingStatuses)
//
// // Create new statuses, and cache and save them in the database.
// // It shouldn’t happen that an Article in the database has no corresponding ArticleStatus,
// // but the case should be handled anyway.
//
// let articlesNeedingStatusesCreated = articlesMissingStatuses(articlesNeedingStatuses)
// if articlesNeedingStatusesCreated.isEmpty {
// return
// }
// createAndSaveStatusesForArticles(articlesNeedingStatusesCreated, database)
//
// assertNoMissingStatuses(articles)
// }
// func ensureStatusesForParsedArticles(_ parsedArticles: [ParsedItem], _ callback: @escaping RSVoidCompletionBlock) {
//
// // 1. Check cache for statuses
// // 2. Fetch statuses not found in cache
// // 3. Create, save, and cache statuses not found in database
//
// var articleIDs = Set(parsedArticles.map { $0.articleID })
// articleIDs = articleIDsMissingStatuses(articleIDs)
// if articleIDs.isEmpty {
// callback()
// return
// }
//
// queue.fetch { (database: FMDatabase!) -> Void in
//
// let statuses = self.fetchStatusesForArticleIDs(articleIDs, database: database)
//
// DispatchQueue.main.async {
//
// self.cache.addObjectsNotCached(Array(statuses))
//
// let newArticleIDs = self.articleIDsMissingStatuses(articleIDs)
// if !newArticleIDs.isEmpty {
// self.createAndSaveStatusesForArticleIDs(newArticleIDs)
// }
//
// callback()
// }
// }
// }
}
private extension StatusesTable {
@ -157,46 +74,6 @@ private extension StatusesTable {
}
}
// MARK: Fetching
// func fetchAndCacheStatusesForArticles(_ articles: Set<Article>, _ database: FMDatabase) {
//
// fetchAndCacheStatusesForArticleIDs(articles.articleIDs(), database)
// }
//
// func fetchAndCacheStatusesForArticleIDs(_ articleIDs: Set<String>, _ database: FMDatabase) {
//
// if let statuses = fetchStatusesForArticleIDs(articleIDs, database) {
// cache.addObjectsNotCached(Array(statuses))
// }
// }
//
// func fetchStatusesForArticleIDs(_ articleIDs: Set<String>, _ database: FMDatabase) -> Set<ArticleStatus>? {
//
// guard let resultSet = selectRowsWhere(key: DatabaseKey.articleID, inValues: Array(articleIDs), in: database) else {
// return nil
// }
// return articleStatusesWithResultSet(resultSet)
// }
//
// func articleStatusesWithResultSet(_ resultSet: FMResultSet) -> Set<ArticleStatus> {
//
// return resultSet.mapToSet(articleStatusWithRow)
// }
//
// func articleStatusWithRow(_ row: FMResultSet) -> ArticleStatus? {
//
// guard let articleID = row.string(forColumn: DatabaseKey.articleID) else {
// return nil
// }
// if let cachedStatus = cache[articleID] {
// return cachedStatus
// }
// let status = ArticleStatus(articleID: articleID, row: row)
// cache[articleID] = status
// return status
// }
// MARK: Creating
func saveStatuses(_ statuses: Set<ArticleStatus>, _ database: FMDatabase) {
@ -205,6 +82,12 @@ private extension StatusesTable {
insertRows(statusArray, insertType: .orIgnore, in: database)
}
func cacheStatuses(_ statuses: [ArticleStatus]) {
let databaseObjects = statuses.map { $0 as DatabaseObject }
cache.addObjectsNotCached(databaseObjects)
}
func createAndSaveStatusesForArticles(_ articles: Set<Article>, _ database: FMDatabase) {
let articleIDs = Set(articles.map { $0.articleID })
@ -215,31 +98,9 @@ private extension StatusesTable {
let now = Date()
let statuses = articleIDs.map { ArticleStatus(articleID: $0, dateArrived: now) }
let databaseObjects = statuses.map { $0 as DatabaseObject }
cache.addObjectsNotCached(databaseObjects)
cacheStatuses(statuses)
saveStatuses(Set(statuses), database)
}
// MARK: Utilities
// func articleIDsMissingCachedStatuses(_ articleIDs: Set<String>) -> Set<String> {
//
// return Set(articleIDs.filter { !cache.objectWithIDIsCached($0) })
// }
//
// func articlesMissingStatuses(_ articles: Set<Article>) -> Set<Article> {
//
// return articles.withNilProperty(\Article.status)
// }
}
//extension ParsedItem {
//
// var articleID: String {
// get {
// return "\(feedURL) \(uniqueID)" //Must be same as Article.articleID
// }
// }
//}

8
Frameworks/RSParser/Feeds/ParsedItem.swift
View File

@ -10,7 +10,8 @@ import Foundation
public struct ParsedItem: Hashable {
public let uniqueID: String
public let syncServiceID: String? //Nil when not syncing
public let uniqueID: String //RSS guid, for instance; may be calculated
public let feedURL: String
public let url: String?
public let externalURL: String?
@ -27,8 +28,9 @@ public struct ParsedItem: Hashable {
public let attachments: [ParsedAttachment]?
public let hashValue: Int
init(uniqueID: String, feedURL: String, url: String?, externalURL: String?, title: String?, contentHTML: String?, contentText: String?, summary: String?, imageURL: String?, bannerImageURL: String?, datePublished: Date?, dateModified: Date?, authors: [ParsedAuthor]?, tags: [String]?, attachments: [ParsedAttachment]?) {
init(syncServiceID: String?, uniqueID: String, feedURL: String, url: String?, externalURL: String?, title: String?, contentHTML: String?, contentText: String?, summary: String?, imageURL: String?, bannerImageURL: String?, datePublished: Date?, dateModified: Date?, authors: [ParsedAuthor]?, tags: [String]?, attachments: [ParsedAttachment]?) {
self.syncServiceID = syncServiceID
self.uniqueID = uniqueID
self.feedURL = feedURL
self.url = url
@ -44,7 +46,7 @@ public struct ParsedItem: Hashable {
self.authors = authors
self.tags = tags
self.attachments = attachments
self.hashValue = uniqueID.hashValue
self.hashValue = articleID.hashValue
}
public static func ==(lhs: ParsedItem, rhs: ParsedItem) -> Bool {

BIN
ToDo.ooutline
View File

Binary file not shown.