diff --git a/Frameworks/Database/AttachmentsTable.swift b/Frameworks/Database/AttachmentsTable.swift index c58d6b9d6..df94e41a0 100644 --- a/Frameworks/Database/AttachmentsTable.swift +++ b/Frameworks/Database/AttachmentsTable.swift @@ -28,6 +28,8 @@ import Data // Because: // * They don’t take up much space. // * It seriously cuts down on the number of database reads and writes. +// +// CREATE TABLE if not EXISTS attachments(databaseID TEXT NOT NULL PRIMARY KEY, articleID TEXT NOT NULL, url TEXT NOT NULL, mimeType TEXT, title TEXT, sizeInBytes INTEGER, durationInSeconds INTEGER); final class AttachmentsTable: DatabaseTable { diff --git a/Frameworks/Database/AuthorsTable.swift b/Frameworks/Database/AuthorsTable.swift index 3a03cc4f4..09320ba35 100644 --- a/Frameworks/Database/AuthorsTable.swift +++ b/Frameworks/Database/AuthorsTable.swift @@ -23,7 +23,6 @@ final class AuthorsTable: DatabaseTable { let queue: RSDatabaseQueue private let cache = ObjectCache(keyPathForID: \Author.databaseID) private var articleIDToAuthorsCache = [String: Set]() - private var articleIDsWithNoAuthors = Set() private let authorsLookupTable = LookupTable(name: DatabaseTableName.authorsLookup, primaryKey: DatabaseKey.authorID, foreignKey: DatabaseKey.articleID) init(name: String, queue: RSDatabaseQueue) { @@ -36,15 +35,15 @@ final class AuthorsTable: DatabaseTable { attachCachedAuthors(articles) - let articlesNeedingAuthors = articlesMissingAuthors(articles) - if articlesNeedingAuthors.isEmpty { + let articlesMissingAuthors = articlesNeedingAuthors(articles) + if articlesMissingAuthors.isEmpty { return } - let articleIDs = Set(articlesNeedingAuthors.map { $0.databaseID }) + let articleIDs = Set(articlesMissingAuthors.map { $0.databaseID }) let authorTable = fetchAuthorsForArticleIDs(articleIDs, database) - for article in articlesNeedingAuthors { + for article in articlesMissingAuthors { let articleID = article.databaseID @@ -70,24 +69,16 @@ private extension AuthorsTable { } } - func articlesMissingAuthors(_ articles: Set
) -> Set
{ + func articlesNeedingAuthors(_ articles: Set
) -> Set
{ - return articles.filter{ (article) -> Bool in - - if let _ = article.authors { - return false - } - if articleIDsWithNoAuthors.contains(article.databaseID) { - return false - } - - return true - } + // If article.authors is nil and article is not known to have zero authors, include it in the set. + let articlesWithNoAuthors = articles.withNilProperty(\Article.authors) + return Set(articlesWithNoAuthors.filter { !articleIDsWithNoAuthors.contains($0.databaseID) }) } func fetchAuthorsForArticleIDs(_ articleIDs: Set, _ database: FMDatabase) -> [String: Set]? { - let lookupValues = authorsLookupTable.fetchLookupValues(articleIDs, database: database) + let lookupValueDictionary = authorsLookupTable.fetchLookupTableDictionary(articleIDs, database) let authorIDs = Set(lookupValues.map { $0.primaryID }) if authorIDs.isEmpty { return nil diff --git a/Frameworks/Database/Extensions/Article+Database.swift b/Frameworks/Database/Extensions/Article+Database.swift index 811613d32..da741dfe8 100644 --- a/Frameworks/Database/Extensions/Article+Database.swift +++ b/Frameworks/Database/Extensions/Article+Database.swift @@ -48,3 +48,16 @@ extension Article { return d.copy() as! NSDictionary } } + +extension Set where Element == Article { + + func withNilProperty(_ keyPath: KeyPath) -> Set
{ + + return Set(filter{ $0[keyPath: keyPath] == nil }) + } + + func articleIDs() -> Set { + + return Set(map { $0.databaseID }) + } +} diff --git a/Frameworks/Database/StatusesTable.swift b/Frameworks/Database/StatusesTable.swift index 69b49d9f9..e76b80cbf 100644 --- a/Frameworks/Database/StatusesTable.swift +++ b/Frameworks/Database/StatusesTable.swift @@ -121,7 +121,7 @@ private extension StatusesTable { func fetchAndCacheStatusesForArticles(_ articles: Set
, _ database: FMDatabase) { - fetchAndCacheStatusesForArticleIDs(articleIDsFromArticles(articles), database) + fetchAndCacheStatusesForArticleIDs(articles.articleIDs(), database) } func fetchAndCacheStatusesForArticleIDs(_ articleIDs: Set, _ database: FMDatabase) { @@ -210,11 +210,6 @@ private extension StatusesTable { // MARK: Utilities - func articleIDsFromArticles(_ articles: Set
) -> Set { - - return Set(articles.map { $0.databaseID }) - } - func articleIDsMissingCachedStatuses(_ articleIDs: Set) -> Set { return Set(articleIDs.filter { !cache.objectWithIDIsCached($0) }) @@ -222,13 +217,7 @@ private extension StatusesTable { func articlesMissingStatuses(_ articles: Set
) -> Set
{ - let missing = articles.flatMap { (article) -> Article? in - if article.status == nil { - return article - } - return nil - } - return Set(missing) + return articles.withNilProperty(\Article.status) } } diff --git a/Frameworks/Database/TagsTable.swift b/Frameworks/Database/TagsTable.swift index fad223d3e..329f8936f 100644 --- a/Frameworks/Database/TagsTable.swift +++ b/Frameworks/Database/TagsTable.swift @@ -13,10 +13,6 @@ import Data // Article->tags is a many-to-many relationship. // Since a tag is just a simple string, the tags table and the lookup table are the same table. // -// Tags — and the non-existence of tags — are cached, once fetched, for the lifetime of the run. -// This uses some extra memory but cuts way down on the amount of database time spent -// maintaining the tags table. -// // CREATE TABLE if not EXISTS tags(tagName TEXT NOT NULL, articleID TEXT NOT NULL, PRIMARY KEY(tagName, articleID)); // CREATE INDEX if not EXISTS tags_tagName_index on tags (tagName COLLATE NOCASE); @@ -26,194 +22,214 @@ final class TagsTable: DatabaseTable { let name: String let queue: RSDatabaseQueue - - private var articleIDCache = [String: TagNameSet]() // articleID: tags - private var articleIDsWithNoTags = Set + let lookupTable: LookupTable init(name: String, queue: RSDatabaseQueue) { self.name = name self.queue = queue + self.lookupTable = LookupTable(name: DatabaseTableName.tags, primaryKey: DatabaseKey.tagName, foreignKey: DatabaseKey.articleID) } - func saveTagsForArticles(_ articles: Set
) { + func attachTags(_ articles: Set
, _ database: FMDatabase) { - var articlesToSaveTags = Set
() - var articlesToRemoveTags = Set
() - - articles.forEach { (oneArticle) in - - if articleTagsMatchCache(oneArticle) { - return - } - if let tags = oneArticle.tags { - articlesToSaveTags.insert(oneArticle) - } - else { - articlesToRemoveTags.insert(oneArticle) - } + guard let lookupTableDictionary = lookupTable.fetchLookupTableDictionary(articleIDs, database) else { + return } - if !articlesToSaveTags.isEmpty { - updateTagsForArticles(articlesToSaveTags) - } - - if !articlesToRemoveTags.isEmpty { - removeArticleFromTags(articlesToRemoveTags) + for article in articles { + if let lookupValues = lookupTableDictionary[article.databaseID] { + article.tags = lookupValues.tags() + } } } + +// func saveTagsForArticles(_ articles: Set
) { +// +// var articlesToSaveTags = Set
() +// var articlesToRemoveTags = Set
() +// +// articles.forEach { (oneArticle) in +// +// if articleTagsMatchCache(oneArticle) { +// return +// } +// if let tags = oneArticle.tags { +// articlesToSaveTags.insert(oneArticle) +// } +// else { +// articlesToRemoveTags.insert(oneArticle) +// } +// } +// +// if !articlesToSaveTags.isEmpty { +// updateTagsForArticles(articlesToSaveTags) +// } +// +// if !articlesToRemoveTags.isEmpty { +// removeArticleFromTags(articlesToRemoveTags) +// } +// } } private extension TagsTable { - func cacheTagsForArticle(_ article: Article, tags: TagNameSet) { +// func cacheTagsForArticle(_ article: Article, tags: TagNameSet) { +// +// articleIDsWithNoTags.remove(article.articleID) +// articleIDCache[article.articleID] = tags +// } +// +// func cachedTagsForArticleID(_ articleID: String) -> TagNameSet? { +// +// return articleIDsCache[articleID] +// } +// +// func articleTagsMatchCache(_ article: Article) -> Bool { +// +// if let tags = article.tags { +// return tags == articleIDCache[article.articleID] +// } +// return articleIDIsKnowToHaveNoTags(article.articleID) +// } +// +// func articleIDIsKnownToHaveNoTags(_ articleID: String) -> Bool { +// +// return articleIDsWithNoTags.contains(articleID) +// } +// +// func removeTagsFromCacheForArticleID(_ articleID: String) { +// +// articleIDsCache[oneArticleID] = nil +// articleIDsWithNoTags.insert(oneArticleID) +// } +// +// func removeArticleFromTags(_ articles: Set
) { +// +// var articleIDsToRemove = [String]() +// +// articles.forEach { (oneArticle) in +// let oneArticleID = oneArticle.articleID +// if articleIDIsKnownToHaveNoTags(oneArticle) { +// return +// } +// articleIDsToRemove += oneArticleID +// removeTagsFromCacheForArticleID(oneArticleID) +// } +// +// if !articleIDsToRemove.isEmpty { +// queue.update { (database) in +// database.rs_deleteRowsWhereKey(DatabaseKey.articleID, inValues: articleIDsToRemove, tableName: DatabaseTableName.tags) +// } +// } +// } +// +// typealias TagsTable = [String: TagNameSet] // [articleID: Set] +// +// func updateTagsForArticles(_ articles: Set
) { +// +// var tagsForArticleIDs = TagsTable() +// articles.forEach { (oneArticle) +// if let tags = oneArticle.tags { +// cacheTagsForArticle(oneArticle, tags) +// tagsForArticleIDs[oneArticle.articleID] = oneArticle.tags +// } +// else { +// assertionFailure("article must have tags") +// } +// } +// +// if tagsForArticleIDs.isEmpty { // Shouldn’t be empty +// return +// } +// let articleIDs = tagsForArticleIDs.keys +// +// queue.update { (database) in +// +// let existingTags = self.fetchTagsForArticleIDs(articleIDs, database: database) +// self.syncIncomingAndExistingTags(incomingTags: tagsForArticleIDs, existingTags: existingTags, database: database) +// } +// } +// +// func syncIncomingAndExistingTags(incomingTags: TagsTable, existingTags: TagsTable, database: database) { +// +// for (oneArticleID, oneTagNames) in incomingTags { +// if let existingTagNames = existingTags[oneArticleID] { +// syncIncomingAndExistingTagsForArticleID(oneArticleID, incomingTagNames: oneTagNames, existingTagNames: existingTagNames, database: database) +// } +// else { +// saveIncomingTagsForArticleID(oneArticleID, tagNames: oneTagNames, database: database) +// } +// } +// } +// +// func saveIncomingTagsForArticleID(_ articleID: String, tagNames: TagNameSet, database: FMDatabase) { +// +// // No existing tags in database. Simple save. +// +// for oneTagName in tagNames { +// let oneDictionary = [DatabaseTableName.articleID: articleID, DatabaseTableName.tagName: oneTagName] +// database.rs_insertRow(with: oneDictionary, insertType: .OrIgnore, tableName: DatabaseTableName.tags) +// } +// } +// +// func syncingIncomingAndExistingTagsForArticleID(_ articleID: String, incomingTagNames: TagNameSet, existingTagNames: TagNameSet, database: FMDatabase) { +// +// if incomingTagNames == existingTagNames { +// return +// } +// +// var tagsToRemove = TagNameSet() +// for oneExistingTagName in existingTagNames { +// if !incomingTagNames.contains(oneExistingTagName) { +// tagsToRemove.insert(oneExistingTagName) +// } +// } +// +// var tagsToAdd = TagNameSet() +// for oneIncomingTagName in incomingTagNames { +// if !existingTagNames.contains(oneIncomingTagName) { +// tagsToAdd.insert(oneIncomingTagName) +// } +// } +// +// if !tagsToRemove.isEmpty { +// let placeholders = NSString.rs_SQLValueListWithPlaceholders +// let sql = "delete from \(DatabaseTableName.tags) where \(DatabaseKey.articleID) = ? and \(DatabaseKey.tagName) in " +// database.executeUpdate(sql, withArgumentsIn: [articleID, ]) +// } +// } +// +// func fetchTagsForArticleIDs(_ articleIDs: Set, database: FMDatabase) -> TagsTable { +// +// var tagSpecifiers = TagsTable() +// +// guard let rs = database.rs_selectRowsWhereKey(DatabaseKey.articleID, inValues: Array(articleIDs), tableName: DatabaseTableName.tags) else { +// return tagSpecifiers +// } +// +// while rs.next() { +// +// guard let oneTagName = rs.string(forColumn: DatabaseKey.tagName), let oneArticleID = rs.string(forColumn: DatabaseKey.articleID) else { +// continue +// } +// if tagSpecifiers[oneArticleID] == nil { +// tagSpecifiers[oneArticleID] = Set([oneTagName]) +// } +// else { +// tagSpecifiers[oneArticleID]!.insert(oneTagName) +// } +// } +// +// return tagSpecifiers +// } +} - articleIDsWithNoTags.remove(article.articleID) - articleIDCache[article.articleID] = tags - } +private extension Set where Element == LookupValue { - func cachedTagsForArticleID(_ articleID: String) -> TagNameSet? { + func tags() -> Set { - return articleIDsCache[articleID] - } - - func articleTagsMatchCache(_ article: Article) -> Bool { - - if let tags = article.tags { - return tags == articleIDCache[article.articleID] - } - return articleIDIsKnowToHaveNoTags(article.articleID) - } - - func articleIDIsKnownToHaveNoTags(_ articleID: String) -> Bool { - - return articleIDsWithNoTags.contains(articleID) - } - - func removeTagsFromCacheForArticleID(_ articleID: String) { - - articleIDsCache[oneArticleID] = nil - articleIDsWithNoTags.insert(oneArticleID) - } - - func removeArticleFromTags(_ articles: Set
) { - - var articleIDsToRemove = [String]() - - articles.forEach { (oneArticle) in - let oneArticleID = oneArticle.articleID - if articleIDIsKnownToHaveNoTags(oneArticle) { - return - } - articleIDsToRemove += oneArticleID - removeTagsFromCacheForArticleID(oneArticleID) - } - - if !articleIDsToRemove.isEmpty { - queue.update { (database) in - database.rs_deleteRowsWhereKey(DatabaseKey.articleID, inValues: articleIDsToRemove, tableName: DatabaseTableName.tags) - } - } - } - - typealias TagsTable = [String: TagNameSet] // [articleID: Set] - - func updateTagsForArticles(_ articles: Set
) { - - var tagsForArticleIDs = TagsTable() - articles.forEach { (oneArticle) - if let tags = oneArticle.tags { - cacheTagsForArticle(oneArticle, tags) - tagsForArticleIDs[oneArticle.articleID] = oneArticle.tags - } - else { - assertionFailure("article must have tags") - } - } - - if tagsForArticleIDs.isEmpty { // Shouldn’t be empty - return - } - let articleIDs = tagsForArticleIDs.keys - - queue.update { (database) in - - let existingTags = self.fetchTagsForArticleIDs(articleIDs, database: database) - self.syncIncomingAndExistingTags(incomingTags: tagsForArticleIDs, existingTags: existingTags, database: database) - } - } - - func syncIncomingAndExistingTags(incomingTags: TagsTable, existingTags: TagsTable, database: database) { - - for (oneArticleID, oneTagNames) in incomingTags { - if let existingTagNames = existingTags[oneArticleID] { - syncIncomingAndExistingTagsForArticleID(oneArticleID, incomingTagNames: oneTagNames, existingTagNames: existingTagNames, database: database) - } - else { - saveIncomingTagsForArticleID(oneArticleID, tagNames: oneTagNames, database: database) - } - } - } - - func saveIncomingTagsForArticleID(_ articleID: String, tagNames: TagNameSet, database: FMDatabase) { - - // No existing tags in database. Simple save. - - for oneTagName in tagNames { - let oneDictionary = [DatabaseTableName.articleID: articleID, DatabaseTableName.tagName: oneTagName] - database.rs_insertRow(with: oneDictionary, insertType: .OrIgnore, tableName: DatabaseTableName.tags) - } - } - - func syncingIncomingAndExistingTagsForArticleID(_ articleID: String, incomingTagNames: TagNameSet, existingTagNames: TagNameSet, database: FMDatabase) { - - if incomingTagNames == existingTagNames { - return - } - - var tagsToRemove = TagNameSet() - for oneExistingTagName in existingTagNames { - if !incomingTagNames.contains(oneExistingTagName) { - tagsToRemove.insert(oneExistingTagName) - } - } - - var tagsToAdd = TagNameSet() - for oneIncomingTagName in incomingTagNames { - if !existingTagNames.contains(oneIncomingTagName) { - tagsToAdd.insert(oneIncomingTagName) - } - } - - if !tagsToRemove.isEmpty { - let placeholders = NSString.rs_SQLValueListWithPlaceholders - let sql = "delete from \(DatabaseTableName.tags) where \(DatabaseKey.articleID) = ? and \(DatabaseKey.tagName) in " - database.executeUpdate(sql, withArgumentsIn: [articleID, ]) - } - } - - func fetchTagsForArticleIDs(_ articleIDs: Set, database: FMDatabase) -> TagsTable { - - var tagSpecifiers = TagsTable() - - guard let rs = database.rs_selectRowsWhereKey(DatabaseKey.articleID, inValues: Array(articleIDs), tableName: DatabaseTableName.tags) else { - return tagSpecifiers - } - - while rs.next() { - - guard let oneTagName = rs.string(forColumn: DatabaseKey.tagName), let oneArticleID = rs.string(forColumn: DatabaseKey.articleID) else { - continue - } - if tagSpecifiers[oneArticleID] == nil { - tagSpecifiers[oneArticleID] = Set([oneTagName]) - } - else { - tagSpecifiers[oneArticleID]!.insert(oneTagName) - } - } - - return tagSpecifiers + return Set(flatMap{ $0.primaryID }) } } diff --git a/Frameworks/RSDatabase/RSDatabase/LookupTable.swift b/Frameworks/RSDatabase/RSDatabase/LookupTable.swift index 167e9723c..a984d5a9c 100644 --- a/Frameworks/RSDatabase/RSDatabase/LookupTable.swift +++ b/Frameworks/RSDatabase/RSDatabase/LookupTable.swift @@ -11,12 +11,19 @@ import Foundation // Implement a lookup table for a many-to-many relationship. // Example: CREATE TABLE if not EXISTS authorLookup (authorID TEXT NOT NULL, articleID TEXT NOT NULL, PRIMARY KEY(authorID, articleID)); // authorID is primaryKey; articleID is foreignKey. +// +// foreignIDsWithNoRelationship: caches the foreignIDs where it’s known that there’s no relationship. +// lookupsByForeignID: caches the LookupValues for a foreignID. -public struct LookupTable { +typealias LookupTableDictionary = [String: Set] // key is foreignID + +public final class LookupTable { let name: String let primaryKey: String let foreignKey: String + private var foreignIDsWithNoRelationship = Set() + private var lookupsByForeignID = LookupTableDictionary() public init(name: String, primaryKey: String, foreignKey: String) { @@ -25,17 +32,94 @@ public struct LookupTable { self.foreignKey = foreignKey } - public func fetchLookupValues(_ foreignIDs: Set, database: FMDatabase) -> Set { + public func fetchLookupTableDictionary(_ foreignIDs: Set, _ database: FMDatabase) -> LookupTableDictionary? { - guard let resultSet = database.rs_selectRowsWhereKey(foreignKey, inValues: Array(foreignIDs), tableName: name) else { - return Set() + let foreignIDsToLookup = foreignIDs.subtracting(foreignIDsWithNoRelationship) + if foreignIDsToLookup.isEmpty { + return nil } - return lookupValuesWithResultSet(resultSet) + + var lookupValues = Set() + var foreignIDsToFetchFromDatabase = Set() + + // Pull from cache. + for oneForeignID in foreignIDsToLookup { + if let cachedLookups = lookupsByForeignID[oneForeignID] { + lookupValues.formUnion(cachedLookups) + } + else { + foreignIDsToFetchFromDatabase.insert(oneForeignID) + } + } + + if !foreignIDsToFetchFromDatabase.isEmpty { + if let resultSet = database.rs_selectRowsWhereKey(foreignKey, inValues: Array(foreignIDsToLookup), tableName: name) { + lookupValues.formUnion(lookupValuesWithResultSet(resultSet)) + } + } + + cacheNotFoundForeignIDs(lookupValues, foreignIDsToFetchFromDatabase) + cacheLookupValues(lookupValues) + + return lookupTableDictionary(with: lookupValues) + } + + public func removeLookupsForForeignIDs(_ foreignIDs: Set, _ database: FMDatabase) { + + let foreignIDsToRemove = foreignIDs.subtracting(foreignIDsWithNoRelationship) + if foreignIDsToRemove.isEmpty { + return + } + + for oneForeignID in foreignIDsToRemove { + lookupsByForeignID[oneForeignID] = nil + } + foreignIDsWithNoRelationship.formUnion(foreignIDsToRemove) + + database.rs_deleteRowsWhereKey(foreignKey, inValues: Array(foreignIDsToRemove), tableName: name) } } private extension LookupTable { + func addToLookupTableDictionary(_ lookupValues: Set, _ table: inout LookupTableDictionary) { + + for lookupValue in lookupValues { + let foreignID = lookupValue.foreignID + let primaryID = lookupValue.primaryID + if table[foreignID] == nil { + table[foreignID] = Set([primaryID]) + } + else { + table[foreignID]!.insert(primaryID) + } + } + } + + func lookupTableDictionary(with lookupValues: Set) -> LookupTableDictionary { + + var d = LookupTableDictionary() + addToLookupTableDictionary(lookupValues, &d) + return d + } + + func cacheLookupValues(_ lookupValues: Set) { + + addToLookupTableDictionary(lookupValues, &lookupsByForeignID) + } + + func cacheNotFoundForeignIDs(_ lookupValues: Set, _ foreignIDs: Set) { + + // Note where nothing was found, and cache the foreignID in foreignIDsWithNoRelationship. + + let foundForeignIDs = Set(lookupValues.map { $0.foreignID }) + for foreignID in foreignIDs { + if !foundForeignIDs.contains(foreignID) { + foreignIDsWithNoRelationship.insert(foreignID) + } + } + } + func lookupValuesWithResultSet(_ resultSet: FMResultSet) -> Set { return resultSet.mapToSet(lookupValueWithRow) @@ -71,3 +155,4 @@ public struct LookupValue: Hashable { return lhs.primaryID == rhs.primaryID && lhs.foreignID == rhs.foreignID } } +