Make attachments use a DatabaseLookupTable.

This commit is contained in:
Brent Simmons 2017-08-20 17:46:15 -07:00
parent 213b1d7a6f
commit c164c29cde
13 changed files with 77 additions and 320 deletions

View File

@ -12,7 +12,7 @@ public final class Article: Hashable {
weak var account: Account?
public let databaseID: String
public let articleID: String
public let feedID: String // Likely a URL, but not necessarily
public let uniqueID: String // Unique per feed (RSS guid, for example)
public var title: String?

View File

@ -18,7 +18,7 @@ public struct Author: Hashable {
public let emailAddress: String?
public let hashValue: Int
public init?(databaseID: String?, name: String?, url: String?, avatarURL: String?, emailAddress: String?) {
public init?(authorID: String?, name: String?, url: String?, avatarURL: String?, emailAddress: String?) {
if name == nil && url == nil && emailAddress == nil {
return nil
@ -34,11 +34,11 @@ public struct Author: Hashable {
s += emailAddress ?? ""
self.hashValue = s.hashValue
if let databaseID = databaseID {
self.databaseID = databaseID
if let authorID = authorID {
self.authorID = authorID
}
else {
self.databaseID = databaseIDWithString(s)
self.authorID = databaseIDWithString(s)
}
}

View File

@ -10,244 +10,34 @@ import Foundation
import RSDatabase
import Data
// Attachments are treated as atomic.
// If an attachment in a feed changes any of its values,
// its actually saved as a new attachment and the old one is deleted.
// (This is rare compared to an article in a feed changing its text, for instance.)
//
// Article -> Attachment is one-to-many.
// Attachment -> Article is one-to-one.
// A given attachment can be owned by one and only one Article.
// An attachment with the same exact values (except for articleID) might exist.
// (That would be quite rare. But its by design.)
//
// All the functions here must be called only from inside the database serial queue.
// (The serial queue makes locking unnecessary.)
//
// Attachments are cached, for the lifetime of the app run, once fetched or saved.
// Because:
// * They dont take up much space.
// * It seriously cuts down on the number of database reads and writes.
//
// CREATE TABLE if not EXISTS attachments(databaseID TEXT NOT NULL PRIMARY KEY, articleID TEXT NOT NULL, url TEXT NOT NULL, mimeType TEXT, title TEXT, sizeInBytes INTEGER, durationInSeconds INTEGER);
final class AttachmentsTable: DatabaseTable {
struct AttachmentsTable: DatabaseTable {
let name: String
private let cacheByArticleID = ObjectCache<Attachment>(keyPathForID: \Attachment.articleID)
private let cacheByDatabaseID = ObjectCache<Attachment>(keyPathForID: \Attachment.databaseID)
let databaseIDKey = DatabaseKey.attachmentID
private let cache = DatabaseObjectCache()
init(name: String) {
self.name = name
}
private var cachedAttachments = [String: Attachment]() // Attachment.databaseID key
private var cachedAttachmentsByArticle = [String: Set<Attachment>]() // Article.databaseID key
private var articlesWithNoAttachments = Set<String>() // Article.databaseID
func fetchAttachmentsForArticles(_ articles: Set<Article>, database: FMDatabase) {
}
func saveAttachmentsForArticles(_ articles: Set<Article>, database: FMDatabase) {
// This is complex and overly long because its optimized for fewest database hits.
var articlesWithPossiblyAllAttachmentsDeleted = Set<Article>()
var attachmentsToSave = Set<Attachment>()
var attachmentsToDelete = Set<Attachment>()
func reconcileAttachments(incomingAttachments: Set<Attachment>, existingAttachments: Set<Attachment>) {
for oneIncomingAttachment in incomingAttachments { // Add some.
if !existingAttachments.contains(oneIncomingAttachment) {
attachmentsToSave.insert(oneIncomingAttachment)
}
}
for oneExistingAttachment in existingAttachments { // Delete some.
if !incomingAttachments.contains(oneExistingAttachment) {
attachmentsToDelete.insert(oneExistingAttachment)
}
}
}
for oneArticle in articles {
if let oneAttachments = oneArticle.attachments, !oneAttachments.isEmpty {
// If it matches the cache, then do nothing.
if let oneCachedAttachments = cachedAttachmentsByArticle(oneArticle.databaseID) {
if oneCachedAttachments == oneAttachments {
continue
}
// There is a cache and it doesnt match.
reconcileAttachments(incomingAttachments: oneAttachments, existingAttachments: oneCachedAttachments)
}
else { // no cache, but article has attachments
if let resultSet = table.selectRowsWhere(key: DatabaseKey.articleID, equals: oneArticle.databaseID, in: database) {
let existingAttachments = attachmentsWithResultSet(resultSet)
if existingAttachments != oneAttachments { // Dont match?
reconcileAttachments(incomingAttachments: oneAttachments, existingAttachments: existingAttachments)
}
}
else {
// Nothing in database. Just save.
attachmentsToSave.formUnion(oneAttachments)
}
}
cacheAttachmentsForArticle(oneArticle)
}
else {
// No attachments: might need to delete them all from database
if !articlesWithNoAttachments.contains(oneArticle.databaseID) {
articlesWithPossiblyAllAttachmentsDeleted.insert(oneArticle)
uncacheAttachmentsForArticle(oneArticle)
}
}
}
deleteAttachmentsForArticles(articlesWithPossiblyAllAttachmentsDeleted, database)
deleteAttachments(attachmentsToDelete, database)
saveAttachments(attachmentsToSave, database)
// MARK: DatabaseTable Methods
func objectWithRow(_ row: FMResultSet) -> DatabaseObject? {
return attachmentWithRow(row) as DatabaseObject
}
}
private extension AttachmentsTable {
func deleteAttachmentsForArticles(_ articles: Set<Article>, _ database: FMDatabase) {
if articles.isEmpty {
return
}
articles.forEach { uncacheAttachmentsForArticle($0) }
let articleIDs = articles.map { $0.databaseID }
deleteRowsWhere(key: DatabaseKey.articleID, equalsAnyValue: articlesIDs, in: database)
}
func deleteAttachments(_ attachments: Set<Attachment>, _ database: FMDatabase) {
if attachments.isEmpty {
return
}
let databaseIDs = attachments.map { $0.databaseID }
deleteRowsWhere(key: DatabaseKey.databaseID, equalsAnyValue: databaseIDs, in: database)
}
func saveAttachments(_ attachments: Set<Attachment>, _ database: FMDatabase) {
if attachments.isEmpty {
return
}
}
func addCachedAttachmentsToArticle(_ article: Article) {
if let _ = article.attachments {
return
}
if let attachments = cachedAttachmentsByArticle[article.databaseID] {
article.attachments = attachments
}
}
func fetchAttachmentsForArticle(_ article: Article, database: FMDatabase) {
if articlesWithNoAttachments.contains(article.databaseID) {
return
}
addCachedAttachmentsToArticle(article)
if let _ = article.attachments {
return
}
}
func uncacheAttachmentsForArticle(_ article: Article) {
assert(article.attachments == nil || article.attachments.isEmpty)
articlesWithNoAttachments.insert(article.databaseID)
cachedAttachmentsByArticle[article.databaseID] = nil
}
func cacheAttachmentsForArticle(_ article: Article) {
guard let attachments = article.attachments, !attachments.isEmpty else {
assertionFailure("article.attachments must not be empty")
}
articlesWithNoAttachments.remove(article.databaseID)
cachedAttachmentsByArticle[article.databaseID] = attachments
cacheAttachment(attachments)
}
func cachedAttachmentForDatabaseID(_ databaseID: String) -> Attachment? {
return cachedAttachments[databaseID]
}
func cacheAttachments(_ attachments: Set<Attachment>) {
attachments.forEach { cacheAttachment($) }
}
func cacheAttachment(_ attachment: Attachment) {
cachedAttachments[attachment.databaseID] = attachment
}
func uncacheAttachments(_ attachments: Set<Attachment>) {
attachments.removeO
attachments.forEach { uncacheAttachment($0) }
}
func uncacheAttachment(_ attachment: Attachment) {
cachedAttachments[attachment.databaseID] = nil
}
func saveAttachmentsForArticle(_ article: Article, database: FMDatabase) {
if let attachments = article.attachments {
}
else {
if articlesWithNoAttachments.contains(article.databaseID) {
return
}
articlesWithNoAttachments.insert(article.databaseID)
cachedAttachmentsByArticle[article.databaseID] = nil
deleteAttachmentsForArticleID(article.databaseID)
}
}
func attachmentsWithResultSet(_ resultSet: FMResultSet) -> Set<Attachment> {
return resultSet.mapToSet(attachmentWithRow)
}
func attachmentWithRow(_ row: FMResultSet) -> Attachment? {
let databaseID = row.string(forColumn: DatabaseKey.databaseID)
if let cachedAttachment = cachedAttachmentForDatabaseID(databaseID) {
let attachmentID = row.string(forColumn: DatabaseKey.attachmentID)
if let cachedAttachment = cache(attachmentID) {
return cachedAttachment
}
return Attachment(databaseID: databaseID, row: row)
return Attachment(attachmentID: attachmentID, row: row)
}
}

View File

@ -13,14 +13,15 @@ import Data
// article->authors is a many-to-many relationship.
// Theres a lookup table relating authorID and articleID.
//
// CREATE TABLE if not EXISTS authors (databaseID TEXT NOT NULL PRIMARY KEY, name TEXT, url TEXT, avatarURL TEXT, emailAddress TEXT);
// CREATE TABLE if not EXISTS authors (authorID TEXT NOT NULL PRIMARY KEY, name TEXT, url TEXT, avatarURL TEXT, emailAddress TEXT);
// CREATE TABLE if not EXISTS authorLookup (authorID TEXT NOT NULL, articleID TEXT NOT NULL, PRIMARY KEY(authorID, articleID));
struct AuthorsTable: DatabaseTable {
let name: String
private let cache = ObjectCache<Author>(keyPathForID: \Author.databaseID)
let databaseIDKey = DatabaseKey.authorID
private let cache = DatabaseObjectCache()
init(name: String) {
@ -29,98 +30,29 @@ struct AuthorsTable: DatabaseTable {
// MARK: DatabaseTable Methods
func fetchObjectsWithIDs(_ databaseIDs: Set<String>, in database: FMDatabase) -> [DatabaseObject] {
func objectWithRow(_ row: FMResultSet) -> DatabaseObject? {
return authorWithRow(row) as DatabaseObject
}
func save(_ objects: [DatabaseObject], in database: FMDatabase) {
<#code#>
}
}
private extension AuthorsTable {
func attachCachedAuthors(_ articles: Set<Article>) {
for article in articles {
if let authors = articleIDToAuthorsCache[article.databaseID] {
article.authors = Array(authors)
}
}
}
func articlesNeedingAuthors(_ articles: Set<Article>) -> Set<Article> {
// If article.authors is nil and article is not known to have zero authors, include it in the set.
let articlesWithNoAuthors = articles.withNilProperty(\Article.authors)
return Set(articlesWithNoAuthors.filter { !articleIDsWithNoAuthors.contains($0.databaseID) })
}
func fetchAuthorsForArticleIDs(_ articleIDs: Set<String>, _ database: FMDatabase) -> [String: Set<Author>]? {
let lookupTableDictionary = authorsLookupTable.fetchLookupTableDictionary(articleIDs, database)
let authorIDs = authorsLookupTable.primaryIDsInLookupTableDictionary(lookupTableDictionary)
if authorIDs.isEmpty {
return nil
}
guard let resultSet = selectRowsWhere(key: DatabaseKey.databaseID, inValues: Array(authorIDs), in: database) else {
return nil
}
let authors = authorsWithResultSet(resultSet)
if authors.isEmpty {
return nil
}
return authorTableWithLookupValues(lookupValues)
}
func authorTableWithLookupValues(_ lookupValues: Set<LookupValue>) -> [String: Set<Author>] {
var authorTable = [String: Set<Author>]()
for lookupValue in lookupValues {
let authorID = lookupValue.primaryID
guard let author = cache[authorID] else {
continue
}
let articleID = lookupValue.foreignID
if authorTable[articleID] == nil {
authorTable[articleID] = Set([author])
}
else {
authorTable[articleID]!.insert(author)
}
}
return authorTable
}
func authorsWithResultSet(_ resultSet: FMResultSet) -> Set<Author> {
return resultSet.mapToSet(authorWithRow)
}
func authorWithRow(_ row: FMResultSet) -> Author? {
guard let databaseID = row.string(forColumn: DatabaseKey.databaseID) else {
guard let authorID = row.string(forColumn: DatabaseKey.authorID) else {
return nil
}
if let cachedAuthor = cache[databaseID] {
if let cachedAuthor = cache[authorID] {
return cachedAuthor
}
guard let author = Author(databaseID: databaseID, row: row) else {
guard let author = Author(authorID: authorID, row: row) else {
return nil
}
cache[databaseID] = author
cache[authorID] = author
return author
}
}

View File

@ -12,10 +12,11 @@ public struct DatabaseTableName {
static let articles = "articles"
static let authors = "authors"
static let authorsLookup = "authorLookup"
static let authorsLookup = "authorsLookup"
static let statuses = "statuses"
static let tags = "tags"
static let attachments = "attachments"
static let attachmentsLookup = "attachmentsLookup"
}
public struct DatabaseKey {
@ -49,6 +50,7 @@ public struct DatabaseKey {
static let dateArrived = "dateArrived"
// Attachment
static let attachmentID = "attachmentID"
static let mimeType = "mimeType"
static let sizeInBytes = "sizeInBytes"
static let durationInSeconds = "durationInSeconds"

View File

@ -2,16 +2,15 @@ CREATE TABLE if not EXISTS articles (articleID TEXT NOT NULL PRIMARY KEY, feedID
CREATE TABLE if not EXISTS statuses (articleID TEXT NOT NULL PRIMARY KEY, read BOOL NOT NULL DEFAULT 0, starred BOOL NOT NULL DEFAULT 0, userDeleted BOOL NOT NULL DEFAULT 0, dateArrived DATE NOT NULL DEFAULT 0, accountInfo BLOB);
CREATE TABLE if not EXISTS authors (databaseID TEXT NOT NULL PRIMARY KEY, name TEXT, url TEXT, avatarURL TEXT, emailAddress TEXT);
CREATE TABLE if not EXISTS authorLookup (authorID TEXT NOT NULL, articleID TEXT NOT NULL, PRIMARY KEY(authorID, articleID));
CREATE TABLE if not EXISTS authors (authorID TEXT NOT NULL PRIMARY KEY, name TEXT, url TEXT, avatarURL TEXT, emailAddress TEXT);
CREATE TABLE if not EXISTS authorsLookup (authorID TEXT NOT NULL, articleID TEXT NOT NULL, PRIMARY KEY(authorID, articleID));
CREATE TABLE if not EXISTS tags(tagName TEXT NOT NULL, articleID TEXT NOT NULL, PRIMARY KEY(tagName, articleID));
CREATE TABLE if not EXISTS attachments(databaseID TEXT NOT NULL PRIMARY KEY, articleID TEXT NOT NULL, url TEXT NOT NULL, mimeType TEXT, title TEXT, sizeInBytes INTEGER, durationInSeconds INTEGER);
CREATE TABLE if not EXISTS attachments(attachmentID TEXT NOT NULL PRIMARY KEY, url TEXT NOT NULL, mimeType TEXT, title TEXT, sizeInBytes INTEGER, durationInSeconds INTEGER);
CREATE TABLE if not EXISTS attachmentsLookup(attachmentID TEXT NOT NULL, articleID TEXT NOT NULL, PRIMARY KEY(attachmentID, articleID));
CREATE INDEX if not EXISTS articles_feedID_index on articles (feedID);
CREATE INDEX if not EXISTS tags_tagName_index on tags (tagName COLLATE NOCASE);
CREATE INDEX if not EXISTS attachments_articleID_index on attachments (articleID);

View File

@ -27,10 +27,9 @@ final class Database {
private let queue: RSDatabaseQueue
private let databaseFile: String
private let articlesTable: ArticlesTable
private let authorsTable: AuthorsTable
private let authorsLookupTable: DatabaseLookupTable
private let attachmentsTable: AttachmentsTable
private let statusesTable: StatusesTable
private let authorsLookupTable: DatabaseLookupTable
private let attachmentsLookupTable: DatabaseLookupTable
private let tagsLookupTable: DatabaseLookupTable
private var articleArrivalCutoffDate = NSDate.rs_dateWithNumberOfDays(inThePast: 3 * 31)!
private let minimumNumberOfArticles = 10
@ -43,15 +42,17 @@ final class Database {
self.queue = RSDatabaseQueue(filepath: databaseFile, excludeFromBackup: false)
self.articlesTable = ArticlesTable(name: DatabaseTableName.articles, queue: queue)
self.attachmentsTable = AttachmentsTable(name: DatabaseTableName.attachments)
self.statusesTable = StatusesTable(name: DatabaseTableName.statuses)
self.authorsTable = AuthorsTable(name: DatabaseTableName.authors)
let authorsTable = AuthorsTable(name: DatabaseTableName.authors)
self.authorsLookupTable = DatabaseLookupTable(name: DatabaseTableName.authorsLookup, objectIDKey: DatabaseKey.articleID, relatedObjectIDKey: DatabaseKey.authorID, relatedTable: authorsTable, relationshipName: RelationshipName.authors)
let tagsTable = TagsTable(name: DatabaseTableName.tags)
self.tagsLookupTable = DatabaseLookupTable(name: DatabaseTableName.tags, objectIDKey: DatabaseKey.articleID, relatedObjectIDKey: DatabaseKey.tagName, relatedTable: tagsTable, relationshipName: RelationshipName.tags)
let attachmentsTable = AttachmentsTable(name: DatabaseTableName.attachments)
self.attachmentsLookupTable = DatabaseLookupTable(name: DatabaseTableName.tags, objectIDKey: DatabaseKey.articleID, relatedObjectIDKey: DatabaseKey.tagName, relatedTable: tagsTable, relationshipName: RelationshipName.tags)
let createStatementsPath = Bundle(for: type(of: self)).path(forResource: "CreateStatements", ofType: "sql")!
let createStatements = try! NSString(contentsOfFile: createStatementsPath, encoding: String.Encoding.utf8.rawValue)
queue.createTables(usingStatements: createStatements as String)

View File

@ -49,6 +49,15 @@ extension Article {
}
}
extension Article: DatabaseObject {
var databaseID: String {
get {
return articleID
}
}
}
extension Set where Element == Article {
func withNilProperty<T>(_ keyPath: KeyPath<Article,T?>) -> Set<Article> {

View File

@ -23,7 +23,7 @@ extension Author {
}
}
extension Author: DatabaseObject {
public extension Author: DatabaseObject {
var databaseID: String {
get {

View File

@ -18,7 +18,7 @@ import Data
final class StatusesTable: DatabaseTable {
let name: String
private let cache = ObjectCache<ArticleStatus>(keyPathForID: \ArticleStatus.articleID)
private let cache = DatabaseObjectCache()
init(name: String) {

View File

@ -20,7 +20,7 @@ import Data
struct TagsTable: DatabaseTable {
let name: String
let databaseIDKey = DatabaseKey.tagName
init(name: String) {
self.name = name
@ -34,6 +34,11 @@ struct TagsTable: DatabaseTable {
return databaseIDs.map{ $0 as DatabaseObject }
}
func objectWithRow(_ row: FMResultSet) -> DatabaseObject? {
return nil //unused
}
func save(_ objects: [DatabaseObject], in database: FMDatabase) {
// Nothing to do, since tags are saved in the lookup table, not in a separate table.

View File

@ -11,13 +11,32 @@ import Foundation
public protocol DatabaseTable {
var name: String { get }
var databaseIDKey: String { get}
func fetchObjectsWithIDs(_ databaseIDs: Set<String>, in database: FMDatabase) -> [DatabaseObject]
func objectsWithResultSet(_ resultSet: FMResultSet) -> [DatabaseObject]
func objectWithRow(_ row: FMResultSet) -> DatabaseObject?
func save(_ objects: [DatabaseObject], in database: FMDatabase)
}
public extension DatabaseTable {
// MARK: Default implementations
func fetchObjectsWithIDs(_ databaseIDs: Set<String>, in database: FMDatabase) -> [DatabaseObject] {
guard let resultSet = selectRowsWhere(key: databaseIDKey, inValues: Array(databaseIDs), in: database) else {
return [DatabaseObject]()
}
return objectsWithResultSet(resultSet)
}
func objectsWithResultSet(_ resultSet: FMResultSet) -> [DatabaseObject] {
return resultSet.flatMap(objectWithRow)
}
// MARK: Fetching
public func selectRowsWhere(key: String, equals value: Any, in database: FMDatabase) -> FMResultSet? {

View File

@ -31,7 +31,7 @@ public final class DatabaseLookupTable {
self.cache = DatabaseLookupTableCache(relationshipName)
}
public func attachRelationships(to objects: [DatabaseObject], in database: FMDatabase) {
public func attachRelatedObjects(to objects: [DatabaseObject], in database: FMDatabase) {
let objectsThatMayHaveRelatedObjects = cache.objectsThatMayHaveRelatedObjects(objects)
if objectsThatMayHaveRelatedObjects.isEmpty {
@ -54,7 +54,7 @@ public final class DatabaseLookupTable {
cache.update(with: objectsNeedingFetching)
}
public func saveRelationships(for objects: [DatabaseObject], in database: FMDatabase) {
public func saveRelatedObjects(for objects: [DatabaseObject], in database: FMDatabase) {
var objectsWithNoRelationships = [DatabaseObject]()
var objectsWithRelationships = [DatabaseObject]()