FreshRSS
/
NetNewsWire
mirror of https://github.com/Ranchero-Software/NetNewsWire.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make ArticlesDatabase an actor. No serial dispatch queue.

This commit is contained in:
Brent Simmons 2024-03-12 23:01:35 -07:00
parent 78047fcaf7
commit 9b1aa8fc7f
20 changed files with 1153 additions and 1710 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Account/Sources/Account/Account.swift
View File

@ -321,21 +321,12 @@ public final class Account: DisplayNameProvider, UnreadCountProvider, Container,
feedMetadataFile.load()
opmlFile.load()
var shouldHandleRetentionPolicyChange = false
if type == .onMyMac {
let didHandlePolicyChange = metadata.performedApril2020RetentionPolicyChange ?? false
shouldHandleRetentionPolicyChange = !didHandlePolicyChange
}
Task {
await self.database.cleanupDatabaseAtStartup(subscribedToFeedIDs: self.flattenedFeeds().feedIDs())
DispatchQueue.main.async {
if shouldHandleRetentionPolicyChange {
// Handle one-time database changes made necessary by April 2020 retention policy change.
self.database.performApril2020RetentionPolicyChange()
self.metadata.performedApril2020RetentionPolicyChange = true
Task { @MainActor in
self.fetchAllUnreadCounts()
}
self.database.cleanupDatabaseAtStartup(subscribedToFeedIDs: self.flattenedFeeds().feedIDs())
self.fetchAllUnreadCounts()
}
self.delegate.accountDidInitialize(self)
@ -884,7 +875,10 @@ public final class Account: DisplayNameProvider, UnreadCountProvider, Container,
/// Empty caches that can reasonably be emptied. Call when the app goes in the background, for instance.
func emptyCaches() {
database.emptyCaches()
Task.detached {
await self.database.emptyCaches()
}
}
// MARK: - Container

9
Account/Sources/Account/AccountMetadata.swift
View File

@ -24,7 +24,6 @@ final class AccountMetadata: Codable {
case lastArticleFetchEndTime
case endpointURL
case externalID
case performedApril2020RetentionPolicyChange
}
var name: String? {
@ -83,14 +82,6 @@ final class AccountMetadata: Codable {
}
}
var performedApril2020RetentionPolicyChange: Bool? {
didSet {
if performedApril2020RetentionPolicyChange != oldValue {
valueDidChange(.performedApril2020RetentionPolicyChange)
}
}
}
var externalID: String? {
didSet {
if externalID != oldValue {

399
ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabase.swift
View File

@ -7,22 +7,12 @@
//
import Foundation
import RSCore
import Database
import RSParser
import FMDB
import Articles
// This file is the entirety of the public API for ArticlesDatabase.framework.
// Everything else is implementation.
// Main thread only.
import RSParser
public typealias UnreadCountDictionary = [String: Int] // feedID: unreadCount
public typealias UnreadCountDictionaryCompletionResult = Result<UnreadCountDictionary,DatabaseError>
public typealias UnreadCountDictionaryCompletionBlock = (UnreadCountDictionaryCompletionResult) -> Void
public typealias SingleUnreadCountResult = Result<Int, DatabaseError>
public typealias SingleUnreadCountCompletionBlock = (SingleUnreadCountResult) -> Void
public struct ArticleChanges {
public let newArticles: Set<Article>?
@ -34,251 +24,278 @@ public struct ArticleChanges {
self.updatedArticles = Set<Article>()
self.deletedArticles = Set<Article>()
}
public init(newArticles: Set<Article>?, updatedArticles: Set<Article>?, deletedArticles: Set<Article>?) {
self.newArticles = newArticles
self.updatedArticles = updatedArticles
self.deletedArticles = deletedArticles
}
}
public typealias UpdateArticlesResult = Result<ArticleChanges, DatabaseError>
public typealias UpdateArticlesCompletionBlock = (UpdateArticlesResult) -> Void
public typealias ArticleSetResult = Result<Set<Article>, DatabaseError>
public typealias ArticleSetResultBlock = (ArticleSetResult) -> Void
public typealias ArticleIDsResult = Result<Set<String>, DatabaseError>
public typealias ArticleIDsCompletionBlock = (ArticleIDsResult) -> Void
public typealias ArticleStatusesResult = Result<Set<ArticleStatus>, DatabaseError>
public typealias ArticleStatusesResultBlock = (ArticleStatusesResult) -> Void
public final class ArticlesDatabase {
/// Fetch articles and unread counts. Save articles. Mark as read/unread and starred/unstarred.
public actor ArticlesDatabase {
public enum RetentionStyle {
case feedBased // Local and iCloud: article retention is defined by contents of feed
case syncSystem // Feedbin, Feedly, etc.: article retention is defined by external system
/// Local and iCloud: article retention is defined by contents of feed
case feedBased
/// Feedbin, Feedly, etc.: article retention is defined by external system
case syncSystem
}
private let articlesTable: ArticlesTable
private let queue: DatabaseQueue
private let operationQueue = MainThreadOperationQueue()
private var database: FMDatabase?
private var databasePath: String
private let retentionStyle: RetentionStyle
private let articlesTable: ArticlesTable
public init(databaseFilePath: String, accountID: String, retentionStyle: RetentionStyle) {
let queue = DatabaseQueue(databasePath: databaseFilePath)
self.queue = queue
self.articlesTable = ArticlesTable(name: DatabaseTableName.articles, accountID: accountID, queue: queue, retentionStyle: retentionStyle)
public init(databasePath: String, accountID: String, retentionStyle: RetentionStyle) {
let database = FMDatabase.openAndSetUpDatabase(path: databasePath)
database.runCreateStatements(ArticlesDatabase.creationStatements)
self.database = database
self.databasePath = databasePath
self.retentionStyle = retentionStyle
self.articlesTable = ArticlesTable(accountID: accountID, retentionStyle: retentionStyle)
try! queue.runCreateStatements(ArticlesDatabase.tableCreationStatements)
queue.runInDatabase { databaseResult in
let database = databaseResult.database!
if !self.articlesTable.containsColumn("searchRowID", in: database) {
database.executeStatements("ALTER TABLE articles add column searchRowID INTEGER;")
}
database.executeStatements("CREATE INDEX if not EXISTS articles_searchRowID on articles(searchRowID);")
database.executeStatements("DROP TABLE if EXISTS tags;DROP INDEX if EXISTS tags_tagName_index;DROP INDEX if EXISTS articles_feedID_index;DROP INDEX if EXISTS statuses_read_index;DROP TABLE if EXISTS attachments;DROP TABLE if EXISTS attachmentsLookup;")
// Migrate from older schemas
database.beginTransaction()
if !database.columnExists("searchRowID", inTableWithName: DatabaseTableName.articles) {
database.executeStatements("ALTER TABLE articles add column searchRowID INTEGER;")
}
database.executeStatements("CREATE INDEX if not EXISTS articles_searchRowID on articles(searchRowID);")
database.executeStatements("DROP TABLE if EXISTS tags;DROP INDEX if EXISTS tags_tagName_index;DROP INDEX if EXISTS articles_feedID_index;DROP INDEX if EXISTS statuses_read_index;DROP TABLE if EXISTS attachments;DROP TABLE if EXISTS attachmentsLookup;")
database.commit()
DispatchQueue.main.async {
self.articlesTable.indexUnindexedArticles()
Task {
await self.indexUnindexedArticles()
}
}
// MARK: - Fetching Articles
// MARK: - Articles
public func fetchArticles(_ feedID: String) throws -> Set<Article> {
return try articlesTable.fetchArticles(feedID)
}
public func fetchArticles(_ feedIDs: Set<String>) throws -> Set<Article> {
return try articlesTable.fetchArticles(feedIDs)
public func articles(feedID: String) throws -> Set<Article> {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.articles(feedID: feedID, database: database)
}
public func fetchArticles(articleIDs: Set<String>) throws -> Set<Article> {
return try articlesTable.fetchArticles(articleIDs: articleIDs)
public func articles(feedIDs: Set<String>) throws -> Set<Article> {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.articles(feedIDs: feedIDs, database: database)
}
public func fetchUnreadArticles(_ feedIDs: Set<String>, _ limit: Int?) throws -> Set<Article> {
return try articlesTable.fetchUnreadArticles(feedIDs, limit)
public func articles(articleIDs: Set<String>) throws -> Set<Article> {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.articles(articleIDs: articleIDs, database: database)
}
public func fetchTodayArticles(_ feedIDs: Set<String>, _ limit: Int?) throws -> Set<Article> {
return try articlesTable.fetchArticlesSince(feedIDs, todayCutoffDate(), limit)
public func unreadArticles(feedIDs: Set<String>, limit: Int?) throws -> Set<Article> {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.unreadArticles(feedIDs: feedIDs, limit: limit, database: database)
}
public func fetchStarredArticles(_ feedIDs: Set<String>, _ limit: Int?) throws -> Set<Article> {
return try articlesTable.fetchStarredArticles(feedIDs, limit)
public func todayArticles(feedIDs: Set<String>, limit: Int?) throws -> Set<Article> {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.todayArticles(feedIDs: feedIDs, cutoffDate: todayCutoffDate(), limit: limit, database: database)
}
public func fetchArticlesMatching(_ searchString: String, _ feedIDs: Set<String>) throws -> Set<Article> {
return try articlesTable.fetchArticlesMatching(searchString, feedIDs)
public func starredArticles(feedIDs: Set<String>, limit: Int?) throws -> Set<Article> {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.starredArticles(feedIDs: feedIDs, limit: limit, database: database)
}
public func fetchArticlesMatchingWithArticleIDs(_ searchString: String, _ articleIDs: Set<String>) throws -> Set<Article> {
return try articlesTable.fetchArticlesMatchingWithArticleIDs(searchString, articleIDs)
public func articlesMatching(searchString: String, feedIDs: Set<String>) throws -> Set<Article> {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.articlesMatching(searchString: searchString, feedIDs: feedIDs, database: database)
}
// MARK: - Fetching Articles Async
public func articlesMatching(searchString: String, articleIDs: Set<String>) throws -> Set<Article> {
public func fetchArticlesAsync(_ feedID: String, _ completion: @escaping ArticleSetResultBlock) {
articlesTable.fetchArticlesAsync(feedID, completion)
}
public func fetchArticlesAsync(_ feedIDs: Set<String>, _ completion: @escaping ArticleSetResultBlock) {
articlesTable.fetchArticlesAsync(feedIDs, completion)
}
public func fetchArticlesAsync(articleIDs: Set<String>, _ completion: @escaping ArticleSetResultBlock) {
articlesTable.fetchArticlesAsync(articleIDs: articleIDs, completion)
}
public func fetchUnreadArticlesAsync(_ feedIDs: Set<String>, _ limit: Int?, _ completion: @escaping ArticleSetResultBlock) {
articlesTable.fetchUnreadArticlesAsync(feedIDs, limit, completion)
}
public func fetchTodayArticlesAsync(_ feedIDs: Set<String>, _ limit: Int?, _ completion: @escaping ArticleSetResultBlock) {
articlesTable.fetchArticlesSinceAsync(feedIDs, todayCutoffDate(), limit, completion)
}
public func fetchedStarredArticlesAsync(_ feedIDs: Set<String>, _ limit: Int?, _ completion: @escaping ArticleSetResultBlock) {
articlesTable.fetchStarredArticlesAsync(feedIDs, limit, completion)
}
public func fetchArticlesMatchingAsync(_ searchString: String, _ feedIDs: Set<String>, _ completion: @escaping ArticleSetResultBlock) {
articlesTable.fetchArticlesMatchingAsync(searchString, feedIDs, completion)
}
public func fetchArticlesMatchingWithArticleIDsAsync(_ searchString: String, _ articleIDs: Set<String>, _ completion: @escaping ArticleSetResultBlock) {
articlesTable.fetchArticlesMatchingWithArticleIDsAsync(searchString, articleIDs, completion)
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.articlesMatching(searchString: searchString, articleIDs: articleIDs, database: database)
}
// MARK: - Unread Counts
/// Fetch all non-zero unread counts.
public func fetchAllUnreadCounts(_ completion: @escaping UnreadCountDictionaryCompletionBlock) {
let operation = FetchAllUnreadCountsOperation(databaseQueue: queue)
operationQueue.cancelOperations(named: operation.name!)
operation.completionBlock = { operation in
let fetchOperation = operation as! FetchAllUnreadCountsOperation
completion(fetchOperation.result)
public func allUnreadCounts() throws -> UnreadCountDictionary {
guard let database else {
throw DatabaseError.suspended
}
operationQueue.add(operation)
return articlesTable.allUnreadCounts(database: database)
}
/// Fetch unread count for a single feed.
public func fetchUnreadCount(_ feedID: String, _ completion: @escaping SingleUnreadCountCompletionBlock) {
let operation = FetchFeedUnreadCountOperation(feedID: feedID, databaseQueue: queue, cutoffDate: articlesTable.articleCutoffDate)
operation.completionBlock = { operation in
let fetchOperation = operation as! FetchFeedUnreadCountOperation
completion(fetchOperation.result)
public func unreadCount(feedID: String) throws -> Int? {
guard let database else {
throw DatabaseError.suspended
}
operationQueue.add(operation)
return articlesTable.unreadCount(feedID: feedID, database: database)
}
/// Fetch non-zero unread counts for given feedIDs.
public func fetchUnreadCounts(for feedIDs: Set<String>, _ completion: @escaping UnreadCountDictionaryCompletionBlock) {
let operation = FetchUnreadCountsForFeedsOperation(feedIDs: feedIDs, databaseQueue: queue)
operation.completionBlock = { operation in
let fetchOperation = operation as! FetchUnreadCountsForFeedsOperation
completion(fetchOperation.result)
public func unreadCounts(feedIDs: Set<String>) throws -> UnreadCountDictionary {
guard let database else {
throw DatabaseError.suspended
}
operationQueue.add(operation)
return articlesTable.unreadCounts(feedIDs: feedIDs, database: database)
}
public func fetchUnreadCountForToday(for feedIDs: Set<String>, completion: @escaping SingleUnreadCountCompletionBlock) {
fetchUnreadCount(for: feedIDs, since: todayCutoffDate(), completion: completion)
public func unreadCountForToday(feedIDs: Set<String>) throws -> Int? {
try unreadCount(feedIDs: feedIDs, since: todayCutoffDate())
}
public func fetchUnreadCount(for feedIDs: Set<String>, since: Date, completion: @escaping SingleUnreadCountCompletionBlock) {
articlesTable.fetchUnreadCount(feedIDs, since, completion)
public func unreadCount(feedIDs: Set<String>, since: Date) throws -> Int? {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.unreadCount(feedIDs: feedIDs, since: since, database: database)
}
public func fetchStarredAndUnreadCount(for feedIDs: Set<String>, completion: @escaping SingleUnreadCountCompletionBlock) {
articlesTable.fetchStarredAndUnreadCount(feedIDs, completion)
public func starredAndUnreadCount(feedIDs: Set<String>) throws -> Int? {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.starredAndUnreadCount(feedIDs: feedIDs, database: database)
}
// MARK: - Saving, Updating, and Deleting Articles
/// Update articles and save new ones — for feed-based systems (local and iCloud).
public func update(with parsedItems: Set<ParsedItem>, feedID: String, deleteOlder: Bool, completion: @escaping UpdateArticlesCompletionBlock) {
public func update(parsedItems: Set<ParsedItem>, feedID: String, deleteOlder: Bool) throws -> ArticleChanges {
precondition(retentionStyle == .feedBased)
articlesTable.update(parsedItems, feedID, deleteOlder, completion)
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.update(parsedItems: parsedItems, feedID: feedID, deleteOlder: deleteOlder, database: database)
}
/// Update articles and save new ones — for sync systems (Feedbin, Feedly, etc.).
public func update(feedIDsAndItems: [String: Set<ParsedItem>], defaultRead: Bool, completion: @escaping UpdateArticlesCompletionBlock) {
public func update(feedIDsAndItems: [String: Set<ParsedItem>], defaultRead: Bool) throws -> ArticleChanges {
precondition(retentionStyle == .syncSystem)
articlesTable.update(feedIDsAndItems, defaultRead, completion)
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.update(feedIDsAndItems: feedIDsAndItems, read: defaultRead, database: database)
}
/// Delete articles
public func delete(articleIDs: Set<String>, completion: DatabaseCompletionBlock?) {
articlesTable.delete(articleIDs: articleIDs, completion: completion)
public func delete(articleIDs: Set<String>) throws {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.delete(articleIDs: articleIDs, database: database)
}
// MARK: - Status
/// Fetch the articleIDs of unread articles.
public func fetchUnreadArticleIDsAsync(completion: @escaping ArticleIDsCompletionBlock) {
articlesTable.fetchUnreadArticleIDsAsync(completion)
public func unreadArticleIDs() throws -> Set<String>? {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.unreadArticleIDs(database: database)
}
/// Fetch the articleIDs of starred articles.
public func fetchStarredArticleIDsAsync(completion: @escaping ArticleIDsCompletionBlock) {
articlesTable.fetchStarredArticleIDsAsync(completion)
public func starredArticleIDs() throws -> Set<String>? {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.starredArticleIDs(database: database)
}
/// Fetch articleIDs for articles that we should have, but don’t. These articles are either (starred) or (newer than the article cutoff date).
public func fetchArticleIDsForStatusesWithoutArticlesNewerThanCutoffDate(_ completion: @escaping ArticleIDsCompletionBlock) {
articlesTable.fetchArticleIDsForStatusesWithoutArticlesNewerThanCutoffDate(completion)
public func articleIDsForStatusesWithoutArticlesNewerThanCutoffDate() throws -> Set<String>? {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.articleIDsForStatusesWithoutArticlesNewerThanCutoffDate(database: database)
}
public func mark(_ articles: Set<Article>, statusKey: ArticleStatus.Key, flag: Bool, completion: @escaping ArticleStatusesResultBlock) {
return articlesTable.mark(articles, statusKey, flag, completion)
public func mark(articles: Set<Article>, statusKey: ArticleStatus.Key, flag: Bool) throws -> Set<ArticleStatus>? {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.mark(articles: articles, statusKey: statusKey, flag: flag, database: database)
}
public func markAndFetchNew(articleIDs: Set<String>, statusKey: ArticleStatus.Key, flag: Bool, completion: @escaping ArticleIDsCompletionBlock) {
articlesTable.markAndFetchNew(articleIDs, statusKey, flag, completion)
public func markAndFetchNew(articleIDs: Set<String>, statusKey: ArticleStatus.Key, flag: Bool) throws -> Set<String> {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.markAndFetchNew(articleIDs: articleIDs, statusKey: statusKey, flag: flag, database: database)
}
/// Create statuses for specified articleIDs. For existing statuses, don’t do anything.
/// For newly-created statuses, mark them as read and not-starred.
public func createStatusesIfNeeded(articleIDs: Set<String>, completion: @escaping DatabaseCompletionBlock) {
articlesTable.createStatusesIfNeeded(articleIDs, completion)
public func createStatusesIfNeeded(articleIDs: Set<String>) throws {
guard let database else {
throw DatabaseError.suspended
}
return articlesTable.createStatusesIfNeeded(articleIDs: articleIDs, database: database)
}
#if os(iOS)
// MARK: - Suspend and Resume (for iOS)
/// Cancel current operations and close the database.
public func cancelAndSuspend() {
cancelOperations()
suspend()
}
/// Close the database and stop running database calls.
/// Any pending calls will complete first.
public func suspend() {
operationQueue.suspend()
queue.suspend()
#if os(iOS)
database?.close()
database = nil
#endif
}
/// Open the database and allow for running database calls again.
public func resume() {
queue.resume()
operationQueue.resume()
}
func resume() {
#if os(iOS)
if database == nil {
self.database = FMDatabase.openAndSetUpDatabase(path: databasePath)
}
#endif
}
// MARK: - Caches
/// Call to free up some memory. Should be done when the app is backgrounded, for instance.
/// This does not empty *all* caches — just the ones that are empty-able.
public func emptyCaches() {
articlesTable.emptyCaches()
}
@ -289,30 +306,17 @@ public final class ArticlesDatabase {
/// This prevents the database from growing forever. If we didn’t do this:
/// 1) The database would grow to an inordinate size, and
/// 2) the app would become very slow.
public func cleanupDatabaseAtStartup(subscribedToFeedIDs: Set<String>) {
if retentionStyle == .syncSystem {
articlesTable.deleteOldArticles()
}
articlesTable.deleteArticlesNotInSubscribedToFeedIDs(subscribedToFeedIDs)
articlesTable.deleteOldStatuses()
}
public func cleanupDatabaseAtStartup(subscribedToFeedIDs: Set<String>) throws {
/// Do database cleanups made necessary by the retention policy change in April 2020.
///
/// The retention policy for feed-based systems changed in April 2020:
/// we keep articles only for as long as they’re in the feed.
/// This change could result in a bunch of older articles suddenly
/// appearing as unread articles.
///
/// These are articles that were in the database,
/// but weren’t appearing in the UI because they were beyond the 90-day window.
/// (The previous retention policy used a 90-day window.)
///
/// This function marks everything as read that’s beyond that 90-day window.
/// It’s intended to be called only once on an account.
public func performApril2020RetentionPolicyChange() {
precondition(retentionStyle == .feedBased)
articlesTable.markOlderStatusesAsRead()
guard let database else {
throw DatabaseError.suspended
}
if retentionStyle == .syncSystem {
articlesTable.deleteOldArticles(database: database)
}
articlesTable.deleteArticlesNotInSubscribedToFeedIDs(subscribedToFeedIDs, database: database)
articlesTable.deleteOldStatuses(database: database)
}
}
@ -320,31 +324,40 @@ public final class ArticlesDatabase {
private extension ArticlesDatabase {
static let tableCreationStatements = """
CREATE TABLE if not EXISTS articles (articleID TEXT NOT NULL PRIMARY KEY, feedID TEXT NOT NULL, uniqueID TEXT NOT NULL, title TEXT, contentHTML TEXT, contentText TEXT, url TEXT, externalURL TEXT, summary TEXT, imageURL TEXT, bannerImageURL TEXT, datePublished DATE, dateModified DATE, searchRowID INTEGER);
static let creationStatements = """
CREATE TABLE if not EXISTS articles (articleID TEXT NOT NULL PRIMARY KEY, feedID TEXT NOT NULL, uniqueID TEXT NOT NULL, title TEXT, contentHTML TEXT, contentText TEXT, url TEXT, externalURL TEXT, summary TEXT, imageURL TEXT, bannerImageURL TEXT, datePublished DATE, dateModified DATE, searchRowID INTEGER);
CREATE TABLE if not EXISTS statuses (articleID TEXT NOT NULL PRIMARY KEY, read BOOL NOT NULL DEFAULT 0, starred BOOL NOT NULL DEFAULT 0, dateArrived DATE NOT NULL DEFAULT 0);
CREATE TABLE if not EXISTS statuses (articleID TEXT NOT NULL PRIMARY KEY, read BOOL NOT NULL DEFAULT 0, starred BOOL NOT NULL DEFAULT 0, dateArrived DATE NOT NULL DEFAULT 0);
CREATE TABLE if not EXISTS authors (authorID TEXT NOT NULL PRIMARY KEY, name TEXT, url TEXT, avatarURL TEXT, emailAddress TEXT);
CREATE TABLE if not EXISTS authorsLookup (authorID TEXT NOT NULL, articleID TEXT NOT NULL, PRIMARY KEY(authorID, articleID));
CREATE TABLE if not EXISTS authors (authorID TEXT NOT NULL PRIMARY KEY, name TEXT, url TEXT, avatarURL TEXT, emailAddress TEXT);
CREATE TABLE if not EXISTS authorsLookup (authorID TEXT NOT NULL, articleID TEXT NOT NULL, PRIMARY KEY(authorID, articleID));
CREATE INDEX if not EXISTS articles_feedID_datePublished_articleID on articles (feedID, datePublished, articleID);
CREATE INDEX if not EXISTS articles_feedID_datePublished_articleID on articles (feedID, datePublished, articleID);
CREATE INDEX if not EXISTS statuses_starred_index on statuses (starred);
CREATE INDEX if not EXISTS statuses_starred_index on statuses (starred);
CREATE VIRTUAL TABLE if not EXISTS search using fts4(title, body);
CREATE VIRTUAL TABLE if not EXISTS search using fts4(title, body);
CREATE TRIGGER if not EXISTS articles_after_delete_trigger_delete_search_text after delete on articles begin delete from search where rowid = OLD.searchRowID; end;
"""
CREATE TRIGGER if not EXISTS articles_after_delete_trigger_delete_search_text after delete on articles begin delete from search where rowid = OLD.searchRowID; end;
"""
func todayCutoffDate() -> Date {
// 24 hours previous. This is used by the Today smart feed, which should not actually empty out at midnight.
return Date(timeIntervalSinceNow: -(60 * 60 * 24)) // This does not need to be more precise.
}
// MARK: - Operations
func indexUnindexedArticles() {
func cancelOperations() {
operationQueue.cancelAllOperations()
guard let database else {
return // not an error in this case
}
let didIndexArticles = articlesTable.indexUnindexedArticles(database: database)
if didIndexArticles {
// Indexing happens in bunches. Continue until there are no more articles to index.
Task {
self.indexUnindexedArticles()
}
}
}
}

332
ArticlesDatabase/Sources/ArticlesDatabase/ArticlesDatabaseCompatibility.swift Normal file
View File

@ -0,0 +1,332 @@
//
// ArticlesDatabase.swift
// NetNewsWire
//
// Created by Brent Simmons on 7/20/15.
// Copyright © 2015 Ranchero Software, LLC. All rights reserved.
//
import Foundation
import RSCore
import Database
import RSParser
import Articles
// This file exists for compatibility — it provides nonisolated functions and callback-based APIs.
// It will go away as we adopt structured concurrency.
public typealias UnreadCountDictionaryCompletionResult = Result<UnreadCountDictionary,DatabaseError>
public typealias UnreadCountDictionaryCompletionBlock = (UnreadCountDictionaryCompletionResult) -> Void
public typealias SingleUnreadCountResult = Result<Int, DatabaseError>
public typealias SingleUnreadCountCompletionBlock = (SingleUnreadCountResult) -> Void
public typealias UpdateArticlesResult = Result<ArticleChanges, DatabaseError>
public typealias UpdateArticlesCompletionBlock = (UpdateArticlesResult) -> Void
public typealias ArticleSetResult = Result<Set<Article>, DatabaseError>
public typealias ArticleSetResultBlock = (ArticleSetResult) -> Void
public typealias ArticleIDsResult = Result<Set<String>, DatabaseError>
public typealias ArticleIDsCompletionBlock = (ArticleIDsResult) -> Void
public typealias ArticleStatusesResult = Result<Set<ArticleStatus>, DatabaseError>
public typealias ArticleStatusesResultBlock = (ArticleStatusesResult) -> Void
public extension ArticlesDatabase {
// MARK: - Fetching Articles Async
nonisolated func fetchArticlesAsync(_ feedID: String, _ completion: @escaping ArticleSetResultBlock) {
Task {
do {
let articles = try await articles(feedID: feedID)
completion(.success(articles))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchArticlesAsync(_ feedIDs: Set<String>, _ completion: @escaping ArticleSetResultBlock) {
Task {
do {
let articles = try await articles(feedIDs: feedIDs)
completion(.success(articles))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchArticlesAsync(articleIDs: Set<String>, _ completion: @escaping ArticleSetResultBlock) {
Task {
do {
let articles = try await articles(articleIDs: articleIDs)
completion(.success(articles))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchUnreadArticlesAsync(_ feedIDs: Set<String>, _ limit: Int?, _ completion: @escaping ArticleSetResultBlock) {
Task {
do {
let articles = try await unreadArticles(feedIDs: feedIDs, limit: limit)
completion(.success(articles))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchTodayArticlesAsync(_ feedIDs: Set<String>, _ limit: Int?, _ completion: @escaping ArticleSetResultBlock) {
Task {
do {
let articles = try await todayArticles(feedIDs: feedIDs, limit: limit)
completion(.success(articles))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchedStarredArticlesAsync(_ feedIDs: Set<String>, _ limit: Int?, _ completion: @escaping ArticleSetResultBlock) {
Task {
do {
let articles = try await starredArticles(feedIDs: feedIDs, limit: limit)
completion(.success(articles))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchArticlesMatchingAsync(_ searchString: String, _ feedIDs: Set<String>, _ completion: @escaping ArticleSetResultBlock) {
Task {
do {
let articles = try await articlesMatching(searchString: searchString, feedIDs: feedIDs)
completion(.success(articles))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchArticlesMatchingWithArticleIDsAsync(_ searchString: String, _ articleIDs: Set<String>, _ completion: @escaping ArticleSetResultBlock) {
Task {
do {
let articles = try await articlesMatching(searchString: searchString, articleIDs: articleIDs)
completion(.success(articles))
} catch {
completion(.failure(.suspended))
}
}
}
// MARK: - Unread Counts
/// Fetch all non-zero unread counts.
nonisolated func fetchAllUnreadCounts(_ completion: @escaping UnreadCountDictionaryCompletionBlock) {
Task {
do {
let unreadCountDictionary = try await allUnreadCounts()
completion(.success(unreadCountDictionary))
} catch {
completion(.failure(.suspended))
}
}
}
/// Fetch unread count for a single feed.
nonisolated func fetchUnreadCount(_ feedID: String, _ completion: @escaping SingleUnreadCountCompletionBlock) {
Task {
do {
let unreadCount = try await unreadCount(feedID: feedID) ?? 0
completion(.success(unreadCount))
} catch {
completion(.failure(.suspended))
}
}
}
/// Fetch non-zero unread counts for given feedIDs.
nonisolated func fetchUnreadCounts(for feedIDs: Set<String>, _ completion: @escaping UnreadCountDictionaryCompletionBlock) {
Task {
do {
let unreadCountDictionary = try await unreadCounts(feedIDs: feedIDs)
completion(.success(unreadCountDictionary))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchUnreadCountForToday(for feedIDs: Set<String>, completion: @escaping SingleUnreadCountCompletionBlock) {
Task {
do {
let unreadCount = try await unreadCountForToday(feedIDs: feedIDs)!
completion(.success(unreadCount))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchUnreadCount(for feedIDs: Set<String>, since: Date, completion: @escaping SingleUnreadCountCompletionBlock) {
Task {
do {
let unreadCount = try await unreadCount(feedIDs: feedIDs, since: since)!
completion(.success(unreadCount))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func fetchStarredAndUnreadCount(for feedIDs: Set<String>, completion: @escaping SingleUnreadCountCompletionBlock) {
Task {
do {
let unreadCount = try await starredAndUnreadCount(feedIDs: feedIDs)!
completion(.success(unreadCount))
} catch {
completion(.failure(.suspended))
}
}
}
// MARK: - Saving, Updating, and Deleting Articles
/// Update articles and save new ones — for feed-based systems (local and iCloud).
nonisolated func update(with parsedItems: Set<ParsedItem>, feedID: String, deleteOlder: Bool, completion: @escaping UpdateArticlesCompletionBlock) {
Task {
do {
let articleChanges = try await update(parsedItems: parsedItems, feedID: feedID, deleteOlder: deleteOlder)
completion(.success(articleChanges))
} catch {
completion(.failure(.suspended))
}
}
}
/// Update articles and save new ones — for sync systems (Feedbin, Feedly, etc.).
nonisolated func update(feedIDsAndItems: [String: Set<ParsedItem>], defaultRead: Bool, completion: @escaping UpdateArticlesCompletionBlock) {
Task {
do {
let articleChanges = try await update(feedIDsAndItems: feedIDsAndItems, defaultRead: defaultRead)
completion(.success(articleChanges))
} catch {
completion(.failure(.suspended))
}
}
}
/// Delete articles
nonisolated func delete(articleIDs: Set<String>, completion: DatabaseCompletionBlock?) {
Task {
do {
try await delete(articleIDs: articleIDs)
completion?(nil)
} catch {
completion?(.suspended)
}
}
}
// MARK: - Status
/// Fetch the articleIDs of unread articles.
nonisolated func fetchUnreadArticleIDsAsync(completion: @escaping ArticleIDsCompletionBlock) {
Task {
do {
let articleIDs = try await unreadArticleIDs()!
completion(.success(articleIDs))
} catch {
completion(.failure(.suspended))
}
}
}
/// Fetch the articleIDs of starred articles.
nonisolated func fetchStarredArticleIDsAsync(completion: @escaping ArticleIDsCompletionBlock) {
Task {
do {
let articleIDs = try await starredArticleIDs()!
completion(.success(articleIDs))
} catch {
completion(.failure(.suspended))
}
}
}
/// Fetch articleIDs for articles that we should have, but don’t. These articles are either (starred) or (newer than the article cutoff date).
nonisolated func fetchArticleIDsForStatusesWithoutArticlesNewerThanCutoffDate(_ completion: @escaping ArticleIDsCompletionBlock) {
Task {
do {
let articleIDs = try await articleIDsForStatusesWithoutArticlesNewerThanCutoffDate()!
completion(.success(articleIDs))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func mark(_ articles: Set<Article>, statusKey: ArticleStatus.Key, flag: Bool, completion: @escaping ArticleStatusesResultBlock) {
Task {
do {
let statuses = try await mark(articles: articles, statusKey: statusKey, flag: flag)!
completion(.success(statuses))
} catch {
completion(.failure(.suspended))
}
}
}
nonisolated func markAndFetchNew(articleIDs: Set<String>, statusKey: ArticleStatus.Key, flag: Bool, completion: @escaping ArticleIDsCompletionBlock) {
Task {
do {
let statuses = try await markAndFetchNew(articleIDs: articleIDs, statusKey: statusKey, flag: flag)
completion(.success(statuses))
} catch {
completion(.failure(.suspended))
}
}
}
/// Create statuses for specified articleIDs. For existing statuses, don’t do anything.
/// For newly-created statuses, mark them as read and not-starred.
nonisolated func createStatusesIfNeeded(articleIDs: Set<String>, completion: @escaping DatabaseCompletionBlock) {
Task {
do {
try await createStatusesIfNeeded(articleIDs: articleIDs)
completion(nil)
} catch {
completion(.suspended)
}
}
}
}

1127
ArticlesDatabase/Sources/ArticlesDatabase/ArticlesTable.swift
View File

File diff suppressed because it is too large Load Diff

6
ArticlesDatabase/Sources/ArticlesDatabase/AuthorsTable.swift
View File

@ -20,13 +20,9 @@ import FMDB
final class AuthorsTable: DatabaseRelatedObjectsTable {
let name: String
let name = DatabaseTableName.authors
let databaseIDKey = DatabaseKey.authorID
var cache = DatabaseObjectCache()
init(name: String) {
self.name = name
}
// MARK: - DatabaseRelatedObjectsTable

1
ArticlesDatabase/Sources/ArticlesDatabase/Constants.swift
View File

@ -16,6 +16,7 @@ struct DatabaseTableName {
static let authors = "authors"
static let authorsLookup = "authorsLookup"
static let statuses = "statuses"
static let search = "search"
}
struct DatabaseKey {

5
ArticlesDatabase/Sources/ArticlesDatabase/Extensions/Article+Database.swift
View File

@ -108,11 +108,6 @@ extension Article {
return d.count < 1 ? nil : d
}
// static func articlesWithParsedItems(_ parsedItems: Set<ParsedItem>, _ accountID: String, _ feedID: String, _ statusesDictionary: [String: ArticleStatus]) -> Set<Article> {
// let maximumDateAllowed = Date().addingTimeInterval(60 * 60 * 24) // Allow dates up to about 24 hours ahead of now
// return Set(parsedItems.map{ Article(parsedItem: $0, maximumDateAllowed: maximumDateAllowed, accountID: accountID, feedID: feedID, status: statusesDictionary[$0.articleID]!) })
// }
private static func _maximumDateAllowed() -> Date {
return Date().addingTimeInterval(60 * 60 * 24) // Allow dates up to about 24 hours ahead of now
}

75
ArticlesDatabase/Sources/ArticlesDatabase/Operations/FetchAllUnreadCountsOperation.swift
View File

@ -1,75 +0,0 @@
//
// FetchAllUnreadCountsOperation.swift
// ArticlesDatabase
//
// Created by Brent Simmons on 1/26/20.
// Copyright © 2020 Ranchero Software. All rights reserved.
//
import Foundation
import RSCore
import Database
import FMDB
public final class FetchAllUnreadCountsOperation: MainThreadOperation {
var result: UnreadCountDictionaryCompletionResult = .failure(.suspended)
// MainThreadOperation
public var isCanceled = false
public var id: Int?
public weak var operationDelegate: MainThreadOperationDelegate?
public var name: String? = "FetchAllUnreadCountsOperation"
public var completionBlock: MainThreadOperation.MainThreadOperationCompletionBlock?
private let queue: DatabaseQueue
init(databaseQueue: DatabaseQueue) {
self.queue = databaseQueue
}
public func run() {
queue.runInDatabase { databaseResult in
if self.isCanceled {
self.informOperationDelegateOfCompletion()
return
}
switch databaseResult {
case .success(let database):
self.fetchUnreadCounts(database)
case .failure:
self.informOperationDelegateOfCompletion()
}
}
}
}
private extension FetchAllUnreadCountsOperation {
func fetchUnreadCounts(_ database: FMDatabase) {
let sql = "select distinct feedID, count(*) from articles natural join statuses where read=0 group by feedID;"
guard let resultSet = database.executeQuery(sql, withArgumentsIn: nil) else {
informOperationDelegateOfCompletion()
return
}
var unreadCountDictionary = UnreadCountDictionary()
while resultSet.next() {
if isCanceled {
resultSet.close()
informOperationDelegateOfCompletion()
return
}
let unreadCount = resultSet.long(forColumnIndex: 1)
if let feedID = resultSet.string(forColumnIndex: 0) {
unreadCountDictionary[feedID] = unreadCount
}
}
resultSet.close()
result = .success(unreadCountDictionary)
informOperationDelegateOfCompletion()
}
}

75
ArticlesDatabase/Sources/ArticlesDatabase/Operations/FetchFeedUnreadCountOperation.swift
View File

@ -1,75 +0,0 @@
//
// FetchFeedUnreadCountOperation.swift
// ArticlesDatabase
//
// Created by Brent Simmons on 1/27/20.
// Copyright © 2020 Ranchero Software. All rights reserved.
//
import Foundation
import RSCore
import Database
import FMDB
/// Fetch the unread count for a single feed.
public final class FetchFeedUnreadCountOperation: MainThreadOperation {
var result: SingleUnreadCountResult = .failure(.suspended)
// MainThreadOperation
public var isCanceled = false
public var id: Int?
public weak var operationDelegate: MainThreadOperationDelegate?
public var name: String? = "FetchFeedUnreadCountOperation"
public var completionBlock: MainThreadOperation.MainThreadOperationCompletionBlock?
private let queue: DatabaseQueue
private let cutoffDate: Date
private let feedID: String
init(feedID: String, databaseQueue: DatabaseQueue, cutoffDate: Date) {
self.feedID = feedID
self.queue = databaseQueue
self.cutoffDate = cutoffDate
}
public func run() {
queue.runInDatabase { databaseResult in
if self.isCanceled {
self.informOperationDelegateOfCompletion()
return
}
switch databaseResult {
case .success(let database):
self.fetchUnreadCount(database)
case .failure:
self.informOperationDelegateOfCompletion()
}
}
}
}
private extension FetchFeedUnreadCountOperation {
func fetchUnreadCount(_ database: FMDatabase) {
let sql = "select count(*) from articles natural join statuses where feedID=? and read=0;"
guard let resultSet = database.executeQuery(sql, withArgumentsIn: [feedID]) else {
informOperationDelegateOfCompletion()
return
}
if isCanceled {
informOperationDelegateOfCompletion()
return
}
if resultSet.next() {
let unreadCount = resultSet.long(forColumnIndex: 0)
result = .success(unreadCount)
}
resultSet.close()
informOperationDelegateOfCompletion()
}
}

86
ArticlesDatabase/Sources/ArticlesDatabase/Operations/FetchUnreadCountsForFeedsOperation.swift
View File

@ -1,86 +0,0 @@
//
// FetchUnreadCountsForFeedsOperation.swift
// ArticlesDatabase
//
// Created by Brent Simmons on 2/1/20.
// Copyright © 2020 Ranchero Software. All rights reserved.
//
import Foundation
import RSCore
import Database
import FMDB
/// Fetch the unread counts for a number of feeds.
public final class FetchUnreadCountsForFeedsOperation: MainThreadOperation {
var result: UnreadCountDictionaryCompletionResult = .failure(.suspended)
// MainThreadOperation
public var isCanceled = false
public var id: Int?
public weak var operationDelegate: MainThreadOperationDelegate?
public var name: String? = "FetchUnreadCountsForFeedsOperation"
public var completionBlock: MainThreadOperation.MainThreadOperationCompletionBlock?
private let queue: DatabaseQueue
private let feedIDs: Set<String>
init(feedIDs: Set<String>, databaseQueue: DatabaseQueue) {
self.feedIDs = feedIDs
self.queue = databaseQueue
}
public func run() {
queue.runInDatabase { databaseResult in
if self.isCanceled {
self.informOperationDelegateOfCompletion()
return
}
switch databaseResult {
case .success(let database):
self.fetchUnreadCounts(database)
case .failure:
self.informOperationDelegateOfCompletion()
}
}
}
}
private extension FetchUnreadCountsForFeedsOperation {
func fetchUnreadCounts(_ database: FMDatabase) {
let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(feedIDs.count))!
let sql = "select distinct feedID, count(*) from articles natural join statuses where feedID in \(placeholders) and read=0 group by feedID;"
let parameters = Array(feedIDs) as [Any]
guard let resultSet = database.executeQuery(sql, withArgumentsIn: parameters) else {
informOperationDelegateOfCompletion()
return
}
if isCanceled {
resultSet.close()
informOperationDelegateOfCompletion()
return
}
var unreadCountDictionary = UnreadCountDictionary()
while resultSet.next() {
if isCanceled {
resultSet.close()
informOperationDelegateOfCompletion()
return
}
let unreadCount = resultSet.long(forColumnIndex: 1)
if let feedID = resultSet.string(forColumnIndex: 0) {
unreadCountDictionary[feedID] = unreadCount
}
}
resultSet.close()
result = .success(unreadCountDictionary)
informOperationDelegateOfCompletion()
}
}

95
ArticlesDatabase/Sources/ArticlesDatabase/SearchTable.swift
View File

@ -77,34 +77,14 @@ final class ArticleSearchInfo: Hashable {
}
}
final class SearchTable: DatabaseTable {
final class SearchTable {
let name = "search"
private let queue: DatabaseQueue
private weak var articlesTable: ArticlesTable?
init(queue: DatabaseQueue, articlesTable: ArticlesTable) {
self.queue = queue
self.articlesTable = articlesTable
}
func ensureIndexedArticles(for articleIDs: Set<String>) {
guard !articleIDs.isEmpty else {
return
}
queue.runInTransaction { databaseResult in
if let database = databaseResult.database {
self.ensureIndexedArticles(articleIDs, database)
}
}
}
let name = DatabaseTableName.search
/// Add to, or update, the search index for articles with specified IDs.
func ensureIndexedArticles(_ articleIDs: Set<String>, _ database: FMDatabase) {
guard let articlesTable = articlesTable else {
return
}
guard let articleSearchInfos = articlesTable.fetchArticleSearchInfos(articleIDs, in: database) else {
func ensureIndexedArticles(articleIDs: Set<String>, database: FMDatabase) {
guard let articleSearchInfos = fetchArticleSearchInfos(articleIDs: articleIDs, database: database) else {
return
}
@ -117,13 +97,15 @@ final class SearchTable: DatabaseTable {
/// Index new articles.
func indexNewArticles(_ articles: Set<Article>, _ database: FMDatabase) {
let articleSearchInfos = Set(articles.map{ ArticleSearchInfo(article: $0) })
performInitialIndexForArticles(articleSearchInfos, database)
}
/// Index updated articles.
func indexUpdatedArticles(_ articles: Set<Article>, _ database: FMDatabase) {
ensureIndexedArticles(articles.articleIDs(), database)
ensureIndexedArticles(articleIDs: articles.articleIDs(), database: database)
}
}
@ -132,17 +114,22 @@ final class SearchTable: DatabaseTable {
private extension SearchTable {
func performInitialIndexForArticles(_ articles: Set<ArticleSearchInfo>, _ database: FMDatabase) {
articles.forEach { performInitialIndex($0, database) }
for article in articles {
performInitialIndex(article, database)
}
}
func performInitialIndex(_ article: ArticleSearchInfo, _ database: FMDatabase) {
let rowid = insert(article, database)
articlesTable?.updateRowsWithValue(rowid, valueKey: DatabaseKey.searchRowID, whereKey: DatabaseKey.articleID, matches: [article.articleID], database: database)
database.updateRowsWithValue(rowid, valueKey: DatabaseKey.searchRowID, whereKey: DatabaseKey.articleID, equals: article.articleID, tableName: DatabaseTableName.articles)
}
func insert(_ article: ArticleSearchInfo, _ database: FMDatabase) -> Int {
let rowDictionary: DatabaseDictionary = [DatabaseKey.body: article.bodyForIndex, DatabaseKey.title: article.title ?? ""]
insertRow(rowDictionary, insertType: .normal, in: database)
database.insertRow(rowDictionary, insertType: .normal, tableName: name)
return Int(database.lastInsertRowId())
}
@ -206,7 +193,7 @@ private extension SearchTable {
if article.bodyForIndex != searchInfo.body {
updateDictionary[DatabaseKey.body] = article.bodyForIndex
}
updateRowsWithDictionary(updateDictionary, whereKey: DatabaseKey.rowID, matches: searchInfo.rowID, database: database)
database.updateRowsWithDictionary(updateDictionary, whereKey: DatabaseKey.rowID, equals: searchInfo.rowID, tableName: name)
}
private func fetchSearchInfos(_ articles: Set<ArticleSearchInfo>, _ database: FMDatabase) -> Set<SearchInfo>? {
@ -221,4 +208,52 @@ private extension SearchTable {
}
return resultSet.mapToSet { SearchInfo(row: $0) }
}
func fetchArticleSearchInfos(articleIDs: Set<String>, database: FMDatabase) -> Set<ArticleSearchInfo>? {
let parameters = articleIDs.map { $0 as AnyObject }
let placeholders = NSString.rs_SQLValueList(withPlaceholders: UInt(articleIDs.count))!
guard let resultSet = database.executeQuery(articleSearchInfosQuery(with: placeholders), withArgumentsIn: parameters) else {
return nil
}
let articleSearchInfo = resultSet.mapToSet { (row) -> ArticleSearchInfo? in
let articleID = row.string(forColumn: DatabaseKey.articleID)!
let title = row.string(forColumn: DatabaseKey.title)
let contentHTML = row.string(forColumn: DatabaseKey.contentHTML)
let contentText = row.string(forColumn: DatabaseKey.contentText)
let summary = row.string(forColumn: DatabaseKey.summary)
let authorsNames = row.string(forColumn: DatabaseKey.authors)
let searchRowIDObject = row.object(forColumnName: DatabaseKey.searchRowID)
var searchRowID: Int? = nil
if searchRowIDObject != nil && !(searchRowIDObject is NSNull) {
searchRowID = Int(row.longLongInt(forColumn: DatabaseKey.searchRowID))
}
return ArticleSearchInfo(articleID: articleID, title: title, contentHTML: contentHTML, contentText: contentText, summary: summary, authorsNames: authorsNames, searchRowID: searchRowID)
}
return articleSearchInfo
}
private func articleSearchInfosQuery(with placeholders: String) -> String {
return """
SELECT
art.articleID,
art.title,
art.contentHTML,
art.contentText,
art.summary,
art.searchRowID,
(SELECT GROUP_CONCAT(name, ' ')
FROM authorsLookup as autL
JOIN authors as aut ON autL.authorID = aut.authorID
WHERE art.articleID = autL.articleID
GROUP BY autl.articleID) as authors
FROM articles as art
WHERE articleID in \(placeholders);
"""
}
}

123
ArticlesDatabase/Sources/ArticlesDatabase/StatusesTable.swift
View File

@ -16,24 +16,20 @@ import FMDB
//
// CREATE TABLE if not EXISTS statuses (articleID TEXT NOT NULL PRIMARY KEY, read BOOL NOT NULL DEFAULT 0, starred BOOL NOT NULL DEFAULT 0, dateArrived DATE NOT NULL DEFAULT 0);
final class StatusesTable: DatabaseTable {
final class StatusesTable {
let name = DatabaseTableName.statuses
private let cache = StatusCache()
private let queue: DatabaseQueue
init(queue: DatabaseQueue) {
self.queue = queue
}
// MARK: - Creating/Updating
@discardableResult
func ensureStatusesForArticleIDs(_ articleIDs: Set<String>, _ read: Bool, _ database: FMDatabase) -> ([String: ArticleStatus], Set<String>) {
#if DEBUG
// Check for missing statuses — this asserts that all the passed-in articleIDs exist in the statuses table.
defer {
if let resultSet = self.selectRowsWhere(key: DatabaseKey.articleID, inValues: Array(articleIDs), in: database) {
if let resultSet = database.selectRowsWhere(key: DatabaseKey.articleID, equalsAnyValue: Array(articleIDs), tableName: name) {
let fetchedStatuses = resultSet.mapToSet(statusWithRow)
let fetchedArticleIDs = Set(fetchedStatuses.map{ $0.articleID })
assert(fetchedArticleIDs == articleIDs)
@ -94,96 +90,29 @@ final class StatusesTable: DatabaseTable {
// MARK: - Fetching
func fetchUnreadArticleIDs() throws -> Set<String> {
return try fetchArticleIDs("select articleID from statuses where read=0;")
}
func articleIDs(key: ArticleStatus.Key, value: Bool, database: FMDatabase) -> Set<String>? {
func fetchStarredArticleIDs() throws -> Set<String> {
return try fetchArticleIDs("select articleID from statuses where starred=1;")
}
func fetchArticleIDsAsync(_ statusKey: ArticleStatus.Key, _ value: Bool, _ completion: @escaping ArticleIDsCompletionBlock) {
queue.runInDatabase { databaseResult in
var sql = "select articleID from statuses where \(key.rawValue)="
sql += value ? "1" : "0"
sql += ";"
func makeDatabaseCalls(_ database: FMDatabase) {
var sql = "select articleID from statuses where \(statusKey.rawValue)="
sql += value ? "1" : "0"
sql += ";"
guard let resultSet = database.executeQuery(sql, withArgumentsIn: nil) else {
DispatchQueue.main.async {
completion(.success(Set<String>()))
}
return
}
let articleIDs = resultSet.mapToSet{ $0.string(forColumnIndex: 0) }
DispatchQueue.main.async {
completion(.success(articleIDs))
}
}
switch databaseResult {
case .success(let database):
makeDatabaseCalls(database)
case .failure(let databaseError):
DispatchQueue.main.async {
completion(.failure(databaseError))
}
}
}
}
func fetchArticleIDsForStatusesWithoutArticlesNewerThan(_ cutoffDate: Date, _ completion: @escaping ArticleIDsCompletionBlock) {
queue.runInDatabase { databaseResult in
var error: DatabaseError?
var articleIDs = Set<String>()
func makeDatabaseCall(_ database: FMDatabase) {
let sql = "select articleID from statuses s where (starred=1 or dateArrived>?) and not exists (select 1 from articles a where a.articleID = s.articleID);"
if let resultSet = database.executeQuery(sql, withArgumentsIn: [cutoffDate]) {
articleIDs = resultSet.mapToSet(self.articleIDWithRow)
}
}
switch databaseResult {
case .success(let database):
makeDatabaseCall(database)
case .failure(let databaseError):
error = databaseError
}
if let error = error {
DispatchQueue.main.async {
completion(.failure(error))
}
}
else {
DispatchQueue.main.async {
completion(.success(articleIDs))
}
}
}
}
func fetchArticleIDs(_ sql: String) throws -> Set<String> {
var error: DatabaseError?
var articleIDs = Set<String>()
queue.runInDatabaseSync { databaseResult in
switch databaseResult {
case .success(let database):
if let resultSet = database.executeQuery(sql, withArgumentsIn: nil) {
articleIDs = resultSet.mapToSet(self.articleIDWithRow)
}
case .failure(let databaseError):
error = databaseError
}
guard let resultSet = database.executeQuery(sql, withArgumentsIn: nil) else {
return nil
}
if let error = error {
throw(error)
let articleIDs = resultSet.mapToSet{ $0.string(forColumnIndex: 0) }
return articleIDs
}
func articleIDsForStatusesWithoutArticlesNewerThan(cutoffDate: Date, database: FMDatabase) -> Set<String>? {
let sql = "select articleID from statuses s where (starred=1 or dateArrived>?) and not exists (select 1 from articles a where a.articleID = s.articleID);"
guard let resultSet = database.executeQuery(sql, withArgumentsIn: [cutoffDate]) else {
return nil
}
let articleIDs = resultSet.mapToSet(articleIDWithRow)
return articleIDs
}
@ -228,7 +157,8 @@ final class StatusesTable: DatabaseTable {
// MARK: - Cleanup
func removeStatuses(_ articleIDs: Set<String>, _ database: FMDatabase) {
deleteRowsWhere(key: DatabaseKey.articleID, equalsAnyValue: Array(articleIDs), in: database)
database.deleteRowsWhere(key: DatabaseKey.articleID, equalsAnyValue: Array(articleIDs), tableName: name)
}
}
@ -246,7 +176,7 @@ private extension StatusesTable {
func saveStatuses(_ statuses: Set<ArticleStatus>, _ database: FMDatabase) {
let statusArray = statuses.map { $0.databaseDictionary()! }
self.insertRows(statusArray, insertType: .orIgnore, in: database)
database.insertRows(statusArray, insertType: .orIgnore, tableName: name)
}
func createAndSaveStatusesForArticleIDs(_ articleIDs: Set<String>, _ read: Bool, _ database: FMDatabase) {
@ -258,7 +188,7 @@ private extension StatusesTable {
}
func fetchAndCacheStatusesForArticleIDs(_ articleIDs: Set<String>, _ database: FMDatabase) {
guard let resultSet = self.selectRowsWhere(key: DatabaseKey.articleID, inValues: Array(articleIDs), in: database) else {
guard let resultSet = database.selectRowsWhere(key: DatabaseKey.articleID, equalsAnyValue: Array(articleIDs), tableName: name) else {
return
}
@ -269,7 +199,8 @@ private extension StatusesTable {
// MARK: - Marking
func markArticleIDs(_ articleIDs: Set<String>, _ statusKey: ArticleStatus.Key, _ flag: Bool, _ database: FMDatabase) {
updateRowsWithValue(NSNumber(value: flag), valueKey: statusKey.rawValue, whereKey: DatabaseKey.articleID, matches: Array(articleIDs), database: database)
database.updateRowsWithValue(NSNumber(value: flag), valueKey: statusKey.rawValue, whereKey: DatabaseKey.articleID, equalsAnyValue: Array(articleIDs), tableName: name)
}
}

32
Database/Sources/Database/Database.swift
View File

@ -7,17 +7,12 @@
//
import Foundation
import FMDB
public enum DatabaseError: Error, Sendable {
case suspended // On iOS, to support background refreshing, a database may be suspended.
}
/// Result type that provides an FMDatabase or a DatabaseError.
public typealias DatabaseResult = Result<FMDatabase, DatabaseError>
/// Block that executes database code or handles DatabaseQueueError.
public typealias DatabaseBlock = (DatabaseResult) -> Void
// Compatibility — to be removed once we switch to structured concurrency
/// Completion block that provides an optional DatabaseError.
public typealias DatabaseCompletionBlock = @Sendable (DatabaseError?) -> Void
@ -27,28 +22,3 @@ public typealias DatabaseIntResult = Result<Int, DatabaseError>
/// Completion block for DatabaseIntResult.
public typealias DatabaseIntCompletionBlock = @Sendable (DatabaseIntResult) -> Void
// MARK: - Extensions
public extension DatabaseResult {
/// Convenience for getting the database from a DatabaseResult.
var database: FMDatabase? {
switch self {
case .success(let database):
return database
case .failure:
return nil
}
}
/// Convenience for getting the error from a DatabaseResult.
var error: DatabaseError? {
switch self {
case .success:
return nil
case .failure(let error):
return error
}
}
}

259
Database/Sources/Database/DatabaseQueue.swift
View File

@ -1,259 +0,0 @@
//
// DatabaseQueue.swift
// RSDatabase
//
// Created by Brent Simmons on 11/13/19.
// Copyright © 2019 Brent Simmons. All rights reserved.
//
import Foundation
import SQLite3
import FMDB
/// Manage a serial queue and a SQLite database.
/// It replaces RSDatabaseQueue, which is deprecated.
/// Main-thread only.
/// Important note: on iOS, the queue can be suspended
/// in order to support background refreshing.
public final class DatabaseQueue {
/// Check to see if the queue is suspended. Read-only.
/// Calling suspend() and resume() will change the value of this property.
/// This will return true only on iOS — on macOS it’s always false.
public var isSuspended: Bool {
#if os(iOS)
precondition(Thread.isMainThread)
return _isSuspended
#else
return false
#endif
}
private var _isSuspended = true
private var isCallingDatabase = false
private let database: FMDatabase
private let databasePath: String
private let serialDispatchQueue: DispatchQueue
private let targetDispatchQueue: DispatchQueue
#if os(iOS)
private let databaseLock = NSLock()
#endif
/// When init returns, the database will not be suspended: it will be ready for database calls.
public init(databasePath: String) {
precondition(Thread.isMainThread)
self.serialDispatchQueue = DispatchQueue(label: "DatabaseQueue (Serial) - \(databasePath)", attributes: .initiallyInactive)
self.targetDispatchQueue = DispatchQueue(label: "DatabaseQueue (Target) - \(databasePath)")
self.serialDispatchQueue.setTarget(queue: self.targetDispatchQueue)
self.serialDispatchQueue.activate()
self.databasePath = databasePath
self.database = FMDatabase(path: databasePath)!
openDatabase()
_isSuspended = false
}
// MARK: - Suspend and Resume
/// Close the SQLite database and don’t allow database calls until resumed.
/// This is for iOS, where we need to close the SQLite database in some conditions.
///
/// After calling suspend, if you call into the database before calling resume,
/// your code will not run, and runInDatabaseSync and runInTransactionSync will
/// both throw DatabaseQueueError.isSuspended.
///
/// On Mac, suspend() and resume() are no-ops, since there isn’t a need for them.
public func suspend() {
#if os(iOS)
precondition(Thread.isMainThread)
guard !_isSuspended else {
return
}
_isSuspended = true
serialDispatchQueue.suspend()
targetDispatchQueue.async {
self.lockDatabase()
self.database.close()
self.unlockDatabase()
DispatchQueue.main.async {
self.serialDispatchQueue.resume()
}
}
#endif
}
/// Open the SQLite database. Allow database calls again.
/// This is also for iOS only.
public func resume() {
#if os(iOS)
precondition(Thread.isMainThread)
guard _isSuspended else {
return
}
serialDispatchQueue.suspend()
targetDispatchQueue.sync {
if _isSuspended {
lockDatabase()
openDatabase()
unlockDatabase()
_isSuspended = false
}
}
serialDispatchQueue.resume()
#endif
}
// MARK: - Make Database Calls
/// Run a DatabaseBlock synchronously. This call will block the main thread
/// potentially for a while, depending on how long it takes to execute
/// the DatabaseBlock *and* depending on how many other calls have been
/// scheduled on the queue. Use sparingly — prefer async versions.
public func runInDatabaseSync(_ databaseBlock: DatabaseBlock) {
precondition(Thread.isMainThread)
serialDispatchQueue.sync {
self._runInDatabase(self.database, databaseBlock, false)
}
}
/// Run a DatabaseBlock asynchronously.
public func runInDatabase(_ databaseBlock: @escaping DatabaseBlock) {
precondition(Thread.isMainThread)
serialDispatchQueue.async {
self._runInDatabase(self.database, databaseBlock, false)
}
}
/// Run a DatabaseBlock wrapped in a transaction synchronously.
/// Transactions help performance significantly when updating the database.
/// Nevertheless, it’s best to avoid this because it will block the main thread —
/// prefer the async `runInTransaction` instead.
public func runInTransactionSync(_ databaseBlock: @escaping DatabaseBlock) {
precondition(Thread.isMainThread)
serialDispatchQueue.sync {
self._runInDatabase(self.database, databaseBlock, true)
}
}
/// Run a DatabaseBlock wrapped in a transaction asynchronously.
/// Transactions help performance significantly when updating the database.
public func runInTransaction(_ databaseBlock: @escaping DatabaseBlock) {
precondition(Thread.isMainThread)
serialDispatchQueue.async {
self._runInDatabase(self.database, databaseBlock, true)
}
}
/// Run all the lines that start with "create".
/// Use this to create tables, indexes, etc.
public func runCreateStatements(_ statements: String) throws {
precondition(Thread.isMainThread)
var error: DatabaseError? = nil
runInDatabaseSync { result in
switch result {
case .success(let database):
statements.enumerateLines { (line, stop) in
if line.lowercased().hasPrefix("create") {
database.executeStatements(line)
}
stop = false
}
case .failure(let databaseError):
error = databaseError
}
}
if let error = error {
throw(error)
}
}
/// Compact the database. This should be done from time to time —
/// weekly-ish? — to keep up the performance level of a database.
/// Generally a thing to do at startup, if it’s been a while
/// since the last vacuum() call. You almost certainly want to call
/// vacuumIfNeeded instead.
public func vacuum() {
precondition(Thread.isMainThread)
runInDatabase { result in
result.database?.executeStatements("vacuum;")
}
}
/// Vacuum the database if it’s been more than `daysBetweenVacuums` since the last vacuum.
/// Normally you would call this right after initing a DatabaseQueue.
///
/// - Returns: true if database will be vacuumed.
@discardableResult
public func vacuumIfNeeded(daysBetweenVacuums: Int) -> Bool {
precondition(Thread.isMainThread)
let defaultsKey = "DatabaseQueue-LastVacuumDate-\(databasePath)"
let minimumVacuumInterval = TimeInterval(daysBetweenVacuums * (60 * 60 * 24)) // Doesn’t have to be precise
let now = Date()
let cutoffDate = now - minimumVacuumInterval
if let lastVacuumDate = UserDefaults.standard.object(forKey: defaultsKey) as? Date {
if lastVacuumDate < cutoffDate {
vacuum()
UserDefaults.standard.set(now, forKey: defaultsKey)
return true
}
return false
}
// Never vacuumed — almost certainly a new database.
// Just set the LastVacuumDate pref to now and skip vacuuming.
UserDefaults.standard.set(now, forKey: defaultsKey)
return false
}
}
private extension DatabaseQueue {
func lockDatabase() {
#if os(iOS)
databaseLock.lock()
#endif
}
func unlockDatabase() {
#if os(iOS)
databaseLock.unlock()
#endif
}
func _runInDatabase(_ database: FMDatabase, _ databaseBlock: DatabaseBlock, _ useTransaction: Bool) {
lockDatabase()
defer {
unlockDatabase()
}
precondition(!isCallingDatabase)
isCallingDatabase = true
autoreleasepool {
if _isSuspended {
databaseBlock(.failure(.suspended))
}
else {
if useTransaction {
database.beginTransaction()
}
databaseBlock(.success(database))
if useTransaction {
database.commit()
}
}
}
isCallingDatabase = false
}
func openDatabase() {
database.open()
database.executeStatements("PRAGMA synchronous = 1;")
database.setShouldCacheStatements(true)
}
}

139
Database/Sources/Database/DatabaseTable.swift
View File

@ -1,139 +0,0 @@
//
// DatabaseTable.swift
// RSDatabase
//
// Created by Brent Simmons on 7/16/17.
// Copyright © 2017 Ranchero Software, LLC. All rights reserved.
//
import Foundation
import FMDB
public protocol DatabaseTable {
var name: String { get }
}
public extension DatabaseTable {
// MARK: Fetching
func selectRowsWhere(key: String, equals value: Any, in database: FMDatabase) -> FMResultSet? {
return database.rs_selectRowsWhereKey(key, equalsValue: value, tableName: name)
}
func selectSingleRowWhere(key: String, equals value: Any, in database: FMDatabase) -> FMResultSet? {
return database.rs_selectSingleRowWhereKey(key, equalsValue: value, tableName: name)
}
func selectRowsWhere(key: String, inValues values: [Any], in database: FMDatabase) -> FMResultSet? {
if values.isEmpty {
return nil
}
return database.rs_selectRowsWhereKey(key, inValues: values, tableName: name)
}
// MARK: Deleting
func deleteRowsWhere(key: String, equalsAnyValue values: [Any], in database: FMDatabase) {
if values.isEmpty {
return
}
database.rs_deleteRowsWhereKey(key, inValues: values, tableName: name)
}
// MARK: Updating
func updateRowsWithValue(_ value: Any, valueKey: String, whereKey: String, matches: [Any], database: FMDatabase) {
let _ = database.rs_updateRows(withValue: value, valueKey: valueKey, whereKey: whereKey, inValues: matches, tableName: self.name)
}
func updateRowsWithDictionary(_ dictionary: DatabaseDictionary, whereKey: String, matches: Any, database: FMDatabase) {
let _ = database.rs_updateRows(with: dictionary, whereKey: whereKey, equalsValue: matches, tableName: self.name)
}
// MARK: Saving
func insertRows(_ dictionaries: [DatabaseDictionary], insertType: RSDatabaseInsertType, in database: FMDatabase) {
dictionaries.forEach { (oneDictionary) in
let _ = database.rs_insertRow(with: oneDictionary, insertType: insertType, tableName: self.name)
}
}
func insertRow(_ rowDictionary: DatabaseDictionary, insertType: RSDatabaseInsertType, in database: FMDatabase) {
insertRows([rowDictionary], insertType: insertType, in: database)
}
// MARK: Counting
func numberWithCountResultSet(_ resultSet: FMResultSet) -> Int {
guard resultSet.next() else {
return 0
}
return Int(resultSet.int(forColumnIndex: 0))
}
func numberWithSQLAndParameters(_ sql: String, _ parameters: [Any], in database: FMDatabase) -> Int {
if let resultSet = database.executeQuery(sql, withArgumentsIn: parameters) {
return numberWithCountResultSet(resultSet)
}
return 0
}
// MARK: Mapping
func mapResultSet<T>(_ resultSet: FMResultSet, _ completion: (_ resultSet: FMResultSet) -> T?) -> [T] {
var objects = [T]()
while resultSet.next() {
if let obj = completion(resultSet) {
objects += [obj]
}
}
return objects
}
// MARK: Columns
func containsColumn(_ columnName: String, in database: FMDatabase) -> Bool {
if let resultSet = database.executeQuery("select * from \(name) limit 1;", withArgumentsIn: nil) {
if let columnMap = resultSet.columnNameToIndexMap {
if let _ = columnMap[columnName.lowercased()] {
return true
}
}
}
return false
}
}
public extension FMResultSet {
func compactMap<T>(_ completion: (_ row: FMResultSet) -> T?) -> [T] {
var objects = [T]()
while next() {
if let obj = completion(self) {
objects += [obj]
}
}
close()
return objects
}
func mapToSet<T>(_ completion: (_ row: FMResultSet) -> T?) -> Set<T> {
return Set(compactMap(completion))
}
}

42
Database/Sources/Database/FMDatabase+Extras.swift
View File

@ -46,7 +46,47 @@ public extension FMDatabase {
func insertRows(_ dictionaries: [DatabaseDictionary], insertType: RSDatabaseInsertType, tableName: String) {
for dictionary in dictionaries {
_ = rs_insertRow(with: dictionary, insertType: insertType, tableName: tableName)
insertRow(dictionary, insertType: insertType, tableName: tableName)
}
}
func insertRow(_ dictionary: DatabaseDictionary, insertType: RSDatabaseInsertType, tableName: String) {
rs_insertRow(with: dictionary, insertType: insertType, tableName: tableName)
}
func updateRowsWithValue(_ value: Any, valueKey: String, whereKey: String, equalsAnyValue values: [Any], tableName: String) {
rs_updateRows(withValue: value, valueKey: valueKey, whereKey: whereKey, inValues: values, tableName: tableName)
}
func updateRowsWithValue(_ value: Any, valueKey: String, whereKey: String, equals match: Any, tableName: String) {
updateRowsWithValue(value, valueKey: valueKey, whereKey: whereKey, equalsAnyValue: [match], tableName: tableName)
}
func updateRowsWithDictionary(_ dictionary: [String: Any], whereKey: String, equals value: Any, tableName: String) {
rs_updateRows(with: dictionary, whereKey: whereKey, equalsValue: value, tableName: tableName)
}
func deleteRowsWhere(key: String, equalsAnyValue values: [Any], tableName: String) {
rs_deleteRowsWhereKey(key, inValues: values, tableName: tableName)
}
func selectRowsWhere(key: String, equalsAnyValue values: [Any], tableName: String) -> FMResultSet? {
rs_selectRowsWhereKey(key, inValues: values, tableName: tableName)
}
func count(sql: String, parameters: [Any]?, tableName: String) -> Int? {
guard let resultSet = executeQuery(sql, withArgumentsIn: parameters) else {
return nil
}
let count = resultSet.intWithCountResult()
return count
}
}

22
Database/Sources/Database/FMResultSet+Extras.swift
View File

@ -16,7 +16,27 @@ public extension FMResultSet {
return nil
}
return Int(long(forColumnIndex: 0))
let count = Int(long(forColumnIndex: 0))
close()
return count
}
func compactMap<T>(_ completion: (_ row: FMResultSet) -> T?) -> [T] {
var objects = [T]()
while next() {
if let obj = completion(self) {
objects += [obj]
}
}
close()
return objects
}
func mapToSet<T>(_ completion: (_ row: FMResultSet) -> T?) -> Set<T> {
return Set(compactMap(completion))
}
}

7
Database/Sources/Database/Related Objects/DatabaseRelatedObjectsTable.swift
View File

@ -11,8 +11,9 @@ import FMDB
// Protocol for a database table for related objects — authors and attachments in NetNewsWire, for instance.
public protocol DatabaseRelatedObjectsTable: DatabaseTable {
public protocol DatabaseRelatedObjectsTable {
var name: String { get }
var databaseIDKey: String { get}
var cache: DatabaseObjectCache { get }
@ -49,7 +50,7 @@ public extension DatabaseRelatedObjectsTable {
return cachedObjects
}
guard let resultSet = selectRowsWhere(key: databaseIDKey, inValues: Array(databaseIDsToFetch), in: database) else {
guard let resultSet = database.selectRowsWhere(key: databaseIDKey, equalsAnyValue: Array(databaseIDsToFetch), tableName: name) else {
return cachedObjects
}
@ -76,7 +77,7 @@ public extension DatabaseRelatedObjectsTable {
cache.add(objectsToSave)
if let databaseDictionaries = objectsToSave.databaseDictionaries() {
insertRows(databaseDictionaries, insertType: .orIgnore, in: database)
database.insertRows(databaseDictionaries, insertType: .orIgnore, tableName: name)
}
}

7
Shared/Secrets.swift.gyb
View File

@ -19,10 +19,10 @@ salt = [byte for byte in os.urandom(64)]
}%
import Secrets
public struct Secrets: SecretsProvider {
public final class Secrets: SecretsProvider {
% for secret in secrets:
public var ${snake_to_camel(secret)}: String {
public lazy var ${snake_to_camel(secret)}: String = {
let encoded: [UInt8] = [
% for chunk in chunks(encode(os.environ.get(secret) or "", salt), 8):
${"".join(["0x%02x, " % byte for byte in chunk])}
@ -30,7 +30,7 @@ public struct Secrets: SecretsProvider {
]
return decode(encoded, salt: salt)
}
}()
% end
%{
@ -48,5 +48,4 @@ public struct Secrets: SecretsProvider {
element ^ salt[offset % salt.count]
}, as: UTF8.self)
}
}