FreshRSS
/
NetNewsWire
mirror of https://github.com/Ranchero-Software/NetNewsWire.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add articles download for Feedbin

This commit is contained in:
Maurice Parker 2019-05-12 18:32:32 -05:00
parent 4ed1b8a66a
commit 9c676f29f8
11 changed files with 322 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Frameworks/Account/Account.swift
View File

@ -568,10 +568,13 @@ public final class Account: DisplayNameProvider, UnreadCountProvider, Container,
}
func update(_ feed: Feed, with parsedFeed: ParsedFeed, _ completion: @escaping (() -> Void)) {
feed.takeSettings(from: parsedFeed)
update(feed, parsedItems: parsedFeed.items, completion)
}
func update(_ feed: Feed, parsedItems: Set<ParsedItem>, _ completion: @escaping (() -> Void)) {
database.update(feedID: feed.feedID, parsedFeed: parsedFeed) { (newArticles, updatedArticles) in
database.update(feedID: feed.feedID, parsedItems: parsedItems) { (newArticles, updatedArticles) in
var userInfo = [String: Any]()
if let newArticles = newArticles, !newArticles.isEmpty {
@ -587,8 +590,9 @@ public final class Account: DisplayNameProvider, UnreadCountProvider, Container,
NotificationCenter.default.post(name: .AccountDidDownloadArticles, object: self, userInfo: userInfo)
}
}
// MARK: - Container
public func flattenedFeeds() -> Set<Feed> {

4
Frameworks/Account/Account.xcodeproj/project.pbxproj
View File

@ -30,6 +30,7 @@
51D5875B227F630B00900287 /* tags_add.json in Resources */ = {isa = PBXBuildFile; fileRef = 51D58758227F630B00900287 /* tags_add.json */; };
51D5875C227F630B00900287 /* tags_initial.json in Resources */ = {isa = PBXBuildFile; fileRef = 51D58759227F630B00900287 /* tags_initial.json */; };
51D5875E227F643C00900287 /* AccountFolderSyncTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 51D5875D227F643C00900287 /* AccountFolderSyncTest.swift */; };
51E490362288C37100C791F0 /* FeedbinDate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 51E490352288C37100C791F0 /* FeedbinDate.swift */; };
841973FE1F6DD1BC006346C4 /* RSCore.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 841973EF1F6DD19E006346C4 /* RSCore.framework */; };
841973FF1F6DD1C5006346C4 /* RSParser.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 841973FA1F6DD1AC006346C4 /* RSParser.framework */; };
841974011F6DD1EC006346C4 /* Folder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 841974001F6DD1EC006346C4 /* Folder.swift */; };
@ -126,6 +127,7 @@
51D58758227F630B00900287 /* tags_add.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = tags_add.json; sourceTree = "<group>"; };
51D58759227F630B00900287 /* tags_initial.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = tags_initial.json; sourceTree = "<group>"; };
51D5875D227F643C00900287 /* AccountFolderSyncTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountFolderSyncTest.swift; sourceTree = "<group>"; };
51E490352288C37100C791F0 /* FeedbinDate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FeedbinDate.swift; sourceTree = "<group>"; };
841973E81F6DD19E006346C4 /* RSCore.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = RSCore.xcodeproj; path = ../RSCore/RSCore.xcodeproj; sourceTree = "<group>"; };
841973F41F6DD1AC006346C4 /* RSParser.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = RSParser.xcodeproj; path = ../RSParser/RSParser.xcodeproj; sourceTree = "<group>"; };
841974001F6DD1EC006346C4 /* Folder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Folder.swift; sourceTree = "<group>"; };
@ -246,6 +248,7 @@
children = (
5144EA4D227B829A00D19003 /* FeedbinAccountDelegate.swift */,
5144EA48227B497600D19003 /* FeedbinAPICaller.swift */,
51E490352288C37100C791F0 /* FeedbinDate.swift */,
84CAD7151FDF2E22000F0755 /* FeedbinEntry.swift */,
5133230F22810E5700C30F19 /* FeedbinIcon.swift */,
84245C841FDDD8CB0074AFBB /* FeedbinSubscription.swift */,
@ -508,6 +511,7 @@
841974251F6DDCE4006346C4 /* AccountDelegate.swift in Sources */,
5165D73122837F3400D9D53D /* InitialFeedDownloader.swift in Sources */,
846E77541F6F00E300A165E2 /* AccountManager.swift in Sources */,
51E490362288C37100C791F0 /* FeedbinDate.swift in Sources */,
5165D72922835F7A00D9D53D /* FeedSpecifier.swift in Sources */,
844B297D2106C7EC004020B3 /* Feed.swift in Sources */,
84B2D4D02238CD8A00498ADA /* FeedMetadata.swift in Sources */,

9
Frameworks/Account/AccountMetadata.swift
View File

@ -20,6 +20,7 @@ final class AccountMetadata: Codable {
case isActive
case username
case conditionalGetInfo
case lastArticleFetch
}
var name: String? {
@ -53,6 +54,14 @@ final class AccountMetadata: Codable {
}
}
}
var lastArticleFetch: Date? {
didSet {
if lastArticleFetch != oldValue {
valueDidChange(.lastArticleFetch)
}
}
}
weak var delegate: AccountMetadataDelegate?

57
Frameworks/Account/Feedbin/FeedbinAPICaller.swift
View File

@ -28,7 +28,6 @@ final class FeedbinAPICaller: NSObject {
static let taggings = "taggings"
static let icons = "icons"
}
private let feedbinBaseURL = URL(string: "https://api.feedbin.com/v2/")!
private var transport: Transport!
@ -298,6 +297,62 @@ final class FeedbinAPICaller: NSObject {
}
func retrieveEntries(_ feedID: String, completion: @escaping (Result<([FeedbinEntry]?, String?), Error>) -> Void) {
let since: Date = {
if let lastArticleFetch = accountMetadata?.lastArticleFetch {
return lastArticleFetch
} else {
return Calendar.current.date(byAdding: .month, value: -3, to: Date()) ?? Date()
}
}()
let sinceString = FeedbinDate.formatter.string(from: since)
var callURL = URLComponents(url: feedbinBaseURL.appendingPathComponent("feeds/\(feedID)/entries.json"), resolvingAgainstBaseURL: false)!
callURL.queryItems = [URLQueryItem(name: "since", value: sinceString)]
let request = URLRequest(url: callURL.url!, credentials: credentials)
transport.send(request: request, resultType: [FeedbinEntry].self) { result in
switch result {
case .success(let (response, entries)):
let pagingInfo = HTTPLinkPagingInfo(urlResponse: response)
completion(.success((entries, pagingInfo.nextPage)))
case .failure(let error):
completion(.failure(error))
}
}
}
func retrieveEntries(page: String, completion: @escaping (Result<([FeedbinEntry]?, String?), Error>) -> Void) {
guard let callURL = URL(string: page) else {
completion(.success((nil, nil)))
return
}
let request = URLRequest(url: callURL, credentials: credentials)
transport.send(request: request, resultType: [FeedbinEntry].self) { result in
switch result {
case .success(let (response, entries)):
let pagingInfo = HTTPLinkPagingInfo(urlResponse: response)
completion(.success((entries, pagingInfo.nextPage)))
case .failure(let error):
completion(.failure(error))
}
}
}
}
// MARK: Private

310
Frameworks/Account/Feedbin/FeedbinAccountDelegate.swift
View File

@ -71,19 +71,34 @@ final class FeedbinAccountDelegate: AccountDelegate {
var refreshProgress = DownloadProgress(numberOfTasks: 0)
func refreshAll(for account: Account, completion: (() -> Void)? = nil) {
refreshFolders(account) { [weak self] result in
refreshAccount(account) { [weak self] result in
switch result {
case .success():
DispatchQueue.main.async {
completion?()
self?.refreshArticles(account) { result in
switch result {
case .success():
DispatchQueue.main.async {
completion?()
}
case .failure(let error):
DispatchQueue.main.async {
completion?()
self?.handleError(error)
}
}
}
case .failure(let error):
DispatchQueue.main.async {
completion?()
self?.handleError(error)
}
}
}
}
func importOPML(for account:Account, opmlFile: URL, completion: @escaping (Result<Void, Error>) -> Void) {
@ -383,7 +398,7 @@ private extension FeedbinAccountDelegate {
#endif
}
func refreshFolders(_ account: Account, completion: @escaping (Result<Void, Error>) -> Void) {
func refreshAccount(_ account: Account, completion: @escaping (Result<Void, Error>) -> Void) {
caller.retrieveTags { [weak self] result in
switch result {
@ -399,109 +414,6 @@ private extension FeedbinAccountDelegate {
}
func importOPMLItems(_ account: Account, items: [RSOPMLItem], parentFolder: Folder?) {
items.forEach { (item) in
if let feedSpecifier = item.feedSpecifier {
importFeedSpecifier(account, feedSpecifier: feedSpecifier, parentFolder: parentFolder)
return
}
guard let folderName = item.titleFromAttributes else {
// Folder doesn’t have a name, so it won’t be created, and its items will go one level up.
if let itemChildren = item.children {
importOPMLItems(account, items: itemChildren, parentFolder: parentFolder)
}
return
}
if let folder = account.ensureFolder(with: folderName) {
if let itemChildren = item.children {
importOPMLItems(account, items: itemChildren, parentFolder: folder)
}
}
}
}
func importFeedSpecifier(_ account: Account, feedSpecifier: RSOPMLFeedSpecifier, parentFolder: Folder?) {
caller.createSubscription(url: feedSpecifier.feedURL) { [weak self] result in
switch result {
case .success(let subResult):
switch subResult {
case .created(let sub):
DispatchQueue.main.async {
let feed = account.createFeed(with: sub.name, url: sub.url, feedID: String(sub.feedID), homePageURL: sub.homePageURL)
feed.subscriptionID = String(sub.subscriptionID)
self?.importFeedSpecifierPostProcess(account: account, sub: sub, feedSpecifier: feedSpecifier, feed: feed, parentFolder: parentFolder)
}
default:
break
}
case .failure(let error):
guard let self = self else { return }
os_log(.error, log: self.log, "Create feed on OPML import failed: %@.", error.localizedDescription)
}
}
}
func importFeedSpecifierPostProcess(account: Account, sub: FeedbinSubscription, feedSpecifier: RSOPMLFeedSpecifier, feed: Feed, parentFolder: Folder?) {
// Rename the feed if its name in the OPML file doesn't match the found name
if sub.name != feedSpecifier.title, let newName = feedSpecifier.title {
self.caller.renameSubscription(subscriptionID: String(sub.subscriptionID), newName: newName) { [weak self] result in
switch result {
case .success:
DispatchQueue.main.async {
feed.editedName = newName
}
case .failure(let error):
guard let self = self else { return }
os_log(.error, log: self.log, "Rename feed on OPML import failed: %@.", error.localizedDescription)
}
}
}
// Move the new feed if it is in a folder
if let folder = parentFolder, let feedID = Int(feed.feedID) {
self.caller.createTagging(feedID: feedID, name: folder.name ?? "") { [weak self] result in
switch result {
case .success(let taggingID):
DispatchQueue.main.async {
self?.saveFolderRelationship(for: feed, withFolderName: folder.name ?? "", id: String(taggingID))
folder.addFeed(feed)
}
case .failure(let error):
guard let self = self else { return }
os_log(.error, log: self.log, "Move feed to folder on OPML import failed: %@.", error.localizedDescription)
}
}
} else {
DispatchQueue.main.async {
account.addFeed(feed)
}
}
}
func syncFolders(_ account: Account, _ tags: [FeedbinTag]?) {
guard let tags = tags else { return }
@ -731,6 +643,109 @@ private extension FeedbinAccountDelegate {
}
func importOPMLItems(_ account: Account, items: [RSOPMLItem], parentFolder: Folder?) {
items.forEach { (item) in
if let feedSpecifier = item.feedSpecifier {
importFeedSpecifier(account, feedSpecifier: feedSpecifier, parentFolder: parentFolder)
return
}
guard let folderName = item.titleFromAttributes else {
// Folder doesn’t have a name, so it won’t be created, and its items will go one level up.
if let itemChildren = item.children {
importOPMLItems(account, items: itemChildren, parentFolder: parentFolder)
}
return
}
if let folder = account.ensureFolder(with: folderName) {
if let itemChildren = item.children {
importOPMLItems(account, items: itemChildren, parentFolder: folder)
}
}
}
}
func importFeedSpecifier(_ account: Account, feedSpecifier: RSOPMLFeedSpecifier, parentFolder: Folder?) {
caller.createSubscription(url: feedSpecifier.feedURL) { [weak self] result in
switch result {
case .success(let subResult):
switch subResult {
case .created(let sub):
DispatchQueue.main.async {
let feed = account.createFeed(with: sub.name, url: sub.url, feedID: String(sub.feedID), homePageURL: sub.homePageURL)
feed.subscriptionID = String(sub.subscriptionID)
self?.importFeedSpecifierPostProcess(account: account, sub: sub, feedSpecifier: feedSpecifier, feed: feed, parentFolder: parentFolder)
}
default:
break
}
case .failure(let error):
guard let self = self else { return }
os_log(.error, log: self.log, "Create feed on OPML import failed: %@.", error.localizedDescription)
}
}
}
func importFeedSpecifierPostProcess(account: Account, sub: FeedbinSubscription, feedSpecifier: RSOPMLFeedSpecifier, feed: Feed, parentFolder: Folder?) {
// Rename the feed if its name in the OPML file doesn't match the found name
if sub.name != feedSpecifier.title, let newName = feedSpecifier.title {
self.caller.renameSubscription(subscriptionID: String(sub.subscriptionID), newName: newName) { [weak self] result in
switch result {
case .success:
DispatchQueue.main.async {
feed.editedName = newName
}
case .failure(let error):
guard let self = self else { return }
os_log(.error, log: self.log, "Rename feed on OPML import failed: %@.", error.localizedDescription)
}
}
}
// Move the new feed if it is in a folder
if let folder = parentFolder, let feedID = Int(feed.feedID) {
self.caller.createTagging(feedID: feedID, name: folder.name ?? "") { [weak self] result in
switch result {
case .success(let taggingID):
DispatchQueue.main.async {
self?.saveFolderRelationship(for: feed, withFolderName: folder.name ?? "", id: String(taggingID))
folder.addFeed(feed)
}
case .failure(let error):
guard let self = self else { return }
os_log(.error, log: self.log, "Move feed to folder on OPML import failed: %@.", error.localizedDescription)
}
}
} else {
DispatchQueue.main.async {
account.addFeed(feed)
}
}
}
func processRestoredFeed(for account: Account, feed: Feed, editedName: String?, folder: Folder?, completion: @escaping (Result<Void, Error>) -> Void) {
if let folder = folder {
@ -840,5 +855,86 @@ private extension FeedbinAccountDelegate {
completion(.success(feed))
}
}
func refreshArticles(_ account: Account, completion: @escaping (Result<Void, Error>) -> Void) {
os_log(.debug, log: log, "Refreshing articles...")
for feed in account.flattenedFeeds() {
caller.retrieveEntries(feed.feedID) { [weak self] result in
switch result {
case .success(let (entries, page)):
self?.processEntries(account: account, entries: entries, completion: completion)
self?.refreshArticles(account, page: page)
case .failure(let error):
guard let self = self else { return }
os_log(.error, log: self.log, "Refresh articles failed: %@.", error.localizedDescription)
}
}
}
}
func refreshArticles(_ account: Account, page: String?) {
guard let page = page else {
return
}
caller.retrieveEntries(page: page) { [weak self] result in
switch result {
case .success(let (entries, nextPage)):
self?.processEntries(account: account, entries: entries, completion: nil)
self?.refreshArticles(account, page: nextPage)
case .failure(let error):
guard let self = self else { return }
os_log(.error, log: self.log, "Refresh articles for additional pages failed: %@.", error.localizedDescription)
}
}
}
func processEntries(account: Account, entries: [FeedbinEntry]?, completion: ((Result<Void, Error>) -> Void)?) {
let parsedItems = mapEntriesToParsedItems(entries: entries)
let parsedMap = Dictionary(grouping: parsedItems, by: { item in item.feedURL } )
for (feedID, mapItems) in parsedMap {
if let feed = account.idToFeedDictionary[feedID] {
DispatchQueue.main.async {
account.update(feed, parsedItems: Set(mapItems)) {
completion?(.success(()))
}
}
}
}
}
func mapEntriesToParsedItems(entries: [FeedbinEntry]?) -> Set<ParsedItem> {
guard let entries = entries else {
return Set<ParsedItem>()
}
let parsedItems: [ParsedItem] = entries.map { entry in
let authors = Set([ParsedAuthor(name: entry.authorName, url: nil, avatarURL: nil, emailAddress: nil)])
return ParsedItem(syncServiceID: String(entry.articleID), uniqueID: String(entry.articleID), feedURL: String(entry.feedID), url: nil, externalURL: entry.url, title: entry.title, contentHTML: entry.contentHTML, contentText: nil, summary: entry.summary, imageURL: nil, bannerImageURL: nil, datePublished: entry.parseDatePublished(), dateModified: nil, authors: authors, tags: nil, attachments: nil)
}
return Set(parsedItems)
}
}

21
Frameworks/Account/Feedbin/FeedbinDate.swift Normal file
View File

@ -0,0 +1,21 @@
//
// FeedbinDate.swift
// Account
//
// Created by Maurice Parker on 5/12/19.
// Copyright © 2019 Ranchero Software, LLC. All rights reserved.
//
import Foundation
struct FeedbinDate {
public static var formatter: DateFormatter = {
let formatter = DateFormatter()
formatter.dateFormat = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"
formatter.locale = Locale(identifier: "en_US")
formatter.timeZone = TimeZone(abbreviation: "GMT")
return formatter
}()
}

18
Frameworks/Account/Feedbin/FeedbinEntry.swift
View File

@ -18,10 +18,9 @@ struct FeedbinEntry: Codable {
let url: String?
let authorName: String?
let contentHTML: String?
let contentDiffHTML: String?
let summary: String?
let datePublished: Date?
let dateArrived: Date?
let datePublished: String?
let dateArrived: String?
enum CodingKeys: String, CodingKey {
case articleID = "id"
@ -30,10 +29,21 @@ struct FeedbinEntry: Codable {
case url = "url"
case authorName = "author"
case contentHTML = "content"
case contentDiffHTML = "content_diff"
case summary = "summary"
case datePublished = "published"
case dateArrived = "created_at"
}
// Feedbin dates can't be decoded by the JSONDecoding 8601 decoding strategy. Feedbin
// requires a very specific date formatter to work and even then it fails occasionally.
// Rather than loose all the entries we only lose the one date by decoding as a string
// and letting the one date fail when parsed.
func parseDatePublished() -> Date? {
if datePublished != nil {
return FeedbinDate.formatter.date(from: datePublished!)
} else {
return nil
}
}
}

4
Frameworks/ArticlesDatabase/ArticlesDatabase.swift
View File

@ -90,8 +90,8 @@ public final class ArticlesDatabase {
// MARK: - Saving and Updating Articles
public func update(feedID: String, parsedFeed: ParsedFeed, completion: @escaping UpdateArticlesWithFeedCompletionBlock) {
return articlesTable.update(feedID, parsedFeed, completion)
public func update(feedID: String, parsedItems: Set<ParsedItem>, completion: @escaping UpdateArticlesWithFeedCompletionBlock) {
return articlesTable.update(feedID, parsedItems, completion)
}
// MARK: - Status

8
Frameworks/ArticlesDatabase/ArticlesTable.swift
View File

@ -119,9 +119,9 @@ final class ArticlesTable: DatabaseTable {
// MARK: Updating
func update(_ feedID: String, _ parsedFeed: ParsedFeed, _ completion: @escaping UpdateArticlesWithFeedCompletionBlock) {
func update(_ feedID: String, _ parsedItems: Set<ParsedItem>, _ completion: @escaping UpdateArticlesWithFeedCompletionBlock) {
if parsedFeed.items.isEmpty {
if parsedItems.isEmpty {
completion(nil, nil)
return
}
@ -135,14 +135,14 @@ final class ArticlesTable: DatabaseTable {
// 7. Call back with new and updated Articles.
// 8. Update search index.
let articleIDs = Set(parsedFeed.items.map { $0.articleID })
let articleIDs = Set(parsedItems.map { $0.articleID })
self.queue.update { (database) in
let statusesDictionary = self.statusesTable.ensureStatusesForArticleIDs(articleIDs, database) //1
assert(statusesDictionary.count == articleIDs.count)
let allIncomingArticles = Article.articlesWithParsedItems(parsedFeed.items, self.accountID, feedID, statusesDictionary) //2
let allIncomingArticles = Article.articlesWithParsedItems(parsedItems, self.accountID, feedID, statusesDictionary) //2
if allIncomingArticles.isEmpty {
self.callUpdateArticlesCompletionBlock(nil, nil, completion)
return

2
submodules/RSParser

@ -1 +1 @@
Subproject commit 915e867d1529d87ae6dd6d616e35372b5305869f
Subproject commit 7ee920bc6a8fd71b6c00e87fc1c473e5b28d41c6

2
submodules/RSWeb

@ -1 +1 @@
Subproject commit 9e8ef66b5cba0316926f243d0465dfbb1fdc307e
Subproject commit 6b0839c66cf772fb08aabd1b1d9c882897ad1adb