FreshRSS/NetNewsWire
1
0
Fork 0
mirror of https://github.com/Ranchero-Software/NetNewsWire.git synced 2025-01-26 17:05:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

Parse articles from story river

Browse Source
This commit is contained in:
Anh Do 2020-03-13 18:18:47 -04:00
parent d37f70d2dd
commit 175cd0e798
No known key found for this signature in database
GPG Key ID: 451E3092F917B62D
6 changed files with 186 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Frameworks/Account/Account.xcodeproj/project.pbxproj
View File

@ -10,6 +10,8 @@
179DB02FFBC17AC9798F0EBC /* NewsBlurArticle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB7399814F6FB3247825C /* NewsBlurArticle.swift */; };
179DB28CF49F73A945EBF5DB /* NewsBlurLoginResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB088236E3236010462E8 /* NewsBlurLoginResponse.swift */; };
179DB49A960F8B78C4924458 /* NewsBlurGenericCodingKeys.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB66D933E976C29159DEE /* NewsBlurGenericCodingKeys.swift */; };
179DB61D33CD8DC94C90F7ED /* NewsBlurDate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DBB17C42E6E434EDC29FA /* NewsBlurDate.swift */; };
179DBED55C9B4D6A413486C1 /* NewsBlurUnreadArticle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB818180A51098A9816B2 /* NewsBlurUnreadArticle.swift */; };
179DBF4DE2562D4C532F6008 /* NewsBlurSubscription.swift in Sources */ = {isa = PBXBuildFile; fileRef = 179DB1B909672E0E807B5E8C /* NewsBlurSubscription.swift */; };
3B3A33E7238D3D6800314204 /* Secrets.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B3A33E6238D3D6800314204 /* Secrets.swift */; };
3B826DA72385C81C00FC1ADB /* FeedWranglerAuthorizationResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B826D9E2385C81C00FC1ADB /* FeedWranglerAuthorizationResult.swift */; };
@ -230,6 +232,8 @@
179DB1B909672E0E807B5E8C /* NewsBlurSubscription.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurSubscription.swift; sourceTree = "<group>"; };
179DB66D933E976C29159DEE /* NewsBlurGenericCodingKeys.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurGenericCodingKeys.swift; sourceTree = "<group>"; };
179DB7399814F6FB3247825C /* NewsBlurArticle.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurArticle.swift; sourceTree = "<group>"; };
179DB818180A51098A9816B2 /* NewsBlurUnreadArticle.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurUnreadArticle.swift; sourceTree = "<group>"; };
179DBB17C42E6E434EDC29FA /* NewsBlurDate.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NewsBlurDate.swift; sourceTree = "<group>"; };
3B3A33E6238D3D6800314204 /* Secrets.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Secrets.swift; path = ../../Shared/Secrets.swift; sourceTree = "<group>"; };
3B826D9E2385C81C00FC1ADB /* FeedWranglerAuthorizationResult.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FeedWranglerAuthorizationResult.swift; sourceTree = "<group>"; };
3B826D9F2385C81C00FC1ADB /* FeedWranglerFeedItem.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FeedWranglerFeedItem.swift; sourceTree = "<group>"; };
@ -454,6 +458,8 @@
179DB1B909672E0E807B5E8C /* NewsBlurSubscription.swift */,
179DB7399814F6FB3247825C /* NewsBlurArticle.swift */,
179DB66D933E976C29159DEE /* NewsBlurGenericCodingKeys.swift */,
179DB818180A51098A9816B2 /* NewsBlurUnreadArticle.swift */,
179DBB17C42E6E434EDC29FA /* NewsBlurDate.swift */,
);
path = Models;
sourceTree = "<group>";
@ -1147,6 +1153,8 @@
179DBF4DE2562D4C532F6008 /* NewsBlurSubscription.swift in Sources */,
179DB02FFBC17AC9798F0EBC /* NewsBlurArticle.swift in Sources */,
179DB49A960F8B78C4924458 /* NewsBlurGenericCodingKeys.swift in Sources */,
179DBED55C9B4D6A413486C1 /* NewsBlurUnreadArticle.swift in Sources */,
179DB61D33CD8DC94C90F7ED /* NewsBlurDate.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};

55
Frameworks/Account/NewsBlur/Models/NewsBlurArticle.swift
View File

@ -10,41 +10,36 @@ import Foundation
import RSCore
import RSParser
typealias NewsBlurArticleHash = NewsBlurUnreadArticleHashesResponse.ArticleHash
typealias NewsBlurArticle = NewsBlurArticlesResponse.Article
struct NewsBlurUnreadArticleHashesResponse: Decodable {
let subscriptions: [String: [ArticleHash]]
struct NewsBlurArticlesResponse: Decodable {
let articles: [Article]
struct ArticleHash: Hashable, Codable {
var hash: String
var timestamp: Date
struct Article: Decodable {
let articleId: String
let feedId: Int
let title: String?
let url: String?
let authorName: String?
let contentHTML: String?
let datePublished: Date
}
}
extension NewsBlurUnreadArticleHashesResponse {
extension NewsBlurArticlesResponse {
private enum CodingKeys: String, CodingKey {
case feeds = "unread_feed_story_hashes"
}
init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
// Parse subscriptions
var subscriptions: [String: [ArticleHash]] = [:]
let subscriptionContainer = try container.nestedContainer(keyedBy: NewsBlurGenericCodingKeys.self, forKey: .feeds)
try subscriptionContainer.allKeys.forEach { key in
subscriptions[key.stringValue] = []
var hashArrayContainer = try subscriptionContainer.nestedUnkeyedContainer(forKey: key)
while !hashArrayContainer.isAtEnd {
var hashContainer = try hashArrayContainer.nestedUnkeyedContainer()
let hash = try hashContainer.decode(String.self)
let timestamp = try hashContainer.decode(Date.self)
let articleHash = ArticleHash(hash: hash, timestamp: timestamp)
subscriptions[key.stringValue]?.append(articleHash)
}
}
self.subscriptions = subscriptions
case articles = "stories"
}
}
extension NewsBlurArticlesResponse.Article {
private enum CodingKeys: String, CodingKey {
case articleId = "story_hash"
case feedId = "story_feed_id"
case title = "story_title"
case url = "story_permalink"
case authorName = "story_authors"
case contentHTML = "story_content"
case datePublished = "story_date"
}
}

20
Frameworks/Account/NewsBlur/Models/NewsBlurDate.swift Normal file
View File

@ -0,0 +1,20 @@
//
// NewsBlurDate.swift
// Account
//
// Created by Anh Quang Do on 2020-03-13.
// Copyright (c) 2020 Ranchero Software, LLC. All rights reserved.
//
import Foundation
struct NewsBlurDate {
static let yyyyMMddHHmmss: DateFormatter = {
let formatter = DateFormatter()
formatter.calendar = Calendar(identifier: .iso8601)
formatter.locale = Locale(identifier: "en_US_POSIX")
formatter.timeZone = TimeZone(abbreviation: "GMT")
formatter.dateFormat = "yyyy-MM-dd HH:mm:ss"
return formatter
}()
}

50
Frameworks/Account/NewsBlur/Models/NewsBlurUnreadArticle.swift Normal file
View File

@ -0,0 +1,50 @@
//
// NewsBlurUnreadArticle.swift
// Account
//
// Created by Anh Quang Do on 2020-03-13.
// Copyright (c) 2020 Ranchero Software, LLC. All rights reserved.
//
import Foundation
import RSCore
import RSParser
typealias NewsBlurArticleHash = NewsBlurUnreadArticleHashesResponse.ArticleHash
struct NewsBlurUnreadArticleHashesResponse: Decodable {
let subscriptions: [String: [ArticleHash]]
struct ArticleHash: Hashable, Codable {
var hash: String
var timestamp: Date
}
}
extension NewsBlurUnreadArticleHashesResponse {
private enum CodingKeys: String, CodingKey {
case feeds = "unread_feed_story_hashes"
}
init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
// Parse subscriptions
var subscriptions: [String: [ArticleHash]] = [:]
let subscriptionContainer = try container.nestedContainer(keyedBy: NewsBlurGenericCodingKeys.self, forKey: .feeds)
try subscriptionContainer.allKeys.forEach { key in
subscriptions[key.stringValue] = []
var hashArrayContainer = try subscriptionContainer.nestedUnkeyedContainer(forKey: key)
while !hashArrayContainer.isAtEnd {
var hashContainer = try hashArrayContainer.nestedUnkeyedContainer()
let hash = try hashContainer.decode(String.self)
let timestamp = try hashContainer.decode(Date.self)
let articleHash = ArticleHash(hash: hash, timestamp: timestamp)
subscriptions[key.stringValue]?.append(articleHash)
}
}
self.subscriptions = subscriptions
}
}

34
Frameworks/Account/NewsBlur/NewsBlurAPICaller.swift
View File

@ -84,7 +84,7 @@ final class NewsBlurAPICaller: NSObject {
}
}
func retrieveSubscriptions(completion: @escaping (Result<[NewsBlurSubscription], Error>) -> Void) {
func retrieveSubscriptions(completion: @escaping (Result<[NewsBlurSubscription]?, Error>) -> Void) {
let url = baseURL
.appendingPathComponent("reader/feeds")
.appendingQueryItems([
@ -101,14 +101,14 @@ final class NewsBlurAPICaller: NSObject {
transport.send(request: request, resultType: NewsBlurFeedsResponse.self) { result in
switch result {
case .success((_, let payload)):
completion(.success(payload?.subscriptions ?? []))
completion(.success(payload?.subscriptions))
case .failure(let error):
completion(.failure(error))
}
}
}
func retrieveUnreadArticleHashes(completion: @escaping (Result<[NewsBlurArticleHash], Error>) -> Void) {
func retrieveUnreadArticleHashes(completion: @escaping (Result<[NewsBlurArticleHash]?, Error>) -> Void) {
let url = baseURL
.appendingPathComponent("reader/unread_story_hashes")
.appendingQueryItems([
@ -124,11 +124,29 @@ final class NewsBlurAPICaller: NSObject {
transport.send(request: request, resultType: NewsBlurUnreadArticleHashesResponse.self, dateDecoding: .secondsSince1970) { result in
switch result {
case .success((_, let payload)):
guard let subscriptions = payload?.subscriptions else {
completion(.success([]))
return
}
completion(.success(subscriptions.values.flatMap { $0 }))
completion(.success(payload?.subscriptions.values.flatMap { $0 }))
case .failure(let error):
completion(.failure(error))
}
}
}
func retrieveArticles(hashes: [NewsBlurArticleHash], completion: @escaping (Result<[NewsBlurArticle]?, Error>) -> Void) {
let url = baseURL
.appendingPathComponent("reader/river_stories")
.appendingQueryItem(.init(name: "include_hidden", value: "true"))?
.appendingQueryItems(hashes.map { URLQueryItem(name: "h", value: $0.hash) })
guard let callURL = url else {
completion(.failure(TransportError.noURL))
return
}
let request = URLRequest(url: callURL, credentials: credentials)
transport.send(request: request, resultType: NewsBlurArticlesResponse.self, dateDecoding: .formatted(NewsBlurDate.yyyyMMddHHmmss)) { result in
switch result {
case .success((_, let payload)):
completion(.success(payload?.articles))
case .failure(let error):
completion(.failure(error))
}

60
Frameworks/Account/NewsBlur/NewsBlurAccountDelegate.swift
View File

@ -123,15 +123,50 @@ final class NewsBlurAccountDelegate: AccountDelegate {
completion(.success(()))
}
func refreshArticles(for account: Account, completion: @escaping (Result<[NewsBlurArticleHash], Error>) -> Void) {
func refreshArticles(for account: Account, completion: @escaping (Result<Void, Error>) -> Void) {
os_log(.debug, log: log, "Refreshing articles...")
os_log(.debug, log: log, "Refreshing unread articles...")
caller.retrieveUnreadArticleHashes { result in
switch result {
case .success(let articleHashes):
print(articleHashes)
self.refreshProgress.completeTask()
self.refreshUnreadArticles(for: account, hashes: articleHashes, updateFetchDate: nil, completion: completion)
case .failure(let error):
break
completion(.failure(error))
}
}
}
func refreshUnreadArticles(for account: Account, hashes: [NewsBlurArticleHash]?, updateFetchDate: Date?, completion: @escaping (Result<Void, Error>) -> Void) {
guard let hashes = hashes, !hashes.isEmpty else {
if let lastArticleFetch = updateFetchDate {
self.accountMetadata?.lastArticleFetchStartTime = lastArticleFetch
self.accountMetadata?.lastArticleFetchEndTime = Date()
}
completion(.success(()))
return
}
let numberOfArticles = min(hashes.count, 100) // api limit
let hashesToFetch = Array(hashes[..<numberOfArticles])
caller.retrieveArticles(hashes: hashesToFetch) { result in
switch result {
case .success(let articles):
self.processArticles(account: account, articles: articles) { error in
self.refreshProgress.completeTask()
if let error = error {
completion(.failure(error))
return
}
self.refreshUnreadArticles(for: account, hashes: Array(hashes[numberOfArticles...]), updateFetchDate: updateFetchDate, completion: completion)
}
case .failure(let error):
completion(.failure(error))
}
}
}
@ -139,6 +174,25 @@ final class NewsBlurAccountDelegate: AccountDelegate {
func refreshMissingArticles(for account: Account, completion: @escaping (Result<Void, Error>)-> Void) {
completion(.success(()))
}
func processArticles(account: Account, articles: [NewsBlurArticle]?, completion: @escaping DatabaseCompletionBlock) {
let parsedItems = mapArticlesToParsedItems(articles: articles)
let webFeedIDsAndItems = Dictionary(grouping: parsedItems, by: { item in item.feedURL } ).mapValues { Set($0) }
account.update(webFeedIDsAndItems: webFeedIDsAndItems, defaultRead: true, completion: completion)
}
func mapArticlesToParsedItems(articles: [NewsBlurArticle]?) -> Set<ParsedItem> {
guard let articles = articles else {
return Set<ParsedItem>()
}
let parsedItems: [ParsedItem] = articles.map { article in
let author = Set([ParsedAuthor(name: article.authorName, url: nil, avatarURL: nil, emailAddress: nil)])
return ParsedItem(syncServiceID: article.articleId, uniqueID: String(article.articleId), feedURL: String(article.feedId), url: article.url, externalURL: nil, title: article.title, contentHTML: article.contentHTML, contentText: nil, summary: nil, imageURL: nil, bannerImageURL: nil, datePublished: article.datePublished, dateModified: nil, authors: author, tags: nil, attachments: nil)
}
return Set(parsedItems)
}
func importOPML(for account: Account, opmlFile: URL, completion: @escaping (Result<Void, Error>) -> ()) {
completion(.success(()))