FreshRSS
/
NetNewsWire
mirror of https://github.com/Ranchero-Software/NetNewsWire.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Request article IDs and content.

This commit is contained in:
Jeremy Beker 2019-06-10 16:53:35 -04:00
parent 6b147e7dc9
commit 9144ee71e5
No known key found for this signature in database
GPG Key ID: CD5EE767A4A34FD0
5 changed files with 314 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

230
Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAPICaller.swift
View File

@ -108,6 +108,36 @@ final class GoogleReaderCompatibleAPICaller: NSObject {
}
func requestAuthorizationToken(endpoint: URL, completion: @escaping (Result<String, Error>) -> Void) {
guard let credentials = credentials else {
completion(.failure(CredentialsError.incompleteCredentials))
return
}
let request = URLRequest(url: endpoint.appendingPathComponent("/reader/api/0/token"), credentials: credentials)
transport.send(request: request) { result in
switch result {
case .success(let (_, data)):
guard let resultData = data else {
completion(.failure(TransportError.noData))
break
}
// Convert the return data to UTF8 and then parse out the Auth token
guard let rawData = String(data: resultData, encoding: .utf8) else {
completion(.failure(TransportError.noData))
break
}
completion(.success(rawData))
case .failure(let error):
completion(.failure(error))
}
}
}
func importOPML(opmlData: Data, completion: @escaping (Result<GoogleReaderCompatibleImportResult, Error>) -> Void) {
let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("imports.json")
@ -412,24 +442,51 @@ final class GoogleReaderCompatibleAPICaller: NSObject {
return
}
let concatIDs = articleIDs.reduce("") { param, articleID in return param + ",\(articleID)" }
let paramIDs = String(concatIDs.dropFirst())
guard let baseURL = APIBaseURL else {
completion(.failure(CredentialsError.incompleteCredentials))
return
}
var callComponents = URLComponents(url: GoogleReaderCompatibleBaseURL.appendingPathComponent("entries.json"), resolvingAgainstBaseURL: false)!
callComponents.queryItems = [URLQueryItem(name: "ids", value: paramIDs), URLQueryItem(name: "mode", value: "extended")]
let request = URLRequest(url: callComponents.url!, credentials: credentials)
transport.send(request: request, resultType: [GoogleReaderCompatibleEntry].self) { result in
self.requestAuthorizationToken(endpoint: baseURL) { (result) in
switch result {
case .success(let (_, entries)):
completion(.success((entries)))
case .success(let token):
// Do POST asking for data about all the new articles
var request = URLRequest(url: baseURL.appendingPathComponent("/reader/api/0/stream/items/contents"), credentials: self.credentials)
request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.httpMethod = "POST"
// Get ids from above into hex representation of value
let idsToFetch = articleIDs.map({ (reference) -> String in
return "i=\(reference)"
}).joined(separator:"&")
let postData = "T=\(token)&output=json&\(idsToFetch)".data(using: String.Encoding.utf8)
//let postData = "T=\(token)&output=json&i=1349530380539369".data(using: String.Encoding.utf8)
self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: GoogleReaderCompatibleEntryWrapper.self, completion: { (result) in
switch result {
case .success(let (response, entryWrapper)):
guard let entryWrapper = entryWrapper else {
completion(.failure(GoogleReaderCompatibleAccountDelegateError.invalidResponse))
return
}
let dateInfo = HTTPDateInfo(urlResponse: response)
self.accountMetadata?.lastArticleFetch = dateInfo?.date
completion(.success((entryWrapper.entries)))
case .failure(let error):
completion(.failure(error))
}
})
case .failure(let error):
completion(.failure(error))
}
}
}
func retrieveEntries(feedID: String, completion: @escaping (Result<([GoogleReaderCompatibleEntry]?, String?), Error>) -> Void) {
@ -459,30 +516,96 @@ final class GoogleReaderCompatibleAPICaller: NSObject {
func retrieveEntries(completion: @escaping (Result<([GoogleReaderCompatibleEntry]?, String?, Int?), Error>) -> Void) {
guard let baseURL = APIBaseURL else {
completion(.failure(CredentialsError.incompleteCredentials))
return
}
let since: Date = {
if let lastArticleFetch = accountMetadata?.lastArticleFetch {
if let lastArticleFetch = self.accountMetadata?.lastArticleFetch {
return lastArticleFetch
} else {
return Calendar.current.date(byAdding: .month, value: -3, to: Date()) ?? Date()
}
}()
let sinceString = GoogleReaderCompatibleDate.formatter.string(from: since)
var callComponents = URLComponents(url: GoogleReaderCompatibleBaseURL.appendingPathComponent("entries.json"), resolvingAgainstBaseURL: false)!
callComponents.queryItems = [URLQueryItem(name: "since", value: sinceString), URLQueryItem(name: "per_page", value: "100"), URLQueryItem(name: "mode", value: "extended")]
let request = URLRequest(url: callComponents.url!, credentials: credentials)
let sinceString = since.timeIntervalSince1970
transport.send(request: request, resultType: [GoogleReaderCompatibleEntry].self) { result in
// Add query string for getting JSON (probably should break this out as I will be doing it a lot)
guard var components = URLComponents(url: baseURL.appendingPathComponent("/reader/api/0/stream/items/ids"), resolvingAgainstBaseURL: false) else {
completion(.failure(TransportError.noURL))
return
}
components.queryItems = [
URLQueryItem(name: "o", value: String(sinceString)),
URLQueryItem(name: "n", value: "10000"),
URLQueryItem(name: "output", value: "json"),
URLQueryItem(name: "xt", value: "user/-/state/com.google/read"),
URLQueryItem(name: "s", value: "user/-/state/com.google/reading-list")
]
guard let callURL = components.url else {
completion(.failure(TransportError.noURL))
return
}
let conditionalGet = accountMetadata?.conditionalGetInfo[ConditionalGetKeys.unreadEntries]
let request = URLRequest(url: callURL, credentials: credentials, conditionalGet: conditionalGet)
self.transport.send(request: request, resultType: GoogleReaderCompatibleReferenceWrapper.self) { result in
switch result {
case .success(let (response, entries)):
case .success(let (_, entries)):
let dateInfo = HTTPDateInfo(urlResponse: response)
self.accountMetadata?.lastArticleFetch = dateInfo?.date
guard let entries = entries else {
completion(.failure(GoogleReaderCompatibleAccountDelegateError.invalidResponse))
return
}
self.requestAuthorizationToken(endpoint: baseURL) { (result) in
switch result {
case .success(let token):
// Do POST asking for data about all the new articles
var request = URLRequest(url: baseURL.appendingPathComponent("/reader/api/0/stream/items/contents"), credentials: self.credentials)
request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.httpMethod = "POST"
// Get ids from above into hex representation of value
let idsToFetch = entries.itemRefs.map({ (reference) -> String in
let idValue = Int(reference.itemId)!
let idHexString = String(idValue, radix: 16, uppercase: false)
return "i=\(idHexString)"
}).joined(separator:"&")
let postData = "T=\(token)&output=json&\(idsToFetch)".data(using: String.Encoding.utf8)
//let postData = "T=\(token)&output=json&i=1349530380539369".data(using: String.Encoding.utf8)
let pagingInfo = HTTPLinkPagingInfo(urlResponse: response)
let lastPageNumber = self.extractPageNumber(link: pagingInfo.lastPage)
completion(.success((entries, pagingInfo.nextPage, lastPageNumber)))
self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: GoogleReaderCompatibleEntryWrapper.self, completion: { (result) in
switch result {
case .success(let (response, entryWrapper)):
guard let entryWrapper = entryWrapper else {
completion(.failure(GoogleReaderCompatibleAccountDelegateError.invalidResponse))
return
}
let dateInfo = HTTPDateInfo(urlResponse: response)
self.accountMetadata?.lastArticleFetch = dateInfo?.date
completion(.success((entryWrapper.entries, nil, nil)))
case .failure(let error):
completion(.failure(error))
}
})
case .failure(let error):
completion(.failure(error))
}
}
//completion(.success((entries, pagingInfo.nextPage, lastPageNumber)))
case .failure(let error):
self.accountMetadata?.lastArticleFetch = nil
@ -491,6 +614,13 @@ final class GoogleReaderCompatibleAPICaller: NSObject {
}
}
func retrieveEntries(page: String, completion: @escaping (Result<([GoogleReaderCompatibleEntry]?, String?), Error>) -> Void) {
@ -522,16 +652,46 @@ final class GoogleReaderCompatibleAPICaller: NSObject {
func retrieveUnreadEntries(completion: @escaping (Result<[Int]?, Error>) -> Void) {
let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json")
guard let baseURL = APIBaseURL else {
completion(.failure(CredentialsError.incompleteCredentials))
return
}
// Add query string for getting JSON (probably should break this out as I will be doing it a lot)
guard var components = URLComponents(url: baseURL.appendingPathComponent("/reader/api/0/stream/items/ids"), resolvingAgainstBaseURL: false) else {
completion(.failure(TransportError.noURL))
return
}
components.queryItems = [
URLQueryItem(name: "s", value: "user/-/state/com.google/reading-list"),
URLQueryItem(name: "n", value: "10000"),
URLQueryItem(name: "xt", value: "user/-/state/com.google/read"),
URLQueryItem(name: "output", value: "json")
]
guard let callURL = components.url else {
completion(.failure(TransportError.noURL))
return
}
let conditionalGet = accountMetadata?.conditionalGetInfo[ConditionalGetKeys.unreadEntries]
let request = URLRequest(url: callURL, credentials: credentials, conditionalGet: conditionalGet)
transport.send(request: request, resultType: [Int].self) { result in
transport.send(request: request, resultType: GoogleReaderCompatibleReferenceWrapper.self) { result in
switch result {
case .success(let (response, unreadEntries)):
guard let itemRefs = unreadEntries?.itemRefs else {
completion(.success([]))
return
}
let itemIds = itemRefs.map{ Int($0.itemId)! }
self.storeConditionalGet(key: ConditionalGetKeys.unreadEntries, headers: response.allHeaderFields)
completion(.success(unreadEntries))
completion(.success(itemIds))
case .failure(let error):
completion(.failure(error))
}
@ -541,17 +701,17 @@ final class GoogleReaderCompatibleAPICaller: NSObject {
}
func createUnreadEntries(entries: [Int], completion: @escaping (Result<Void, Error>) -> Void) {
let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json")
let request = URLRequest(url: callURL, credentials: credentials)
let payload = GoogleReaderCompatibleUnreadEntry(unreadEntries: entries)
transport.send(request: request, method: HTTPMethod.post, payload: payload, completion: completion)
// let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json")
// let request = URLRequest(url: callURL, credentials: credentials)
// let payload = GoogleReaderCompatibleUnreadEntry(unreadEntries: entries)
// transport.send(request: request, method: HTTPMethod.post, payload: payload, completion: completion)
}
func deleteUnreadEntries(entries: [Int], completion: @escaping (Result<Void, Error>) -> Void) {
let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json")
let request = URLRequest(url: callURL, credentials: credentials)
let payload = GoogleReaderCompatibleUnreadEntry(unreadEntries: entries)
transport.send(request: request, method: HTTPMethod.delete, payload: payload, completion: completion)
// let callURL = GoogleReaderCompatibleBaseURL.appendingPathComponent("unread_entries.json")
// let request = URLRequest(url: callURL, credentials: credentials)
// let payload = GoogleReaderCompatibleUnreadEntry(unreadEntries: entries)
// transport.send(request: request, method: HTTPMethod.delete, payload: payload, completion: completion)
}
func retrieveStarredEntries(completion: @escaping (Result<[Int]?, Error>) -> Void) {

43
Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleAccountDelegate.swift
View File

@ -21,6 +21,7 @@ import os.log
public enum GoogleReaderCompatibleAccountDelegateError: String, Error {
case invalidParameter = "There was an invalid parameter passed."
case invalidResponse = "There was an invalid response from the server."
}
final class GoogleReaderCompatibleAccountDelegate: AccountDelegate {
@ -98,14 +99,14 @@ final class GoogleReaderCompatibleAccountDelegate: AccountDelegate {
self.refreshArticles(account) {
// self.refreshArticleStatus(for: account) {
// self.refreshMissingArticles(account) {
self.refreshArticleStatus(for: account) {
self.refreshMissingArticles(account) {
self.refreshProgress.clear()
DispatchQueue.main.async {
completion(.success(()))
}
// }
// }
}
}
}
case .failure(let error):
@ -178,18 +179,18 @@ final class GoogleReaderCompatibleAccountDelegate: AccountDelegate {
}
group.enter()
caller.retrieveStarredEntries() { result in
switch result {
case .success(let articleIDs):
self.syncArticleStarredState(account: account, articleIDs: articleIDs)
group.leave()
case .failure(let error):
os_log(.info, log: self.log, "Retrieving starred entries failed: %@.", error.localizedDescription)
group.leave()
}
}
// group.enter()
// caller.retrieveStarredEntries() { result in
// switch result {
// case .success(let articleIDs):
// self.syncArticleStarredState(account: account, articleIDs: articleIDs)
// group.leave()
// case .failure(let error):
// os_log(.info, log: self.log, "Retrieving starred entries failed: %@.", error.localizedDescription)
// group.leave()
// }
//
// }
group.notify(queue: DispatchQueue.main) {
os_log(.debug, log: self.log, "Done refreshing article statuses.")
@ -970,7 +971,7 @@ private extension GoogleReaderCompatibleAccountDelegate {
func processEntries(account: Account, entries: [GoogleReaderCompatibleEntry]?, completion: @escaping (() -> Void)) {
let parsedItems = mapEntriesToParsedItems(entries: entries)
let parsedItems = mapEntriesToParsedItems(account: account, entries: entries)
let parsedMap = Dictionary(grouping: parsedItems, by: { item in item.feedURL } )
let group = DispatchGroup()
@ -997,15 +998,17 @@ private extension GoogleReaderCompatibleAccountDelegate {
}
func mapEntriesToParsedItems(entries: [GoogleReaderCompatibleEntry]?) -> Set<ParsedItem> {
func mapEntriesToParsedItems(account: Account, entries: [GoogleReaderCompatibleEntry]?) -> Set<ParsedItem> {
guard let entries = entries else {
return Set<ParsedItem>()
}
let parsedItems: [ParsedItem] = entries.map { entry in
let authors = Set([ParsedAuthor(name: entry.authorName, url: entry.jsonFeed?.jsonFeedAuthor?.url, avatarURL: entry.jsonFeed?.jsonFeedAuthor?.avatarURL, emailAddress: nil)])
return ParsedItem(syncServiceID: String(entry.articleID), uniqueID: String(entry.articleID), feedURL: String(entry.feedID), url: nil, externalURL: entry.url, title: entry.title, contentHTML: entry.contentHTML, contentText: nil, summary: entry.summary, imageURL: nil, bannerImageURL: nil, datePublished: entry.parseDatePublished(), dateModified: nil, authors: authors, tags: nil, attachments: nil)
// let authors = Set([ParsedAuthor(name: entry.authorName, url: entry.jsonFeed?.jsonFeedAuthor?.url, avatarURL: entry.jsonFeed?.jsonFeedAuthor?.avatarURL, emailAddress: nil)])
// let feed = account.idToFeedDictionary[entry.origin.streamId!]! // TODO clean this up
return ParsedItem(syncServiceID: String(entry.articleID), uniqueID: String(entry.articleID), feedURL: entry.origin.streamId!, url: nil, externalURL: entry.alternates.first?.url, title: entry.title, contentHTML: entry.summary.content, contentText: nil, summary: entry.summary.content, imageURL: nil, bannerImageURL: nil, datePublished: entry.parseDatePublished(), dateModified: nil, authors: nil, tags: nil, attachments: nil)
}
return Set(parsedItems)

121
Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleEntry.swift
View File

@ -10,58 +10,103 @@ import Foundation
import RSParser
import RSCore
struct GoogleReaderCompatibleEntryWrapper: Codable {
let id: String
let updated: Int
let entries: [GoogleReaderCompatibleEntry]
enum CodingKeys: String, CodingKey {
case id = "id"
case updated = "updated"
case entries = "items"
}
}
/* {
"id": "tag:google.com,2005:reader/item/00058a3b5197197b",
"crawlTimeMsec": "1559362260113",
"timestampUsec": "1559362260113787",
"published": 1554845280,
"title": "",
"summary": {
"content": "\n<p>Found an old screenshot of NetNewsWire 1.0 for iPhone!</p>\n\n<p><img src=\"https://nnw.ranchero.com/uploads/2019/c07c0574b1.jpg\" alt=\"Netnewswire 1.0 for iPhone screenshot showing the list of feeds.\" title=\"NewsGator got renamed to Sitrion, years later, and then renamed again as Limeade.\" border=\"0\" width=\"260\" height=\"320\"></p>\n"
},
"alternate": [
{
"href": "https://nnw.ranchero.com/2019/04/09/found-an-old.html"
}
],
"categories": [
"user/-/state/com.google/reading-list",
"user/-/label/Uncategorized"
],
"origin": {
"streamId": "feed/130",
"title": "NetNewsWire"
}
}
*/
struct GoogleReaderCompatibleEntry: Codable {
let articleID: Int
let feedID: Int
let articleID: String
let title: String?
let url: String?
let authorName: String?
let contentHTML: String?
let summary: String?
let datePublished: String?
let dateArrived: String?
let jsonFeed: GoogleReaderCompatibleEntryJSONFeed?
let publishedTimestamp: Double?
let crawledTimestamp: String?
let timestampUsec: String?
let summary: GoogleReaderCompatibleArticleSummary
let alternates: [GoogleReaderCompatibleAlternateLocation]
let categories: [String]
let origin: GoogleReaderCompatibleEntryOrigin
enum CodingKeys: String, CodingKey {
case articleID = "id"
case feedID = "feed_id"
case title = "title"
case url = "url"
case authorName = "author"
case contentHTML = "content"
case summary = "summary"
case datePublished = "published"
case dateArrived = "created_at"
case jsonFeed = "json_feed"
}
// GoogleReaderCompatible dates can't be decoded by the JSONDecoding 8601 decoding strategy. GoogleReaderCompatible
// requires a very specific date formatter to work and even then it fails occasionally.
// Rather than loose all the entries we only lose the one date by decoding as a string
// and letting the one date fail when parsed.
func parseDatePublished() -> Date? {
if datePublished != nil {
return GoogleReaderCompatibleDate.formatter.date(from: datePublished!)
} else {
return nil
}
case alternates = "alternate"
case categories = "categories"
case publishedTimestamp = "published"
case crawledTimestamp = "crawlTimeMsec"
case origin = "origin"
case timestampUsec = "timestampUsec"
}
}
struct GoogleReaderCompatibleEntryJSONFeed: Codable {
let jsonFeedAuthor: GoogleReaderCompatibleEntryJSONFeedAuthor?
enum CodingKeys: String, CodingKey {
case jsonFeedAuthor = "author"
func parseDatePublished() -> Date? {
guard let unixTime = publishedTimestamp else {
return nil
}
return Date(timeIntervalSince1970: unixTime)
}
}
struct GoogleReaderCompatibleEntryJSONFeedAuthor: Codable {
struct GoogleReaderCompatibleArticleSummary: Codable {
let content: String?
enum CodingKeys: String, CodingKey {
case content = "content"
}
}
struct GoogleReaderCompatibleAlternateLocation: Codable {
let url: String?
let avatarURL: String?
enum CodingKeys: String, CodingKey {
case url = "url"
case avatarURL = "avatar"
case url = "href"
}
}
struct GoogleReaderCompatibleEntryOrigin: Codable {
let streamId: String?
let title: String?
enum CodingKeys: String, CodingKey {
case streamId = "streamId"
case title = "title"
}
}

16
Frameworks/Account/GoogleReaderCompatible/GoogleReaderCompatibleUnreadEntry.swift
View File

@ -8,12 +8,20 @@
import Foundation
struct GoogleReaderCompatibleUnreadEntry: Codable {
let unreadEntries: [Int]
struct GoogleReaderCompatibleReferenceWrapper: Codable {
let itemRefs: [GoogleReaderCompatibleReference]
enum CodingKeys: String, CodingKey {
case unreadEntries = "unread_entries"
case itemRefs = "itemRefs"
}
}
struct GoogleReaderCompatibleReference: Codable {
let itemId: String
enum CodingKeys: String, CodingKey {
case itemId = "id"
}
}

2
submodules/RSWeb

@ -1 +1 @@
Subproject commit cf3a30eb3833d9dd423fed003393e6e3c1a360d4
Subproject commit 142cb8ccc491201e3de35c0b5d76d23d785f1978