FreshRSS/NetNewsWire
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change the article fetch strategy so that we only fetch 200 at a time and continue fetching until we have them all

Browse Source
This commit is contained in:
Maurice Parker 2020-10-26 19:56:01 -05:00
parent cc269a8089
commit 0d5b3ea48b
1 changed files with 83 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
Account/Sources/Account/ReaderAPI/ReaderAPICaller.swift
View File

@ -571,19 +571,24 @@ final class ReaderAPICaller: NSObject {
self.requestAuthorizationToken(endpoint: baseURL) { (result) in self.requestAuthorizationToken(endpoint: baseURL) { (result) in
switch result { switch result {
case .success(let token): case .success(let token):
// Do POST asking for data about all the new articles
var request = URLRequest(url: baseURL.appendingPathComponent(ReaderAPIEndpoints.contents.rawValue), credentials: self.credentials) var request = URLRequest(url: baseURL.appendingPathComponent(ReaderAPIEndpoints.contents.rawValue), credentials: self.credentials)
self.addVariantHeaders(&request) self.addVariantHeaders(&request)
request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type") request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.httpMethod = "POST" request.httpMethod = "POST"
// Get ids from above into hex representation of value let chunkedArticleIds = articleIDs.chunked(into: 200)
let idsToFetch = articleIDs.map({ (reference) -> String in let group = DispatchGroup()
return "i=tag:google.com,2005:reader/item/\(reference)" var groupEntries = [ReaderAPIEntry]()
var groupError: Error? = nil
for articleIDChunk in chunkedArticleIds {
let itemFetchParameters = articleIDChunk.map({ articleID -> String in
return "i=tag:google.com,2005:reader/item/\(articleID)"
}).joined(separator:"&") }).joined(separator:"&")
let postData = "T=\(token)&output=json&\(idsToFetch)".data(using: String.Encoding.utf8) let postData = "T=\(token)&output=json&\(itemFetchParameters)".data(using: String.Encoding.utf8)
group.enter()
self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: ReaderAPIEntryWrapper.self, completion: { (result) in self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: ReaderAPIEntryWrapper.self, completion: { (result) in
switch result { switch result {
case .success(let (_, entryWrapper)): case .success(let (_, entryWrapper)):
@ -591,13 +596,22 @@ final class ReaderAPICaller: NSObject {
completion(.failure(ReaderAPIAccountDelegateError.invalidResponse)) completion(.failure(ReaderAPIAccountDelegateError.invalidResponse))
return return
} }
groupEntries.append(contentsOf: entryWrapper.entries)
completion(.success((entryWrapper.entries))) group.leave()
case .failure(let error): case .failure(let error):
completion(.failure(error)) groupError = error
group.leave()
} }
}) })
}
group.notify(queue: DispatchQueue.main) {
if let error = groupError {
completion(.failure(error))
} else {
completion(.success(groupEntries))
}
}
case .failure(let error): case .failure(let error):
completion(.failure(error)) completion(.failure(error))
@ -619,7 +633,7 @@ final class ReaderAPICaller: NSObject {
.appendingPathComponent(ReaderAPIEndpoints.itemIds.rawValue) .appendingPathComponent(ReaderAPIEndpoints.itemIds.rawValue)
.appendingQueryItems([ .appendingQueryItems([
URLQueryItem(name: "s", value: webFeedID), URLQueryItem(name: "s", value: webFeedID),
URLQueryItem(name: "ot", value: String(since.timeIntervalSince1970)), URLQueryItem(name: "ot", value: String(Int(since.timeIntervalSince1970))),
URLQueryItem(name: "output", value: "json") URLQueryItem(name: "output", value: "json")
]) ])
@ -683,11 +697,11 @@ final class ReaderAPICaller: NSObject {
} }
}() }()
let sinceString = since.timeIntervalSince1970 let sinceTimeInterval = since.timeIntervalSince1970
let url = baseURL let url = baseURL
.appendingPathComponent(ReaderAPIEndpoints.itemIds.rawValue) .appendingPathComponent(ReaderAPIEndpoints.itemIds.rawValue)
.appendingQueryItems([ .appendingQueryItems([
URLQueryItem(name: "ot", value: String(sinceString)), URLQueryItem(name: "ot", value: String(Int(sinceTimeInterval))),
URLQueryItem(name: "n", value: "1000"), URLQueryItem(name: "n", value: "1000"),
URLQueryItem(name: "output", value: "json"), URLQueryItem(name: "output", value: "json"),
URLQueryItem(name: "s", value: ReaderStreams.readingList.rawValue) URLQueryItem(name: "s", value: ReaderStreams.readingList.rawValue)
@ -698,66 +712,74 @@ final class ReaderAPICaller: NSObject {
return return
} }
let conditionalGet = accountMetadata?.conditionalGetInfo[ConditionalGetKeys.unreadEntries] var request = URLRequest(url: callURL, credentials: credentials)
var request = URLRequest(url: callURL, credentials: credentials, conditionalGet: conditionalGet)
addVariantHeaders(&request) addVariantHeaders(&request)
self.transport.send(request: request, resultType: ReaderAPIReferenceWrapper.self) { result in self.transport.send(request: request, resultType: ReaderAPIReferenceWrapper.self) { result in
switch result { switch result {
case .success(let (_, entries)): case .success(let (response, entries)):
guard let entriesItemRefs = entries?.itemRefs else { guard let entriesItemRefs = entries?.itemRefs, entriesItemRefs.count > 0 else {
completion(.failure(ReaderAPIAccountDelegateError.invalidResponse))
return
}
guard entriesItemRefs.count > 0 else {
completion(.success((nil, nil, nil))) completion(.success((nil, nil, nil)))
return return
} }
// This needs to be moved when we fix paging for item ids
let dateInfo = HTTPDateInfo(urlResponse: response)
self.accountMetadata?.lastArticleFetchStartTime = dateInfo?.date
self.accountMetadata?.lastArticleFetchEndTime = Date()
self.requestAuthorizationToken(endpoint: baseURL) { (result) in self.requestAuthorizationToken(endpoint: baseURL) { (result) in
switch result { switch result {
case .success(let token): case .success(let token):
// Do POST asking for data about all the new articles
var request = URLRequest(url: baseURL.appendingPathComponent(ReaderAPIEndpoints.contents.rawValue), credentials: self.credentials) var request = URLRequest(url: baseURL.appendingPathComponent(ReaderAPIEndpoints.contents.rawValue), credentials: self.credentials)
self.addVariantHeaders(&request) self.addVariantHeaders(&request)
request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type") request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.httpMethod = "POST" request.httpMethod = "POST"
// Get ids from above into hex representation of value let chunkedItemRefs = entriesItemRefs.chunked(into: 200)
let idsToFetch = entriesItemRefs.map({ (reference) -> String in let group = DispatchGroup()
var groupEntries = [ReaderAPIEntry]()
var groupError: Error? = nil
for itemRefsChunk in chunkedItemRefs {
let itemFetchParameters = itemRefsChunk.map({ itemRef -> String in
if self.variant == .theOldReader { if self.variant == .theOldReader {
return "i=tag:google.com,2005:reader/item/\(reference.itemId)" return "i=tag:google.com,2005:reader/item/\(itemRef.itemId)"
} else { } else {
let idValue = Int(reference.itemId)! let idValue = Int(itemRef.itemId)!
let idHexString = String(idValue, radix: 16, uppercase: false) let idHexString = String(idValue, radix: 16, uppercase: false)
return "i=tag:google.com,2005:reader/item/\(idHexString)" return "i=tag:google.com,2005:reader/item/\(idHexString)"
} }
}).joined(separator:"&") }).joined(separator:"&")
let postData = "T=\(token)&output=json&\(idsToFetch)".data(using: String.Encoding.utf8) let postData = "T=\(token)&output=json&\(itemFetchParameters)".data(using: String.Encoding.utf8)
group.enter()
self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: ReaderAPIEntryWrapper.self, completion: { (result) in self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: ReaderAPIEntryWrapper.self, completion: { (result) in
switch result { switch result {
case .success(let (response, entryWrapper)): case .success(let (_, entryWrapper)):
guard let entryWrapper = entryWrapper else { guard let entryWrapper = entryWrapper else {
completion(.failure(ReaderAPIAccountDelegateError.invalidResponse)) completion(.failure(ReaderAPIAccountDelegateError.invalidResponse))
return return
} }
groupEntries.append(contentsOf: entryWrapper.entries)
let dateInfo = HTTPDateInfo(urlResponse: response) group.leave()
self.accountMetadata?.lastArticleFetchStartTime = dateInfo?.date
self.accountMetadata?.lastArticleFetchEndTime = Date()
completion(.success((entryWrapper.entries, nil, nil)))
case .failure(let error): case .failure(let error):
completion(.failure(error)) groupError = error
group.leave()
} }
}) })
}
group.notify(queue: DispatchQueue.main) {
if let error = groupError {
completion(.failure(error))
} else {
completion(.success((groupEntries, nil, nil)))
}
}
case .failure(let error): case .failure(let error):
completion(.failure(error)) completion(.failure(error))
} }
@ -917,7 +939,7 @@ final class ReaderAPICaller: NSObject {
let url = baseURL let url = baseURL
.appendingPathComponent(ReaderAPIEndpoints.itemIds.rawValue) .appendingPathComponent(ReaderAPIEndpoints.itemIds.rawValue)
.appendingQueryItems([ .appendingQueryItems([
URLQueryItem(name: "s", value: ReaderState.starred), URLQueryItem(name: "s", value: ReaderState.starred.rawValue),
URLQueryItem(name: "n", value: "1000"), URLQueryItem(name: "n", value: "1000"),
URLQueryItem(name: "output", value: "json") URLQueryItem(name: "output", value: "json")
]) ])
@ -951,8 +973,6 @@ final class ReaderAPICaller: NSObject {
} }
} }
// MARK: Private // MARK: Private