FreshRSS
/
NetNewsWire
mirror of https://github.com/Ranchero-Software/NetNewsWire.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change the article fetch strategy so that we only fetch 200 at a time and continue fetching until we have them all

This commit is contained in:
Maurice Parker 2020-10-26 19:56:01 -05:00
parent cc269a8089
commit 0d5b3ea48b
1 changed files with 83 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
Account/Sources/Account/ReaderAPI/ReaderAPICaller.swift
View File

@ -571,19 +571,24 @@ final class ReaderAPICaller: NSObject {
self.requestAuthorizationToken(endpoint: baseURL) { (result) in
switch result {
case .success(let token):
// Do POST asking for data about all the new articles
var request = URLRequest(url: baseURL.appendingPathComponent(ReaderAPIEndpoints.contents.rawValue), credentials: self.credentials)
self.addVariantHeaders(&request)
request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.httpMethod = "POST"
// Get ids from above into hex representation of value
let idsToFetch = articleIDs.map({ (reference) -> String in
return "i=tag:google.com,2005:reader/item/\(reference)"
let chunkedArticleIds = articleIDs.chunked(into: 200)
let group = DispatchGroup()
var groupEntries = [ReaderAPIEntry]()
var groupError: Error? = nil
for articleIDChunk in chunkedArticleIds {
let itemFetchParameters = articleIDChunk.map({ articleID -> String in
return "i=tag:google.com,2005:reader/item/\(articleID)"
}).joined(separator:"&")
let postData = "T=\(token)&output=json&\(idsToFetch)".data(using: String.Encoding.utf8)
let postData = "T=\(token)&output=json&\(itemFetchParameters)".data(using: String.Encoding.utf8)
group.enter()
self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: ReaderAPIEntryWrapper.self, completion: { (result) in
switch result {
case .success(let (_, entryWrapper)):
@ -591,13 +596,22 @@ final class ReaderAPICaller: NSObject {
completion(.failure(ReaderAPIAccountDelegateError.invalidResponse))
return
}
completion(.success((entryWrapper.entries)))
groupEntries.append(contentsOf: entryWrapper.entries)
group.leave()
case .failure(let error):
completion(.failure(error))
groupError = error
group.leave()
}
})
}
group.notify(queue: DispatchQueue.main) {
if let error = groupError {
completion(.failure(error))
} else {
completion(.success(groupEntries))
}
}
case .failure(let error):
completion(.failure(error))
@ -619,7 +633,7 @@ final class ReaderAPICaller: NSObject {
.appendingPathComponent(ReaderAPIEndpoints.itemIds.rawValue)
.appendingQueryItems([
URLQueryItem(name: "s", value: webFeedID),
URLQueryItem(name: "ot", value: String(since.timeIntervalSince1970)),
URLQueryItem(name: "ot", value: String(Int(since.timeIntervalSince1970))),
URLQueryItem(name: "output", value: "json")
])
@ -683,11 +697,11 @@ final class ReaderAPICaller: NSObject {
}
}()
let sinceString = since.timeIntervalSince1970
let sinceTimeInterval = since.timeIntervalSince1970
let url = baseURL
.appendingPathComponent(ReaderAPIEndpoints.itemIds.rawValue)
.appendingQueryItems([
URLQueryItem(name: "ot", value: String(sinceString)),
URLQueryItem(name: "ot", value: String(Int(sinceTimeInterval))),
URLQueryItem(name: "n", value: "1000"),
URLQueryItem(name: "output", value: "json"),
URLQueryItem(name: "s", value: ReaderStreams.readingList.rawValue)
@ -698,66 +712,74 @@ final class ReaderAPICaller: NSObject {
return
}
let conditionalGet = accountMetadata?.conditionalGetInfo[ConditionalGetKeys.unreadEntries]
var request = URLRequest(url: callURL, credentials: credentials, conditionalGet: conditionalGet)
var request = URLRequest(url: callURL, credentials: credentials)
addVariantHeaders(&request)
self.transport.send(request: request, resultType: ReaderAPIReferenceWrapper.self) { result in
switch result {
case .success(let (_, entries)):
case .success(let (response, entries)):
guard let entriesItemRefs = entries?.itemRefs else {
completion(.failure(ReaderAPIAccountDelegateError.invalidResponse))
return
}
guard entriesItemRefs.count > 0 else {
guard let entriesItemRefs = entries?.itemRefs, entriesItemRefs.count > 0 else {
completion(.success((nil, nil, nil)))
return
}
// This needs to be moved when we fix paging for item ids
let dateInfo = HTTPDateInfo(urlResponse: response)
self.accountMetadata?.lastArticleFetchStartTime = dateInfo?.date
self.accountMetadata?.lastArticleFetchEndTime = Date()
self.requestAuthorizationToken(endpoint: baseURL) { (result) in
switch result {
case .success(let token):
// Do POST asking for data about all the new articles
var request = URLRequest(url: baseURL.appendingPathComponent(ReaderAPIEndpoints.contents.rawValue), credentials: self.credentials)
self.addVariantHeaders(&request)
request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type")
request.httpMethod = "POST"
// Get ids from above into hex representation of value
let idsToFetch = entriesItemRefs.map({ (reference) -> String in
let chunkedItemRefs = entriesItemRefs.chunked(into: 200)
let group = DispatchGroup()
var groupEntries = [ReaderAPIEntry]()
var groupError: Error? = nil
for itemRefsChunk in chunkedItemRefs {
let itemFetchParameters = itemRefsChunk.map({ itemRef -> String in
if self.variant == .theOldReader {
return "i=tag:google.com,2005:reader/item/\(reference.itemId)"
return "i=tag:google.com,2005:reader/item/\(itemRef.itemId)"
} else {
let idValue = Int(reference.itemId)!
let idValue = Int(itemRef.itemId)!
let idHexString = String(idValue, radix: 16, uppercase: false)
return "i=tag:google.com,2005:reader/item/\(idHexString)"
}
}).joined(separator:"&")
let postData = "T=\(token)&output=json&\(idsToFetch)".data(using: String.Encoding.utf8)
let postData = "T=\(token)&output=json&\(itemFetchParameters)".data(using: String.Encoding.utf8)
group.enter()
self.transport.send(request: request, method: HTTPMethod.post, data: postData!, resultType: ReaderAPIEntryWrapper.self, completion: { (result) in
switch result {
case .success(let (response, entryWrapper)):
case .success(let (_, entryWrapper)):
guard let entryWrapper = entryWrapper else {
completion(.failure(ReaderAPIAccountDelegateError.invalidResponse))
return
}
let dateInfo = HTTPDateInfo(urlResponse: response)
self.accountMetadata?.lastArticleFetchStartTime = dateInfo?.date
self.accountMetadata?.lastArticleFetchEndTime = Date()
completion(.success((entryWrapper.entries, nil, nil)))
groupEntries.append(contentsOf: entryWrapper.entries)
group.leave()
case .failure(let error):
completion(.failure(error))
groupError = error
group.leave()
}
})
}
group.notify(queue: DispatchQueue.main) {
if let error = groupError {
completion(.failure(error))
} else {
completion(.success((groupEntries, nil, nil)))
}
}
case .failure(let error):
completion(.failure(error))
}
@ -917,7 +939,7 @@ final class ReaderAPICaller: NSObject {
let url = baseURL
.appendingPathComponent(ReaderAPIEndpoints.itemIds.rawValue)
.appendingQueryItems([
URLQueryItem(name: "s", value: ReaderState.starred),
URLQueryItem(name: "s", value: ReaderState.starred.rawValue),
URLQueryItem(name: "n", value: "1000"),
URLQueryItem(name: "output", value: "json")
])
@ -951,8 +973,6 @@ final class ReaderAPICaller: NSObject {
}
}
// MARK: Private