NetNewsWire/Frameworks/Account/FeedFinder/FeedFinder.swift

//
//  FeedFinder.swift
//  NetNewsWire
//
//  Created by Brent Simmons on 8/2/16.
//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
//

import Foundation
import RSParser
import RSWeb
import RSCore

class FeedFinder {
	
	static func find(url: URL, completion: @escaping (Result<Set<FeedSpecifier>, Error>) -> Void) {
		downloadAddingToCache(url) { (data, response, error) in
			if response?.forcedStatusCode == 404 {
				completion(.failure(AccountError.createErrorNotFound))
				return
			}
			
			if let error = error {
				completion(.failure(error))
				return
			}
			
			guard let data = data, let response = response else {
				completion(.failure(AccountError.createErrorNotFound))
				return
			}
			
			if !response.statusIsOK || data.isEmpty {
				completion(.failure(AccountError.createErrorNotFound))
				return
			}
			
			if FeedFinder.isFeed(data, url.absoluteString) {
				let feedSpecifier = FeedSpecifier(title: nil, urlString: url.absoluteString, source: .UserEntered)
				completion(.success(Set([feedSpecifier])))
				return
			}
			
			if !FeedFinder.isHTML(data) {
				completion(.failure(AccountError.createErrorNotFound))
				return
			}
			
			FeedFinder.findFeedsInHTMLPage(htmlData: data, urlString: url.absoluteString, completion: completion)
		}
	}
}

private extension FeedFinder {

	static func addFeedSpecifier(_ feedSpecifier: FeedSpecifier, feedSpecifiers: inout [String: FeedSpecifier]) {
		// If there’s an existing feed specifier, merge the two so that we have the best data. If one has a title and one doesn’t, use that non-nil title. Use the better source.

		if let existingFeedSpecifier = feedSpecifiers[feedSpecifier.urlString] {
			let mergedFeedSpecifier = existingFeedSpecifier.feedSpecifierByMerging(feedSpecifier)
			feedSpecifiers[feedSpecifier.urlString] = mergedFeedSpecifier
		}
		else {
			feedSpecifiers[feedSpecifier.urlString] = feedSpecifier
		}
	}

	static func findFeedsInHTMLPage(htmlData: Data, urlString: String, completion: @escaping (Result<Set<FeedSpecifier>, Error>) -> Void) {
		// Feeds in the <head> section we automatically assume are feeds.
		// If there are none from the <head> section,
		// then possible feeds in <body> section are downloaded individually
		// and added once we determine they are feeds.

		let possibleFeedSpecifiers = possibleFeedsInHTMLPage(htmlData: htmlData, urlString: urlString)
		var feedSpecifiers = [String: FeedSpecifier]()
		var feedSpecifiersToDownload = Set<FeedSpecifier>()

		var didFindFeedInHTMLHead = false

		for oneFeedSpecifier in possibleFeedSpecifiers {
			if oneFeedSpecifier.source == .HTMLHead {
				addFeedSpecifier(oneFeedSpecifier, feedSpecifiers: &feedSpecifiers)
				didFindFeedInHTMLHead = true
			}
			else {
				if feedSpecifiers[oneFeedSpecifier.urlString] == nil {
					feedSpecifiersToDownload.insert(oneFeedSpecifier)
				}
			}
		}

		if didFindFeedInHTMLHead {
			completion(.success(Set(feedSpecifiers.values)))
			return
		}
		else if feedSpecifiersToDownload.isEmpty {
			completion(.failure(AccountError.createErrorNotFound))
			return
		}
		else {
			downloadFeedSpecifiers(feedSpecifiersToDownload, feedSpecifiers: feedSpecifiers, completion: completion)
		}
	}

	static func possibleFeedsInHTMLPage(htmlData: Data, urlString: String) -> Set<FeedSpecifier> {
		let parserData = ParserData(url: urlString, data: htmlData)
		var feedSpecifiers = HTMLFeedFinder(parserData: parserData).feedSpecifiers

		if feedSpecifiers.isEmpty {
			// Odds are decent it’s a WordPress site, and just adding /feed/ will work.
			// It’s also fairly common for /index.xml to work.
			if let url = URL(string: urlString) {
				let feedURL = url.appendingPathComponent("feed", isDirectory: true)
				let wordpressFeedSpecifier = FeedSpecifier(title: nil, urlString: feedURL.absoluteString, source: .HTMLLink)
				feedSpecifiers.insert(wordpressFeedSpecifier)

				let indexXMLURL = url.appendingPathComponent("index.xml", isDirectory: false)
				let indexXMLFeedSpecifier = FeedSpecifier(title: nil, urlString: indexXMLURL.absoluteString, source: .HTMLLink)
				feedSpecifiers.insert(indexXMLFeedSpecifier)
			}
		}

		return feedSpecifiers
	}

	static func isHTML(_ data: Data) -> Bool {
		return data.isProbablyHTML
	}

	static func downloadFeedSpecifiers(_ downloadFeedSpecifiers: Set<FeedSpecifier>, feedSpecifiers: [String: FeedSpecifier], completion: @escaping (Result<Set<FeedSpecifier>, Error>) -> Void) {

		var resultFeedSpecifiers = feedSpecifiers
		let group = DispatchGroup()
		
		for downloadFeedSpecifier in downloadFeedSpecifiers {
			guard let url = URL(string: downloadFeedSpecifier.urlString) else {
				continue
			}
			
			group.enter()
			downloadUsingCache(url) { (data, response, error) in
				if let data = data, let response = response, response.statusIsOK, error == nil {
					if self.isFeed(data, downloadFeedSpecifier.urlString) {
						addFeedSpecifier(downloadFeedSpecifier, feedSpecifiers: &resultFeedSpecifiers)
					}
				}
				group.leave()
			}
			
		}

		group.notify(queue: DispatchQueue.main) {
			completion(.success(Set(resultFeedSpecifiers.values)))
		}
	}

	static func isFeed(_ data: Data, _ urlString: String) -> Bool {
		let parserData = ParserData(url: urlString, data: data)
		return FeedParser.canParse(parserData)
	}
}
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+								//
 								//  FeedFinder.swift
-												Update code formatting in some files in Account.framework.

											
										
										
											2019-07-09 07:58:19 +02:00
+								//  NetNewsWire
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+								//
 								//  Created by Brent Simmons on 8/2/16.
-												Replace Ranchero Software with Ranchero Software, LLC.

											
										
										
											2017-05-29 22:17:58 +02:00
+								//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+								//
 								import Foundation
-												Commit major surgery but leave it unfinished. Everything is broken.

											
										
										
											2017-07-02 02:22:19 +02:00
+								import RSParser
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+								import RSWeb
 								import RSCore
-												Use synthesized Hashable, Equatable. Remove some extraneous public declarations.

											
										
										
											2018-08-26 02:03:10 +02:00
+								class FeedFinder {
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
 									static func find(url: URL, completion: @escaping (Result<Set<FeedSpecifier>, Error>) -> Void) {
-												Avoid cached response when adding new feed

											
										
										
											2020-04-11 01:39:52 +02:00
+										downloadAddingToCache(url) { (data, response, error) in
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+											if response?.forcedStatusCode == 404 {
 												completion(.failure(AccountError.createErrorNotFound))
 												return
 											}
 											if let error = error {
 												completion(.failure(error))
 												return
 											}
 											guard let data = data, let response = response else {
 												completion(.failure(AccountError.createErrorNotFound))
 												return
 											}
 											if !response.statusIsOK || data.isEmpty {
 												completion(.failure(AccountError.createErrorNotFound))
 												return
 											}
 											if FeedFinder.isFeed(data, url.absoluteString) {
 												let feedSpecifier = FeedSpecifier(title: nil, urlString: url.absoluteString, source: .UserEntered)
 												completion(.success(Set([feedSpecifier])))
 												return
 											}
 											if !FeedFinder.isHTML(data) {
 												completion(.failure(AccountError.createErrorNotFound))
 												return
 											}
 											FeedFinder.findFeedsInHTMLPage(htmlData: data, urlString: url.absoluteString, completion: completion)
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+										}
 									}
 								}
 								private extension FeedFinder {
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+									static func addFeedSpecifier(_ feedSpecifier: FeedSpecifier, feedSpecifiers: inout [String: FeedSpecifier]) {
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+										// If there’s an existing feed specifier, merge the two so that we have the best data. If one has a title and one doesn’t, use that non-nil title. Use the better source.
 										if let existingFeedSpecifier = feedSpecifiers[feedSpecifier.urlString] {
 											let mergedFeedSpecifier = existingFeedSpecifier.feedSpecifierByMerging(feedSpecifier)
 											feedSpecifiers[feedSpecifier.urlString] = mergedFeedSpecifier
 										}
 										else {
 											feedSpecifiers[feedSpecifier.urlString] = feedSpecifier
 										}
 									}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+									static func findFeedsInHTMLPage(htmlData: Data, urlString: String, completion: @escaping (Result<Set<FeedSpecifier>, Error>) -> Void) {
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+										// Feeds in the <head> section we automatically assume are feeds.
 										// If there are none from the <head> section,
 										// then possible feeds in <body> section are downloaded individually
 										// and added once we determine they are feeds.
 										let possibleFeedSpecifiers = possibleFeedsInHTMLPage(htmlData: htmlData, urlString: urlString)
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+										var feedSpecifiers = [String: FeedSpecifier]()
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+										var feedSpecifiersToDownload = Set<FeedSpecifier>()
 										var didFindFeedInHTMLHead = false
 										for oneFeedSpecifier in possibleFeedSpecifiers {
 											if oneFeedSpecifier.source == .HTMLHead {
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+												addFeedSpecifier(oneFeedSpecifier, feedSpecifiers: &feedSpecifiers)
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+												didFindFeedInHTMLHead = true
 											}
 											else {
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+												if feedSpecifiers[oneFeedSpecifier.urlString] == nil {
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+													feedSpecifiersToDownload.insert(oneFeedSpecifier)
 												}
 											}
 										}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+										if didFindFeedInHTMLHead {
 											completion(.success(Set(feedSpecifiers.values)))
 											return
-												Update code formatting in some files in Account.framework.

											
										
										
											2019-07-09 07:58:19 +02:00
+										}
 										else if feedSpecifiersToDownload.isEmpty {
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+											completion(.failure(AccountError.createErrorNotFound))
 											return
-												Update code formatting in some files in Account.framework.

											
										
										
											2019-07-09 07:58:19 +02:00
+										}
 										else {
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+											downloadFeedSpecifiers(feedSpecifiersToDownload, feedSpecifiers: feedSpecifiers, completion: completion)
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+										}
 									}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+									static func possibleFeedsInHTMLPage(htmlData: Data, urlString: String) -> Set<FeedSpecifier> {
-												Commit major surgery but leave it unfinished. Everything is broken.

											
										
										
											2017-07-02 02:22:19 +02:00
+										let parserData = ParserData(url: urlString, data: htmlData)
 										var feedSpecifiers = HTMLFeedFinder(parserData: parserData).feedSpecifiers
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
 										if feedSpecifiers.isEmpty {
 											// Odds are decent it’s a WordPress site, and just adding /feed/ will work.
-												Check /index.xml when finding a feed when there are no other leads. Fix #235.

											
										
										
											2018-01-06 07:54:28 +01:00
+											// It’s also fairly common for /index.xml to work.
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+											if let url = URL(string: urlString) {
 												let feedURL = url.appendingPathComponent("feed", isDirectory: true)
 												let wordpressFeedSpecifier = FeedSpecifier(title: nil, urlString: feedURL.absoluteString, source: .HTMLLink)
 												feedSpecifiers.insert(wordpressFeedSpecifier)
-												Check /index.xml when finding a feed when there are no other leads. Fix #235.

											
										
										
											2018-01-06 07:54:28 +01:00
 												let indexXMLURL = url.appendingPathComponent("index.xml", isDirectory: false)
 												let indexXMLFeedSpecifier = FeedSpecifier(title: nil, urlString: indexXMLURL.absoluteString, source: .HTMLLink)
 												feedSpecifiers.insert(indexXMLFeedSpecifier)
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+											}
 										}
 										return feedSpecifiers
 									}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+									static func isHTML(_ data: Data) -> Bool {
-												Updates for Data+RSCore changes

											
										
										
											2020-01-18 08:00:56 +01:00
+										return data.isProbablyHTML
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+									}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+									static func downloadFeedSpecifiers(_ downloadFeedSpecifiers: Set<FeedSpecifier>, feedSpecifiers: [String: FeedSpecifier], completion: @escaping (Result<Set<FeedSpecifier>, Error>) -> Void) {
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+										var resultFeedSpecifiers = feedSpecifiers
 										let group = DispatchGroup()
 										for downloadFeedSpecifier in downloadFeedSpecifiers {
 											guard let url = URL(string: downloadFeedSpecifier.urlString) else {
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+												continue
 											}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
 											group.enter()
-												Switch from download to downloadUsingCache where appropriate.

											
										
										
											2017-11-23 19:29:00 +01:00
+											downloadUsingCache(url) { (data, response, error) in
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+												if let data = data, let response = response, response.statusIsOK, error == nil {
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+													if self.isFeed(data, downloadFeedSpecifier.urlString) {
 														addFeedSpecifier(downloadFeedSpecifier, feedSpecifiers: &resultFeedSpecifiers)
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+													}
 												}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+												group.leave()
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+											}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+										}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+										group.notify(queue: DispatchQueue.main) {
 											completion(.success(Set(resultFeedSpecifiers.values)))
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+										}
 									}
-												Refactor feed finder to make it threadsafe

											
										
										
											2019-05-31 00:35:08 +02:00
+									static func isFeed(_ data: Data, _ urlString: String) -> Bool {
-												Commit major surgery but leave it unfinished. Everything is broken.

											
										
										
											2017-07-02 02:22:19 +02:00
+										let parserData = ParserData(url: urlString, data: data)
 										return FeedParser.canParse(parserData)
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+									}
 								}