NetNewsWire/Shared/FeedFinder/FeedFinder.swift

//
//  FeedFinder.swift
//  FeedFinder
//
//  Created by Brent Simmons on 8/2/16.
//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
//

import Foundation
import RSParser
import RSWeb
import RSCore

protocol FeedFinderDelegate: class {

	func feedFinder(_: FeedFinder, didFindFeeds: Set<FeedSpecifier>)
}

class FeedFinder {

	private weak var delegate: FeedFinderDelegate?
	private var feedSpecifiers = [String: FeedSpecifier]()
	private var didNotifyDelegate = false

	var initialDownloadError: Error?
	var initialDownloadStatusCode = -1

	init(url: URL, delegate: FeedFinderDelegate) {

		self.delegate = delegate

		DispatchQueue.main.async() { () -> Void in

			self.findFeeds(url)
		}
	}

	deinit {
		notifyDelegateIfNeeded()
	}
}

private extension FeedFinder {

	func addFeedSpecifier(_ feedSpecifier: FeedSpecifier) {

		// If there’s an existing feed specifier, merge the two so that we have the best data. If one has a title and one doesn’t, use that non-nil title. Use the better source.

		if let existingFeedSpecifier = feedSpecifiers[feedSpecifier.urlString] {
			let mergedFeedSpecifier = existingFeedSpecifier.feedSpecifierByMerging(feedSpecifier)
			feedSpecifiers[feedSpecifier.urlString] = mergedFeedSpecifier
		}
		else {
			feedSpecifiers[feedSpecifier.urlString] = feedSpecifier
		}
	}

	func findFeedsInHTMLPage(htmlData: Data, urlString: String) {

		// Feeds in the <head> section we automatically assume are feeds.
		// If there are none from the <head> section,
		// then possible feeds in <body> section are downloaded individually
		// and added once we determine they are feeds.

		let possibleFeedSpecifiers = possibleFeedsInHTMLPage(htmlData: htmlData, urlString: urlString)
		var feedSpecifiersToDownload = Set<FeedSpecifier>()

		var didFindFeedInHTMLHead = false

		for oneFeedSpecifier in possibleFeedSpecifiers {
			if oneFeedSpecifier.source == .HTMLHead {
				addFeedSpecifier(oneFeedSpecifier)
				didFindFeedInHTMLHead = true
			}
			else {
				if !feedSpecifiersContainsURLString(oneFeedSpecifier.urlString) {
					feedSpecifiersToDownload.insert(oneFeedSpecifier)
				}
			}
		}

		if didFindFeedInHTMLHead || feedSpecifiersToDownload.isEmpty {
			stopFinding()
		}
		else {
			downloadFeedSpecifiers(feedSpecifiersToDownload)
		}
	}

	func possibleFeedsInHTMLPage(htmlData: Data, urlString: String) -> Set<FeedSpecifier> {

		let parserData = ParserData(url: urlString, data: htmlData)
		var feedSpecifiers = HTMLFeedFinder(parserData: parserData).feedSpecifiers

		if feedSpecifiers.isEmpty {
			// Odds are decent it’s a WordPress site, and just adding /feed/ will work.
			// It’s also fairly common for /index.xml to work.
			if let url = URL(string: urlString) {
				let feedURL = url.appendingPathComponent("feed", isDirectory: true)
				let wordpressFeedSpecifier = FeedSpecifier(title: nil, urlString: feedURL.absoluteString, source: .HTMLLink)
				feedSpecifiers.insert(wordpressFeedSpecifier)

				let indexXMLURL = url.appendingPathComponent("index.xml", isDirectory: false)
				let indexXMLFeedSpecifier = FeedSpecifier(title: nil, urlString: indexXMLURL.absoluteString, source: .HTMLLink)
				feedSpecifiers.insert(indexXMLFeedSpecifier)
			}
		}

		return feedSpecifiers
	}

	func feedSpecifiersContainsURLString(_ urlString: String) -> Bool {

		if let _ = feedSpecifiers[urlString] {
			return true
		}
		return false
	}

	func isHTML(_ data: Data) -> Bool {

		return (data as NSData).rs_dataIsProbablyHTML()
	}

	func findFeeds(_ initialURL: URL) {

		downloadInitialFeed(initialURL)
	}

	func downloadInitialFeed(_ initialURL: URL) {

		downloadUsingCache(initialURL) { (data, response, error) in

			self.initialDownloadStatusCode = response?.forcedStatusCode ?? -1

			if let error = error {
				self.initialDownloadError = error
				self.stopFinding()
				return
			}
			guard let data = data, let response = response else {
				self.stopFinding()
				return
			}

			if !response.statusIsOK || data.isEmpty {
				self.stopFinding()
				return
			}

			if self.isFeed(data, initialURL.absoluteString) {
				let feedSpecifier = FeedSpecifier(title: nil, urlString: initialURL.absoluteString, source: .UserEntered)
				self.addFeedSpecifier(feedSpecifier)
				self.stopFinding()
				return
			}

			if !self.isHTML(data) {
				self.stopFinding()
				return
			}

			self.findFeedsInHTMLPage(htmlData: data, urlString: initialURL.absoluteString)
		}
	}

	func downloadFeedSpecifiers(_ feedSpecifiers: Set<FeedSpecifier>) {

		var pendingDownloads = feedSpecifiers

		for oneFeedSpecifier in feedSpecifiers {

			guard let url = URL(string: oneFeedSpecifier.urlString) else {
				pendingDownloads.remove(oneFeedSpecifier)
				continue
			}

			downloadUsingCache(url) { (data, response, error) in

				pendingDownloads.remove(oneFeedSpecifier)

				if let data = data, let response = response, response.statusIsOK, error == nil {
					if self.isFeed(data, oneFeedSpecifier.urlString) {
						self.addFeedSpecifier(oneFeedSpecifier)
					}
				}

				if pendingDownloads.isEmpty {
					self.stopFinding()
				}
			}
		}
	}

	func stopFinding() {

		notifyDelegateIfNeeded()
	}

	func notifyDelegateIfNeeded() {

		if !didNotifyDelegate {
			delegate?.feedFinder(self, didFindFeeds: Set(feedSpecifiers.values))
			didNotifyDelegate = true
		}
	}

	func isFeed(_ data: Data, _ urlString: String) -> Bool {

		let parserData = ParserData(url: urlString, data: data)
		return FeedParser.canParse(parserData)
	}
}
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+								//
 								//  FeedFinder.swift
-												Rename RSFeedFinder references to FeedFinder.

											
										
										
											2018-06-21 22:18:28 +02:00
+								//  FeedFinder
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+								//
 								//  Created by Brent Simmons on 8/2/16.
-												Replace Ranchero Software with Ranchero Software, LLC.

											
										
										
											2017-05-29 22:17:58 +02:00
+								//  Copyright © 2016 Ranchero Software, LLC. All rights reserved.
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+								//
 								import Foundation
-												Commit major surgery but leave it unfinished. Everything is broken.

											
										
										
											2017-07-02 02:22:19 +02:00
+								import RSParser
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+								import RSWeb
 								import RSCore
-												Use synthesized Hashable, Equatable. Remove some extraneous public declarations.

											
										
										
											2018-08-26 02:03:10 +02:00
+								protocol FeedFinderDelegate: class {
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
 									func feedFinder(_: FeedFinder, didFindFeeds: Set<FeedSpecifier>)
 								}
-												Use synthesized Hashable, Equatable. Remove some extraneous public declarations.

											
										
										
											2018-08-26 02:03:10 +02:00
+								class FeedFinder {
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
-												Remove some old instances of fileprivate.

											
										
										
											2019-02-10 07:22:12 +01:00
+									private weak var delegate: FeedFinderDelegate?
 									private var feedSpecifiers = [String: FeedSpecifier]()
 									private var didNotifyDelegate = false
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
-												Use synthesized Hashable, Equatable. Remove some extraneous public declarations.

											
										
										
											2018-08-26 02:03:10 +02:00
+									var initialDownloadError: Error?
 									var initialDownloadStatusCode = -1
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
-												Use synthesized Hashable, Equatable. Remove some extraneous public declarations.

											
										
										
											2018-08-26 02:03:10 +02:00
+									init(url: URL, delegate: FeedFinderDelegate) {
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
 										self.delegate = delegate
 										DispatchQueue.main.async() { () -> Void in
 											self.findFeeds(url)
 										}
 									}
 									deinit {
 										notifyDelegateIfNeeded()
 									}
 								}
 								private extension FeedFinder {
 									func addFeedSpecifier(_ feedSpecifier: FeedSpecifier) {
 										// If there’s an existing feed specifier, merge the two so that we have the best data. If one has a title and one doesn’t, use that non-nil title. Use the better source.
 										if let existingFeedSpecifier = feedSpecifiers[feedSpecifier.urlString] {
 											let mergedFeedSpecifier = existingFeedSpecifier.feedSpecifierByMerging(feedSpecifier)
 											feedSpecifiers[feedSpecifier.urlString] = mergedFeedSpecifier
 										}
 										else {
 											feedSpecifiers[feedSpecifier.urlString] = feedSpecifier
 										}
 									}
 									func findFeedsInHTMLPage(htmlData: Data, urlString: String) {
 										// Feeds in the <head> section we automatically assume are feeds.
 										// If there are none from the <head> section,
 										// then possible feeds in <body> section are downloaded individually
 										// and added once we determine they are feeds.
 										let possibleFeedSpecifiers = possibleFeedsInHTMLPage(htmlData: htmlData, urlString: urlString)
 										var feedSpecifiersToDownload = Set<FeedSpecifier>()
 										var didFindFeedInHTMLHead = false
 										for oneFeedSpecifier in possibleFeedSpecifiers {
 											if oneFeedSpecifier.source == .HTMLHead {
 												addFeedSpecifier(oneFeedSpecifier)
 												didFindFeedInHTMLHead = true
 											}
 											else {
 												if !feedSpecifiersContainsURLString(oneFeedSpecifier.urlString) {
 													feedSpecifiersToDownload.insert(oneFeedSpecifier)
 												}
 											}
 										}
 										if didFindFeedInHTMLHead || feedSpecifiersToDownload.isEmpty {
 											stopFinding()
 										}
 										else {
 											downloadFeedSpecifiers(feedSpecifiersToDownload)
 										}
 									}
 									func possibleFeedsInHTMLPage(htmlData: Data, urlString: String) -> Set<FeedSpecifier> {
-												Commit major surgery but leave it unfinished. Everything is broken.

											
										
										
											2017-07-02 02:22:19 +02:00
+										let parserData = ParserData(url: urlString, data: htmlData)
 										var feedSpecifiers = HTMLFeedFinder(parserData: parserData).feedSpecifiers
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
 										if feedSpecifiers.isEmpty {
 											// Odds are decent it’s a WordPress site, and just adding /feed/ will work.
-												Check /index.xml when finding a feed when there are no other leads. Fix #235.

											
										
										
											2018-01-06 07:54:28 +01:00
+											// It’s also fairly common for /index.xml to work.
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+											if let url = URL(string: urlString) {
 												let feedURL = url.appendingPathComponent("feed", isDirectory: true)
 												let wordpressFeedSpecifier = FeedSpecifier(title: nil, urlString: feedURL.absoluteString, source: .HTMLLink)
 												feedSpecifiers.insert(wordpressFeedSpecifier)
-												Check /index.xml when finding a feed when there are no other leads. Fix #235.

											
										
										
											2018-01-06 07:54:28 +01:00
 												let indexXMLURL = url.appendingPathComponent("index.xml", isDirectory: false)
 												let indexXMLFeedSpecifier = FeedSpecifier(title: nil, urlString: indexXMLURL.absoluteString, source: .HTMLLink)
 												feedSpecifiers.insert(indexXMLFeedSpecifier)
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+											}
 										}
 										return feedSpecifiers
 									}
 									func feedSpecifiersContainsURLString(_ urlString: String) -> Bool {
 										if let _ = feedSpecifiers[urlString] {
 											return true
 										}
 										return false
 									}
 									func isHTML(_ data: Data) -> Bool {
 										return (data as NSData).rs_dataIsProbablyHTML()
 									}
 									func findFeeds(_ initialURL: URL) {
 										downloadInitialFeed(initialURL)
 									}
 									func downloadInitialFeed(_ initialURL: URL) {
-												Switch from download to downloadUsingCache where appropriate.

											
										
										
											2017-11-23 19:29:00 +01:00
+										downloadUsingCache(initialURL) { (data, response, error) in
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
 											self.initialDownloadStatusCode = response?.forcedStatusCode ?? -1
 											if let error = error {
 												self.initialDownloadError = error
 												self.stopFinding()
 												return
 											}
 											guard let data = data, let response = response else {
 												self.stopFinding()
 												return
 											}
 											if !response.statusIsOK || data.isEmpty {
 												self.stopFinding()
 												return
 											}
 											if self.isFeed(data, initialURL.absoluteString) {
 												let feedSpecifier = FeedSpecifier(title: nil, urlString: initialURL.absoluteString, source: .UserEntered)
 												self.addFeedSpecifier(feedSpecifier)
 												self.stopFinding()
 												return
 											}
 											if !self.isHTML(data) {
 												self.stopFinding()
 												return
 											}
 											self.findFeedsInHTMLPage(htmlData: data, urlString: initialURL.absoluteString)
 										}
 									}
 									func downloadFeedSpecifiers(_ feedSpecifiers: Set<FeedSpecifier>) {
 										var pendingDownloads = feedSpecifiers
 										for oneFeedSpecifier in feedSpecifiers {
 											guard let url = URL(string: oneFeedSpecifier.urlString) else {
 												pendingDownloads.remove(oneFeedSpecifier)
 												continue
 											}
-												Switch from download to downloadUsingCache where appropriate.

											
										
										
											2017-11-23 19:29:00 +01:00
+											downloadUsingCache(url) { (data, response, error) in
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
 												pendingDownloads.remove(oneFeedSpecifier)
 												if let data = data, let response = response, response.statusIsOK, error == nil {
 													if self.isFeed(data, oneFeedSpecifier.urlString) {
 														self.addFeedSpecifier(oneFeedSpecifier)
 													}
 												}
 												if pendingDownloads.isEmpty {
 													self.stopFinding()
 												}
 											}
 										}
 									}
 									func stopFinding() {
 										notifyDelegateIfNeeded()
 									}
 									func notifyDelegateIfNeeded() {
 										if !didNotifyDelegate {
-												Fix a memory leak that happened every time you added a feed. Use a weak delegate to avoid a retain cycle.

											
										
										
											2017-12-14 06:19:12 +01:00
+											delegate?.feedFinder(self, didFindFeeds: Set(feedSpecifiers.values))
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+											didNotifyDelegate = true
 										}
 									}
 									func isFeed(_ data: Data, _ urlString: String) -> Bool {
-												Commit major surgery but leave it unfinished. Everything is broken.

											
										
										
											2017-07-02 02:22:19 +02:00
 										let parserData = ParserData(url: urlString, data: data)
 										return FeedParser.canParse(parserData)
-												Add RSFeedFinder framework.

											
										
										
											2017-05-22 22:27:54 +02:00
+									}
 								}