diff --git a/app/build.gradle b/app/build.gradle index f61e1e0f..9f5cdbbe 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -31,8 +31,8 @@ android { testApplicationId "ac.mdiq.podcini.tests" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" - versionCode 3020252 - versionName "6.6.7" + versionCode 3020253 + versionName "6.7.0" applicationId "ac.mdiq.podcini.R" def commit = "" diff --git a/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/DownloadRequest.kt b/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/DownloadRequest.kt index 88d613ab..80e7821b 100644 --- a/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/DownloadRequest.kt +++ b/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/DownloadRequest.kt @@ -3,6 +3,8 @@ package ac.mdiq.podcini.net.download.service import ac.mdiq.podcini.net.utils.UrlChecker.prepareUrl import ac.mdiq.podcini.storage.model.Feed import ac.mdiq.podcini.storage.model.EpisodeMedia +import ac.mdiq.podcini.util.Logd +import ac.mdiq.podcini.util.showStackTrace import android.os.Bundle import android.os.Parcel import android.os.Parcelable @@ -120,6 +122,8 @@ class DownloadRequest private constructor( } fun setLastModified(lastModified: String?): DownloadRequest { + Logd("DownloadRequest", "setLastModified: $lastModified") +// showStackTrace() this.lastModified = lastModified return this } @@ -143,7 +147,6 @@ class DownloadRequest private constructor( this.feedfileId = media.id this.feedfileType = media.getTypeAsInt() } - constructor(destination: String, feed: Feed) { this.destination = destination this.source = when { @@ -156,27 +159,22 @@ class DownloadRequest private constructor( this.feedfileType = feed.getTypeAsInt() arguments.putInt(REQUEST_ARG_PAGE_NR, feed.pageNr) } - fun withInitiatedByUser(initiatedByUser: Boolean): Builder { this.initiatedByUser = initiatedByUser return this } - fun setForce(force: Boolean) { if (force) lastModified = null } - fun lastModified(lastModified: String?): Builder { this.lastModified = lastModified return this } - fun withAuthentication(username: String?, password: String?): Builder { this.username = username this.password = password return this } - fun build(): DownloadRequest { return DownloadRequest(this) } diff --git a/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/DownloadServiceInterfaceImpl.kt b/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/DownloadServiceInterfaceImpl.kt index 9029cb5a..0a2514d3 100644 --- a/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/DownloadServiceInterfaceImpl.kt +++ b/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/DownloadServiceInterfaceImpl.kt @@ -134,8 +134,7 @@ class DownloadServiceInterfaceImpl : DownloadServiceInterface() { } progressUpdaterThread.start() var result: Result - try { - result = performDownload(media, request) + try { result = performDownload(media, request) } catch (e: Exception) { e.printStackTrace() result = Result.failure() @@ -170,32 +169,23 @@ class DownloadServiceInterfaceImpl : DownloadServiceInterface() { } val dest = File(request.destination) if (!dest.exists()) { - try { - dest.createNewFile() - } catch (e: IOException) { - Log.e(TAG, "performDownload Unable to create file") - } + try { dest.createNewFile() } catch (e: IOException) { Log.e(TAG, "performDownload Unable to create file") } } if (dest.exists()) { try { var episode = realm.query(Episode::class).query("id == ${media.id}").first().find() if (episode != null) { - episode = upsertBlk(episode) { - it.media?.setfileUrlOrNull(request.destination) - } + episode = upsertBlk(episode) { it.media?.setfileUrlOrNull(request.destination) } EventFlow.postEvent(FlowEvent.EpisodeMediaEvent.updated(episode)) } else Log.e(TAG, "performDownload media.episode is null") - } catch (e: Exception) { - Log.e(TAG, "performDownload Exception in writeFileUrl: " + e.message) - } + } catch (e: Exception) { Log.e(TAG, "performDownload Exception in writeFileUrl: " + e.message) } } downloader = DefaultDownloaderFactory().create(request) if (downloader == null) { Log.e(TAG, "performDownload Unable to create downloader") return Result.failure() } - try { - downloader!!.call() + try { downloader!!.call() } catch (e: Exception) { Log.e(TAG, "failed performDownload exception on downloader!!.call() ${e.message}") LogsAndStats.addDownloadStatus(downloader!!.result) @@ -328,8 +318,7 @@ class DownloadServiceInterfaceImpl : DownloadServiceInterface() { durationStr = mmr.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) if (durationStr != null) it.media?.setDuration(durationStr!!.toInt()) } - } catch (e: NumberFormatException) { - Logd(TAG, "Invalid file duration: $durationStr") + } catch (e: NumberFormatException) { Logd(TAG, "Invalid file duration: $durationStr") } catch (e: Exception) { Log.e(TAG, "Get duration failed", e) it.media?.setDuration(30000) diff --git a/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/HttpDownloader.kt b/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/HttpDownloader.kt index 5017349d..e6238a1c 100644 --- a/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/HttpDownloader.kt +++ b/app/src/main/kotlin/ac/mdiq/podcini/net/download/service/HttpDownloader.kt @@ -195,8 +195,7 @@ class HttpDownloader(request: DownloadRequest) : Downloader(request) { @Throws(IOException::class) private fun newCall(httpReq: Request.Builder): Response { var httpClient = getHttpClient() - try { - return httpClient.newCall(httpReq.build()).execute() + try { return httpClient.newCall(httpReq.build()).execute() } catch (e: IOException) { Log.e(TAG, e.toString()) if (e.message != null && e.message!!.contains("PROTOCOL_ERROR")) { diff --git a/app/src/main/kotlin/ac/mdiq/podcini/net/feed/FeedUpdateManager.kt b/app/src/main/kotlin/ac/mdiq/podcini/net/feed/FeedUpdateManager.kt index 60383b80..7cafbaea 100644 --- a/app/src/main/kotlin/ac/mdiq/podcini/net/feed/FeedUpdateManager.kt +++ b/app/src/main/kotlin/ac/mdiq/podcini/net/feed/FeedUpdateManager.kt @@ -148,25 +148,25 @@ object FeedUpdateManager { @UnstableApi override fun doWork(): Result { ClientConfigurator.initialize(applicationContext) - val toUpdate: MutableList + val feedsToUpdate: MutableList val feedId = inputData.getLong(EXTRA_FEED_ID, -1L) var allAreLocal = true var force = false if (feedId == -1L) { // Update all - toUpdate = Feeds.getFeedList().toMutableList() - val itr = toUpdate.iterator() + feedsToUpdate = Feeds.getFeedList().toMutableList() + val itr = feedsToUpdate.iterator() while (itr.hasNext()) { val feed = itr.next() if (feed.preferences?.keepUpdated == false) itr.remove() if (!feed.isLocalFeed) allAreLocal = false } - toUpdate.shuffle() // If the worker gets cancelled early, every feed has a chance to be updated + feedsToUpdate.shuffle() // If the worker gets cancelled early, every feed has a chance to be updated } else { val feed = Feeds.getFeed(feedId) ?: return Result.success() Logd(TAG, "doWork feed.downloadUrl: ${feed.downloadUrl}") if (!feed.isLocalFeed) allAreLocal = false - toUpdate = ArrayList() - toUpdate.add(feed) // Needs to be updatable, so no singletonList + feedsToUpdate = mutableListOf(feed) +// feedsToUpdate.add(feed) // Needs to be updatable, so no singletonList force = true } if (!inputData.getBoolean(EXTRA_EVEN_ON_MOBILE, false) && !allAreLocal) { @@ -175,10 +175,10 @@ object FeedUpdateManager { return Result.retry() } } - refreshFeeds(toUpdate, force) + refreshFeeds(feedsToUpdate, force) notificationManager.cancel(R.id.notification_updating_feeds) - autodownloadEpisodeMedia(applicationContext, toUpdate.toList()) - toUpdate.clear() + autodownloadEpisodeMedia(applicationContext, feedsToUpdate.toList()) + feedsToUpdate.clear() return Result.success() } private fun createNotification(toUpdate: List?): Notification { @@ -203,7 +203,7 @@ object FeedUpdateManager { return Futures.immediateFuture(ForegroundInfo(R.id.notification_updating_feeds, createNotification(null))) } @UnstableApi - private fun refreshFeeds(toUpdate: MutableList, force: Boolean) { + private fun refreshFeeds(feedsToUpdate: MutableList, force: Boolean) { if (Build.VERSION.SDK_INT >= 33 && ActivityCompat.checkSelfPermission(this.applicationContext, Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) { // TODO: Consider calling @@ -219,10 +219,10 @@ object FeedUpdateManager { return } var i = 0 - while (i < toUpdate.size) { + while (i < feedsToUpdate.size) { if (isStopped) return - notificationManager.notify(R.id.notification_updating_feeds, createNotification(toUpdate)) - val feed = unmanaged(toUpdate[i++]) + notificationManager.notify(R.id.notification_updating_feeds, createNotification(feedsToUpdate)) + val feed = unmanaged(feedsToUpdate[i++]) try { Logd(TAG, "updating local feed? ${feed.isLocalFeed} ${feed.title}") when { diff --git a/app/src/main/kotlin/ac/mdiq/podcini/playback/service/PlaybackService.kt b/app/src/main/kotlin/ac/mdiq/podcini/playback/service/PlaybackService.kt index d4a9bd7f..2af57abd 100644 --- a/app/src/main/kotlin/ac/mdiq/podcini/playback/service/PlaybackService.kt +++ b/app/src/main/kotlin/ac/mdiq/podcini/playback/service/PlaybackService.kt @@ -2097,15 +2097,8 @@ class PlaybackService : MediaLibraryService() { private var positionSaverFuture: ScheduledFuture<*>? = null private var widgetUpdaterFuture: ScheduledFuture<*>? = null private var sleepTimerFuture: ScheduledFuture<*>? = null - -// @Volatile -// private var chapterLoaderFuture: Disposable? = null - private var sleepTimer: SleepTimer? = null - /** - * Returns true if the sleep timer is currently active. - */ @get:Synchronized val isSleepTimerActive: Boolean get() = sleepTimerFuture?.isCancelled == false && sleepTimerFuture?.isDone == false && (sleepTimer?.getWaitingTime() ?: 0) > 0 @@ -2124,16 +2117,10 @@ class PlaybackService : MediaLibraryService() { val isWidgetUpdaterActive: Boolean get() = widgetUpdaterFuture != null && !widgetUpdaterFuture!!.isCancelled && !widgetUpdaterFuture!!.isDone - /** - * Returns true if the position saver is currently running. - */ @get:Synchronized val isPositionSaverActive: Boolean get() = positionSaverFuture != null && !positionSaverFuture!!.isCancelled && !positionSaverFuture!!.isDone - /** - * Starts the position saver task. If the position saver is already active, nothing will happen. - */ @Synchronized fun startPositionSaver() { if (!isPositionSaverActive) { @@ -2145,9 +2132,6 @@ class PlaybackService : MediaLibraryService() { } else Logd(TAG, "Call to startPositionSaver was ignored.") } - /** - * Cancels the position saver. If the position saver is not running, nothing will happen. - */ @Synchronized fun cancelPositionSaver() { if (isPositionSaverActive) { @@ -2156,9 +2140,6 @@ class PlaybackService : MediaLibraryService() { } } - /** - * Starts the widget updater task. If the widget updater is already active, nothing will happen. - */ @Synchronized fun startWidgetUpdater() { if (!isWidgetUpdaterActive && !schedExecutor.isShutdown) { @@ -2184,13 +2165,11 @@ class PlaybackService : MediaLibraryService() { * Starts a new sleep timer with the given waiting time. If another sleep timer is already active, it will be * cancelled first. * After waitingTime has elapsed, onSleepTimerExpired() will be called. - * * @throws java.lang.IllegalArgumentException if waitingTime <= 0 */ @Synchronized fun setSleepTimer(waitingTime: Long) { require(waitingTime > 0) { "Waiting time <= 0" } - Logd(TAG, "Setting sleep timer to $waitingTime milliseconds") if (isSleepTimerActive) sleepTimerFuture!!.cancel(true) sleepTimer = SleepTimer(waitingTime) @@ -2198,9 +2177,6 @@ class PlaybackService : MediaLibraryService() { EventFlow.postEvent(FlowEvent.SleepTimerUpdatedEvent.justEnabled(waitingTime)) } - /** - * Disables the sleep timer. If the sleep timer is not active, nothing will happen. - */ @Synchronized fun disableSleepTimer() { if (isSleepTimerActive) { @@ -2209,9 +2185,6 @@ class PlaybackService : MediaLibraryService() { } } - /** - * Restarts the sleep timer. If the sleep timer is not active, nothing will happen. - */ @Synchronized fun restartSleepTimer() { if (isSleepTimerActive) { @@ -2353,8 +2326,7 @@ class PlaybackService : MediaLibraryService() { fun onChapterLoaded(media: Playable?) } - internal class ShakeListener(private val mContext: Context, private val mSleepTimer: SleepTimer) : - SensorEventListener { + internal class ShakeListener(private val mContext: Context, private val mSleepTimer: SleepTimer) : SensorEventListener { private var mAccelerometer: Sensor? = null private var mSensorMgr: SensorManager? = null @@ -2389,14 +2361,10 @@ class PlaybackService : MediaLibraryService() { } } override fun onAccuracyChanged(sensor: Sensor, accuracy: Int) {} - companion object { - private val TAG: String = ShakeListener::class.simpleName ?: "Anonymous" - } } companion object { private val TAG: String = TaskManager::class.simpleName ?: "Anonymous" - private const val SCHED_EX_POOL_SIZE = 2 private const val SLEEP_TIMER_UPDATE_INTERVAL = 10000L // in millisoconds diff --git a/app/src/main/kotlin/ac/mdiq/podcini/storage/database/Feeds.kt b/app/src/main/kotlin/ac/mdiq/podcini/storage/database/Feeds.kt index 32e0ffec..e4787eb5 100644 --- a/app/src/main/kotlin/ac/mdiq/podcini/storage/database/Feeds.kt +++ b/app/src/main/kotlin/ac/mdiq/podcini/storage/database/Feeds.kt @@ -9,6 +9,10 @@ import ac.mdiq.podcini.preferences.UserPreferences.isAutoDelete import ac.mdiq.podcini.preferences.UserPreferences.isAutoDeleteLocal import ac.mdiq.podcini.storage.database.Episodes.deleteEpisodes import ac.mdiq.podcini.storage.database.Feeds.EpisodeAssistant.searchEpisodeByIdentifyingValue +import ac.mdiq.podcini.storage.database.Feeds.EpisodeDuplicateGuesser.canonicalizeTitle +import ac.mdiq.podcini.storage.database.Feeds.EpisodeDuplicateGuesser.datesLookSimilar +import ac.mdiq.podcini.storage.database.Feeds.EpisodeDuplicateGuesser.durationsLookSimilar +import ac.mdiq.podcini.storage.database.Feeds.EpisodeDuplicateGuesser.mimeTypeLooksSimilar import ac.mdiq.podcini.storage.database.LogsAndStats.addDownloadStatus import ac.mdiq.podcini.storage.database.Queues.addToQueueSync import ac.mdiq.podcini.storage.database.Queues.removeFromAllQueuesQuiet @@ -192,6 +196,142 @@ object Feeds { * I.e. episodes are removed from the database if they are not in this episode list. * @return The updated Feed from the database if it already existed, or the new Feed from the parameters otherwise. */ +// @Synchronized +// fun updateFeed0(context: Context, newFeed: Feed, removeUnlistedItems: Boolean): Feed? { +// Logd(TAG, "updateFeed called") +// var resultFeed: Feed? +// val unlistedItems: MutableList = ArrayList() +// +// // Look up feed in the feedslist +// val savedFeed = searchFeedByIdentifyingValueOrID(newFeed, true) +// if (savedFeed == null) { +// Logd(TAG, "Found no existing Feed with title ${newFeed.title}. Adding as new one.") +// Logd(TAG, "newFeed.episodes: ${newFeed.episodes.size}") +// resultFeed = newFeed +// try { +// addNewFeedsSync(context, newFeed) +// // Update with default values that are set in database +// resultFeed = searchFeedByIdentifyingValueOrID(newFeed) +// if (removeUnlistedItems) runBlocking { deleteEpisodes(context, unlistedItems).join() } +// } catch (e: InterruptedException) { e.printStackTrace() +// } catch (e: ExecutionException) { e.printStackTrace() } +// return resultFeed +// } +// +// Logd(TAG, "Feed with title " + newFeed.title + " already exists. Syncing new with existing one.") +// newFeed.episodes.sortWith(EpisodePubdateComparator()) +// if (newFeed.pageNr == savedFeed.pageNr) { +// if (savedFeed.compareWithOther(newFeed)) { +// Logd(TAG, "Feed has updated attribute values. Updating old feed's attributes") +// savedFeed.updateFromOther(newFeed) +// } +// } else { +// Logd(TAG, "New feed has a higher page number.") +// savedFeed.nextPageLink = newFeed.nextPageLink +// } +//// appears not useful +//// if (savedFeed.preferences != null && savedFeed.preferences!!.compareWithOther(newFeed.preferences)) { +//// Logd(TAG, "Feed has updated preferences. Updating old feed's preferences") +//// savedFeed.preferences!!.updateFromOther(newFeed.preferences) +//// } +// val priorMostRecent = savedFeed.mostRecentItem +// val priorMostRecentDate: Date? = priorMostRecent?.getPubDate() +// var idLong = Feed.newId() +// // Look for new or updated Items +// for (idx in newFeed.episodes.indices) { +// val episode = newFeed.episodes[idx] +// val possibleDuplicate = EpisodeAssistant.searchEpisodeGuessDuplicate(newFeed.episodes, episode) +// if (!newFeed.isLocalFeed && possibleDuplicate != null && episode !== possibleDuplicate) { +// // Canonical episode is the first one returned (usually oldest) +// addDownloadStatus(DownloadResult(savedFeed.id, episode.title ?: "", DownloadError.ERROR_PARSER_EXCEPTION_DUPLICATE, false, +// """ +// The podcast host appears to have added the same episode twice. Podcini still refreshed the feed and attempted to repair it. +// +// Original episode: +// ${EpisodeAssistant.duplicateEpisodeDetails(episode)} +// +// Second episode that is also in the feed: +// ${EpisodeAssistant.duplicateEpisodeDetails(possibleDuplicate)} +// """.trimIndent())) +// continue +// } +// var oldItem = searchEpisodeByIdentifyingValue(savedFeed.episodes, episode) +// if (!newFeed.isLocalFeed && oldItem == null) { +// oldItem = EpisodeAssistant.searchEpisodeGuessDuplicate(savedFeed.episodes, episode) +// if (oldItem != null) { +// Logd(TAG, "Repaired duplicate: $oldItem, $episode") +// addDownloadStatus(DownloadResult(savedFeed.id, +// episode.title ?: "", DownloadError.ERROR_PARSER_EXCEPTION_DUPLICATE, false, +// """ +// The podcast host changed the ID of an existing episode instead of just updating the episode itself. Podcini still refreshed the feed and attempted to repair it. +// +// Original episode: +// ${EpisodeAssistant.duplicateEpisodeDetails(oldItem)} +// +// Now the feed contains: +// ${EpisodeAssistant.duplicateEpisodeDetails(episode)} +// """.trimIndent())) +// oldItem.identifier = episode.identifier +// if (needSynch() && oldItem.isPlayed() && oldItem.media != null) { +// val durs = oldItem.media!!.getDuration() / 1000 +// val action = EpisodeAction.Builder(oldItem, EpisodeAction.PLAY) +// .currentTimestamp() +// .started(durs) +// .position(durs) +// .total(durs) +// .build() +// SynchronizationQueueSink.enqueueEpisodeActionIfSyncActive(context, action) +// } +// } +// } +// if (oldItem != null) oldItem.updateFromOther(episode) +// else { +// Logd(TAG, "Found new episode: ${episode.title}") +// episode.feed = savedFeed +// episode.id = idLong++ +// episode.feedId = savedFeed.id +// if (episode.media != null) { +// episode.media!!.id = episode.id +// if (!savedFeed.hasVideoMedia && episode.media!!.getMediaType() == MediaType.VIDEO) savedFeed.hasVideoMedia = true +// } +// if (idx >= savedFeed.episodes.size) savedFeed.episodes.add(episode) +// else savedFeed.episodes.add(idx, episode) +// +// val pubDate = episode.getPubDate() +// if (pubDate == null || priorMostRecentDate == null || priorMostRecentDate.before(pubDate) || priorMostRecentDate == pubDate) { +// Logd(TAG, "Marking episode published on $pubDate new, prior most recent date = $priorMostRecentDate") +// episode.setNew() +// if (savedFeed.preferences?.autoAddNewToQueue == true) { +// val q = savedFeed.preferences?.queue +// if (q != null) runOnIOScope { addToQueueSync(false, episode, q) } +// } +// } +// } +// } +// // identify episodes to be removed +// if (removeUnlistedItems) { +// val it = savedFeed.episodes.toMutableList().iterator() +// while (it.hasNext()) { +// val feedItem = it.next() +// if (searchEpisodeByIdentifyingValue(newFeed.episodes, feedItem) == null) { +// unlistedItems.add(feedItem) +// it.remove() +// } +// } +// } +// // update attributes +// savedFeed.lastUpdate = newFeed.lastUpdate +// savedFeed.type = newFeed.type +// savedFeed.lastUpdateFailed = false +// resultFeed = savedFeed +// try { +// upsertBlk(savedFeed) {} +// if (removeUnlistedItems) runBlocking { deleteEpisodes(context, unlistedItems).join() } +// } catch (e: InterruptedException) { e.printStackTrace() +// } catch (e: ExecutionException) { e.printStackTrace() } +// return resultFeed +// } + @Synchronized fun updateFeed(context: Context, newFeed: Feed, removeUnlistedItems: Boolean): Feed? { Logd(TAG, "updateFeed called") @@ -233,27 +373,32 @@ object Feeds { val priorMostRecent = savedFeed.mostRecentItem val priorMostRecentDate: Date? = priorMostRecent?.getPubDate() var idLong = Feed.newId() + Logd(TAG, "updateFeed building newFeedAssistant") + val newFeedAssistant = FeedAssistant(newFeed, savedFeed.id) + Logd(TAG, "updateFeed building savedFeedAssistant") + val savedFeedAssistant = FeedAssistant(savedFeed) + // Look for new or updated Items for (idx in newFeed.episodes.indices) { val episode = newFeed.episodes[idx] - val possibleDuplicate = EpisodeAssistant.searchEpisodeGuessDuplicate(newFeed.episodes, episode) - if (!newFeed.isLocalFeed && possibleDuplicate != null && episode !== possibleDuplicate) { - // Canonical episode is the first one returned (usually oldest) - addDownloadStatus(DownloadResult(savedFeed.id, episode.title ?: "", DownloadError.ERROR_PARSER_EXCEPTION_DUPLICATE, false, - """ - The podcast host appears to have added the same episode twice. Podcini still refreshed the feed and attempted to repair it. - - Original episode: - ${EpisodeAssistant.duplicateEpisodeDetails(episode)} - - Second episode that is also in the feed: - ${EpisodeAssistant.duplicateEpisodeDetails(possibleDuplicate)} - """.trimIndent())) - continue - } - var oldItem = searchEpisodeByIdentifyingValue(savedFeed.episodes, episode) +// val possibleDuplicate = EpisodeAssistant.searchEpisodeGuessDuplicate(newFeed.episodes, episode) +// if (!newFeed.isLocalFeed && possibleDuplicate != null && episode !== possibleDuplicate) { +// // Canonical episode is the first one returned (usually oldest) +// addDownloadStatus(DownloadResult(savedFeed.id, episode.title ?: "", DownloadError.ERROR_PARSER_EXCEPTION_DUPLICATE, false, +// """ +// The podcast host appears to have added the same episode twice. Podcini still refreshed the feed and attempted to repair it. +// +// Original episode: +// ${EpisodeAssistant.duplicateEpisodeDetails(episode)} +// +// Second episode that is also in the feed: +// ${EpisodeAssistant.duplicateEpisodeDetails(possibleDuplicate)} +// """.trimIndent())) +// continue +// } + var oldItem = savedFeedAssistant.searchEpisodeByIdentifyingValue(episode) if (!newFeed.isLocalFeed && oldItem == null) { - oldItem = EpisodeAssistant.searchEpisodeGuessDuplicate(savedFeed.episodes, episode) + oldItem = savedFeedAssistant.searchEpisodeGuessDuplicate(episode) if (oldItem != null) { Logd(TAG, "Repaired duplicate: $oldItem, $episode") addDownloadStatus(DownloadResult(savedFeed.id, @@ -304,17 +449,21 @@ object Feeds { } } } + savedFeedAssistant.clear() + // identify episodes to be removed if (removeUnlistedItems) { val it = savedFeed.episodes.toMutableList().iterator() while (it.hasNext()) { val feedItem = it.next() - if (searchEpisodeByIdentifyingValue(newFeed.episodes, feedItem) == null) { + if (newFeedAssistant.searchEpisodeByIdentifyingValue(feedItem) == null) { unlistedItems.add(feedItem) it.remove() } } } + newFeedAssistant.clear() + // update attributes savedFeed.lastUpdate = newFeed.lastUpdate savedFeed.type = newFeed.type @@ -421,7 +570,7 @@ object Feeds { return !feed.isLocalFeed || isAutoDeleteLocal } - fun getYoutubeSyndicate(video: Boolean, music: Boolean): Feed { + private fun getYoutubeSyndicate(video: Boolean, music: Boolean): Feed { var feedId: Long = if (video) 1 else 2 if (music) feedId += 2 // music feed takes ids 3 and 4 var feed = getFeed(feedId, true) @@ -467,6 +616,104 @@ object Feeds { } } + class FeedAssistant(val feed: Feed, val feedId: Long = 0L) { + val map = mutableMapOf() + + init { + for (e in feed.episodes) { + if (!e.identifier.isNullOrEmpty()) { + if (map.containsKey(e.identifier!!)) { + // TODO: add addDownloadStatus + Logd(TAG, "FeedAssistant init identifier duplicate: ${e.identifier} ${e.title}") + addDownloadStatus(e, map[e.identifier!!]!!) + continue + } + map[e.identifier!!] = e + } + val idv = e.identifyingValue + if (idv != e.identifier && !idv.isNullOrEmpty()) { + if (map.containsKey(idv)) { + // TODO: add addDownloadStatus + Logd(TAG, "FeedAssistant init identifyingValue duplicate: $idv ${e.title}") + addDownloadStatus(e, map[idv]!!) + continue + } + map[idv] = e + } + val url = e.media?.getStreamUrl() + if (url != idv && !url.isNullOrEmpty()) { + if (map.containsKey(url)) { + // TODO: add addDownloadStatus + Logd(TAG, "FeedAssistant init url duplicate: $url ${e.title}") + addDownloadStatus(e, map[url]!!) + continue + } + map[url] = e + } + val title = canonicalizeTitle(e.title) + if (title != idv && title.isNotEmpty()) { + if (map.containsKey(title)) { + // TODO: add addDownloadStatus + val episode = map[title] + if (episode != null) { + val media1 = episode.media + val media2 = e.media + if (media1 != null && media2 != null && datesLookSimilar(episode, e) && durationsLookSimilar(media1, media2) && mimeTypeLooksSimilar(media1, media2)) { + Logd(TAG, "FeedAssistant init title duplicate: $title ${e.title}") + addDownloadStatus(e, episode) + continue + } + } + } +// TODO: does it mean there are duplicate titles? + map[title] = e + } + } + } + + private fun addDownloadStatus(episode: Episode, possibleDuplicate: Episode) { + val feedId_ = if (feedId > 0) feedId else feed.id + addDownloadStatus(DownloadResult(feedId_, episode.title ?: "", DownloadError.ERROR_PARSER_EXCEPTION_DUPLICATE, false, + """ + The podcast host appears to have added the same episode twice. Podcini still refreshed the feed and attempted to repair it. + + Original episode: + ${EpisodeAssistant.duplicateEpisodeDetails(episode)} + + Second episode that is also in the feed: + ${EpisodeAssistant.duplicateEpisodeDetails(possibleDuplicate)} + """.trimIndent())) + } + + fun searchEpisodeByIdentifyingValue(item: Episode): Episode? { + return map[item.identifyingValue] + } + + fun searchEpisodeGuessDuplicate(item: Episode): Episode? { + var episode = map[item.identifier] + if (episode != null) return episode + val url = item.media?.getStreamUrl() + if (!url.isNullOrEmpty()) { + episode = map[url] + if (episode != null) return episode + } + val title = canonicalizeTitle(item.title) + if (title.isNotEmpty()) { + episode = map[title] + if (episode != null) { + val media1 = episode.media + val media2 = item.media + if (media1 == null || media2 == null) return null + if (datesLookSimilar(episode, item) && durationsLookSimilar(media1, media2) && mimeTypeLooksSimilar(media1, media2)) return episode + } + } + return null + } + + fun clear() { + map.clear() + } + } private object EpisodeAssistant { fun searchEpisodeByIdentifyingValue(episodes: List?, searchItem: Episode): Episode? { if (episodes.isNullOrEmpty()) return null @@ -479,16 +726,16 @@ object Feeds { * Guess if one of the episodes could actually mean the searched episode, even if it uses another identifying value. * This is to work around podcasters breaking their GUIDs. */ - fun searchEpisodeGuessDuplicate(episodes: List?, searchItem: Episode): Episode? { - if (episodes.isNullOrEmpty()) return null - for (episode in episodes) { - if (EpisodeDuplicateGuesser.sameAndNotEmpty(episode.identifier, searchItem.identifier)) return episode - } - for (episode in episodes) { - if (EpisodeDuplicateGuesser.seemDuplicates(episode, searchItem)) return episode - } - return null - } +// fun searchEpisodeGuessDuplicate(episodes: List?, searchItem: Episode): Episode? { +// if (episodes.isNullOrEmpty()) return null +// for (episode in episodes) { +// if (EpisodeDuplicateGuesser.sameAndNotEmpty(episode.identifier, searchItem.identifier)) return episode +// } +// for (episode in episodes) { +// if (EpisodeDuplicateGuesser.seemDuplicates(episode, searchItem)) return episode +// } +// return null +// } fun duplicateEpisodeDetails(episode: Episode): String { return (""" Title: ${episode.title} @@ -506,6 +753,7 @@ object Feeds { * even if their feed explicitly says that the episodes are different. */ object EpisodeDuplicateGuesser { + // only used in test fun seemDuplicates(item1: Episode, item2: Episode): Boolean { if (sameAndNotEmpty(item1.identifier, item2.identifier)) return true val media1 = item1.media @@ -514,21 +762,21 @@ object Feeds { if (sameAndNotEmpty(media1.getStreamUrl(), media2.getStreamUrl())) return true return (titlesLookSimilar(item1, item2) && datesLookSimilar(item1, item2) && durationsLookSimilar(media1, media2) && mimeTypeLooksSimilar(media1, media2)) } - fun sameAndNotEmpty(string1: String?, string2: String?): Boolean { + private fun sameAndNotEmpty(string1: String?, string2: String?): Boolean { if (string1.isNullOrEmpty() || string2.isNullOrEmpty()) return false return string1 == string2 } - private fun datesLookSimilar(item1: Episode, item2: Episode): Boolean { + internal fun datesLookSimilar(item1: Episode, item2: Episode): Boolean { if (item1.getPubDate() == null || item2.getPubDate() == null) return false val dateFormat = DateFormat.getDateInstance(DateFormat.SHORT, Locale.US) // MM/DD/YY val dateOriginal = dateFormat.format(item2.getPubDate()!!) val dateNew = dateFormat.format(item1.getPubDate()!!) return dateOriginal == dateNew // Same date; time is ignored. } - private fun durationsLookSimilar(media1: EpisodeMedia, media2: EpisodeMedia): Boolean { + internal fun durationsLookSimilar(media1: EpisodeMedia, media2: EpisodeMedia): Boolean { return abs((media1.getDuration() - media2.getDuration()).toDouble()) < 10 * 60L * 1000L } - private fun mimeTypeLooksSimilar(media1: EpisodeMedia, media2: EpisodeMedia): Boolean { + internal fun mimeTypeLooksSimilar(media1: EpisodeMedia, media2: EpisodeMedia): Boolean { var mimeType1 = media1.mimeType var mimeType2 = media2.mimeType if (mimeType1 == null || mimeType2 == null) return true @@ -541,7 +789,7 @@ object Feeds { private fun titlesLookSimilar(item1: Episode, item2: Episode): Boolean { return sameAndNotEmpty(canonicalizeTitle(item1.title), canonicalizeTitle(item2.title)) } - private fun canonicalizeTitle(title: String?): String { + internal fun canonicalizeTitle(title: String?): String { if (title == null) return "" return title .trim { it <= ' ' } diff --git a/app/src/main/kotlin/ac/mdiq/podcini/storage/model/MediaType.kt b/app/src/main/kotlin/ac/mdiq/podcini/storage/model/MediaType.kt index 7cc54b3f..4a008fd0 100644 --- a/app/src/main/kotlin/ac/mdiq/podcini/storage/model/MediaType.kt +++ b/app/src/main/kotlin/ac/mdiq/podcini/storage/model/MediaType.kt @@ -4,11 +4,17 @@ enum class MediaType { AUDIO, VIDEO, UNKNOWN; companion object { - private val AUDIO_APPLICATION_MIME_STRINGS: Set = HashSet(mutableListOf( +// private val AUDIO_APPLICATION_MIME_STRINGS: Set = HashSet(mutableListOf( +// "application/ogg", +// "application/opus", +// "application/x-flac" +// )) + + private val AUDIO_APPLICATION_MIME_STRINGS: HashSet = hashSetOf( "application/ogg", "application/opus", "application/x-flac" - )) + ) fun fromMimeType(mimeType: String?): MediaType { return when { diff --git a/changelog.md b/changelog.md index 72ff8f3e..01219e6b 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,7 @@ +# 6.7.0 + +* largely improved efficiency of podcasts refresh, no more massive list searches + # 6.6.7 * volume adaptation numbers were changed to 0.2, 0.5, 1, 1.6, 2.4, 3.6 to avoid much distortion diff --git a/fastlane/metadata/android/en-US/changelogs/3020253.txt b/fastlane/metadata/android/en-US/changelogs/3020253.txt new file mode 100644 index 00000000..5175df0c --- /dev/null +++ b/fastlane/metadata/android/en-US/changelogs/3020253.txt @@ -0,0 +1,3 @@ + Version 6.7.0: + +* largely improved efficiency of podcasts refresh, no more massive list searches