Merge pull request #5876 from TeamNewPipe/terminated-channels

Better error handling of terminated channels

app/build.gradle

@@ -183,7 +183,7 @@ dependencies {
 /** NewPipe libraries **/
     // You can use a local version by uncommenting a few lines in settings.gradle
     implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
-    implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.21.4'
+    implementation 'com.github.TeamNewPipe:NewPipeExtractor:d4186d100b6c6dddfcf3cf4b004f5960a8bf441d'
 
 /** Checkstyle **/
     checkstyle "com.puppycrawl.tools:checkstyle:${checkstyleVersion}"

app/src/main/java/org/schabi/newpipe/error/ErrorInfo.kt

@@ -6,6 +6,7 @@ import kotlinx.android.parcel.Parcelize
 import org.schabi.newpipe.R
 import org.schabi.newpipe.extractor.Info
 import org.schabi.newpipe.extractor.NewPipe
+import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException
 import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException
 import org.schabi.newpipe.extractor.exceptions.ExtractionException
@@ -95,6 +96,7 @@ class ErrorInfo(
             action: UserAction
         ): Int {
             return when {
+                throwable is AccountTerminatedException -> R.string.account_terminated
                 throwable is ContentNotAvailableException -> R.string.content_not_available
                 throwable != null && throwable.isNetworkRelated -> R.string.network_error
                 throwable is ContentNotSupportedException -> R.string.content_not_supported

app/src/main/java/org/schabi/newpipe/error/ErrorPanelHelper.kt

@@ -13,6 +13,8 @@ import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers
 import io.reactivex.rxjava3.disposables.Disposable
 import org.schabi.newpipe.MainActivity
 import org.schabi.newpipe.R
+import org.schabi.newpipe.extractor.NewPipe
+import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException
 import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException
 import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException
@@ -22,9 +24,11 @@ import org.schabi.newpipe.extractor.exceptions.PrivateContentException
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException
 import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException
 import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException
+import org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty
 import org.schabi.newpipe.ktx.animate
 import org.schabi.newpipe.ktx.isInterruptedCaused
 import org.schabi.newpipe.ktx.isNetworkRelated
+import org.schabi.newpipe.util.ServiceHelper
 import java.util.concurrent.TimeUnit
 
 class ErrorPanelHelper(
@@ -35,6 +39,8 @@ class ErrorPanelHelper(
     private val context: Context = rootView.context!!
     private val errorPanelRoot: View = rootView.findViewById(R.id.error_panel)
     private val errorTextView: TextView = errorPanelRoot.findViewById(R.id.error_message_view)
+    private val errorServiceInfoTextView: TextView = errorPanelRoot.findViewById(R.id.error_message_service_info_view)
+    private val errorServiceExplenationTextView: TextView = errorPanelRoot.findViewById(R.id.error_message_service_explenation_view)
     private val errorButtonAction: Button = errorPanelRoot.findViewById(R.id.error_button_action)
     private val errorButtonRetry: Button = errorPanelRoot.findViewById(R.id.error_button_retry)
 
@@ -70,13 +76,40 @@ class ErrorPanelHelper(
                 errorButtonAction.setOnClickListener(null)
             }
             errorTextView.setText(R.string.recaptcha_request_toast)
+            // additional info is only provided by AccountTerminatedException
+            errorServiceInfoTextView.isVisible = false
+            errorServiceExplenationTextView.isVisible = false
             errorButtonRetry.isVisible = true
+        } else if (errorInfo.throwable is AccountTerminatedException) {
+            errorButtonRetry.isVisible = false
+            errorButtonAction.isVisible = false
+            errorTextView.setText(R.string.account_terminated)
+            if (!isNullOrEmpty((errorInfo.throwable as AccountTerminatedException).message)) {
+                errorServiceInfoTextView.setText(
+                    context.resources.getString(
+                        R.string.service_provides_reason,
+                        NewPipe.getNameOfService(ServiceHelper.getSelectedServiceId(context))
+                    )
+                )
+                errorServiceExplenationTextView.setText(
+                    (errorInfo.throwable as AccountTerminatedException).message
+                )
+                errorServiceInfoTextView.isVisible = true
+                errorServiceExplenationTextView.isVisible = true
+            } else {
+                errorServiceInfoTextView.isVisible = false
+                errorServiceExplenationTextView.isVisible = false
+            }
         } else {
             errorButtonAction.setText(R.string.error_snackbar_action)
             errorButtonAction.setOnClickListener {
                 ErrorActivity.reportError(context, errorInfo)
             }
 
+            // additional info is only provided by AccountTerminatedException
+            errorServiceInfoTextView.isVisible = false
+            errorServiceExplenationTextView.isVisible = false
+
             // hide retry button by default, then show only if not unavailable/unsupported content
             errorButtonRetry.isVisible = false
             errorTextView.setText(

app/src/main/java/org/schabi/newpipe/local/feed/FeedFragment.kt

@@ -28,6 +28,7 @@ import android.view.MenuInflater
 import android.view.MenuItem
 import android.view.View
 import android.view.ViewGroup
+import androidx.annotation.Nullable
 import androidx.appcompat.app.AlertDialog
 import androidx.core.content.edit
 import androidx.core.os.bundleOf
@@ -35,15 +36,25 @@ import androidx.core.view.isVisible
 import androidx.lifecycle.ViewModelProvider
 import androidx.preference.PreferenceManager
 import icepick.State
+import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers
+import io.reactivex.rxjava3.core.Single
+import io.reactivex.rxjava3.disposables.CompositeDisposable
+import io.reactivex.rxjava3.schedulers.Schedulers
+import org.schabi.newpipe.NewPipeDatabase
 import org.schabi.newpipe.R
 import org.schabi.newpipe.database.feed.model.FeedGroupEntity
+import org.schabi.newpipe.database.subscription.SubscriptionEntity
 import org.schabi.newpipe.databinding.FragmentFeedBinding
 import org.schabi.newpipe.error.ErrorInfo
 import org.schabi.newpipe.error.UserAction
+import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException
+import org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty
 import org.schabi.newpipe.fragments.list.BaseListFragment
 import org.schabi.newpipe.ktx.animate
 import org.schabi.newpipe.ktx.animateHideRecyclerViewAllowingScrolling
 import org.schabi.newpipe.local.feed.service.FeedLoadService
+import org.schabi.newpipe.local.subscription.SubscriptionManager
 import org.schabi.newpipe.util.Localization
 import java.time.OffsetDateTime
 
@@ -51,6 +62,8 @@ class FeedFragment : BaseListFragment<FeedState, Unit>() {
     private var _feedBinding: FragmentFeedBinding? = null
     private val feedBinding get() = _feedBinding!!
 
+    private val disposables = CompositeDisposable()
+
     private lateinit var viewModel: FeedViewModel
     @State
     @JvmField
@@ -158,6 +171,7 @@ class FeedFragment : BaseListFragment<FeedState, Unit>() {
     }
 
     override fun onDestroy() {
+        disposables.dispose()
         super.onDestroy()
         activity?.supportActionBar?.subtitle = null
     }
@@ -241,17 +255,23 @@ class FeedFragment : BaseListFragment<FeedState, Unit>() {
             listState = null
         }
 
-        oldestSubscriptionUpdate = loadedState.oldestUpdate
+        val feedsNotLoaded = loadedState.notLoadedCount > 0
-
+        feedBinding.refreshSubtitleText.isVisible = feedsNotLoaded
-        val loadedCount = loadedState.notLoadedCount > 0
+        if (feedsNotLoaded) {
-        feedBinding.refreshSubtitleText.isVisible = loadedCount
-        if (loadedCount) {
             feedBinding.refreshSubtitleText.text = getString(
                 R.string.feed_subscription_not_loaded_count,
                 loadedState.notLoadedCount
             )
         }
 
+        if (oldestSubscriptionUpdate != loadedState.oldestUpdate ||
+            (oldestSubscriptionUpdate == null && loadedState.oldestUpdate == null)
+        ) {
+            // ignore errors if they have already been handled for the current update
+            handleItemsErrors(loadedState.itemsErrors)
+        }
+        oldestSubscriptionUpdate = loadedState.oldestUpdate
+
         if (loadedState.items.isEmpty()) {
             showEmptyState()
         } else {
@@ -269,6 +289,72 @@ class FeedFragment : BaseListFragment<FeedState, Unit>() {
         }
     }
 
+    private fun handleItemsErrors(errors: List<Throwable>) {
+        errors.forEachIndexed() { i, t ->
+            if (t is FeedLoadService.RequestException &&
+                t.cause is ContentNotAvailableException
+            ) {
+                Single.fromCallable {
+                    NewPipeDatabase.getInstance(requireContext()).subscriptionDAO()
+                        .getSubscription(t.subscriptionId)
+                }.subscribeOn(Schedulers.io())
+                    .observeOn(AndroidSchedulers.mainThread())
+                    .subscribe(
+                        {
+                            subscriptionEntity ->
+                            handleFeedNotAvailable(
+                                subscriptionEntity,
+                                t.cause,
+                                errors.subList(i + 1, errors.size)
+                            )
+                        },
+                        { throwable -> throwable.printStackTrace() }
+                    )
+                return // this will be called on the remaining errors by handleFeedNotAvailable()
+            }
+        }
+    }
+
+    private fun handleFeedNotAvailable(
+        subscriptionEntity: SubscriptionEntity,
+        @Nullable cause: Throwable?,
+        nextItemsErrors: List<Throwable>
+    ) {
+        val sharedPreferences = PreferenceManager.getDefaultSharedPreferences(requireContext())
+        val isFastFeedModeEnabled = sharedPreferences.getBoolean(
+            getString(R.string.feed_use_dedicated_fetch_method_key), false
+        )
+
+        val builder = AlertDialog.Builder(requireContext())
+            .setTitle(R.string.feed_load_error)
+            .setPositiveButton(
+                R.string.unsubscribe
+            ) { _, _ ->
+                SubscriptionManager(requireContext()).deleteSubscription(
+                    subscriptionEntity.serviceId, subscriptionEntity.url
+                ).subscribe()
+                handleItemsErrors(nextItemsErrors)
+            }
+            .setNegativeButton(R.string.cancel) { _, _ -> }
+
+        var message = getString(R.string.feed_load_error_account_info, subscriptionEntity.name)
+        if (cause is AccountTerminatedException) {
+            message += "\n" + getString(R.string.feed_load_error_terminated)
+        } else if (cause is ContentNotAvailableException) {
+            if (isFastFeedModeEnabled) {
+                message += "\n" + getString(R.string.feed_load_error_fast_unknown)
+                builder.setNeutralButton(R.string.feed_use_dedicated_fetch_method_disable_button) { _, _ ->
+                    sharedPreferences.edit {
+                        putBoolean(getString(R.string.feed_use_dedicated_fetch_method_key), false)
+                    }
+                }
+            } else if (!isNullOrEmpty(cause.message)) {
+                message += "\n" + cause.message
+            }
+        }
+        builder.setMessage(message).create().show()
+    }
+
     private fun updateRelativeTimeViews() {
         updateRefreshViewState()
         infoListAdapter.notifyDataSetChanged()

app/src/main/java/org/schabi/newpipe/local/feed/service/FeedLoadService.kt

@@ -48,9 +48,7 @@ import org.schabi.newpipe.MainActivity.DEBUG
 import org.schabi.newpipe.R
 import org.schabi.newpipe.database.feed.model.FeedGroupEntity
 import org.schabi.newpipe.extractor.ListInfo
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException
 import org.schabi.newpipe.extractor.stream.StreamInfoItem
-import org.schabi.newpipe.ktx.isNetworkRelated
 import org.schabi.newpipe.local.feed.FeedDatabaseManager
 import org.schabi.newpipe.local.feed.service.FeedEventManager.Event.ErrorResultEvent
 import org.schabi.newpipe.local.feed.service.FeedEventManager.Event.ProgressEvent
@@ -58,7 +56,6 @@ import org.schabi.newpipe.local.feed.service.FeedEventManager.Event.SuccessResul
 import org.schabi.newpipe.local.feed.service.FeedEventManager.postEvent
 import org.schabi.newpipe.local.subscription.SubscriptionManager
 import org.schabi.newpipe.util.ExtractorHelper
-import java.io.IOException
 import java.time.OffsetDateTime
 import java.time.ZoneOffset
 import java.util.concurrent.TimeUnit
@@ -162,7 +159,7 @@ class FeedLoadService : Service() {
     // Loading & Handling
     // /////////////////////////////////////////////////////////////////////////
 
-    private class RequestException(val subscriptionId: Long, message: String, cause: Throwable) : Exception(message, cause) {
+    class RequestException(val subscriptionId: Long, message: String, cause: Throwable) : Exception(message, cause) {
         companion object {
             fun wrapList(subscriptionId: Long, info: ListInfo<StreamInfoItem>): List<Throwable> {
                 val toReturn = ArrayList<Throwable>(info.errors.size)
@@ -209,29 +206,40 @@ class FeedLoadService : Service() {
             .filter { !cancelSignal.get() }
 
             .map { subscriptionEntity ->
+                var error: Throwable? = null
                 try {
                     val listInfo = if (useFeedExtractor) {
                         ExtractorHelper
                             .getFeedInfoFallbackToChannelInfo(subscriptionEntity.serviceId, subscriptionEntity.url)
+                            .onErrorReturn {
+                                error = it // store error, otherwise wrapped into RuntimeException
+                                throw it
+                            }
                             .blockingGet()
                     } else {
                         ExtractorHelper
                             .getChannelInfo(subscriptionEntity.serviceId, subscriptionEntity.url, true)
+                            .onErrorReturn {
+                                error = it // store error, otherwise wrapped into RuntimeException
+                                throw it
+                            }
                             .blockingGet()
                     } as ListInfo<StreamInfoItem>
 
                     return@map Notification.createOnNext(Pair(subscriptionEntity.uid, listInfo))
                 } catch (e: Throwable) {
+                    if (error == null) {
+                        // do this to prevent blockingGet() from wrapping into RuntimeException
+                        error = e
+                    }
+
                     val request = "${subscriptionEntity.serviceId}:${subscriptionEntity.url}"
-                    val wrapper = RequestException(subscriptionEntity.uid, request, e)
+                    val wrapper = RequestException(subscriptionEntity.uid, request, error!!)
                     return@map Notification.createOnError<Pair<Long, ListInfo<StreamInfoItem>>>(wrapper)
                 }
             }
             .sequential()
 
-            .observeOn(AndroidSchedulers.mainThread())
-            .doOnNext(errorHandlingConsumer)
-
             .observeOn(AndroidSchedulers.mainThread())
             .doOnNext(notificationsConsumer)
 
@@ -331,24 +339,6 @@ class FeedLoadService : Service() {
             }
         }
 
-    private val errorHandlingConsumer: Consumer<Notification<Pair<Long, ListInfo<StreamInfoItem>>>>
-        get() = Consumer {
-            if (it.isOnError) {
-                var error = it.error!!
-                if (error is RequestException) error = error.cause!!
-                val cause = error.cause
-
-                when {
-                    error is ReCaptchaException -> throw error
-                    cause is ReCaptchaException -> throw cause
-
-                    error is IOException -> throw error
-                    cause is IOException -> throw cause
-                    error.isNetworkRelated -> throw IOException(error)
-                }
-            }
-        }
-
     private val notificationsConsumer: Consumer<Notification<Pair<Long, ListInfo<StreamInfoItem>>>>
         get() = Consumer { onItemCompleted(it.value?.second?.name) }
 

app/src/main/res/layout/error_panel.xml

@@ -15,7 +15,29 @@
         android:text="@string/general_error"
         android:textSize="16sp"
         android:textStyle="bold"
-        tools:text="Network error" />
+        tools:text="Account terminated" />
+
+    <TextView
+        android:id="@+id/error_message_service_info_view"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:gravity="center"
+        android:layout_marginTop="6dp"
+        android:text="@string/general_error"
+        android:textSize="16sp"
+        tools:text="YouTube provides this reason:" />
+
+    <TextView
+        android:id="@+id/error_message_service_explenation_view"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:gravity="center"
+        android:layout_marginTop="4dp"
+        android:text="@string/general_error"
+        android:textSize="16sp"
+        android:textStyle="italic"
+        tools:text="This account has been terminated because we received multiple third-party claims of copyright infringement regarding material that the user posted." />
+
 
     <Button
         android:id="@+id/error_button_action"

app/src/main/res/values/strings.xml

@@ -693,11 +693,16 @@
     <string name="feed_update_threshold_title">Feed update threshold</string>
     <string name="feed_update_threshold_summary">Time after last update before a subscription is considered outdated — %s</string>
     <string name="feed_update_threshold_option_always_update">Always update</string>
+    <string name="feed_load_error">Error loading feed</string>
+    <string name="feed_load_error_account_info">Could not load feed for \'%s\'.</string>
+    <string name="feed_load_error_terminated">The author\'s account has been terminated.\nNewPipe will not be able to load this feed in the future.\nDo you want to unsubscribe from this channel?</string>
+    <string name="feed_load_error_fast_unknown">The fast feed mode does not provide more info on this.</string>
     <string name="feed_use_dedicated_fetch_method_title">Fetch from dedicated feed when available</string>
     <string name="feed_use_dedicated_fetch_method_summary">Available in some services, it is usually much faster but may return a limited amount of items and often incomplete information (e.g. no duration, item type, no live status).</string>
     <string name="feed_use_dedicated_fetch_method_enable_button">Enable fast mode</string>
     <string name="feed_use_dedicated_fetch_method_disable_button">Disable fast mode</string>
     <string name="feed_use_dedicated_fetch_method_help_text">Do you think feed loading is too slow? If so, try enabling fast loading (you can change it in settings or by pressing the button below).\n\nNewPipe offers two feed loading strategies:\n• Fetching the whole subscription channel, which is slow but complete.\n• Using a dedicated service endpoint, which is fast but usually not complete.\n\nThe difference between the two is that the fast one usually lacks some information, like the item\'s duration or type (can\'t distinguish between live videos and normal ones) and it may return less items.\n\nYouTube is an example of a service that offers this fast method with its RSS feed.\n\nSo the choice boils down to what you prefer: speed or precise information.</string>
+
     <string name="content_not_supported">This content is not yet supported by NewPipe.\n\nIt will hopefully be supported in a future version.</string>
     <string name="detail_sub_channel_thumbnail_view_description">Channel\'s avatar thumbnail</string>
     <string name="channel_created_by">Created by %s</string>
@@ -712,6 +717,8 @@
     <string name="soundcloud_go_plus_content">This is a SoundCloud Go+ track, at least in your country, so it cannot be streamed or downloaded by NewPipe.</string>
     <string name="private_content">This content is private, so it cannot be streamed or downloaded by NewPipe.</string>
     <string name="youtube_music_premium_content">This video is available only to YouTube Music Premium members, so it cannot be streamed or downloaded by NewPipe.</string>
+    <string name="account_terminated">Account terminated</string>
+    <string name="service_provides_reason">%s provides this reason:</string>
     <string name="paid_content">This content is only available to users who have paid, so it cannot be streamed or downloaded by NewPipe.</string>
     <string name="featured">Featured</string>
     <string name="radio">Radio</string>