Merge pull request #3441 from wb9688/nextpage

Next page stuff

app/build.gradle

@@ -163,7 +163,7 @@ dependencies {
         exclude module: 'support-annotations'
     }
 
-    implementation 'com.github.TeamNewPipe:NewPipeExtractor:54d9e5a2f89d6d739dadb6c744efeaf981c85bef'
+    implementation 'com.github.TeamNewPipe:NewPipeExtractor:a70cb0283ffc3bba2709815673a5a7940aab0a3a'
 
     implementation "com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751"
     implementation "org.jsoup:jsoup:1.13.1"

app/src/main/java/org/schabi/newpipe/fragments/list/BaseListInfoFragment.java

@@ -9,6 +9,7 @@ import androidx.annotation.NonNull;
 
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.ListInfo;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.util.Constants;
 import org.schabi.newpipe.views.NewPipeRecyclerView;
 
@@ -30,7 +31,7 @@ public abstract class BaseListInfoFragment<I extends ListInfo>
     protected String url;
 
     protected I currentInfo;
-    protected String currentNextPageUrl;
+    protected Page currentNextPage;
     protected Disposable currentWorker;
 
     @Override
@@ -78,7 +79,7 @@ public abstract class BaseListInfoFragment<I extends ListInfo>
     public void writeTo(final Queue<Object> objectsToSave) {
         super.writeTo(objectsToSave);
         objectsToSave.add(currentInfo);
-        objectsToSave.add(currentNextPageUrl);
+        objectsToSave.add(currentNextPage);
     }
 
     @Override
@@ -86,7 +87,7 @@ public abstract class BaseListInfoFragment<I extends ListInfo>
     public void readFrom(@NonNull final Queue<Object> savedObjects) throws Exception {
         super.readFrom(savedObjects);
         currentInfo = (I) savedObjects.poll();
-        currentNextPageUrl = (String) savedObjects.poll();
+        currentNextPage = (Page) savedObjects.poll();
     }
 
     /*//////////////////////////////////////////////////////////////////////////
@@ -130,7 +131,7 @@ public abstract class BaseListInfoFragment<I extends ListInfo>
                 .subscribe((@NonNull I result) -> {
                     isLoading.set(false);
                     currentInfo = result;
-                    currentNextPageUrl = result.getNextPageUrl();
+                    currentNextPage = result.getNextPage();
                     handleResult(result);
                 }, (@NonNull Throwable throwable) -> onError(throwable));
     }
@@ -182,7 +183,7 @@ public abstract class BaseListInfoFragment<I extends ListInfo>
     @Override
     public void handleNextItems(final ListExtractor.InfoItemsPage result) {
         super.handleNextItems(result);
-        currentNextPageUrl = result.getNextPageUrl();
+        currentNextPage = result.getNextPage();
         infoListAdapter.addInfoItemList(result.getItems());
 
         showListFooter(hasMoreItems());
@@ -190,7 +191,7 @@ public abstract class BaseListInfoFragment<I extends ListInfo>
 
     @Override
     protected boolean hasMoreItems() {
-        return !TextUtils.isEmpty(currentNextPageUrl);
+        return Page.isValid(currentNextPage);
     }
 
     /*//////////////////////////////////////////////////////////////////////////

app/src/main/java/org/schabi/newpipe/fragments/list/channel/ChannelFragment.java

@@ -403,7 +403,7 @@ public class ChannelFragment extends BaseListInfoFragment<ChannelInfo>
 
     @Override
     protected Single<ListExtractor.InfoItemsPage> loadMoreItemsLogic() {
-        return ExtractorHelper.getMoreChannelItems(serviceId, url, currentNextPageUrl);
+        return ExtractorHelper.getMoreChannelItems(serviceId, url, currentNextPage);
     }
 
     @Override
@@ -555,7 +555,7 @@ public class ChannelFragment extends BaseListInfoFragment<ChannelInfo>
             }
         }
         return new ChannelPlayQueue(currentInfo.getServiceId(), currentInfo.getUrl(),
-                currentInfo.getNextPageUrl(), streamItems, index);
+                currentInfo.getNextPage(), streamItems, index);
     }
 
     @Override

app/src/main/java/org/schabi/newpipe/fragments/list/comments/CommentsFragment.java

@@ -71,7 +71,7 @@ public class CommentsFragment extends BaseListInfoFragment<CommentsInfo> {
 
     @Override
     protected Single<ListExtractor.InfoItemsPage> loadMoreItemsLogic() {
-        return ExtractorHelper.getMoreCommentItems(serviceId, currentInfo, currentNextPageUrl);
+        return ExtractorHelper.getMoreCommentItems(serviceId, currentInfo, currentNextPage);
     }
 
     @Override

app/src/main/java/org/schabi/newpipe/fragments/list/kiosk/DefaultKioskFragment.java

@@ -44,7 +44,7 @@ public class DefaultKioskFragment extends KioskFragment {
             name = kioskTranslatedName;
 
             currentInfo = null;
-            currentNextPageUrl = null;
+            currentNextPage = null;
         } catch (ExtractionException e) {
             onUnrecoverableError(e, UserAction.REQUESTED_KIOSK, "none",
                     "Loading default kiosk from selected service", 0);

app/src/main/java/org/schabi/newpipe/fragments/list/kiosk/KioskFragment.java

@@ -150,7 +150,7 @@ public class KioskFragment extends BaseListInfoFragment<KioskInfo> {
 
     @Override
     public Single<ListExtractor.InfoItemsPage> loadMoreItemsLogic() {
-        return ExtractorHelper.getMoreKioskItems(serviceId, url, currentNextPageUrl);
+        return ExtractorHelper.getMoreKioskItems(serviceId, url, currentNextPage);
     }
 
     /*//////////////////////////////////////////////////////////////////////////

app/src/main/java/org/schabi/newpipe/fragments/list/playlist/PlaylistFragment.java

@@ -229,7 +229,7 @@ public class PlaylistFragment extends BaseListInfoFragment<PlaylistInfo> {
 
     @Override
     protected Single<ListExtractor.InfoItemsPage> loadMoreItemsLogic() {
-        return ExtractorHelper.getMorePlaylistItems(serviceId, url, currentNextPageUrl);
+        return ExtractorHelper.getMorePlaylistItems(serviceId, url, currentNextPage);
     }
 
     @Override
@@ -349,7 +349,7 @@ public class PlaylistFragment extends BaseListInfoFragment<PlaylistInfo> {
         return new PlaylistPlayQueue(
                 currentInfo.getServiceId(),
                 currentInfo.getUrl(),
-                currentInfo.getNextPageUrl(),
+                currentInfo.getNextPage(),
                 infoItems,
                 index
         );

app/src/main/java/org/schabi/newpipe/fragments/list/search/SearchFragment.java

@@ -38,6 +38,7 @@ import org.schabi.newpipe.database.history.model.SearchHistoryEntry;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.search.SearchExtractor;
@@ -131,8 +132,7 @@ public class SearchFragment extends BaseListFragment<SearchInfo, ListExtractor.I
 
     private Map<Integer, String> menuItemToFilterName;
     private StreamingService service;
-    private String currentPageUrl;
+    private Page nextPage;
-    private String nextPageUrl;
     private String contentCountry;
     private boolean isSuggestionsEnabled = true;
 
@@ -368,15 +368,13 @@ public class SearchFragment extends BaseListFragment<SearchInfo, ListExtractor.I
     @Override
     public void writeTo(final Queue<Object> objectsToSave) {
         super.writeTo(objectsToSave);
-        objectsToSave.add(currentPageUrl);
+        objectsToSave.add(nextPage);
-        objectsToSave.add(nextPageUrl);
     }
 
     @Override
     public void readFrom(@NonNull final Queue<Object> savedObjects) throws Exception {
         super.readFrom(savedObjects);
-        currentPageUrl = (String) savedObjects.poll();
+        nextPage = (Page) savedObjects.poll();
-        nextPageUrl = (String) savedObjects.poll();
     }
 
     @Override
@@ -859,7 +857,7 @@ public class SearchFragment extends BaseListFragment<SearchInfo, ListExtractor.I
 
     @Override
     protected void loadMoreItems() {
-        if (nextPageUrl == null || nextPageUrl.isEmpty()) {
+        if (!Page.isValid(nextPage)) {
             return;
         }
         isLoading.set(true);
@@ -872,7 +870,7 @@ public class SearchFragment extends BaseListFragment<SearchInfo, ListExtractor.I
                 searchString,
                 asList(contentFilter),
                 sortFilter,
-                nextPageUrl)
+                nextPage)
                 .subscribeOn(Schedulers.io())
                 .observeOn(AndroidSchedulers.mainThread())
                 .doOnEvent((nextItemsResult, throwable) -> isLoading.set(false))
@@ -981,8 +979,7 @@ public class SearchFragment extends BaseListFragment<SearchInfo, ListExtractor.I
         handleSearchSuggestion();
 
         lastSearchedString = searchString;
-        nextPageUrl = result.getNextPageUrl();
+        nextPage = result.getNextPage();
-        currentPageUrl = result.getUrl();
 
         if (infoListAdapter.getItemsList().size() == 0) {
             if (!result.getRelatedItems().isEmpty()) {
@@ -1031,14 +1028,15 @@ public class SearchFragment extends BaseListFragment<SearchInfo, ListExtractor.I
     @Override
     public void handleNextItems(final ListExtractor.InfoItemsPage result) {
         showListFooter(false);
-        currentPageUrl = result.getNextPageUrl();
         infoListAdapter.addInfoItemList(result.getItems());
-        nextPageUrl = result.getNextPageUrl();
+        nextPage = result.getNextPage();
 
         if (!result.getErrors().isEmpty()) {
             showSnackBarError(result.getErrors(), UserAction.SEARCHED,
                     NewPipe.getNameOfService(serviceId),
-                    "\"" + searchString + "\" → page: " + nextPageUrl, 0);
+                    "\"" + searchString + "\" → pageUrl: " + nextPage.getUrl() + ", "
+                            + "pageIds: " + nextPage.getIds() + ", "
+                            + "pageCookies: " + nextPage.getCookies(), 0);
         }
         super.handleNextItems(result);
     }

app/src/main/java/org/schabi/newpipe/player/playqueue/AbstractInfoPlayQueue.java

@@ -5,6 +5,7 @@ import android.util.Log;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.ListInfo;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import java.util.ArrayList;
@@ -21,7 +22,7 @@ abstract class AbstractInfoPlayQueue<T extends ListInfo, U extends InfoItem> ext
 
     final int serviceId;
     final String baseUrl;
-    String nextUrl;
+    Page nextPage;
 
     private transient Disposable fetchReactor;
 
@@ -29,16 +30,16 @@ abstract class AbstractInfoPlayQueue<T extends ListInfo, U extends InfoItem> ext
         this(item.getServiceId(), item.getUrl(), null, Collections.emptyList(), 0);
     }
 
-    AbstractInfoPlayQueue(final int serviceId, final String url, final String nextPageUrl,
+    AbstractInfoPlayQueue(final int serviceId, final String url, final Page nextPage,
                           final List<StreamInfoItem> streams, final int index) {
         super(index, extractListItems(streams));
 
         this.baseUrl = url;
-        this.nextUrl = nextPageUrl;
+        this.nextPage = nextPage;
         this.serviceId = serviceId;
 
         this.isInitial = streams.isEmpty();
-        this.isComplete = !isInitial && (nextPageUrl == null || nextPageUrl.isEmpty());
+        this.isComplete = !isInitial && !Page.isValid(nextPage);
     }
 
     protected abstract String getTag();
@@ -66,7 +67,7 @@ abstract class AbstractInfoPlayQueue<T extends ListInfo, U extends InfoItem> ext
                 if (!result.hasNextPage()) {
                     isComplete = true;
                 }
-                nextUrl = result.getNextPageUrl();
+                nextPage = result.getNextPage();
 
                 append(extractListItems(result.getRelatedItems()));
 
@@ -100,7 +101,7 @@ abstract class AbstractInfoPlayQueue<T extends ListInfo, U extends InfoItem> ext
                 if (!result.hasNextPage()) {
                     isComplete = true;
                 }
-                nextUrl = result.getNextPageUrl();
+                nextPage = result.getNextPage();
 
                 append(extractListItems(result.getItems()));
 

app/src/main/java/org/schabi/newpipe/player/playqueue/ChannelPlayQueue.java

@@ -1,6 +1,7 @@
 package org.schabi.newpipe.player.playqueue;
 
 
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.channel.ChannelInfo;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@@ -17,15 +18,15 @@ public final class ChannelPlayQueue extends AbstractInfoPlayQueue<ChannelInfo, C
     }
 
     public ChannelPlayQueue(final ChannelInfo info) {
-        this(info.getServiceId(), info.getUrl(), info.getNextPageUrl(), info.getRelatedItems(), 0);
+        this(info.getServiceId(), info.getUrl(), info.getNextPage(), info.getRelatedItems(), 0);
     }
 
     public ChannelPlayQueue(final int serviceId,
                             final String url,
-                            final String nextPageUrl,
+                            final Page nextPage,
                             final List<StreamInfoItem> streams,
                             final int index) {
-        super(serviceId, url, nextPageUrl, streams, index);
+        super(serviceId, url, nextPage, streams, index);
     }
 
     @Override
@@ -41,7 +42,7 @@ public final class ChannelPlayQueue extends AbstractInfoPlayQueue<ChannelInfo, C
                     .observeOn(AndroidSchedulers.mainThread())
                     .subscribe(getHeadListObserver());
         } else {
-            ExtractorHelper.getMoreChannelItems(this.serviceId, this.baseUrl, this.nextUrl)
+            ExtractorHelper.getMoreChannelItems(this.serviceId, this.baseUrl, this.nextPage)
                     .subscribeOn(Schedulers.io())
                     .observeOn(AndroidSchedulers.mainThread())
                     .subscribe(getNextPageObserver());

app/src/main/java/org/schabi/newpipe/player/playqueue/PlaylistPlayQueue.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.player.playqueue;
 
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
@@ -16,15 +17,15 @@ public final class PlaylistPlayQueue extends AbstractInfoPlayQueue<PlaylistInfo,
     }
 
     public PlaylistPlayQueue(final PlaylistInfo info) {
-        this(info.getServiceId(), info.getUrl(), info.getNextPageUrl(), info.getRelatedItems(), 0);
+        this(info.getServiceId(), info.getUrl(), info.getNextPage(), info.getRelatedItems(), 0);
     }
 
     public PlaylistPlayQueue(final int serviceId,
                              final String url,
-                             final String nextPageUrl,
+                             final Page nextPage,
                              final List<StreamInfoItem> streams,
                              final int index) {
-        super(serviceId, url, nextPageUrl, streams, index);
+        super(serviceId, url, nextPage, streams, index);
     }
 
     @Override
@@ -40,7 +41,7 @@ public final class PlaylistPlayQueue extends AbstractInfoPlayQueue<PlaylistInfo,
                     .observeOn(AndroidSchedulers.mainThread())
                     .subscribe(getHeadListObserver());
         } else {
-            ExtractorHelper.getMorePlaylistItems(this.serviceId, this.baseUrl, this.nextUrl)
+            ExtractorHelper.getMorePlaylistItems(this.serviceId, this.baseUrl, this.nextPage)
                     .subscribeOn(Schedulers.io())
                     .observeOn(AndroidSchedulers.mainThread())
                     .subscribe(getNextPageObserver());

app/src/main/java/org/schabi/newpipe/util/ExtractorHelper.java

@@ -33,6 +33,7 @@ import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
 import org.schabi.newpipe.extractor.ListInfo;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelInfo;
 import org.schabi.newpipe.extractor.comments.CommentsInfo;
@@ -87,14 +88,13 @@ public final class ExtractorHelper {
                                                            final String searchString,
                                                            final List<String> contentFilter,
                                                            final String sortFilter,
-                                                           final String pageUrl) {
+                                                           final Page page) {
         checkServiceId(serviceId);
         return Single.fromCallable(() ->
                 SearchInfo.getMoreItems(NewPipe.getService(serviceId),
                         NewPipe.getService(serviceId)
                                 .getSearchQHFactory()
-                                .fromQuery(searchString, contentFilter, sortFilter),
+                                .fromQuery(searchString, contentFilter, sortFilter), page));
-                        pageUrl));
 
     }
 
@@ -125,10 +125,10 @@ public final class ExtractorHelper {
     }
 
     public static Single<InfoItemsPage> getMoreChannelItems(final int serviceId, final String url,
-                                                            final String nextStreamsUrl) {
+                                                            final Page nextPage) {
         checkServiceId(serviceId);
         return Single.fromCallable(() ->
-                ChannelInfo.getMoreItems(NewPipe.getService(serviceId), url, nextStreamsUrl));
+                ChannelInfo.getMoreItems(NewPipe.getService(serviceId), url, nextPage));
     }
 
     public static Single<ListInfo<StreamInfoItem>> getFeedInfoFallbackToChannelInfo(
@@ -157,10 +157,10 @@ public final class ExtractorHelper {
 
     public static Single<InfoItemsPage> getMoreCommentItems(final int serviceId,
                                                             final CommentsInfo info,
-                                                            final String nextPageUrl) {
+                                                            final Page nextPage) {
         checkServiceId(serviceId);
         return Single.fromCallable(() ->
-                CommentsInfo.getMoreItems(NewPipe.getService(serviceId), info, nextPageUrl));
+                CommentsInfo.getMoreItems(NewPipe.getService(serviceId), info, nextPage));
     }
 
     public static Single<PlaylistInfo> getPlaylistInfo(final int serviceId, final String url,
@@ -172,10 +172,10 @@ public final class ExtractorHelper {
     }
 
     public static Single<InfoItemsPage> getMorePlaylistItems(final int serviceId, final String url,
-                                                             final String nextStreamsUrl) {
+                                                             final Page nextPage) {
         checkServiceId(serviceId);
         return Single.fromCallable(() ->
-                PlaylistInfo.getMoreItems(NewPipe.getService(serviceId), url, nextStreamsUrl));
+                PlaylistInfo.getMoreItems(NewPipe.getService(serviceId), url, nextPage));
     }
 
     public static Single<KioskInfo> getKioskInfo(final int serviceId, final String url,
@@ -184,12 +184,10 @@ public final class ExtractorHelper {
                 Single.fromCallable(() -> KioskInfo.getInfo(NewPipe.getService(serviceId), url)));
     }
 
-    public static Single<InfoItemsPage> getMoreKioskItems(final int serviceId,
+    public static Single<InfoItemsPage> getMoreKioskItems(final int serviceId, final String url,
-                                                          final String url,
+                                                          final Page nextPage) {
-                                                          final String nextStreamsUrl) {
         return Single.fromCallable(() ->
-                KioskInfo.getMoreItems(NewPipe.getService(serviceId),
+                KioskInfo.getMoreItems(NewPipe.getService(serviceId), url, nextPage));
-                        url, nextStreamsUrl));
     }
 
     /*//////////////////////////////////////////////////////////////////////////
@@ -240,8 +238,8 @@ public final class ExtractorHelper {
      * @param infoType        the {@link InfoItem.InfoType} of the item
      * @return a {@link Single} that loads the item
      */
-    public static <I extends Info> Maybe<I> loadFromCache(final int serviceId, final String url,
+    private static <I extends Info> Maybe<I> loadFromCache(final int serviceId, final String url,
-                                                          final InfoItem.InfoType infoType) {
+                                                           final InfoItem.InfoType infoType) {
         checkServiceId(serviceId);
         return Maybe.defer(() -> {
             //noinspection unchecked