diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index f2bb0e715..c2716efb3 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -32,7 +32,7 @@
android:theme="@style/AppTheme">
+ android:value=".MainActivity" />
diff --git a/app/src/main/java/org/schabi/newpipe/ChannelActivity.java b/app/src/main/java/org/schabi/newpipe/ChannelActivity.java
index 80b75e61f..48afa06df 100644
--- a/app/src/main/java/org/schabi/newpipe/ChannelActivity.java
+++ b/app/src/main/java/org/schabi/newpipe/ChannelActivity.java
@@ -11,10 +11,8 @@ import android.support.v7.widget.LinearLayoutManager;
import android.support.v7.widget.RecyclerView;
import android.support.v7.widget.Toolbar;
import android.util.Log;
-import android.view.LayoutInflater;
import android.view.View;
import android.widget.ImageView;
-import android.widget.LinearLayout;
import android.widget.ProgressBar;
import android.widget.Toast;
@@ -27,12 +25,10 @@ import org.schabi.newpipe.extractor.ChannelInfo;
import org.schabi.newpipe.extractor.ExtractionException;
import org.schabi.newpipe.extractor.ParsingException;
import org.schabi.newpipe.extractor.ServiceList;
-import org.schabi.newpipe.extractor.StreamPreviewInfo;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.info_list.InfoListAdapter;
import java.io.IOException;
-import java.util.ArrayList;
public class ChannelActivity extends AppCompatActivity {
@@ -47,6 +43,7 @@ public class ChannelActivity extends AppCompatActivity {
private int serviceId = -1;
private String channelUrl = "";
private int pageNumber = 0;
+ private boolean hasNextPage = true;
private boolean isLoading = false;
private ImageLoader imageLoader = ImageLoader.getInstance();
@@ -91,22 +88,23 @@ public class ChannelActivity extends AppCompatActivity {
totalItemCount = layoutManager.getItemCount();
pastVisiblesItems = layoutManager.findFirstVisibleItemPosition();
- if ( (visibleItemCount + pastVisiblesItems) >= totalItemCount && !isLoading)
+ if ( (visibleItemCount + pastVisiblesItems) >= totalItemCount
+ && !isLoading
+ && hasNextPage)
{
pageNumber++;
- Log.d(TAG, "bottomn");
+ requestData(true);
}
}
}
});
- requestData(pageNumber);
+ requestData(false);
}
private void updateUi(final ChannelInfo info) {
- isLoading = false;
CollapsingToolbarLayout ctl = (CollapsingToolbarLayout) findViewById(R.id.channel_toolbar_layout);
ProgressBar progressBar = (ProgressBar) findViewById(R.id.progressBar);
ImageView channelBanner = (ImageView) findViewById(R.id.channel_banner_image);
@@ -144,11 +142,9 @@ public class ChannelActivity extends AppCompatActivity {
} else {
feedButton.setVisibility(View.GONE);
}
-
- initVideos(info);
}
- private void initVideos(final ChannelInfo info) {
+ private void addVideos(final ChannelInfo info) {
infoListAdapter.addStreamItemList(info.related_streams);
}
@@ -162,7 +158,7 @@ public class ChannelActivity extends AppCompatActivity {
});
}
- private void requestData(int page) {
+ private void requestData(final boolean onlyVideos) {
// start processing
isLoading = true;
Thread channelExtractorThread = new Thread(new Runnable() {
@@ -173,7 +169,7 @@ public class ChannelActivity extends AppCompatActivity {
try {
StreamingService service = ServiceList.getService(serviceId);
ChannelExtractor extractor = service.getChannelExtractorInstance(
- channelUrl, new Downloader());
+ channelUrl, pageNumber, new Downloader());
final ChannelInfo info = ChannelInfo.getInfo(extractor, new Downloader());
@@ -181,7 +177,12 @@ public class ChannelActivity extends AppCompatActivity {
h.post(new Runnable() {
@Override
public void run() {
- updateUi(info);
+ isLoading = false;
+ if(!onlyVideos) {
+ updateUi(info);
+ }
+ hasNextPage = info.hasNextPage;
+ addVideos(info);
}
});
diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java
index 833891762..04098a4d7 100644
--- a/app/src/main/java/org/schabi/newpipe/Downloader.java
+++ b/app/src/main/java/org/schabi/newpipe/Downloader.java
@@ -5,10 +5,12 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
-import info.guardianproject.netcipher.NetCipher;
/**
* Created by Christian Schabesberger on 28.01.16.
@@ -40,10 +42,26 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
public String download(String siteUrl, String language) throws IOException {
+ Map requestProperties = new HashMap<>();
+ requestProperties.put("Accept-Language", language);
+ return download(siteUrl, requestProperties);
+ }
+
+
+ /**Download the text file at the supplied URL as in download(String),
+ * but set the HTTP header field "Accept-Language" to the supplied string.
+ * @param siteUrl the URL of the text file to return the contents of
+ * @param customProperties set request header properties
+ * @return the contents of the specified text file
+ * @throws IOException*/
+ public String download(String siteUrl, Map customProperties) throws IOException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
- //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
- con.setRequestProperty("Accept-Language", language);
+ Iterator it = customProperties.entrySet().iterator();
+ while(it.hasNext()) {
+ Map.Entry pair = (Map.Entry)it.next();
+ con.setRequestProperty((String)pair.getKey(), (String)pair.getValue());
+ }
return dl(con);
}
diff --git a/app/src/main/java/org/schabi/newpipe/extractor/ChannelExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/ChannelExtractor.java
index bf6292bbf..d8c931d08 100644
--- a/app/src/main/java/org/schabi/newpipe/extractor/ChannelExtractor.java
+++ b/app/src/main/java/org/schabi/newpipe/extractor/ChannelExtractor.java
@@ -29,7 +29,7 @@ public abstract class ChannelExtractor {
private Downloader downloader;
private StreamPreviewInfoCollector previewInfoCollector;
- public ChannelExtractor(UrlIdHandler urlIdHandler, String url, Downloader dl, int serviceId)
+ public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int page, Downloader dl, int serviceId)
throws ExtractionException, IOException {
this.serviceId = serviceId;
this.urlIdHandler = urlIdHandler;
@@ -48,6 +48,7 @@ public abstract class ChannelExtractor {
public abstract String getBannerUrl() throws ParsingException;
public abstract String getFeedUrl() throws ParsingException;
public abstract StreamPreviewInfoCollector getStreams() throws ParsingException;
+ public abstract boolean hasNextPage() throws ParsingException;
public int getServiceId() {
return serviceId;
}
diff --git a/app/src/main/java/org/schabi/newpipe/extractor/ChannelInfo.java b/app/src/main/java/org/schabi/newpipe/extractor/ChannelInfo.java
index 0151d895a..2f856dac0 100644
--- a/app/src/main/java/org/schabi/newpipe/extractor/ChannelInfo.java
+++ b/app/src/main/java/org/schabi/newpipe/extractor/ChannelInfo.java
@@ -39,6 +39,7 @@ public class ChannelInfo {
// importand data
info.service_id = extractor.getServiceId();
info.channel_name = extractor.getChannelName();
+ info.hasNextPage = extractor.hasNextPage();
try {
info.avatar_url = extractor.getAvatarUrl();
@@ -72,6 +73,7 @@ public class ChannelInfo {
public String banner_url = "";
public String feed_url = "";
public List related_streams = null;
+ public boolean hasNextPage = false;
public List errors = new Vector<>();
}
diff --git a/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java b/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java
index a4cde400d..14475dba7 100644
--- a/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java
+++ b/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java
@@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor;
import java.io.IOException;
+import java.util.Map;
/**
* Created by Christian Schabesberger on 28.01.16.
@@ -32,6 +33,14 @@ public interface Downloader {
* @throws IOException*/
String download(String siteUrl, String language) throws IOException;
+ /**Download the text file at the supplied URL as in download(String),
+ * but set the HTTP header field "Accept-Language" to the supplied string.
+ * @param siteUrl the URL of the text file to return the contents of
+ * @param customProperties set request header properties
+ * @return the contents of the specified text file
+ * @throws IOException*/
+ String download(String siteUrl, Map customProperties) throws IOException;
+
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
diff --git a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
index d9acdefff..1805b48db 100644
--- a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
+++ b/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
@@ -40,7 +40,7 @@ public abstract class StreamingService {
public abstract SearchEngine getSearchEngineInstance(Downloader downloader);
public abstract UrlIdHandler getUrlIdHandlerInstance();
public abstract UrlIdHandler getChannelUrlIdHandlerInstance();
- public abstract ChannelExtractor getChannelExtractorInstance(String url, Downloader downloader)
+ public abstract ChannelExtractor getChannelExtractorInstance(String url, int page, Downloader downloader)
throws ExtractionException, IOException;
public final int getServiceId() {
diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
index e697f0d3b..b4f856a77 100644
--- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
+++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
@@ -1,20 +1,9 @@
package org.schabi.newpipe.extractor.services.youtube;
-import android.util.Log;
-/*
-import com.steadystate.css.dom.CSSStyleDeclarationImpl;
-import com.steadystate.css.dom.CSSStyleSheetImpl;
-import com.steadystate.css.parser.CSSOMParser;
-import com.steadystate.css.parser.SACParserCSS3;
-import org.w3c.css.sac.CSSParseException;
-import org.w3c.css.sac.InputSource;
-import org.w3c.dom.css.CSSRule;
-import org.w3c.dom.css.CSSRuleList;
-import org.w3c.dom.css.CSSStyleSheet;
-import java.io.StringReader;
-*/
+import org.json.JSONException;
+import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@@ -30,6 +19,8 @@ import org.schabi.newpipe.extractor.UrlIdHandler;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
/**
* Created by Christian Schabesberger on 25.07.16.
@@ -58,42 +49,79 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
// private CSSOMParser cssParser = new CSSOMParser(new SACParserCSS3());
private Downloader downloader;
- private final Document doc;
- private final String channelUrl;
- private String vUrl ="";
+ private Document doc = null;
+ private boolean isAjaxPage = false;
+ private static String userUrl = "";
+ private static String channelName = "";
+ private static String avatarUrl = "";
+ private static String bannerUrl = "";
+ private static String feedUrl = "";
+ // the fist page is html all other pages are ajax. Every new page can be requested by sending
+ // this request url.
+ private static String nextPageUrl = "";
- public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, Downloader dl, int serviceId)
+ public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int page, Downloader dl, int serviceId)
throws ExtractionException, IOException {
- super(urlIdHandler, url, dl, serviceId);
+ super(urlIdHandler, url, page, dl, serviceId);
- channelUrl = urlIdHandler.cleanUrl(url) ; //+ "/video?veiw=0&flow=list&sort=dd";
+ url = urlIdHandler.cleanUrl(url) ; //+ "/video?veiw=0&flow=list&sort=dd";
downloader = dl;
- // we first need to get the user url. Otherwise we can't find videos
- String channelPageContent = downloader.download(channelUrl);
- Document channelDoc = Jsoup.parse(channelPageContent, channelUrl);
- String userUrl = getUserUrl(channelDoc);
- vUrl = userUrl + "/videos?veiw=0&flow=list&sort=dd";
- String pageContent = downloader.download(vUrl);
- doc = Jsoup.parse(pageContent, vUrl);
+ if(page == 0) {
+ if (isUserUrl(url)) {
+ userUrl = url;
+ } else {
+ // we first need to get the user url. Otherwise we can't find videos
+ String channelPageContent = downloader.download(url);
+ Document channelDoc = Jsoup.parse(channelPageContent, url);
+ userUrl = getUserUrl(channelDoc);
+ }
+
+ userUrl = userUrl + "/videos?veiw=0&flow=list&sort=dd&live_view=500";
+ String pageContent = downloader.download(userUrl);
+ doc = Jsoup.parse(pageContent, userUrl);
+ nextPageUrl = getNextPageUrl(doc);
+ isAjaxPage = false;
+ } else {
+ Map userProperties = new HashMap<>();
+ userProperties.put("Referer", userUrl);
+ String ajaxDataRaw = downloader.download(nextPageUrl, userProperties);
+ JSONObject ajaxData;
+ String htmlDataRaw;
+ try {
+ ajaxData = new JSONObject(ajaxDataRaw);
+ htmlDataRaw = ajaxData.getString("content_html");
+ } catch (JSONException e) {
+ throw new ParsingException("Could not parse json data for next page", e);
+ }
+ doc = Jsoup.parse(htmlDataRaw, nextPageUrl);
+ nextPageUrl = getNextPageUrl(ajaxData);
+ isAjaxPage = true;
+ }
}
@Override
public String getChannelName() throws ParsingException {
- try {
- return doc.select("span[class=\"qualified-channel-title-text\"]").first()
- .select("a").first().text();
- } catch(Exception e) {
- throw new ParsingException("Could not get channel name");
- }
+ try {
+ if(!isAjaxPage) {
+ channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first()
+ .select("a").first().text();
+ }
+ return channelName;
+ } catch(Exception e) {
+ throw new ParsingException("Could not get channel name");
+ }
}
@Override
public String getAvatarUrl() throws ParsingException {
try {
- return doc.select("img[class=\"channel-header-profile-image\"]")
- .first().attr("abs:src");
+ if(!isAjaxPage) {
+ avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]")
+ .first().attr("abs:src");
+ }
+ return avatarUrl;
} catch(Exception e) {
throw new ParsingException("Could not get avatar", e);
}
@@ -101,16 +129,18 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getBannerUrl() throws ParsingException {
-
try {
- Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
- String cssContent = el.html();
- String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
- if(url.contains("s.ytimg.com")) {
- return null;
- } else {
- return url;
+ if(!isAjaxPage) {
+ Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
+ String cssContent = el.html();
+ String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
+ if (url.contains("s.ytimg.com")) {
+ bannerUrl = null;
+ } else {
+ bannerUrl = url;
+ }
}
+ return bannerUrl;
} catch(Exception e) {
throw new ParsingException("Could not get Banner", e);
}
@@ -119,7 +149,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public StreamPreviewInfoCollector getStreams() throws ParsingException {
StreamPreviewInfoCollector collector = getStreamPreviewInfoCollector();
- Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
+ Element ul = null;
+ if(isAjaxPage) {
+ ul = doc.select("body").first();
+ } else {
+ ul = doc.select("ul[id=\"browse-items-primary\"]").first();
+ }
for(final Element li : ul.children()) {
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
@@ -235,6 +270,19 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ParsingException("Could not get thumbnail url", e);
}
}
+
+ private boolean isLiveStream(Element item) {
+ Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
+
+ if(bla == null) {
+ // sometimes livestreams dont have badges but sill are live streams
+ // if video time is not available we most likly have an offline livestream
+ if(item.select("span[class*=\"video-time\"]").first() == null) {
+ return true;
+ }
+ }
+ return bla != null;
+ }
});
}
}
@@ -245,27 +293,49 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Override
public String getFeedUrl() throws ParsingException {
try {
- return doc.select("link[title=\"RSS\"]").first().attr("abs:href");
+ if(!isAjaxPage) {
+ feedUrl = doc.select("link[title=\"RSS\"]").first().attr("abs:href");
+ }
+ return feedUrl;
} catch(Exception e) {
throw new ParsingException("Could not get feed url", e);
}
}
+ @Override
+ public boolean hasNextPage() throws ParsingException {
+ return !nextPageUrl.isEmpty();
+ }
+
private String getUserUrl(Document d) throws ParsingException {
return d.select("span[class=\"qualified-channel-title-text\"]").first()
.select("a").first().attr("abs:href");
}
- private boolean isLiveStream(Element item) {
- Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
+ private boolean isUserUrl(String url) throws ParsingException {
+ return url.contains("/user/");
+ }
- if(bla == null) {
- // sometimes livestreams dont have badges but sill are live streams
- // if video time is not available we most likly have an offline livestream
- if(item.select("span[class*=\"video-time\"]").first() == null) {
- return true;
- }
+ private String getNextPageUrl(Document d) throws ParsingException {
+ try {
+ Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
+ return button.attr("abs:data-uix-load-more-href");
+ } catch(Exception e) {
+ throw new ParsingException("could not load next page url", e);
}
- return bla != null;
+ }
+
+ private String getNextPageUrl(JSONObject ajaxData) throws ParsingException {
+ Document doc = null;
+ try {
+ String docRaw = ajaxData.getString("load_more_widget_html");
+ if(docRaw.isEmpty()) {
+ return "";
+ }
+ doc = Jsoup.parse(docRaw);
+ } catch(JSONException je) {
+ throw new ParsingException("Could not get load_more_widget from ajax response", je);
+ }
+ return getNextPageUrl(doc);
}
}
diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
index 812b74771..9c2198926 100644
--- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
+++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
@@ -70,8 +70,8 @@ public class YoutubeService extends StreamingService {
}
@Override
- public ChannelExtractor getChannelExtractorInstance(String url, Downloader downloader)
+ public ChannelExtractor getChannelExtractorInstance(String url, int page, Downloader downloader)
throws ExtractionException, IOException {
- return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, downloader, getServiceId());
+ return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, page, downloader, getServiceId());
}
}
diff --git a/app/src/main/java/org/schabi/newpipe/info_list/InfoListAdapter.java b/app/src/main/java/org/schabi/newpipe/info_list/InfoListAdapter.java
index fbb220b8d..365ed246e 100644
--- a/app/src/main/java/org/schabi/newpipe/info_list/InfoListAdapter.java
+++ b/app/src/main/java/org/schabi/newpipe/info_list/InfoListAdapter.java
@@ -46,8 +46,10 @@ public class InfoListAdapter extends RecyclerView.Adapter {
}
public void addStreamItemList(List videos) {
- streamList.addAll(videos);
- notifyDataSetChanged();
+ if(videos!= null) {
+ streamList.addAll(videos);
+ notifyDataSetChanged();
+ }
}
public void clearSteamItemList() {
diff --git a/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java b/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java
index 52659f29f..99d2a57dd 100644
--- a/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java
+++ b/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java
@@ -237,7 +237,7 @@ public class SearchInfoItemFragment extends Fragment {
searchView.setSuggestionsAdapter(suggestionListAdapter);
searchView.setOnSuggestionListener(new SearchSuggestionListener(searchView, suggestionListAdapter));
searchView.setOnQueryTextListener(new SearchQueryListener());
- if(!searchQuery.isEmpty()) {
+ if(searchQuery != null && !searchQuery.isEmpty()) {
searchView.setQuery(searchQuery, false);
searchView.setIconifiedByDefault(false);
}