Peertube
/
NewPipe-app-android
mirror of https://github.com/TeamNewPipe/NewPipe/
1
0
Fork 0
Code Issues Releases Wiki Activity

got rid of getVideoInfo() in youtube crawler

This commit is contained in:
Christian Schabesberger 2016-02-02 14:06:09 +01:00
parent fb942912db
commit bad576c23d
4 changed files with 80 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java
View File

@ -115,6 +115,7 @@ public class VideoItemDetailFragment extends Fragment {
VideoInfo videoInfo = videoExtractor.getVideoInfo();
h.post(new VideoResultReturnedRunnable(videoInfo));
h.post(new SetThumbnailRunnable(
//todo: make bitmaps not bypass tor
BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url)
.openConnection()

1
app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java
View File

@ -157,6 +157,7 @@ public class VideoItemListFragment extends ListFragment {
if(!downloadedList.get(i)) {
Bitmap thumbnail;
try {
//todo: make bitmaps not bypass tor
thumbnail = BitmapFactory.decodeStream(
new URL(thumbnailUrlList.get(i)).openConnection().getInputStream());
h.post(new SetThumbnailRunnable(i, thumbnail, requestId));

27
app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java
View File

@ -20,6 +20,9 @@ package org.schabi.newpipe.crawler;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import java.util.List;
import java.util.Vector;
/**Scrapes information from a video streaming service (eg, YouTube).*/
@ -134,6 +137,25 @@ public abstract class VideoExtractor {
videoInfo.dashMpdUrl = getDashMpdUrl();
}
if(videoInfo.average_rating.isEmpty()) {
videoInfo.average_rating = getAverageRating();
}
if(videoInfo.like_count == -1) {
videoInfo.like_count = getLikeCount();
}
if(videoInfo.dislike_count == -1) {
videoInfo.dislike_count = getDislikeCount();
}
if(videoInfo.nextVideo == null) {
videoInfo.nextVideo = getNextVideo();
}
if(videoInfo.relatedVideos == null) {
videoInfo.relatedVideos = getRelatedVideos();
}
//Bitmap thumbnail = null;
//Bitmap uploader_thumbnail = null;
@ -158,4 +180,9 @@ public abstract class VideoExtractor {
public abstract VideoInfo.VideoStream[] getVideoStreams() throws ParsingException;
public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException;
public abstract String getAverageRating() throws ParsingException;
public abstract int getLikeCount() throws ParsingException;
public abstract int getDislikeCount() throws ParsingException;
public abstract VideoPreviewInfo getNextVideo() throws ParsingException;
public abstract Vector<VideoPreviewInfo> getRelatedVideos() throws ParsingException;
}

109
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java
View File

@ -25,6 +25,7 @@ import java.io.IOException;
import java.io.StringReader;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import java.util.regex.Matcher;
@ -389,77 +390,69 @@ public class YoutubeVideoExtractor extends VideoExtractor {
return 0;
}
@Override
public String getAverageRating() throws ParsingException {
try {
return playerArgs.getString("avg_rating");
} catch (JSONException e) {
throw new ParsingException("Could not get Average rating", e);
}
}
@Override
public VideoInfo getVideoInfo() throws CrawlingException {
videoInfo = super.getVideoInfo();
//todo: replace this with a call to getVideoId, if possible
//videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl);
videoInfo.id = getVideoId(pageUrl);
if (videoInfo.audioStreams == null
|| videoInfo.audioStreams.length == 0) {
Log.e(TAG, "uninitialised audio streams!");
}
if (videoInfo.videoStreams == null
|| videoInfo.videoStreams.length == 0) {
Log.e(TAG, "uninitialised video streams!");
}
videoInfo.age_limit = 0;
//average rating
try {
videoInfo.average_rating = playerArgs.getString("avg_rating");
} catch (JSONException e) {
e.printStackTrace();
}
//---------------------------------------
// extracting information from html page
//---------------------------------------
public int getLikeCount() throws ParsingException {
String likesString = "";
String dislikesString = "";
try {
// likes
likesString = doc.select("button.like-button-renderer-like-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", ""));
// dislikes
return Integer.parseInt(likesString.replaceAll("[^\\d]", ""));
} catch (NumberFormatException nfe) {
throw new ParsingException(
"failed to parse likesString \"" + likesString + "\" as integers", nfe);
} catch (Exception e) {
throw new ParsingException("Could not get like count", e);
}
}
@Override
public int getDislikeCount() throws ParsingException {
String dislikesString = "";
try {
dislikesString = doc.select("button.like-button-renderer-dislike-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", ""));
} catch (NumberFormatException nfe) {
Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" +
dislikesString + "\" as integers");
} catch (Exception e) {
// if it fails we know that the video does not offer dislikes.
e.printStackTrace();
videoInfo.like_count = 0;
videoInfo.dislike_count = 0;
return Integer.parseInt(dislikesString.replaceAll("[^\\d]", ""));
} catch(NumberFormatException nfe) {
throw new ParsingException(
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
} catch(Exception e) {
throw new ParsingException("Could not get dislike count", e);
}
}
// next video
videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
@Override
public VideoPreviewInfo getNextVideo() throws ParsingException {
try {
return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
} catch(Exception e) {
throw new ParsingException("Could not get next video", e);
}
}
// related videos
Vector<VideoPreviewInfo> relatedVideos = new Vector<>();
for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
relatedVideos.add(extractVideoPreviewInfo(li));
@Override
public Vector<VideoPreviewInfo> getRelatedVideos() throws ParsingException {
try {
Vector<VideoPreviewInfo> relatedVideos = new Vector<>();
for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
relatedVideos.add(extractVideoPreviewInfo(li));
}
}
return relatedVideos;
} catch(Exception e) {
throw new ParsingException("Could not get related videos", e);
}
//todo: replace conversion
videoInfo.relatedVideos = relatedVideos;
//videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]);
return videoInfo;
}
private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) throws RegexException, DecryptException {