Peertube
/
NewPipe-app-android
mirror of https://github.com/TeamNewPipe/NewPipe/
1
0
Fork 0
Code Issues Releases Wiki Activity

use java error system in the crawler

This commit is contained in:
Christian Schabesberger 2016-01-31 19:57:30 +01:00
parent 46c2db310a
commit 7f12b58722
17 changed files with 877 additions and 673 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java
View File

@ -3,7 +3,9 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.services.youtube.YoutubeSearchEngine;
import org.schabi.newpipe.Downloader;
import java.util.ArrayList;
@ -35,8 +37,9 @@ public class YoutubeSearchEngineTest extends AndroidTestCase {
public void setUp() throws Exception{
super.setUp();
SearchEngine engine = new YoutubeSearchEngine();
result = engine.search("https://www.youtube.com/results?search_query=bla", 0, "de");
suggestionReply = engine.suggestionList("hello");
result = engine.search("https://www.youtube.com/results?search_query=bla",
0, "de", new Downloader());
suggestionReply = engine.suggestionList("hello", new Downloader());
}
public void testIfNoErrorOccur() {

53
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java
View File

@ -1,9 +1,14 @@
package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import android.util.Log;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import java.io.IOException;
/**
* Created by the-scrabi on 30.12.15.
@ -28,58 +33,58 @@ import org.schabi.newpipe.services.VideoInfo;
public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
private YoutubeVideoExtractor extractor;
public void setUp() {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys");
public void setUp() throws IOException, CrawlingException {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys",
new Downloader());
}
public void testGetErrorCode() {
assertEquals(extractor.getErrorCode(), VideoInfo.NO_ERROR);
}
public void testGetErrorMessage() {
assertEquals(extractor.getErrorMessage(), "");
}
public void testGetTimeStamp() {
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() >= 0);
extractor.getTimeStamp() <= 0);
}
public void testGetTitle() {
public void testGetValidTimeStamp() throws CrawlingException, IOException {
YoutubeVideoExtractor extractor =
new YoutubeVideoExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
public void testGetTitle() throws ParsingException {
assertTrue(!extractor.getTitle().isEmpty());
}
public void testGetDescription() {
public void testGetDescription() throws ParsingException {
assertTrue(extractor.getDescription() != null);
}
public void testGetUploader() {
public void testGetUploader() throws ParsingException {
assertTrue(!extractor.getUploader().isEmpty());
}
public void testGetLength() {
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetViews() {
public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetUploadDate() {
public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0);
}
public void testGetThumbnailUrl() {
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains("https://"));
}
public void testGetUploaderThumbnailUrl() {
public void testGetUploaderThumbnailUrl() throws ParsingException {
assertTrue(extractor.getUploaderThumbnailUrl(),
extractor.getUploaderThumbnailUrl().contains("https://"));
}
public void testGetAudioStreams() {
public void testGetAudioStreams() throws ParsingException {
for(VideoInfo.AudioStream s : extractor.getAudioStreams()) {
assertTrue(s.url,
s.url.contains("https://"));
@ -88,7 +93,7 @@ public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
}
}
public void testGetVideoStreams() {
public void testGetVideoStreams() throws ParsingException {
for(VideoInfo.VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains("https://"));

40
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java
View File

@ -2,7 +2,13 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.Downloader;
import java.io.IOException;
/**
* Created by the-scrabi on 30.12.15.
@ -29,31 +35,17 @@ import org.schabi.newpipe.services.VideoInfo;
public class YoutubeVideoExtractorGemaTest extends AndroidTestCase {
// Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail.
private static final boolean testActive = false;
private static final boolean testActive = true;
private YoutubeVideoExtractor extractor;
public void setUp() {
public void testGemaError() throws IOException, CrawlingException {
if(testActive) {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8");
}
}
public void testGetErrorCode() {
if(testActive) {
assertEquals(extractor.getErrorCode(), VideoInfo.ERROR_BLOCKED_BY_GEMA);
} else {
assertTrue(true);
}
}
public void testGetErrorMessage() {
if(testActive) {
assertTrue(extractor.getErrorMessage(),
extractor.getErrorMessage().contains("GEMA"));
} else {
assertTrue(true);
try {
new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8",
new Downloader());
assertTrue("Gema exception not thrown", false);
} catch(YoutubeVideoExtractor.GemaException ge) {
assertTrue(true);
}
}
}
}

54
app/src/main/java/org/schabi/newpipe/Downloader.java
View File

@ -39,42 +39,39 @@ public class Downloader implements org.schabi.newpipe.crawler.Downloader {
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
public String download(String siteUrl, String language) {
String ret = "";
try {
URL url = new URL(siteUrl);
//HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
con.setRequestProperty("Accept-Language", language);
ret = dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
public String download(String siteUrl, String language) throws IOException {
URL url = new URL(siteUrl);
//HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
con.setRequestProperty("Accept-Language", language);
return dl(con);
}
/**Common functionality between download(String url) and download(String url, String language)*/
private static String dl(HttpsURLConnection con) throws IOException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
BufferedReader in = new BufferedReader(
in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
while((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
}
catch(UnknownHostException uhe) {//thrown when there's no internet connection
uhe.printStackTrace();
} catch(UnknownHostException uhe) {//thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) {
throw new IOException(e);
} finally {
if(in != null) {
in.close();
}
}
return response.toString();
@ -84,19 +81,10 @@ public class Downloader implements org.schabi.newpipe.crawler.Downloader {
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
public String download(String siteUrl) {
String ret = "";
try {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
ret = dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
public String download(String siteUrl) throws IOException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
return dl(con);
}
}

258
app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java
View File

@ -31,15 +31,20 @@ import android.widget.TextView;
import android.view.MenuItem;
import android.widget.Toast;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.MalformedInputException;
import java.util.ArrayList;
import java.util.Vector;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.ServiceList;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
/**
@ -68,7 +73,6 @@ public class VideoItemDetailFragment extends Fragment {
* The fragment argument representing the item ID that this fragment
* represents.
*/
//public static final String ARG_ITEM_ID = "item_id";
public static final String VIDEO_URL = "video_url";
public static final String STREAMING_SERVICE = "streaming_service";
public static final String AUTO_PLAY = "auto_play";
@ -87,7 +91,6 @@ public class VideoItemDetailFragment extends Fragment {
private FloatingActionButton playVideoButton;
private final Point initialThumbnailPos = new Point(0, 0);
public interface OnInvokeCreateOptionsMenuListener {
void createOptionsMenu();
}
@ -108,45 +111,64 @@ public class VideoItemDetailFragment extends Fragment {
@Override
public void run() {
try {
this.videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
VideoInfo videoInfo = videoExtractor.getVideoInfo();
h.post(new VideoResultReturnedRunnable(videoInfo));
if (videoInfo.errorCode == VideoInfo.NO_ERROR) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.VIDEO_THUMBNAIL));
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.uploader_thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.CHANNEL_THUMBNAIL));
if (showNextVideoItem) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url)
new URL(videoInfo.nextVideo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.VIDEO_THUMBNAIL));
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.uploader_thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.CHANNEL_THUMBNAIL));
if(showNextVideoItem) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.nextVideo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL));
}
SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL));
}
} catch (Exception e) {
} catch (MalformedInputException e) {
postNewErrorToast(h, R.string.could_not_load_thumbnails);
e.printStackTrace();
} catch (IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
}
// custom service related exceptions
catch (YoutubeVideoExtractor.DecryptException de) {
postNewErrorToast(h, R.string.youtube_signature_decryption_error);
de.printStackTrace();
} catch (YoutubeVideoExtractor.GemaException ge) {
h.post(new Runnable() {
@Override
public void run() {
progressBar.setVisibility(View.GONE);
// This is poor style, but unless we have better error handling in the
// crawler, this may not be better.
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.network_error, Toast.LENGTH_LONG).show();
onErrorBlockedByGema();
}
});
}
// ----------------------------------------
catch(VideoExtractor.ContentNotAvailableException e) {
h.post(new Runnable() {
@Override
public void run() {
onNotSpecifiedContentError();
}
});
e.printStackTrace();
} catch (ParsingException e) {
postNewErrorToast(h, e.getMessage());
e.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace();
}
}
}
@ -213,7 +235,7 @@ public class VideoItemDetailFragment extends Fragment {
private void updateInfo(VideoInfo info) {
currentVideoInfo = info;
Resources res = activity.getResources();
try {
VideoInfoItemViewCreator videoItemViewCreator =
new VideoInfoItemViewCreator(LayoutInflater.from(getActivity()));
@ -226,107 +248,80 @@ public class VideoItemDetailFragment extends Fragment {
TextView thumbsDownView = (TextView) activity.findViewById(R.id.detailThumbsDownCountView);
TextView uploadDateView = (TextView) activity.findViewById(R.id.detailUploadDateView);
TextView descriptionView = (TextView) activity.findViewById(R.id.detailDescriptionView);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
FrameLayout nextVideoFrame = (FrameLayout) activity.findViewById(R.id.detailNextVideoFrame);
RelativeLayout nextVideoRootFrame =
(RelativeLayout) activity.findViewById(R.id.detailNextVideoRootLayout);
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
progressBar.setVisibility(View.GONE);
switch (info.errorCode) {
case VideoInfo.NO_ERROR: {
View nextVideoView = videoItemViewCreator
.getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext());
nextVideoFrame.addView(nextVideoView);
View nextVideoView = videoItemViewCreator
.getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext());
nextVideoFrame.addView(nextVideoView);
Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton);
Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton);
Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton);
Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton);
textContentLayout.setVisibility(View.VISIBLE);
playVideoButton.setVisibility(View.VISIBLE);
if (!showNextVideoItem) {
nextVideoRootFrame.setVisibility(View.GONE);
similarVideosButton.setVisibility(View.GONE);
}
textContentLayout.setVisibility(View.VISIBLE);
playVideoButton.setVisibility(View.VISIBLE);
if (!showNextVideoItem) {
nextVideoRootFrame.setVisibility(View.GONE);
similarVideosButton.setVisibility(View.GONE);
}
videoTitleView.setText(info.title);
uploaderView.setText(info.uploader);
actionBarHandler.setChannelName(info.uploader);
videoTitleView.setText(info.title);
uploaderView.setText(info.uploader);
actionBarHandler.setChannelName(info.uploader);
String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext());
viewCountView.setText(localizedViewCount);
String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext());
viewCountView.setText(localizedViewCount);
String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext());
thumbsUpView.setText(localizedLikeCount);
String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext());
thumbsUpView.setText(localizedLikeCount);
String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext());
thumbsDownView.setText(localizedDislikeCount);
String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext());
thumbsDownView.setText(localizedDislikeCount);
String localizedDate = Localization.localizeDate(info.upload_date, getContext());
uploadDateView.setText(localizedDate);
String localizedDate = Localization.localizeDate(info.upload_date, getContext());
uploadDateView.setText(localizedDate);
descriptionView.setText(Html.fromHtml(info.description));
descriptionView.setText(Html.fromHtml(info.description));
descriptionView.setMovementMethod(LinkMovementMethod.getInstance());
descriptionView.setMovementMethod(LinkMovementMethod.getInstance());
actionBarHandler.setServiceId(streamingServiceId);
actionBarHandler.setVideoInfo(info.webpage_url, info.title);
actionBarHandler.setStartPosition(info.startPosition);
actionBarHandler.setServiceId(streamingServiceId);
actionBarHandler.setVideoInfo(info.webpage_url, info.title);
actionBarHandler.setStartPosition(info.startPosition);
// parse streams
Vector<VideoInfo.VideoStream> streamsToUse = new Vector<>();
for (VideoInfo.VideoStream i : info.videoStreams) {
if (useStream(i, streamsToUse)) {
streamsToUse.add(i);
}
}
VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()];
for (int i = 0; i < streamList.length; i++) {
streamList[i] = streamsToUse.get(i);
}
actionBarHandler.setStreams(streamList, info.audioStreams);
// parse streams
Vector<VideoInfo.VideoStream> streamsToUse = new Vector<>();
for (VideoInfo.VideoStream i : info.videoStreams) {
if (useStream(i, streamsToUse)) {
streamsToUse.add(i);
}
}
VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()];
for (int i = 0; i < streamList.length; i++) {
streamList[i] = streamsToUse.get(i);
}
actionBarHandler.setStreams(streamList, info.audioStreams);
nextVideoButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent detailIntent =
new Intent(getActivity(), VideoItemDetailActivity.class);
nextVideoButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent detailIntent =
new Intent(getActivity(), VideoItemDetailActivity.class);
/*detailIntent.putExtra(
VideoItemDetailFragment.ARG_ITEM_ID, currentVideoInfo.nextVideo.id); */
detailIntent.putExtra(
VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url);
detailIntent.putExtra(
VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url);
detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId);
startActivity(detailIntent);
}
});
detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId);
startActivity(detailIntent);
}
break;
case VideoInfo.ERROR_BLOCKED_BY_GEMA:
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.gruese_die_gema));
backgroundButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent intent = new Intent();
intent.setAction(Intent.ACTION_VIEW);
intent.setData(Uri.parse(activity.getString(R.string.c3s_url)));
activity.startActivity(intent);
}
});
break;
case VideoInfo.ERROR_NO_SPECIFIED_ERROR:
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, info.errorMessage, Toast.LENGTH_LONG)
.show();
break;
default:
Log.e(TAG, "Video Available Status not known.");
}
});
if(autoPlayEnabled) {
actionBarHandler.playVideo();
@ -337,6 +332,37 @@ public class VideoItemDetailFragment extends Fragment {
}
}
private void onErrorBlockedByGema() {
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.gruese_die_gema));
backgroundButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent intent = new Intent();
intent.setAction(Intent.ACTION_VIEW);
intent.setData(Uri.parse(activity.getString(R.string.c3s_url)));
activity.startActivity(intent);
}
});
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.blocked_by_gema, Toast.LENGTH_LONG).show();
}
private void onNotSpecifiedContentError() {
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, R.string.content_not_available, Toast.LENGTH_LONG)
.show();
}
private boolean useStream(VideoInfo.VideoStream stream, Vector<VideoInfo.VideoStream> streams) {
for(VideoInfo.VideoStream i : streams) {
if(i.resolution.equals(stream.resolution)) {
@ -465,4 +491,24 @@ public class VideoItemDetailFragment extends Fragment {
public void setOnInvokeCreateOptionsMenuListener(OnInvokeCreateOptionsMenuListener listener) {
this.onInvokeCreateOptionsMenuListener = listener;
}
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
stringResource, Toast.LENGTH_LONG).show();
}
});
}
private void postNewErrorToast(Handler h, final String message) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
message, Toast.LENGTH_LONG).show();
}
});
}
}

30
app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java
View File

@ -15,10 +15,12 @@ import android.widget.AbsListView;
import android.widget.ListView;
import android.widget.Toast;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import java.util.Vector;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.StreamingService;
@ -116,17 +118,15 @@ public class VideoItemListFragment extends ListFragment {
if(runs) {
h.post(new ResultRunnable(result, requestId));
}
} catch(Exception e) {
} catch(IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
} catch(CrawlingException ce) {
postNewErrorToast(h, R.string.parsing_error);
ce.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace();
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
}
}
}
@ -386,4 +386,14 @@ public class VideoItemListFragment extends ListFragment {
mActivatedPosition = position;
}
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
}
}

37
app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java Normal file
View File

@ -0,0 +1,37 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 30.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* CrawlingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class CrawlingException extends Exception {
public CrawlingException() {}
public CrawlingException(String message) {
super(message);
}
public CrawlingException(Throwable cause) {
super(cause);
}
public CrawlingException(String message, Throwable cause) {
super(message, cause);
}
}

12
app/src/main/java/org/schabi/newpipe/crawler/Downloader.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.crawler;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 28.01.16.
*
@ -26,12 +28,14 @@ public interface Downloader {
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
String download(String siteUrl, String language);
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, String language) throws IOException;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
String download(String siteUrl);
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl) throws IOException;
}

35
app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java Normal file
View File

@ -0,0 +1,35 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 31.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ParsingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ParsingException extends CrawlingException {
public ParsingException() {}
public ParsingException(String message) {
super(message);
}
public ParsingException(Throwable cause) {
super(cause);
}
public ParsingException(String message, Throwable cause) {
super(message, cause);
}
}

7
app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.crawler;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Vector;
@ -31,8 +32,10 @@ public interface SearchEngine {
public final Vector<VideoPreviewInfo> resultList = new Vector<>();
}
ArrayList<String> suggestionList(String query, Downloader dl);
ArrayList<String> suggestionList(String query, Downloader dl)
throws CrawlingException, IOException;
//Result search(String query, int page);
Result search(String query, int page, String contentCountry, Downloader dl);
Result search(String query, int page, String contentCountry, Downloader dl)
throws CrawlingException, IOException;
}

4
app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.crawler;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 23.08.15.
*
@ -25,7 +27,7 @@ public interface StreamingService {
public String name = "";
}
ServiceInfo getServiceInfo();
VideoExtractor getExtractorInstance(String url, Downloader downloader);
VideoExtractor getExtractorInstance(String url, Downloader downloader) throws IOException, CrawlingException;
SearchEngine getSearchEngineInstance();
/**When a VIEW_ACTION is caught this function will test if the url delivered within the calling

178
app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java
View File

@ -22,8 +22,43 @@ package org.schabi.newpipe.crawler;
/**Scrapes information from a video streaming service (eg, YouTube).*/
@SuppressWarnings("ALL")
public abstract class VideoExtractor {
public class ExctractorInitException extends CrawlingException {
public ExctractorInitException() {}
public ExctractorInitException(String message) {
super(message);
}
public ExctractorInitException(Throwable cause) {
super(cause);
}
public ExctractorInitException(String message, Throwable cause) {
super(message, cause);
}
}
public class RegexException extends ParsingException {
public RegexException() {}
public RegexException(String message) {
super(message);
}
}
public class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException() {}
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(Throwable cause) {
super(cause);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}
protected final String pageUrl;
protected VideoInfo videoInfo;
@ -34,7 +69,7 @@ public abstract class VideoExtractor {
/**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/
public VideoInfo getVideoInfo()
public VideoInfo getVideoInfo() throws CrawlingException
{
if(videoInfo == null) {
videoInfo = new VideoInfo();
@ -44,90 +79,83 @@ public abstract class VideoExtractor {
videoInfo.webpage_url = pageUrl;
}
if(getErrorCode() == VideoInfo.NO_ERROR) {
if (videoInfo.title.isEmpty()) {
videoInfo.title = getTitle();
}
if (videoInfo.duration < 1) {
videoInfo.duration = getLength();
}
if (videoInfo.uploader.isEmpty()) {
videoInfo.uploader = getUploader();
}
if (videoInfo.description.isEmpty()) {
videoInfo.description = getDescription();
}
if (videoInfo.view_count == -1) {
videoInfo.view_count = getViews();
}
if (videoInfo.upload_date.isEmpty()) {
videoInfo.upload_date = getUploadDate();
}
if (videoInfo.thumbnail_url.isEmpty()) {
videoInfo.thumbnail_url = getThumbnailUrl();
}
if (videoInfo.id.isEmpty()) {
videoInfo.id = getVideoId(pageUrl);
}
/** Load and extract audio*/
if (videoInfo.audioStreams == null) {
videoInfo.audioStreams = getAudioStreams();
}
/** Extract video stream url*/
if (videoInfo.videoStreams == null) {
videoInfo.videoStreams = getVideoStreams();
}
if (videoInfo.uploader_thumbnail_url.isEmpty()) {
videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl();
}
if (videoInfo.startPosition < 0) {
videoInfo.startPosition = getTimeStamp();
}
if(videoInfo.dashMpdUrl.isEmpty()) {
videoInfo.dashMpdUrl = getDashMpdUrl();
}
} else {
videoInfo.errorCode = getErrorCode();
videoInfo.errorMessage = getErrorMessage();
if (videoInfo.title.isEmpty()) {
videoInfo.title = getTitle();
}
if (videoInfo.duration < 1) {
videoInfo.duration = getLength();
}
if (videoInfo.uploader.isEmpty()) {
videoInfo.uploader = getUploader();
}
if (videoInfo.description.isEmpty()) {
videoInfo.description = getDescription();
}
if (videoInfo.view_count == -1) {
videoInfo.view_count = getViews();
}
if (videoInfo.upload_date.isEmpty()) {
videoInfo.upload_date = getUploadDate();
}
if (videoInfo.thumbnail_url.isEmpty()) {
videoInfo.thumbnail_url = getThumbnailUrl();
}
if (videoInfo.id.isEmpty()) {
videoInfo.id = getVideoId(pageUrl);
}
/** Load and extract audio*/
if (videoInfo.audioStreams == null) {
videoInfo.audioStreams = getAudioStreams();
}
/** Extract video stream url*/
if (videoInfo.videoStreams == null) {
videoInfo.videoStreams = getVideoStreams();
}
if (videoInfo.uploader_thumbnail_url.isEmpty()) {
videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl();
}
if (videoInfo.startPosition < 0) {
videoInfo.startPosition = getTimeStamp();
}
if(videoInfo.dashMpdUrl.isEmpty()) {
videoInfo.dashMpdUrl = getDashMpdUrl();
}
//Bitmap thumbnail = null;
//Bitmap uploader_thumbnail = null;
//int videoAvailableStatus = VIDEO_AVAILABLE;
return videoInfo;
}
//todo: add licence field
public abstract int getErrorCode();
public abstract String getErrorMessage();
//todo: remove these functions, or make them static, otherwise its useles, to have them here
public abstract String getVideoUrl(String videoId);
public abstract String getVideoId(String siteUrl);
public abstract String getVideoId(String siteUrl) throws ParsingException;
///////////////////////////////////////////////////////////////////////////////////////////
public abstract int getTimeStamp();
public abstract String getTitle();
public abstract String getDescription();
public abstract String getUploader();
public abstract int getLength();
public abstract long getViews();
public abstract String getUploadDate();
public abstract String getThumbnailUrl();
public abstract String getUploaderThumbnailUrl();
public abstract VideoInfo.AudioStream[] getAudioStreams();
public abstract VideoInfo.VideoStream[] getVideoStreams();
public abstract String getDashMpdUrl();
public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle() throws ParsingException;
public abstract String getDescription() throws ParsingException;
public abstract String getUploader() throws ParsingException;
public abstract int getLength() throws ParsingException;
public abstract long getViews() throws ParsingException;
public abstract String getUploadDate() throws ParsingException;
public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract VideoInfo.AudioStream[] getAudioStreams() throws ParsingException;
public abstract VideoInfo.VideoStream[] getVideoStreams() throws ParsingException;
public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException;
}

11
app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java
View File

@ -26,14 +26,6 @@ import java.util.List;
@SuppressWarnings("ALL")
public class VideoInfo extends AbstractVideoInfo {
// If a video could not be parsed, this predefined error codes
// will be returned AND can be parsed by the frontend of the app.
// Error codes:
public final static int NO_ERROR = 0x0;
public final static int ERROR_NO_SPECIFIED_ERROR = 0x1;
// GEMA a german music colecting society.
public final static int ERROR_BLOCKED_BY_GEMA = 0x2;
public String uploader_thumbnail_url = "";
public String description = "";
public VideoStream[] videoStreams = null;
@ -43,8 +35,6 @@ public class VideoInfo extends AbstractVideoInfo {
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future.
public String dashMpdUrl = "";
public int errorCode = NO_ERROR;
public String errorMessage = "";
public int duration = -1;
/*YouTube-specific fields
@ -60,7 +50,6 @@ public class VideoInfo extends AbstractVideoInfo {
public VideoInfo() {}
/**Creates a new VideoInfo object from an existing AbstractVideoInfo.
* All the shared properties are copied to the new VideoInfo.*/
@SuppressWarnings("WeakerAccess")

162
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java
View File

@ -6,8 +6,11 @@ import android.util.Log;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
@ -49,8 +52,8 @@ public class YoutubeSearchEngine implements SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
@Override
public Result search(String query, int page, String languageCode, Downloader downloader) {
//String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., "");
public Result search(String query, int page, String languageCode, Downloader downloader) throws IOException, ParsingException {
Result result = new Result();
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("www.youtube.com")
@ -63,22 +66,19 @@ public class YoutubeSearchEngine implements SearchEngine {
String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode);
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode);
}
else {
site = downloader.download(url);
}
try {
Document doc = Jsoup.parse(site, url);
Result result = new Result();
Element list = doc.select("ol[class=\"item-section\"]").first();
Document doc = Jsoup.parse(site, url);
Element list = doc.select("ol[class=\"item-section\"]").first();
int i = 0;
for(Element item : list.children()) {
i++;
for (Element item : list.children()) {
/* First we need to determine which kind of item we are working with.
Youtube depicts five different kinds of items on its search result page. These are
regular videos, playlists, channels, two types of video suggestions, and a "no video
@ -90,57 +90,61 @@ public class YoutubeSearchEngine implements SearchEngine {
playlists now.
*/
Element el;
Element el;
// both types of spell correction item
if(!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) {
result.suggestion = el.select("a").first().text();
// search message item
} else if(!((el = item.select("div[class*=\"search-message\"]").first()) == null)) {
result.errorMessage = el.text();
// both types of spell correction item
if (!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) {
result.suggestion = el.select("a").first().text();
// search message item
} else if (!((el = item.select("div[class*=\"search-message\"]").first()) == null)) {
result.errorMessage = el.text();
// video item type
} else if(!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href");
try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id=m.group(1);
} catch (Exception e) {
//e.printStackTrace();
// video item type
} else if (!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href");
try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id = m.group(1);
} catch (Exception e) {
//e.printStackTrace();
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\"" + el + "\"");
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if(resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\""+el+"\"");
}
} catch(Exception e) {
throw new ParsingException(e);
}
return result;
}
@Override
public ArrayList<String> suggestionList(String query, Downloader dl) {
public ArrayList<String> suggestionList(String query, Downloader dl)
throws IOException, ParsingException {
ArrayList<String> suggestions = new ArrayList<>();
@ -155,36 +159,42 @@ public class YoutubeSearchEngine implements SearchEngine {
.appendQueryParameter("q", query);
String url = builder.build().toString();
String response = dl.download(url);
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8"))));
doc.getDocumentElement().normalize();
}catch (ParserConfigurationException | SAXException | IOException e) {
e.printStackTrace();
}
if(doc!=null){
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8"))));
doc.getDocumentElement().normalize();
} catch (ParserConfigurationException | SAXException | IOException e) {
e.printStackTrace();
}
}else {
Log.e(TAG, "GREAT FUCKING ERROR");
if (doc != null) {
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
} else {
Log.e(TAG, "GREAT FUCKING ERROR");
}
return suggestions;
} catch(Exception e) {
throw new ParsingException(e);
}
return suggestions;
}
}

7
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java
View File

@ -1,10 +1,13 @@
package org.schabi.newpipe.crawler.services.youtube;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.SearchEngine;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 23.08.15.
@ -34,9 +37,9 @@ public class YoutubeService implements StreamingService {
return serviceInfo;
}
@Override
public VideoExtractor getExtractorInstance(String url, Downloader downloader) {
public VideoExtractor getExtractorInstance(String url, Downloader downloader) throws CrawlingException, IOException {
if(acceptUrl(url)) {
return new YoutubeVideoExtractor(url, downloader);
return new YoutubeVideoExtractor(url, downloader) ;
}
else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");

647
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java
View File

@ -12,13 +12,16 @@ import org.jsoup.parser.Parser;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.MediaFormat;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.xmlpull.v1.XmlPullParser;
import java.io.IOException;
import java.io.StringReader;
import java.net.URLDecoder;
import java.util.HashMap;
@ -49,11 +52,28 @@ import java.util.regex.Pattern;
public class YoutubeVideoExtractor extends VideoExtractor {
public class DecryptException extends ParsingException {
DecryptException(Throwable cause) {
super(cause);
}
DecryptException(String message, Throwable cause) {
super(message, cause);
}
}
// special content not available exceptions
public class GemaException extends ContentNotAvailableException {
GemaException(String message) {
super(message);
}
}
// ----------------
private static final String TAG = YoutubeVideoExtractor.class.toString();
private final Document doc;
private JSONObject jsonObj;
private JSONObject playerArgs;
private int errorCode = VideoInfo.NO_ERROR;
private String errorMessage = "";
// static values
@ -64,29 +84,32 @@ public class YoutubeVideoExtractor extends VideoExtractor {
private Downloader downloader;
public YoutubeVideoExtractor(String pageUrl, Downloader dl) {
public YoutubeVideoExtractor(String pageUrl, Downloader dl) throws CrawlingException, IOException {
//most common videoInfo fields are now set in our superclass, for all services
super(pageUrl, dl);
downloader = dl;
String pageContent = downloader.download(cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl);
String ytPlayerConfigRaw;
JSONObject ytPlayerConfig;
//attempt to load the youtube js player JSON arguments
try {
String jsonString = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
//todo: implement this by try and catch. TESTING THE STRING AGAINST EMPTY IS CONSIDERED POOR STYLE !!!
if(jsonString.isEmpty()) {
errorCode = findErrorReason(doc);
return;
ytPlayerConfigRaw = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
ytPlayerConfig = new JSONObject(ytPlayerConfigRaw);
playerArgs = ytPlayerConfig.getJSONObject("args");
} catch (RegexException e) {
String errorReason = findErrorReason(doc);
switch(errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ParsingException("player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
jsonObj = new JSONObject(jsonString);
playerArgs = jsonObj.getJSONObject("args");
} catch (Exception e) {//if this fails, the video is most likely not available.
// Determining why is done later.
videoInfo.errorCode = VideoInfo.ERROR_NO_SPECIFIED_ERROR;
Log.e(TAG, "Could not load JSON data for Youtube video \""+pageUrl+"\". This most likely means the video is unavailable");
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config");
}
//----------------------------------
@ -94,102 +117,92 @@ public class YoutubeVideoExtractor extends VideoExtractor {
//----------------------------------
if (decryptionCode.isEmpty()) {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
JSONObject ytAssets = jsonObj.getJSONObject("assets");
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
JSONObject ytAssets = ytPlayerConfig.getJSONObject("assets");
String playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = "https:" + playerUrl;
}
decryptionCode = loadDecryptionCode(playerUrl);
} catch (Exception e){
Log.e(TAG, "Could not load decryption code for the Youtube service.");
e.printStackTrace();
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
}
@Override
public String getTitle() {
public String getTitle() throws ParsingException {
try {//json player args method
return playerArgs.getString("title");
} catch(JSONException je) {//html <meta> method
je.printStackTrace();
Log.w(TAG, "failed to load title from JSON args; trying to extract it from HTML");
} try { // fall through to fall-back
return doc.select("meta[name=title]").attr("content");
} catch (Exception e) {
Log.e(TAG, "failed permanently to load title.");
e.printStackTrace();
return "";
try { // fall through to fall-back
return doc.select("meta[name=title]").attr("content");
} catch (Exception e) {
throw new ParsingException("failed permanently to load title.", e);
}
}
}
@Override
public String getDescription() {
public String getDescription() throws ParsingException {
try {
return doc.select("p[id=\"eow-description\"]").first().html();
} catch (Exception e) {//todo: add fallback method
Log.e(TAG, "failed to load description.");
e.printStackTrace();
return "";
} catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know
throw new ParsingException("failed to load description.", e);
}
}
@Override
public String getUploader() {
public String getUploader() throws ParsingException {
try {//json player args method
return playerArgs.getString("author");
} catch(JSONException je) {
je.printStackTrace();
Log.w(TAG, "failed to load uploader name from JSON args; trying to extract it from HTML");
Log.w(TAG,
"failed to load uploader name from JSON args; trying to extract it from HTML");
} try {//fall through to fallback HTML method
return doc.select("div.yt-user-info").first().text();
} catch (Exception e) {
e.printStackTrace();
Log.e(TAG, "failed permanently to load uploader name.");
return "";
throw new ParsingException("failed permanently to load uploader name.", e);
}
}
@Override
public int getLength() {
public int getLength() throws ParsingException {
try {
return playerArgs.getInt("length_seconds");
} catch (JSONException je) {//todo: find fallback method
Log.e(TAG, "failed to load video duration from JSON args");
je.printStackTrace();
return -1;
} catch (JSONException e) {//todo: find fallback method
throw new ParsingException("failed to load video duration from JSON args", e);
}
}
@Override
public long getViews() {
public long getViews() throws ParsingException {
try {
String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content");
return Long.parseLong(viewCountString);
} catch (Exception e) {//todo: find fallback method
Log.e(TAG, "failed to number of views");
e.printStackTrace();
return -1;
throw new ParsingException("failed to number of views", e);
}
}
@Override
public String getUploadDate() {
public String getUploadDate() throws ParsingException {
try {
return doc.select("meta[itemprop=datePublished]").attr("content");
} catch (Exception e) {//todo: add fallback method
Log.e(TAG, "failed to get upload date.");
e.printStackTrace();
return "";
throw new ParsingException("failed to get upload date.", e);
}
}
@Override
public String getThumbnailUrl() {
public String getThumbnailUrl() throws ParsingException {
//first attempt getting a small image version
//in the html extracting part we try to get a thumbnail with a higher resolution
// Try to get high resolution thumbnail if it fails use low res from the player instead
@ -197,148 +210,98 @@ public class YoutubeVideoExtractor extends VideoExtractor {
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
} catch(Exception e) {
Log.w(TAG, "Could not find high res Thumbnail. Using low res instead");
//fall through to fallback
} try {
} try { //fall through to fallback
return playerArgs.getString("thumbnail_url");
} catch (JSONException je) {
je.printStackTrace();
Log.w(TAG, "failed to extract thumbnail URL from JSON args; trying to extract it from HTML");
return "";
throw new ParsingException(
"failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je);
}
}
@Override
public String getUploaderThumbnailUrl() {
public String getUploaderThumbnailUrl() throws ParsingException {
try {
return doc.select("a[class*=\"yt-user-photo\"]").first()
.select("img").first()
.attr("abs:data-thumb");
} catch (Exception e) {//todo: add fallback method
Log.e(TAG, "failed to get uploader thumbnail URL.");
e.printStackTrace();
return "";
throw new ParsingException("failed to get uploader thumbnail URL.", e);
}
}
@Override
public String getDashMpdUrl() {
public String getDashMpdUrl() throws ParsingException {
try {
return playerArgs.getString("dashmpd");
} catch(NullPointerException e) {
Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).");
throw new ParsingException(
"Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e);
} catch (Exception e) {
e.printStackTrace();
throw new ParsingException(e);
}
return "";
}
@Override
public VideoInfo.AudioStream[] getAudioStreams() {
public VideoInfo.AudioStream[] getAudioStreams() throws ParsingException {
try {
String dashManifest = playerArgs.getString("dashmpd");
Log.d(TAG, dashManifest);
return parseDashManifest(dashManifest, decryptionCode);
} catch (NullPointerException e) {
Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).");
throw new ParsingException(
"Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e);
} catch (Exception e) {
e.printStackTrace();
throw new ParsingException(e);
}
return new VideoInfo.AudioStream[0];
}
@Override
public VideoInfo.VideoStream[] getVideoStreams() {
public VideoInfo.VideoStream[] getVideoStreams() throws ParsingException {
Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
try{
//------------------------------------
// extract video stream url
//------------------------------------
Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
// The following line belongs to dash audio stuff.
// We can't use dash audio, unless we have exoplayer as our main player.
//String adaptive_fmts = playerArgs.getString("adaptive_fmts");
String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
for(String url_data_str : encoded_url_map.split(",")) {
Map<String, String> tags = new HashMap<>();
for(String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) {
String[] split_tag = raw_tag.split("=");
tags.put(split_tag[0], split_tag[1]);
}
try {
Map<String, String> tags = new HashMap<>();
for (String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) {
String[] split_tag = raw_tag.split("=");
tags.put(split_tag[0], split_tag[1]);
}
int itag = Integer.parseInt(tags.get("itag"));
String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8");
int itag = Integer.parseInt(tags.get("itag"));
String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8");
// if video has a signature: decrypt it and add it to the url
if(tags.get("s") != null) {
streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
}
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature="
+ decryptSignature(tags.get("s"), decryptionCode);
}
if(resolveFormat(itag) != -1) {
videoStreams.add(new VideoInfo.VideoStream(
streamUrl,
resolveFormat(itag),
resolveResolutionString(itag)));
if (resolveFormat(itag) != -1) {
videoStreams.add(new VideoInfo.VideoStream(
streamUrl,
resolveFormat(itag),
resolveResolutionString(itag)));
}
} catch (Exception e) {
Log.w(TAG, "Could not get Video stream.");
e.printStackTrace();
}
}
return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]);
} catch (Exception e) {
Log.e(TAG, "Failed to get video stream");
e.printStackTrace();
return new VideoInfo.VideoStream[0];
throw new ParsingException("Failed to get video streams", e);
}
}
/**These lists only contain itag formats that are supported by the common Android Video player.
However if you are looking for a list showing all itag formats, look at
https://github.com/rg3/youtube-dl/issues/1687 */
@SuppressWarnings("WeakerAccess")
public static int resolveFormat(int itag) {
switch(itag) {
// !!! lists only supported formats !!!
// video
case 17: return MediaFormat.v3GPP.id;
case 18: return MediaFormat.MPEG_4.id;
case 22: return MediaFormat.MPEG_4.id;
case 36: return MediaFormat.v3GPP.id;
case 37: return MediaFormat.MPEG_4.id;
case 38: return MediaFormat.MPEG_4.id;
case 43: return MediaFormat.WEBM.id;
case 44: return MediaFormat.WEBM.id;
case 45: return MediaFormat.WEBM.id;
case 46: return MediaFormat.WEBM.id;
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return -1;
if(videoStreams.isEmpty()) {
throw new ParsingException("Failed to get any video stream");
}
}
@SuppressWarnings("WeakerAccess")
public static String resolveResolutionString(int itag) {
switch(itag) {
case 17: return "144p";
case 18: return "360p";
case 22: return "720p";
case 36: return "240p";
case 37: return "1080p";
case 38: return "1080p";
case 43: return "360p";
case 44: return "480p";
case 45: return "720p";
case 46: return "1080p";
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return null;
}
return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]);
}
@SuppressWarnings("WeakerAccess")
@Override
public String getVideoId(String url) {
public String getVideoId(String url) throws ParsingException {
String id;
String pat;
@ -349,16 +312,16 @@ public class YoutubeVideoExtractor extends VideoExtractor {
pat = "youtu\\.be/([a-zA-Z0-9_-]{11})";
}
else {
Log.e(TAG, "Error could not parse url: " + url);
return "";
throw new ParsingException("Error no suitable url: " + url);
}
id = matchGroup1(pat, url);
if(!id.isEmpty()){
//Log.i(TAG, "string \""+url+"\" matches!");
return id;
} else {
throw new ParsingException("Error could not parse url: " + url);
}
//Log.i(TAG, "string \""+url+"\" does not match.");
return "";
}
@SuppressWarnings("WeakerAccess")
@ -370,116 +333,136 @@ public class YoutubeVideoExtractor extends VideoExtractor {
/**Attempts to parse (and return) the offset to start playing the video from.
* @return the offset (in seconds), or 0 if no timestamp is found.*/
@Override
public int getTimeStamp(){
String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
public int getTimeStamp() throws ParsingException {
//todo: add unit test for timestamp
String timeStamp;
try {
timeStamp = matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
} catch (RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
// -2 because well the testing system will then know its the regex that failed :/
// not good i know
return -2;
}
//TODO: test this
if(!timeStamp.isEmpty()) {
String secondsString = matchGroup1("(\\d{1,3})s", timeStamp);
String minutesString = matchGroup1("(\\d{1,3})m", timeStamp);
String hoursString = matchGroup1("(\\d{1,3})h", timeStamp);
try {
String secondsString = "";
String minutesString = "";
String hoursString = "";
try {
secondsString = matchGroup1("(\\d{1,3})s", timeStamp);
minutesString = matchGroup1("(\\d{1,3})m", timeStamp);
hoursString = matchGroup1("(\\d{1,3})h", timeStamp);
} catch (Exception e) {
//it could be that time is given in another method
if (secondsString.isEmpty() //if nothing was got,
&& minutesString.isEmpty()//treat as unlabelled seconds
&& hoursString.isEmpty()) {
secondsString = matchGroup1("t=(\\d{1,3})", timeStamp);
}
}
if(secondsString.isEmpty()//if nothing was got,
&& minutesString.isEmpty()//treat as unlabelled seconds
&& hoursString.isEmpty())
secondsString = matchGroup1("t=(\\d{1,3})", timeStamp);
int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString));
int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString));
int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString));
int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString));
int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString));
int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString));
int ret = seconds + (60 * minutes) + (3600 * hours);//don't trust BODMAS!
//Log.d(TAG, "derived timestamp value:"+ret);
return ret;
//the ordering varies internationally
} catch (ParsingException e) {
throw new ParsingException("Could not get timestamp.", e);
}
} else {
return -1;
}
}
int ret = seconds + (60*minutes) + (3600*hours);//don't trust BODMAS!
//Log.d(TAG, "derived timestamp value:"+ret);
return ret;
//the ordering varies internationally
}//else, return default 0
@Override
public int getAgeLimit() throws ParsingException {
// Not yet implemented.
// Also you need to be logged in to see age restricted videos on youtube,
// therefore NP is not able to receive such videos.
return 0;
}
@Override
public VideoInfo getVideoInfo() {
//todo: @medovax i like your work, but what the fuck:
public VideoInfo getVideoInfo() throws CrawlingException {
videoInfo = super.getVideoInfo();
if(errorCode == VideoInfo.NO_ERROR) {
//todo: replace this with a call to getVideoId, if possible
videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl);
//todo: replace this with a call to getVideoId, if possible
//videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl);
videoInfo.id = getVideoId(pageUrl);
if (videoInfo.audioStreams == null
|| videoInfo.audioStreams.length == 0) {
Log.e(TAG, "uninitialised audio streams!");
}
if (videoInfo.videoStreams == null
|| videoInfo.videoStreams.length == 0) {
Log.e(TAG, "uninitialised video streams!");
}
videoInfo.age_limit = 0;
//average rating
try {
videoInfo.average_rating = playerArgs.getString("avg_rating");
} catch (JSONException e) {
e.printStackTrace();
}
//---------------------------------------
// extracting information from html page
//---------------------------------------
String likesString = "";
String dislikesString = "";
try {
// likes
likesString = doc.select("button.like-button-renderer-like-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", ""));
// dislikes
dislikesString = doc.select("button.like-button-renderer-dislike-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", ""));
} catch (NumberFormatException nfe) {
Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" +
dislikesString + "\" as integers");
} catch (Exception e) {
// if it fails we know that the video does not offer dislikes.
e.printStackTrace();
videoInfo.like_count = 0;
videoInfo.dislike_count = 0;
}
// next video
videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
// related videos
Vector<VideoPreviewInfo> relatedVideos = new Vector<>();
for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
relatedVideos.add(extractVideoPreviewInfo(li));
}
}
//todo: replace conversion
videoInfo.relatedVideos = relatedVideos;
//videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]);
if (videoInfo.audioStreams == null
|| videoInfo.audioStreams.length == 0) {
Log.e(TAG, "uninitialised audio streams!");
}
if (videoInfo.videoStreams == null
|| videoInfo.videoStreams.length == 0) {
Log.e(TAG, "uninitialised video streams!");
}
videoInfo.age_limit = 0;
//average rating
try {
videoInfo.average_rating = playerArgs.getString("avg_rating");
} catch (JSONException e) {
e.printStackTrace();
}
//---------------------------------------
// extracting information from html page
//---------------------------------------
String likesString = "";
String dislikesString = "";
try {
// likes
likesString = doc.select("button.like-button-renderer-like-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", ""));
// dislikes
dislikesString = doc.select("button.like-button-renderer-dislike-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", ""));
} catch (NumberFormatException nfe) {
Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" +
dislikesString + "\" as integers");
} catch (Exception e) {
// if it fails we know that the video does not offer dislikes.
e.printStackTrace();
videoInfo.like_count = 0;
videoInfo.dislike_count = 0;
}
// next video
videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
// related videos
Vector<VideoPreviewInfo> relatedVideos = new Vector<>();
for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
relatedVideos.add(extractVideoPreviewInfo(li));
}
}
//todo: replace conversion
videoInfo.relatedVideos = relatedVideos;
//videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]);
return videoInfo;
}
@Override
public int getErrorCode() {
return errorCode;
}
@Override
public String getErrorMessage() {
return errorMessage;
}
private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) {
private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) throws RegexException, DecryptException {
if(!dashManifest.contains("/signature/")) {
String encryptedSig = matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest);
String decryptedSig;
@ -487,7 +470,12 @@ public class YoutubeVideoExtractor extends VideoExtractor {
decryptedSig = decryptSignature(encryptedSig, decryptoinCode);
dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
}
String dashDoc = downloader.download(dashManifest);
String dashDoc;
try {
dashDoc = downloader.download(dashManifest);
} catch(IOException ioe) {
throw new DecryptException("Could not get dash mpd", ioe);
}
Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
try {
XmlPullParser parser = Xml.newPullParser();
@ -545,63 +533,67 @@ public class YoutubeVideoExtractor extends VideoExtractor {
/**Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a VideoPreviewInfo object,
* which is a subset of the fields in a full VideoInfo.*/
private VideoPreviewInfo extractVideoPreviewInfo(Element li) {
private VideoPreviewInfo extractVideoPreviewInfo(Element li) throws ParsingException {
VideoPreviewInfo info = new VideoPreviewInfo();
info.webpage_url = li.select("a.content-link").first()
.attr("abs:href");
try {
info.webpage_url = li.select("a.content-link").first()
.attr("abs:href");
info.id = matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url);
//todo: check NullPointerException causing
info.title = li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
info.view_count = Long.parseLong(li.select("span.view-count")
.first().text().replaceAll("[^\\d]", ""));
} catch (NullPointerException e) {//related videos sometimes have no view count
info.view_count = 0;
}
info.uploader = li.select("span.g-hovercard").first().text();
info.duration = li.select("span.video-time").first().text();
Element img = li.select("img").first();
info.thumbnail_url = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (info.thumbnail_url.contains(".gif")) {
info.thumbnail_url = img.attr("data-thumb");
}
if (info.thumbnail_url.startsWith("//")) {
info.thumbnail_url = "https:" + info.thumbnail_url;
}
} catch (Exception e) {
e.printStackTrace();
}
//todo: check NullPointerException causing
info.title = li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
info.view_count = Long.parseLong(li.select("span.view-count")
.first().text().replaceAll("[^\\d]", ""));
} catch (NullPointerException e) {//related videos sometimes have no view count
info.view_count = 0;
}
info.uploader = li.select("span.g-hovercard").first().text();
info.duration = li.select("span.video-time").first().text();
Element img = li.select("img").first();
info.thumbnail_url = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if(info.thumbnail_url.contains(".gif")) {
info.thumbnail_url = img.attr("data-thumb");
}
if(info.thumbnail_url.startsWith("//")) {
info.thumbnail_url = "https:" + info.thumbnail_url;
throw new ParsingException(e);
}
return info;
}
private String loadDecryptionCode(String playerUrl) {
String playerCode = downloader.download(playerUrl);
String decryptionFuncName = "";
String decryptionFunc = "";
private String loadDecryptionCode(String playerUrl) throws DecryptException {
String decryptionFuncName;
String decryptionFunc;
String helperObjectName;
String helperObject = "";
String helperObject;
String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}";
String decryptionCode;
try {
String playerCode = downloader.download(playerUrl);
decryptionFuncName = matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode);
String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") +"=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})";
String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") + "=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})";
decryptionFunc = "var " + matchGroup1(functionPattern, playerCode) + ";";
helperObjectName = matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc);
@ -609,17 +601,20 @@ public class YoutubeVideoExtractor extends VideoExtractor {
String helperPattern = "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
helperObject = matchGroup1(helperPattern, playerCode);
} catch (Exception e) {
e.printStackTrace();
}
callerFunc = callerFunc.replace("%%", decryptionFuncName);
decryptionCode = helperObject + decryptionFunc + callerFunc;
callerFunc = callerFunc.replace("%%", decryptionFuncName);
decryptionCode = helperObject + decryptionFunc + callerFunc;
} catch(IOException ioe) {
throw new DecryptException("Could not load decrypt function", ioe);
} catch(Exception e) {
throw new DecryptException("Could not parse decrypt function ", e);
}
return decryptionCode;
}
private String decryptSignature(String encryptedSig, String decryptionCode) {
private String decryptSignature(String encryptedSig, String decryptionCode)
throws DecryptException{
Context context = Context.enter();
context.setOptimizationLevel(-1);
Object result = null;
@ -629,17 +624,18 @@ public class YoutubeVideoExtractor extends VideoExtractor {
Function decryptionFunc = (Function) scope.get("decrypt", scope);
result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
} catch (Exception e) {
e.printStackTrace();
throw new DecryptException(e);
} finally {
Context.exit();
}
Context.exit();
return (result == null ? "" : result.toString());
}
private String cleanUrl(String complexUrl) {
private String cleanUrl(String complexUrl) throws ParsingException {
return getVideoUrl(getVideoId(complexUrl));
}
private String matchGroup1(String pattern, String input) {
private String matchGroup1(String pattern, String input) throws RegexException {
Pattern pat = Pattern.compile(pattern);
Matcher mat = pat.matcher(input);
boolean foundMatch = mat.find();
@ -647,17 +643,64 @@ public class YoutubeVideoExtractor extends VideoExtractor {
return mat.group(1);
}
else {
Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\"");
new Exception("failed to find pattern \""+pattern+"\"").printStackTrace();
return "";
//Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\"");
throw new RegexException("failed to find pattern \""+pattern+" inside of "+input+"\"");
}
}
private int findErrorReason(Document doc) {
private String findErrorReason(Document doc) {
errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
if(errorMessage.contains("GEMA")) {
return VideoInfo.ERROR_BLOCKED_BY_GEMA;
// Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
return "GEMA";
}
return "";
}
/**These lists only contain itag formats that are supported by the common Android Video player.
However if you are looking for a list showing all itag formats, look at
https://github.com/rg3/youtube-dl/issues/1687 */
@SuppressWarnings("WeakerAccess")
public static int resolveFormat(int itag) {
switch(itag) {
// !!! lists only supported formats !!!
// video
case 17: return MediaFormat.v3GPP.id;
case 18: return MediaFormat.MPEG_4.id;
case 22: return MediaFormat.MPEG_4.id;
case 36: return MediaFormat.v3GPP.id;
case 37: return MediaFormat.MPEG_4.id;
case 38: return MediaFormat.MPEG_4.id;
case 43: return MediaFormat.WEBM.id;
case 44: return MediaFormat.WEBM.id;
case 45: return MediaFormat.WEBM.id;
case 46: return MediaFormat.WEBM.id;
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return -1;
}
}
@SuppressWarnings("WeakerAccess")
public static String resolveResolutionString(int itag) {
switch(itag) {
case 17: return "144p";
case 18: return "360p";
case 22: return "720p";
case 36: return "240p";
case 37: return "1080p";
case 38: return "1080p";
case 43: return "360p";
case 44: return "480p";
case 45: return "720p";
case 46: return "1080p";
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return null;
}
return VideoInfo.ERROR_NO_SPECIFIED_ERROR;
}
}

6
app/src/main/res/values/strings.xml
View File

@ -65,7 +65,13 @@
<string name="background_player_playing_toast">Playing in background</string>
<string name="c3s_url" translatable="false">https://www.c3s.cc/</string>
<string name="play_btn_text">Play</string>
<string name="general_error">Error</string>
<string name="network_error">Network error</string>
<string name="could_not_load_thumbnails">Could not load Thumbnails</string>
<string name="youtube_signature_decryption_error">Could not decrypt video url signature.</string>
<string name="parsing_error">Could not parse website.</string>
<string name="content_not_available">Content not available.</string>
<string name="blocked_by_gema">Blocked by GEMA.</string>
<!-- Content descriptions (for better accessibility) -->
<string name="list_thumbnail_view_description">Video preview thumbnail</string>