fixed some searchengine errors

This commit is contained in:
Christian Schabesberger 2016-02-21 21:05:16 +01:00
parent 2995a7dc8f
commit a1479d04df
16 changed files with 183 additions and 152 deletions

View File

@ -81,6 +81,12 @@ public class YoutubeSearchEngineTest extends AndroidTestCase {
}
}
public void testViewCount() {
for(VideoPreviewInfo i : result.resultList) {
assertTrue(Long.toString(i.view_count), i.view_count != -1);
}
}
public void testIfSuggestionsAreReplied() {
assertEquals(suggestionReply.size() > 0, true);
}

View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ExctractionException;
import org.schabi.newpipe.extractor.ExtractionException;
import org.schabi.newpipe.extractor.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.VideoInfo;
@ -33,7 +33,7 @@ import java.io.IOException;
public class YoutubeStreamExtractorDefaultTest extends AndroidTestCase {
private YoutubeStreamExtractor extractor;
public void setUp() throws IOException, ExctractionException {
public void setUp() throws IOException, ExtractionException {
/* some anonymus video test
extractor = new YoutubeStreamExtractor("https://www.youtube.com/watch?v=FmG385_uUys",
new Downloader()); */
@ -47,7 +47,7 @@ public class YoutubeStreamExtractorDefaultTest extends AndroidTestCase {
extractor.getTimeStamp() <= 0);
}
public void testGetValidTimeStamp() throws ExctractionException, IOException {
public void testGetValidTimeStamp() throws ExtractionException, IOException {
YoutubeStreamExtractor extractor =
new YoutubeStreamExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
assertTrue(Integer.toString(extractor.getTimeStamp()),

View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ExctractionException;
import org.schabi.newpipe.extractor.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamExtractor;
import java.io.IOException;
@ -35,7 +35,7 @@ public class YoutubeStreamExtractorGemaTest extends AndroidTestCase {
// Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail.
private static final boolean testActive = false;
public void testGemaError() throws IOException, ExctractionException {
public void testGemaError() throws IOException, ExtractionException {
if(testActive) {
try {
new YoutubeStreamExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8",

View File

@ -147,6 +147,13 @@ public class VideoItemDetailFragment extends Fragment {
onErrorBlockedByGema();
}
});
} catch(YoutubeStreamExtractor.LiveStreamException e) {
h.post(new Runnable() {
@Override
public void run() {
onNotSpecifiedContentErrorWithMessage(R.string.live_streams_not_supported);
}
});
}
// ----------------------------------------
catch(StreamExtractor.ContentNotAvailableException e) {
@ -186,8 +193,10 @@ public class VideoItemDetailFragment extends Fragment {
@Override
public void onLoadingFailed(String imageUri, View view, FailReason failReason) {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.could_not_load_thumbnails, Toast.LENGTH_LONG).show();
if(getContext() != null) {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.could_not_load_thumbnails, Toast.LENGTH_LONG).show();
}
failReason.getCause().printStackTrace();
}
@ -672,6 +681,15 @@ public class VideoItemDetailFragment extends Fragment {
.show();
}
private void onNotSpecifiedContentErrorWithMessage(int resourceId) {
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, resourceId, Toast.LENGTH_LONG)
.show();
}
private boolean useStream(VideoInfo.VideoStream stream, Vector<VideoInfo.VideoStream> streams) {
for(VideoInfo.VideoStream i : streams) {
if(i.resolution.equals(stream.resolution)) {

View File

@ -16,7 +16,8 @@ import android.widget.Toast;
import java.io.IOException;
import java.util.List;
import org.schabi.newpipe.extractor.ExctractionException;
import org.schabi.newpipe.extractor.ExtractionException;
import org.schabi.newpipe.extractor.ParsingException;
import org.schabi.newpipe.extractor.VideoPreviewInfo;
import org.schabi.newpipe.extractor.SearchEngine;
import org.schabi.newpipe.extractor.StreamingService;
@ -108,14 +109,14 @@ public class VideoItemListFragment extends ListFragment {
SearchEngine.Result result = engine.search(query, page, searchLanguage,
new Downloader());
Log.i(TAG, "language code passed:\""+searchLanguage+"\"");
//Log.i(TAG, "language code passed:\""+searchLanguage+"\"");
if(runs) {
h.post(new ResultRunnable(result, requestId));
}
} catch(IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
} catch(ExctractionException ce) {
} catch(ExtractionException ce) {
postNewErrorToast(h, R.string.parsing_error);
ce.printStackTrace();
} catch(Exception e) {
@ -124,67 +125,7 @@ public class VideoItemListFragment extends ListFragment {
}
}
}
/*
<<<
private class LoadThumbsRunnable implements Runnable {
private final Vector<String> thumbnailUrlList = new Vector<>();
private final Vector<Boolean> downloadedList;
final Handler h = new Handler();
private volatile boolean run = true;
private final int requestId;
public LoadThumbsRunnable(Vector<VideoPreviewInfo> videoList,
Vector<Boolean> downloadedList, int requestId) {
for(VideoPreviewInfo item : videoList) {
thumbnailUrlList.add(item.thumbnail_url);
}
this.downloadedList = downloadedList;
this.requestId = requestId;
}
public void terminate() {
run = false;
}
public boolean isRunning() {
return run;
}
@Override
public void run() {
for(int i = 0; i < thumbnailUrlList.size() && run; i++) {
if(!downloadedList.get(i)) {
Bitmap thumbnail;
try {
//todo: make bitmaps not bypass tor
thumbnail = BitmapFactory.decodeStream(
new URL(thumbnailUrlList.get(i)).openConnection().getInputStream());
h.post(new SetThumbnailRunnable(i, thumbnail, requestId));
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
}
private class SetThumbnailRunnable implements Runnable {
private final int index;
private final Bitmap thumbnail;
private final int requestId;
public SetThumbnailRunnable(int index, Bitmap thumbnail, int requestId) {
this.index = index;
this.thumbnail = thumbnail;
this.requestId = requestId;
}
@Override
public void run() {
if(requestId == currentRequestId) {
videoListAdapter.updateDownloadedThumbnailList(index);
videoListAdapter.setThumbnail(index, thumbnail);
}
}
}
=======
>>>>>>> 6d1b4652fc98e5c2d5e19b0f98ba38a731137a70
*/
public void present(List<VideoPreviewInfo> videoList) {
mode = PRESENT_VIDEOS_MODE;
setListShown(true);
@ -378,7 +319,7 @@ public class VideoItemListFragment extends ListFragment {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.makeText(getActivity(), getString(stringResource),
Toast.LENGTH_SHORT).show();
}
});

View File

@ -4,7 +4,7 @@ package org.schabi.newpipe.extractor;
* Created by Christian Schabesberger on 30.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ExctractionException.java is part of NewPipe.
* ExtractionException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -20,18 +20,18 @@ package org.schabi.newpipe.extractor;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ExctractionException extends Exception {
public ExctractionException() {}
public class ExtractionException extends Exception {
public ExtractionException() {}
public ExctractionException(String message) {
public ExtractionException(String message) {
super(message);
}
public ExctractionException(Throwable cause) {
public ExtractionException(Throwable cause) {
super(cause);
}
public ExctractionException(String message, Throwable cause) {
public ExtractionException(String message, Throwable cause) {
super(message, cause);
}
}

View File

@ -21,7 +21,7 @@ package org.schabi.newpipe.extractor;
*/
public class ParsingException extends ExctractionException {
public class ParsingException extends ExtractionException {
public ParsingException() {}
public ParsingException(String message) {
super(message);

View File

@ -34,9 +34,9 @@ public interface SearchEngine {
}
ArrayList<String> suggestionList(String query, Downloader dl)
throws ExctractionException, IOException;
throws ExtractionException, IOException;
//Result search(String query, int page);
Result search(String query, int page, String contentCountry, Downloader dl)
throws ExctractionException, IOException;
throws ExtractionException, IOException;
}

View File

@ -28,7 +28,7 @@ import java.util.List;
@SuppressWarnings("ALL")
public interface StreamExtractor {
public class ExctractorInitException extends ExctractionException {
public class ExctractorInitException extends ExtractionException {
public ExctractorInitException() {}
public ExctractorInitException(String message) {
super(message);

View File

@ -28,7 +28,7 @@ public interface StreamingService {
}
ServiceInfo getServiceInfo();
StreamExtractor getExtractorInstance(String url, Downloader downloader)
throws IOException, ExctractionException;
throws IOException, ExtractionException;
SearchEngine getSearchEngineInstance();
VideoUrlIdHandler getUrlIdHandler();

View File

@ -31,7 +31,7 @@ public class VideoInfo extends AbstractVideoInfo {
/**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/
public static VideoInfo getVideoInfo(StreamExtractor extractor, Downloader downloader)
throws ExctractionException, IOException {
throws ExtractionException, IOException {
VideoInfo videoInfo = new VideoInfo();
videoInfo = extractImportantData(videoInfo, extractor, downloader);
@ -43,7 +43,7 @@ public class VideoInfo extends AbstractVideoInfo {
private static VideoInfo extractImportantData(
VideoInfo videoInfo, StreamExtractor extractor, Downloader downloader)
throws ExctractionException, IOException {
throws ExtractionException, IOException {
/* ---- importand data, withoug the video can't be displayed goes here: ---- */
// if one of these is not available an exception is ment to be thrown directly into the frontend.
@ -62,7 +62,7 @@ public class VideoInfo extends AbstractVideoInfo {
private static VideoInfo extractStreams(
VideoInfo videoInfo, StreamExtractor extractor, Downloader downloader)
throws ExctractionException, IOException {
throws ExtractionException, IOException {
/* ---- stream extraction goes here ---- */
// At least one type of stream has to be available,
// otherwise an exception will be thrown directly into the frontend.
@ -70,14 +70,14 @@ public class VideoInfo extends AbstractVideoInfo {
try {
videoInfo.dashMpdUrl = extractor.getDashMpdUrl();
} catch(Exception e) {
videoInfo.addException(new ExctractionException("Couldn't get Dash manifest", e));
videoInfo.addException(new ExtractionException("Couldn't get Dash manifest", e));
}
/* Load and extract audio */
try {
videoInfo.audio_streams = extractor.getAudioStreams();
} catch(Exception e) {
videoInfo.addException(new ExctractionException("Couldn't get audio streams", e));
videoInfo.addException(new ExtractionException("Couldn't get audio streams", e));
}
// also try to get streams from the dashMpd
if(videoInfo.dashMpdUrl != null && !videoInfo.dashMpdUrl.isEmpty()) {
@ -91,7 +91,7 @@ public class VideoInfo extends AbstractVideoInfo {
DashMpdParser.getAudioStreams(videoInfo.dashMpdUrl, downloader));
} catch(Exception e) {
videoInfo.addException(
new ExctractionException("Couldn't get audio streams from dash mpd", e));
new ExtractionException("Couldn't get audio streams from dash mpd", e));
}
}
/* Extract video stream url*/
@ -99,14 +99,14 @@ public class VideoInfo extends AbstractVideoInfo {
videoInfo.video_streams = extractor.getVideoStreams();
} catch (Exception e) {
videoInfo.addException(
new ExctractionException("Couldn't get video streams", e));
new ExtractionException("Couldn't get video streams", e));
}
/* Extract video only stream url*/
try {
videoInfo.video_only_streams = extractor.getVideoOnlyStreams();
} catch(Exception e) {
videoInfo.addException(
new ExctractionException("Couldn't get video only streams", e));
new ExtractionException("Couldn't get video only streams", e));
}
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream,
@ -114,7 +114,7 @@ public class VideoInfo extends AbstractVideoInfo {
if((videoInfo.video_streams == null || videoInfo.video_streams.isEmpty())
&& (videoInfo.audio_streams == null || videoInfo.audio_streams.isEmpty())
&& (videoInfo.dashMpdUrl == null || videoInfo.dashMpdUrl.isEmpty())) {
throw new ExctractionException("Could not get any stream. See error variable to get further details.");
throw new ExtractionException("Could not get any stream. See error variable to get further details.");
}
return videoInfo;

View File

@ -7,6 +7,7 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.ParsingException;
import org.schabi.newpipe.extractor.SearchEngine;
import org.schabi.newpipe.extractor.VideoPreviewInfo;
@ -18,8 +19,6 @@ import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@ -101,44 +100,40 @@ public class YoutubeSearchEngine implements SearchEngine {
// video item type
} else if (!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href");
// importand information
resultItem.webpage_url = getWebpageUrl(item);
resultItem.id = (new YoutubeVideoUrlIdHandler()).getVideoId(resultItem.webpage_url);
resultItem.title = getTitle(item);
// optional iformation
//todo: make this a proper error handling
try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id = m.group(1);
resultItem.duration = getDuration(item);
} catch (Exception e) {
//e.printStackTrace();
e.printStackTrace();
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
//todo: test against view_count
String viewCountInfo = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").get(1)
.text();
viewCountInfo = viewCountInfo.substring(0, viewCountInfo.indexOf(' '));
viewCountInfo = viewCountInfo.replaceAll("[,.]", "");
viewCountInfo = viewCountInfo.replaceAll("\\s","");
resultItem.view_count = Long.parseLong(viewCountInfo);
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
try {
resultItem.uploader = getUploader(item);
} catch (Exception e) {
e.printStackTrace();
}
try {
resultItem.upload_date = getUploadDate(item);
} catch (Exception e) {
e.printStackTrace();
}
try {
resultItem.view_count = getViewCount(item);
} catch (Exception e) {
e.printStackTrace();
}
try {
resultItem.thumbnail_url = getThumbnailUrl(item);
} catch (Exception e) {
e.printStackTrace();
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
@ -207,4 +202,61 @@ public class YoutubeSearchEngine implements SearchEngine {
}
}
private String getWebpageUrl(Element item) {
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
Element dl = el.select("h3").first().select("a").first();
return dl.attr("abs:href");
}
private String getTitle(Element item) {
Element el = item.select("div[class*=\"yt-lockup-video\"").first();
Element dl = el.select("h3").first().select("a").first();
return dl.text();
}
private String getDuration(Element item) {
try {
return item.select("span[class=\"video-time\"]").first().text();
} catch(Exception e) {
e.printStackTrace();
}
return "";
}
private String getUploader(Element item) {
return item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
}
private String getUploadDate(Element item) {
return item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
}
private long getViewCount(Element item) throws Parser.RegexException{
String output;
String input = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").get(1)
.text();
output = Parser.matchGroup1("([0-9,\\. ])", input).replace(" ", "");
return Long.parseLong(output);
}
private String getThumbnailUrl(Element item) {
String url;
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (url.contains(".gif")) {
url = te.attr("abs:data-thumb");
}
return url;
}
}

View File

@ -1,6 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.ExctractionException;
import org.schabi.newpipe.extractor.ExtractionException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.StreamExtractor;
import org.schabi.newpipe.extractor.StreamingService;
@ -39,7 +39,7 @@ public class YoutubeService implements StreamingService {
}
@Override
public StreamExtractor getExtractorInstance(String url, Downloader downloader)
throws ExctractionException, IOException {
throws ExtractionException, IOException {
VideoUrlIdHandler urlIdHandler = new YoutubeVideoUrlIdHandler();
if(urlIdHandler.acceptUrl(url)) {
return new YoutubeStreamExtractor(url, downloader) ;

View File

@ -10,7 +10,7 @@ import org.jsoup.nodes.Element;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.ExctractionException;
import org.schabi.newpipe.extractor.ExtractionException;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.Parser;
import org.schabi.newpipe.extractor.ParsingException;
@ -47,6 +47,33 @@ import java.util.Vector;
public class YoutubeStreamExtractor implements StreamExtractor {
// exceptions
public class DecryptException extends ParsingException {
DecryptException(Throwable cause) {
super(cause);
}
DecryptException(String message, Throwable cause) {
super(message, cause);
}
}
// special content not available exceptions
public class GemaException extends ContentNotAvailableException {
GemaException(String message) {
super(message);
}
}
public class LiveStreamException extends ContentNotAvailableException {
LiveStreamException() {
super();
}
}
// ----------------
// Sometimes if the html page of youtube is already downloaded, youtube web page will internally
// download the /get_video_info page. Since a certain date dashmpd url is only available over
// this /get_video_info page, so we always need to download this one to.
@ -138,25 +165,6 @@ public class YoutubeStreamExtractor implements StreamExtractor {
throw new ParsingException("itag=" + Integer.toString(itag) + " not supported");
}
public class DecryptException extends ParsingException {
DecryptException(Throwable cause) {
super(cause);
}
DecryptException(String message, Throwable cause) {
super(message, cause);
}
}
// special content not available exceptions
public class GemaException extends ContentNotAvailableException {
GemaException(String message) {
super(message);
}
}
// ----------------
private static final String TAG = YoutubeStreamExtractor.class.toString();
private final Document doc;
private JSONObject playerArgs;
@ -173,7 +181,7 @@ public class YoutubeStreamExtractor implements StreamExtractor {
private Downloader downloader;
public YoutubeStreamExtractor(String pageUrl, Downloader dl) throws ExctractionException, IOException {
public YoutubeStreamExtractor(String pageUrl, Downloader dl) throws ExtractionException, IOException {
//most common videoInfo fields are now set in our superclass, for all services
downloader = dl;
this.pageUrl = pageUrl;
@ -183,11 +191,13 @@ public class YoutubeStreamExtractor implements StreamExtractor {
JSONObject ytPlayerConfig;
//attempt to load the youtube js player JSON arguments
String ps; //used to determine if this is a livestream or not
try {
ytPlayerConfigRaw =
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
ytPlayerConfig = new JSONObject(ytPlayerConfigRaw);
playerArgs = ytPlayerConfig.getJSONObject("args");
ps = playerArgs.get("ps").toString();
} catch (Parser.RegexException e) {
String errorReason = findErrorReason(doc);
switch(errorReason) {
@ -199,7 +209,10 @@ public class YoutubeStreamExtractor implements StreamExtractor {
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config");
throw new ParsingException("Could not parse yt player config", e);
}
if (ps.equals("live")) {
throw new LiveStreamException();
}

View File

@ -37,7 +37,7 @@ public class YoutubeVideoUrlIdHandler implements VideoUrlIdHandler {
@SuppressWarnings("WeakerAccess")
@Override
public String getVideoId(String url) throws ParsingException {
String id = "";
String id;
if(url.contains("youtube")) {
if(url.contains("attribution_link")) {

View File

@ -84,6 +84,7 @@
<string name="content_not_available">Content not available.</string>
<string name="blocked_by_gema">Blocked by GEMA.</string>
<string name="could_not_setup_download_menu">Could not setup download menu.</string>
<string name="live_streams_not_supported">This is a LIVE STREAM. These are not yet supported.</string>
<!-- Content descriptions (for better accessibility) -->