From 34042314576df4f7717bd3a5f35a8c8d00cb4b22 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Wed, 1 Mar 2017 16:40:04 +0100 Subject: [PATCH 1/5] get rid of xul xml parser --- .../newpipe/extractor/DashMpdParser.java | 82 +++++++------------ 1 file changed, 31 insertions(+), 51 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java b/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java index c8ef7f1ac..527750a86 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java @@ -1,17 +1,21 @@ package org.schabi.newpipe.extractor; -import android.util.Xml; - import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream_info.AudioStream; -import org.xmlpull.v1.XmlPullParser; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.StringReader; +import java.io.InputStream; import java.util.List; import java.util.Vector; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + /** * Created by Christian Schabesberger on 02.02.16. * @@ -55,57 +59,33 @@ public class DashMpdParser { throw new ReCaptchaException("reCaptcha Challenge needed"); } Vector audioStreams = new Vector<>(); + try { - XmlPullParser parser = Xml.newPullParser(); - parser.setInput(new StringReader(dashDoc)); - String tagName = ""; - String currentMimeType = ""; - int currentBandwidth = -1; - int currentSamplingRate = -1; - boolean currentTagIsBaseUrl = false; - for(int eventType = parser.getEventType(); - eventType != XmlPullParser.END_DOCUMENT; - eventType = parser.next() ) { - switch(eventType) { - case XmlPullParser.START_TAG: - tagName = parser.getName(); - if(tagName.equals("AdaptationSet")) { - currentMimeType = parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "mimeType"); - } else if(tagName.equals("Representation") && currentMimeType.contains("audio")) { - currentBandwidth = Integer.parseInt( - parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "bandwidth")); - currentSamplingRate = Integer.parseInt( - parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "audioSamplingRate")); - } else if(tagName.equals("BaseURL")) { - currentTagIsBaseUrl = true; - } - break; + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + InputStream stream = new ByteArrayInputStream(dashDoc.getBytes()); - case XmlPullParser.TEXT: - // actual stream tag - if(currentTagIsBaseUrl && - (currentMimeType.contains("audio"))) { - int format = -1; - if(currentMimeType.equals(MediaFormat.WEBMA.mimeType)) { - format = MediaFormat.WEBMA.id; - } else if(currentMimeType.equals(MediaFormat.M4A.mimeType)) { - format = MediaFormat.M4A.id; - } - audioStreams.add(new AudioStream(parser.getText(), - format, currentBandwidth, currentSamplingRate)); - } - break; - - case XmlPullParser.END_TAG: - if(tagName.equals("AdaptationSet")) { - currentMimeType = ""; - } else if(tagName.equals("BaseURL")) { - currentTagIsBaseUrl = false; - } - break; + Document doc = builder.parse(stream); + NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet"); + for(int i = 0; i < adaptationSetList.getLength(); i++) { + Element adaptationSet = (Element) adaptationSetList.item(i); + String memeType = adaptationSet.getAttribute("mimeType"); + if(memeType.contains("audio")) { + Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0); + String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent(); + int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth")); + int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate")); + int format = -1; + if(memeType.equals(MediaFormat.WEBMA.mimeType)) { + format = MediaFormat.WEBMA.id; + } else if(memeType.equals(MediaFormat.M4A.mimeType)) { + format = MediaFormat.M4A.id; + } + audioStreams.add(new AudioStream(url, format, bandwidth, samplingRate)); } } - } catch(Exception e) { + } + catch(Exception e) { throw new DashMpdParsingException("Could not parse Dash mpd", e); } return audioStreams; From 780f1d5da35bac7498403b550e1bd8199f83417c Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Wed, 1 Mar 2017 18:03:36 +0100 Subject: [PATCH 2/5] made extractor systemindipendent again --- .../extractor/services/youtube/YoutubeStreamExtractor.java | 1 - .../services/youtube/YoutubeStreamUrlIdHandler.java | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 52a42f645..2df361644 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -10,7 +10,6 @@ import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; import org.schabi.newpipe.extractor.AbstractStreamInfo; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.stream_info.AudioStream; diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java index 055cd5970..8c08a6101 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java @@ -1,7 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube; -import android.support.annotation.NonNull; - import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.Parser; @@ -112,7 +110,7 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler { * @return the id of the stream * @throws ParsingException */ - private @NonNull String getRealIdFromSharedLink(String url) throws ParsingException { + private String getRealIdFromSharedLink(String url) throws ParsingException { URI uri; try { uri = new URI(url); @@ -135,7 +133,7 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler { return realId; } - private @NonNull String getSharedId(URI uri) throws ParsingException { + private String getSharedId(URI uri) throws ParsingException { if (!"/shared".equals(uri.getPath())) { throw new ParsingException("Not a shared link: " + uri.toString() + " (path != " + uri.getPath() + ")"); } From cffd049c8a3dc1a58601389873f848912e43c7bb Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Wed, 1 Mar 2017 19:06:27 +0100 Subject: [PATCH 3/5] remove extractor --- .../newpipe/extractor/AbstractStreamInfo.java | 41 - .../newpipe/extractor/DashMpdParser.java | 93 -- .../schabi/newpipe/extractor/Downloader.java | 52 -- .../schabi/newpipe/extractor/InfoItem.java | 33 - .../newpipe/extractor/InfoItemCollector.java | 61 -- .../schabi/newpipe/extractor/MediaFormat.java | 83 -- .../org/schabi/newpipe/extractor/NewPipe.java | 88 -- .../org/schabi/newpipe/extractor/Parser.java | 77 -- .../schabi/newpipe/extractor/ServiceList.java | 13 - .../newpipe/extractor/StreamingService.java | 78 -- .../extractor/SuggestionExtractor.java | 43 - .../newpipe/extractor/UrlIdHandler.java | 35 - .../extractor/channel/ChannelExtractor.java | 62 -- .../extractor/channel/ChannelInfo.java | 86 -- .../extractor/channel/ChannelInfoItem.java | 44 - .../channel/ChannelInfoItemCollector.java | 74 -- .../channel/ChannelInfoItemExtractor.java | 32 - .../exceptions/ExtractionException.java | 33 - .../exceptions/FoundAdException.java | 30 - .../exceptions/ParsingException.java | 31 - .../exceptions/ReCaptchaException.java | 27 - .../search/InfoItemSearchCollector.java | 79 -- .../extractor/search/SearchEngine.java | 53 -- .../extractor/search/SearchResult.java | 56 -- .../youtube/YoutubeChannelExtractor.java | 361 ------- .../YoutubeChannelInfoItemExtractor.java | 83 -- .../youtube/YoutubeChannelUrlIdHandler.java | 47 - .../youtube/YoutubeParsingHelper.java | 66 -- .../services/youtube/YoutubeSearchEngine.java | 119 --- .../services/youtube/YoutubeService.java | 82 -- .../youtube/YoutubeStreamExtractor.java | 878 ------------------ .../YoutubeStreamInfoItemExtractor.java | 186 ---- .../youtube/YoutubeStreamUrlIdHandler.java | 163 ---- .../youtube/YoutubeSuggestionExtractor.java | 99 -- .../extractor/stream_info/AudioStream.java | 46 - .../stream_info/StreamExtractor.java | 104 --- .../extractor/stream_info/StreamInfo.java | 294 ------ .../extractor/stream_info/StreamInfoItem.java | 41 - .../stream_info/StreamInfoItemCollector.java | 102 -- .../stream_info/StreamInfoItemExtractor.java | 36 - .../extractor/stream_info/VideoStream.java | 44 - 41 files changed, 4055 deletions(-) delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/AbstractStreamInfo.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/Downloader.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/InfoItem.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/NewPipe.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/Parser.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/ServiceList.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/exceptions/ExtractionException.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/exceptions/FoundAdException.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/exceptions/ParsingException.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/stream_info/AudioStream.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfo.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItem.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemCollector.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemExtractor.java delete mode 100644 app/src/main/java/org/schabi/newpipe/extractor/stream_info/VideoStream.java diff --git a/app/src/main/java/org/schabi/newpipe/extractor/AbstractStreamInfo.java b/app/src/main/java/org/schabi/newpipe/extractor/AbstractStreamInfo.java deleted file mode 100644 index bfa86b3f7..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/AbstractStreamInfo.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.schabi.newpipe.extractor; - -/** - * Copyright (C) Christian Schabesberger 2016 - * AbstractStreamInfo.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/**Common properties between StreamInfo and StreamInfoItem.*/ -public abstract class AbstractStreamInfo { - public enum StreamType { - NONE, // placeholder to check if stream type was checked or not - VIDEO_STREAM, - AUDIO_STREAM, - LIVE_STREAM, - AUDIO_LIVE_STREAM, - FILE - } - - public StreamType stream_type; - public int service_id = -1; - public String id = ""; - public String title = ""; - public String uploader = ""; - public String thumbnail_url = ""; - public String webpage_url = ""; - public String upload_date = ""; - public long view_count = -1; -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java b/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java deleted file mode 100644 index 527750a86..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.schabi.newpipe.extractor; - -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.stream_info.AudioStream; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Vector; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -/** - * Created by Christian Schabesberger on 02.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * DashMpdParser.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class DashMpdParser { - - private DashMpdParser() { - } - - static class DashMpdParsingException extends ParsingException { - DashMpdParsingException(String message, Exception e) { - super(message, e); - } - } - - public static List getAudioStreams(String dashManifestUrl) - throws DashMpdParsingException, ReCaptchaException { - String dashDoc; - Downloader downloader = NewPipe.getDownloader(); - try { - dashDoc = downloader.download(dashManifestUrl); - } catch(IOException ioe) { - throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe); - } catch (ReCaptchaException e) { - throw new ReCaptchaException("reCaptcha Challenge needed"); - } - Vector audioStreams = new Vector<>(); - - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - InputStream stream = new ByteArrayInputStream(dashDoc.getBytes()); - - Document doc = builder.parse(stream); - NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet"); - for(int i = 0; i < adaptationSetList.getLength(); i++) { - Element adaptationSet = (Element) adaptationSetList.item(i); - String memeType = adaptationSet.getAttribute("mimeType"); - if(memeType.contains("audio")) { - Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0); - String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent(); - int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth")); - int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate")); - int format = -1; - if(memeType.equals(MediaFormat.WEBMA.mimeType)) { - format = MediaFormat.WEBMA.id; - } else if(memeType.equals(MediaFormat.M4A.mimeType)) { - format = MediaFormat.M4A.id; - } - audioStreams.add(new AudioStream(url, format, bandwidth, samplingRate)); - } - } - } - catch(Exception e) { - throw new DashMpdParsingException("Could not parse Dash mpd", e); - } - return audioStreams; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java b/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java deleted file mode 100644 index fe85696b5..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.schabi.newpipe.extractor; - -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; - -import java.io.IOException; -import java.util.Map; - -/** - * Created by Christian Schabesberger on 28.01.16. - * - * Copyright (C) Christian Schabesberger 2016 - * Downloader.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public interface Downloader { - - /**Download the text file at the supplied URL as in download(String), - * but set the HTTP header field "Accept-Language" to the supplied string. - * @param siteUrl the URL of the text file to return the contents of - * @param language the language (usually a 2-character code) to set as the preferred language - * @return the contents of the specified text file - * @throws IOException*/ - String download(String siteUrl, String language) throws IOException, ReCaptchaException; - - /**Download the text file at the supplied URL as in download(String), - * but set the HTTP header field "Accept-Language" to the supplied string. - * @param siteUrl the URL of the text file to return the contents of - * @param customProperties set request header properties - * @return the contents of the specified text file - * @throws IOException*/ - String download(String siteUrl, Map customProperties) throws IOException, ReCaptchaException; - - /**Download (via HTTP) the text file located at the supplied URL, and return its contents. - * Primarily intended for downloading web pages. - * @param siteUrl the URL of the text file to download - * @return the contents of the specified text file - * @throws IOException*/ - String download(String siteUrl) throws IOException, ReCaptchaException; -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/InfoItem.java b/app/src/main/java/org/schabi/newpipe/extractor/InfoItem.java deleted file mode 100644 index a3395e8e4..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/InfoItem.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.schabi.newpipe.extractor; - -/** - * Created by the-scrabi on 11.02.17. - * - * Copyright (C) Christian Schabesberger 2017 - * InfoItem.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public interface InfoItem { - public enum InfoType { - STREAM, - PLAYLIST, - CHANNEL - } - - InfoType infoType(); - String getTitle(); - String getLink(); -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java b/app/src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java deleted file mode 100644 index d3b0927a1..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.schabi.newpipe.extractor; - -import org.schabi.newpipe.extractor.exceptions.ExtractionException; - -import java.util.List; -import java.util.Vector; - -/** - * Created by Christian Schabesberger on 12.02.17. - * - * Copyright (C) Christian Schabesberger 2017 - * InfoItemCollector.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class InfoItemCollector { - private List itemList = new Vector<>(); - private List errors = new Vector<>(); - private int serviceId = -1; - - public InfoItemCollector(int serviceId) { - this.serviceId = serviceId; - } - - public List getItemList() { - return itemList; - } - public List getErrors() { - return errors; - } - protected void addFromCollector(InfoItemCollector otherC) throws ExtractionException { - if(serviceId != otherC.serviceId) { - throw new ExtractionException("Service Id does not equal: " - + NewPipe.getNameOfService(serviceId) - + " and " + NewPipe.getNameOfService(otherC.serviceId)); - } - errors.addAll(otherC.errors); - itemList.addAll(otherC.itemList); - } - protected void addError(Exception e) { - errors.add(e); - } - protected void addItem(InfoItem item) { - itemList.add(item); - } - protected int getServiceId() { - return serviceId; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/app/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java deleted file mode 100644 index 1455fe396..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.schabi.newpipe.extractor; - -/** - * Created by Adam Howard on 08/11/15. - * - * Copyright (c) Christian Schabesberger - * and Adam Howard 2015 - * - * MediaFormat.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/**Static data about various media formats support by Newpipe, eg mime type, extension*/ - -public enum MediaFormat { - //video and audio combined formats - // id name suffix mime type - MPEG_4 (0x0, "MPEG-4", "mp4", "video/mp4"), - v3GPP (0x1, "3GPP", "3gp", "video/3gpp"), - WEBM (0x2, "WebM", "webm", "video/webm"), - // audio formats - M4A (0x3, "m4a", "m4a", "audio/mp4"), - WEBMA (0x4, "WebM", "webm", "audio/webm"); - - public final int id; - @SuppressWarnings("WeakerAccess") - public final String name; - @SuppressWarnings("WeakerAccess") - public final String suffix; - public final String mimeType; - - MediaFormat(int id, String name, String suffix, String mimeType) { - this.id = id; - this.name = name; - this.suffix = suffix; - this.mimeType = mimeType; - } - - /**Return the friendly name of the media format with the supplied id - * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number. - * @return the friendly name of the MediaFormat associated with this ids, - * or an empty String if none match it.*/ - public static String getNameById(int ident) { - for (MediaFormat vf : MediaFormat.values()) { - if(vf.id == ident) return vf.name; - } - return ""; - } - - /**Return the file extension of the media format with the supplied id - * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number. - * @return the file extension of the MediaFormat associated with this ids, - * or an empty String if none match it.*/ - public static String getSuffixById(int ident) { - for (MediaFormat vf : MediaFormat.values()) { - if(vf.id == ident) return vf.suffix; - } - return ""; - } - - /**Return the MIME type of the media format with the supplied id - * @param ident the id of the media format. Currently an arbitrary, NewPipe-specific number. - * @return the MIME type of the MediaFormat associated with this ids, - * or an empty String if none match it.*/ - public static String getMimeById(int ident) { - for (MediaFormat vf : MediaFormat.values()) { - if(vf.id == ident) return vf.mimeType; - } - return ""; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/app/src/main/java/org/schabi/newpipe/extractor/NewPipe.java deleted file mode 100644 index 4e8e4aa3d..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.schabi.newpipe.extractor; - -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.services.youtube.YoutubeService; - -/** - * Created by Christian Schabesberger on 23.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * NewPipe.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/**Provides access to the video streaming services supported by NewPipe. - * Currently only Youtube until the API becomes more stable.*/ - -@SuppressWarnings("ALL") -public class NewPipe { - - private NewPipe() { - } - - private static final String TAG = NewPipe.class.toString(); - - - private static Downloader downloader = null; - - public static StreamingService[] getServices() { - return ServiceList.serviceList; - } - public static StreamingService getService(int serviceId)throws ExtractionException { - for(StreamingService s : ServiceList.serviceList) { - if(s.getServiceId() == serviceId) { - return s; - } - } - return null; - } - public static StreamingService getService(String serviceName) throws ExtractionException { - return ServiceList.serviceList[getIdOfService(serviceName)]; - } - public static String getNameOfService(int id) { - try { - return getService(id).getServiceInfo().name; - } catch (Exception e) { - System.err.println("Service id not known"); - e.printStackTrace(); - return ""; - } - } - public static int getIdOfService(String serviceName) { - for(int i = 0; i < ServiceList.serviceList.length; i++) { - if(ServiceList.serviceList[i].getServiceInfo().name.equals(serviceName)) { - return i; - } - } - return -1; - } - - public static void init(Downloader d) { - downloader = d; - } - - public static Downloader getDownloader() { - return downloader; - } - - public static StreamingService getServiceByUrl(String url) { - for(StreamingService s : ServiceList.serviceList) { - if(s.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) { - return s; - } - } - return null; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/Parser.java b/app/src/main/java/org/schabi/newpipe/extractor/Parser.java deleted file mode 100644 index 8e70f751c..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/Parser.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.schabi.newpipe.extractor; - -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Created by Christian Schabesberger on 02.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * Parser.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/** avoid using regex !!! */ -public class Parser { - - private Parser() { - } - - public static class RegexException extends ParsingException { - public RegexException(String message) { - super(message); - } - } - - public static String matchGroup1(String pattern, String input) throws RegexException { - return matchGroup(pattern, input, 1); - } - - public static String matchGroup(String pattern, String input, int group) throws RegexException { - Pattern pat = Pattern.compile(pattern); - Matcher mat = pat.matcher(input); - boolean foundMatch = mat.find(); - if (foundMatch) { - return mat.group(group); - } - else { - //Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); - if(input.length() > 1024) { - throw new RegexException("failed to find pattern \""+pattern); - } else { - throw new RegexException("failed to find pattern \"" + pattern + " inside of " + input + "\""); - } - } - } - - public static Map compatParseMap(final String input) throws UnsupportedEncodingException { - Map map = new HashMap<>(); - for(String arg : input.split("&")) { - String[] splitArg = arg.split("="); - if(splitArg.length > 1) { - map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8")); - } else { - map.put(splitArg[0], ""); - } - } - return map; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/app/src/main/java/org/schabi/newpipe/extractor/ServiceList.java deleted file mode 100644 index 89b350cb4..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.schabi.newpipe.extractor; - -import org.schabi.newpipe.extractor.services.youtube.YoutubeService; - -/** - * Created by the-scrabi on 18.02.17. - */ - -class ServiceList { - public static final StreamingService[] serviceList = { - new YoutubeService(0) - }; -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java deleted file mode 100644 index ed3c17bfc..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.schabi.newpipe.extractor; - -import org.schabi.newpipe.extractor.channel.ChannelExtractor; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.search.SearchEngine; -import org.schabi.newpipe.extractor.stream_info.StreamExtractor; - -import java.io.IOException; - -/** - * Created by Christian Schabesberger on 23.08.15. - * - * Copyright (C) Christian Schabesberger 2016 - * StreamingService.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public abstract class StreamingService { - public class ServiceInfo { - public String name = ""; - } - - public enum LinkType { - NONE, - STREAM, - CHANNEL, - PLAYLIST - } - - private int serviceId; - - public StreamingService(int id) { - serviceId = id; - } - - public abstract ServiceInfo getServiceInfo(); - - public abstract StreamExtractor getExtractorInstance(String url) - throws IOException, ExtractionException; - public abstract SearchEngine getSearchEngineInstance(); - public abstract UrlIdHandler getStreamUrlIdHandlerInstance(); - public abstract UrlIdHandler getChannelUrlIdHandlerInstance(); - public abstract ChannelExtractor getChannelExtractorInstance(String url, int page) - throws ExtractionException, IOException; - public abstract SuggestionExtractor getSuggestionExtractorInstance(); - - public final int getServiceId() { - return serviceId; - } - - /** - * figure out where the link is pointing to (a channel, video, playlist, etc.) - */ - public final LinkType getLinkTypeByUrl(String url) { - UrlIdHandler sH = getStreamUrlIdHandlerInstance(); - UrlIdHandler cH = getChannelUrlIdHandlerInstance(); - - if(sH.acceptUrl(url)) { - return LinkType.STREAM; - } else if(cH.acceptUrl(url)) { - return LinkType.CHANNEL; - } else { - return LinkType.NONE; - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java deleted file mode 100644 index f198bc5e0..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.schabi.newpipe.extractor; - -import org.schabi.newpipe.extractor.exceptions.ExtractionException; - -import java.io.IOException; -import java.util.List; - -/** - * Created by Christian Schabesberger on 28.09.16. - * - * Copyright (C) Christian Schabesberger 2016 - * SuggestionExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public abstract class SuggestionExtractor { - - private int serviceId; - - public SuggestionExtractor(int serviceId) { - this.serviceId = serviceId; - } - - public abstract List suggestionList( - String query,String contentCountry) - throws ExtractionException, IOException; - - public int getServiceId() { - return serviceId; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java deleted file mode 100644 index 1218e7b8c..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.schabi.newpipe.extractor; - -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -/** - * Created by Christian Schabesberger on 26.07.16. - * - * Copyright (C) Christian Schabesberger 2016 - * UrlIdHandler.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public interface UrlIdHandler { - - String getUrl(String videoId); - String getId(String siteUrl) throws ParsingException; - String cleanUrl(String siteUrl) throws ParsingException; - - /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling - Intent was meant to be watched with this Service. - Return false if this service shall not allow to be called through ACTIONs.*/ - boolean acceptUrl(String videoUrl); -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java deleted file mode 100644 index fe7b9f7eb..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.schabi.newpipe.extractor.channel; - -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector; - -import java.io.IOException; - -/** - * Created by Christian Schabesberger on 25.07.16. - * - * Copyright (C) Christian Schabesberger 2016 - * ChannelExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public abstract class ChannelExtractor { - private int serviceId; - private String url; - private UrlIdHandler urlIdHandler; - private StreamInfoItemCollector previewInfoCollector; - private int page = -1; - - public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int page, int serviceId) - throws ExtractionException, IOException { - this.url = url; - this.page = page; - this.serviceId = serviceId; - this.urlIdHandler = urlIdHandler; - previewInfoCollector = new StreamInfoItemCollector(urlIdHandler, serviceId); - } - - public String getUrl() { return url; } - public UrlIdHandler getUrlIdHandler() { return urlIdHandler; } - public StreamInfoItemCollector getStreamPreviewInfoCollector() { - return previewInfoCollector; - } - - public abstract String getChannelName() throws ParsingException; - public abstract String getAvatarUrl() throws ParsingException; - public abstract String getBannerUrl() throws ParsingException; - public abstract String getFeedUrl() throws ParsingException; - public abstract StreamInfoItemCollector getStreams() throws ParsingException; - public abstract long getSubscriberCount() throws ParsingException; - public abstract boolean hasNextPage() throws ParsingException; - public int getServiceId() { - return serviceId; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java deleted file mode 100644 index e1fd96ada..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.schabi.newpipe.extractor.channel; - -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItem; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector; - -import java.util.List; -import java.util.Vector; - -/** - * Created by Christian Schabesberger on 31.07.16. - * - * Copyright (C) Christian Schabesberger 2016 - * ChannelInfo.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class ChannelInfo { - public void addException(Exception e) { - errors.add(e); - } - - public static ChannelInfo getInfo(ChannelExtractor extractor) - throws ParsingException { - ChannelInfo info = new ChannelInfo(); - - // importand data - info.service_id = extractor.getServiceId(); - info.channel_name = extractor.getChannelName(); - info.hasNextPage = extractor.hasNextPage(); - - try { - info.avatar_url = extractor.getAvatarUrl(); - } catch (Exception e) { - info.errors.add(e); - } - try { - info.banner_url = extractor.getBannerUrl(); - } catch (Exception e) { - info.errors.add(e); - } - try { - info.feed_url = extractor.getFeedUrl(); - } catch(Exception e) { - info.errors.add(e); - } - try { - StreamInfoItemCollector c = extractor.getStreams(); - info.related_streams = c.getItemList(); - info.errors.addAll(c.getErrors()); - } catch(Exception e) { - info.errors.add(e); - } - try { - info.subscriberCount = extractor.getSubscriberCount(); - } catch (Exception e) { - info.errors.add(e); - } - - return info; - } - - public int service_id = -1; - public String channel_name = ""; - public String avatar_url = ""; - public String banner_url = ""; - public String feed_url = ""; - public List related_streams = null; - public long subscriberCount = -1; - public boolean hasNextPage = false; - - public List errors = new Vector<>(); -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java b/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java deleted file mode 100644 index b834d28e6..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.schabi.newpipe.extractor.channel; - -import org.schabi.newpipe.extractor.InfoItem; - -/** - * Created by Christian Schabesberger on 11.02.17. - * - * Copyright (C) Christian Schabesberger 2017 - * ChannelInfoItem.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class ChannelInfoItem implements InfoItem { - - public int serviceId = -1; - public String channelName = ""; - public String thumbnailUrl = ""; - public String webPageUrl = ""; - public String description = ""; - public long subscriberCount = -1; - public int videoAmount = -1; - - public InfoType infoType() { - return InfoType.CHANNEL; - } - public String getTitle() { - return channelName; - } - public String getLink() { - return webPageUrl; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java b/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java deleted file mode 100644 index 525820d44..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.schabi.newpipe.extractor.channel; - -import org.schabi.newpipe.extractor.InfoItemCollector; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItem; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor; - -/** - * Created by Christian Schabesberger on 12.02.17. - * - * Copyright (C) Christian Schabesberger 2017 - * ChannelInfoItemCollector.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class ChannelInfoItemCollector extends InfoItemCollector { - public ChannelInfoItemCollector(int serviceId) { - super(serviceId); - } - - public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException { - ChannelInfoItem resultItem = new ChannelInfoItem(); - // importand information - resultItem.channelName = extractor.getChannelName(); - - resultItem.serviceId = getServiceId(); - resultItem.webPageUrl = extractor.getWebPageUrl(); - - // optional information - try { - resultItem.subscriberCount = extractor.getSubscriberCount(); - } catch (Exception e) { - addError(e); - } - try { - resultItem.videoAmount = extractor.getVideoAmount(); - } catch (Exception e) { - addError(e); - } - try { - resultItem.thumbnailUrl = extractor.getThumbnailUrl(); - } catch (Exception e) { - addError(e); - } - try { - resultItem.description = extractor.getDescription(); - } catch (Exception e) { - addError(e); - } - return resultItem; - } - - public void commit(ChannelInfoItemExtractor extractor) throws ParsingException { - try { - addItem(extract(extractor)); - } catch (Exception e) { - addError(e); - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java deleted file mode 100644 index 864415cdd..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.schabi.newpipe.extractor.channel; - -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -/** - * Created by Christian Schabesberger on 12.02.17. - * - * Copyright (C) Christian Schabesberger 2017 - * ChannelInfoItemExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public interface ChannelInfoItemExtractor { - String getThumbnailUrl() throws ParsingException; - String getChannelName() throws ParsingException; - String getWebPageUrl() throws ParsingException; - String getDescription() throws ParsingException; - long getSubscriberCount() throws ParsingException; - int getVideoAmount() throws ParsingException; -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/exceptions/ExtractionException.java b/app/src/main/java/org/schabi/newpipe/extractor/exceptions/ExtractionException.java deleted file mode 100644 index 311850187..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/exceptions/ExtractionException.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.schabi.newpipe.extractor.exceptions; - -/** - * Created by Christian Schabesberger on 30.01.16. - * - * Copyright (C) Christian Schabesberger 2016 - * ExtractionException.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class ExtractionException extends Exception { - public ExtractionException(String message) { - super(message); - } - public ExtractionException(Throwable cause) { - super(cause); - } - public ExtractionException(String message, Throwable cause) { - super(message, cause); - } -} \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/extractor/exceptions/FoundAdException.java b/app/src/main/java/org/schabi/newpipe/extractor/exceptions/FoundAdException.java deleted file mode 100644 index 4d6d29826..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/exceptions/FoundAdException.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.schabi.newpipe.extractor.exceptions; - -/** - * Created by Christian Schabesberger on 12.09.16. - * - * Copyright (C) Christian Schabesberger 2016 - * FoundAdException.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class FoundAdException extends ParsingException { - public FoundAdException(String message) { - super(message); - } - public FoundAdException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/exceptions/ParsingException.java b/app/src/main/java/org/schabi/newpipe/extractor/exceptions/ParsingException.java deleted file mode 100644 index 41a7acce1..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/exceptions/ParsingException.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.schabi.newpipe.extractor.exceptions; - -/** - * Created by Christian Schabesberger on 31.01.16. - * - * Copyright (C) Christian Schabesberger 2016 - * ParsingException.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - - -public class ParsingException extends ExtractionException { - public ParsingException(String message) { - super(message); - } - public ParsingException(String message, Throwable cause) { - super(message, cause); - } -} \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java b/app/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java deleted file mode 100644 index a28ec99f9..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.schabi.newpipe.extractor.exceptions; - -/** - * Created by beneth on 07.12.16. - * - * Copyright (C) Christian Schabesberger 2016 - * ReCaptchaException.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class ReCaptchaException extends ExtractionException { - public ReCaptchaException(String message) { - super(message); - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java b/app/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java deleted file mode 100644 index b7ec0e3c1..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.schabi.newpipe.extractor.search; - -import org.schabi.newpipe.extractor.InfoItemCollector; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector; -import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor; - -/** - * Created by Christian Schabesberger on 12.02.17. - * - * Copyright (C) Christian Schabesberger 2017 - * InfoItemSearchCollector.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class InfoItemSearchCollector extends InfoItemCollector { - private String suggestion = ""; - private StreamInfoItemCollector streamCollector; - private ChannelInfoItemCollector channelCollector; - - SearchResult result = new SearchResult(); - - InfoItemSearchCollector(UrlIdHandler handler, int serviceId) { - super(serviceId); - streamCollector = new StreamInfoItemCollector(handler, serviceId); - channelCollector = new ChannelInfoItemCollector(serviceId); - } - - public void setSuggestion(String suggestion) { - this.suggestion = suggestion; - } - - public SearchResult getSearchResult() throws ExtractionException { - - addFromCollector(channelCollector); - addFromCollector(streamCollector); - - result.suggestion = suggestion; - result.errors = getErrors(); - return result; - } - - public void commit(StreamInfoItemExtractor extractor) { - try { - result.resultList.add(streamCollector.extract(extractor)); - } catch(FoundAdException ae) { - System.err.println("Found add"); - } catch (Exception e) { - addError(e); - } - } - - public void commit(ChannelInfoItemExtractor extractor) { - try { - result.resultList.add(channelCollector.extract(extractor)); - } catch(FoundAdException ae) { - System.err.println("Found add"); - } catch (Exception e) { - addError(e); - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java b/app/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java deleted file mode 100644 index 9cef050eb..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.schabi.newpipe.extractor.search; - -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector; - -import java.io.IOException; -import java.util.EnumSet; - -/** - * Created by Christian Schabesberger on 10.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * SearchEngine.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public abstract class SearchEngine { - public enum Filter { - STREAM, CHANNEL, PLAY_LIST - } - - public static class NothingFoundException extends ExtractionException { - public NothingFoundException(String message) { - super(message); - } - } - private InfoItemSearchCollector collector; - - public SearchEngine(UrlIdHandler urlIdHandler, int serviceId) { - collector = new InfoItemSearchCollector(urlIdHandler, serviceId); - } - - protected InfoItemSearchCollector getInfoItemSearchCollector() { - return collector; - } - //Result search(String query, int page); - public abstract InfoItemSearchCollector search( - String query, int page, String contentCountry, EnumSet filter) - throws ExtractionException, IOException; -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java b/app/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java deleted file mode 100644 index 155b43747..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.schabi.newpipe.extractor.search; - -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItem; - -import java.io.IOException; -import java.util.EnumSet; -import java.util.List; -import java.util.Vector; - -/** - * Created by Christian Schabesberger on 29.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * SearchResult.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class SearchResult { - public static SearchResult getSearchResult(SearchEngine engine, String query, - int page, String languageCode, EnumSet filter) - throws ExtractionException, IOException { - - SearchResult result = engine - .search(query, page, languageCode, filter) - .getSearchResult(); - if(result.resultList.isEmpty()) { - if(result.suggestion.isEmpty()) { - if(result.errors.isEmpty()) { - throw new ExtractionException("Empty result despite no error"); - } - } else { - // This is used as a fallback. Do not relay on it !!! - throw new SearchEngine.NothingFoundException(result.suggestion); - } - } - return result; - } - - public String suggestion = ""; - public List resultList = new Vector<>(); - public List errors = new Vector<>(); -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java deleted file mode 100644 index 0b8fda717..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java +++ /dev/null @@ -1,361 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - - - -import org.json.JSONException; -import org.json.JSONObject; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.AbstractStreamInfo; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.Parser; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.channel.ChannelExtractor; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor; - - -import java.io.IOException; - -/** - * Created by Christian Schabesberger on 25.07.16. - * - * Copyright (C) Christian Schabesberger 2016 - * YoutubeChannelExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeChannelExtractor extends ChannelExtractor { - - private static final String TAG = YoutubeChannelExtractor.class.toString(); - - // private CSSOMParser cssParser = new CSSOMParser(new SACParserCSS3()); - - private Document doc = null; - - private boolean isAjaxPage = false; - private static String userUrl = ""; - private static String channelName = ""; - private static String avatarUrl = ""; - private static String bannerUrl = ""; - private static String feedUrl = ""; - private static long subscriberCount = -1; - // the fist page is html all other pages are ajax. Every new page can be requested by sending - // this request url. - private static String nextPageUrl = ""; - - public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int page, int serviceId) - throws ExtractionException, IOException { - super(urlIdHandler, url, page, serviceId); - - Downloader downloader = NewPipe.getDownloader(); - - url = urlIdHandler.cleanUrl(url) ; //+ "/video?veiw=0&flow=list&sort=dd"; - - if(page == 0) { - if (isUserUrl(url)) { - userUrl = url; - } else { - // we first need to get the user url. Otherwise we can't find videos - String channelPageContent = downloader.download(url); - Document channelDoc = Jsoup.parse(channelPageContent, url); - userUrl = getUserUrl(channelDoc); - } - - userUrl = userUrl + "/videos?veiw=0&flow=list&sort=dd&live_view=10000"; - String pageContent = downloader.download(userUrl); - doc = Jsoup.parse(pageContent, userUrl); - nextPageUrl = getNextPageUrl(doc); - isAjaxPage = false; - } else { - String ajaxDataRaw = downloader.download(nextPageUrl); - JSONObject ajaxData; - try { - ajaxData = new JSONObject(ajaxDataRaw); - String htmlDataRaw = ajaxData.getString("content_html"); - doc = Jsoup.parse(htmlDataRaw, nextPageUrl); - - String nextPageHtmlDataRaw = ajaxData.getString("load_more_widget_html"); - if(!nextPageHtmlDataRaw.isEmpty()) { - Document nextPageData = Jsoup.parse(nextPageHtmlDataRaw, nextPageUrl); - nextPageUrl = getNextPageUrl(nextPageData); - } else { - nextPageUrl = ""; - } - } catch (JSONException e) { - throw new ParsingException("Could not parse json data for next page", e); - } - isAjaxPage = true; - } - } - - @Override - public String getChannelName() throws ParsingException { - try { - if(!isAjaxPage) { - channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first() - .select("a").first().text(); - } - return channelName; - } catch(Exception e) { - throw new ParsingException("Could not get channel name"); - } - } - - @Override - public String getAvatarUrl() throws ParsingException { - try { - if(!isAjaxPage) { - avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]") - .first().attr("abs:src"); - } - return avatarUrl; - } catch(Exception e) { - throw new ParsingException("Could not get avatar", e); - } - } - - @Override - public String getBannerUrl() throws ParsingException { - try { - if(!isAjaxPage) { - Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first(); - String cssContent = el.html(); - String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent); - - if (url.contains("s.ytimg.com") || url.contains("default_banner")) { - bannerUrl = null; - } else { - bannerUrl = url; - } - } - return bannerUrl; - } catch(Exception e) { - throw new ParsingException("Could not get Banner", e); - } - } - - @Override - public StreamInfoItemCollector getStreams() throws ParsingException { - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); - Element ul; - if(isAjaxPage) { - ul = doc.select("body").first(); - } else { - ul = doc.select("ul[id=\"browse-items-primary\"]").first(); - } - - for(final Element li : ul.children()) { - if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) { - collector.commit(new StreamInfoItemExtractor() { - @Override - public AbstractStreamInfo.StreamType getStreamType() throws ParsingException { - return AbstractStreamInfo.StreamType.VIDEO_STREAM; - } - - @Override - public boolean isAd() throws ParsingException { - if(!li.select("span[class*=\"icon-not-available\"]").isEmpty()) { - return true; - } else { - return false; - } - } - - @Override - public String getWebPageUrl() throws ParsingException { - try { - Element el = li.select("div[class=\"feed-item-dismissable\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.attr("abs:href"); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getTitle() throws ParsingException { - try { - Element el = li.select("div[class=\"feed-item-dismissable\"]").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.text(); - } catch (Exception e) { - throw new ParsingException("Could not get title", e); - } - } - - @Override - public int getDuration() throws ParsingException { - try { - return YoutubeParsingHelper.parseDurationString( - li.select("span[class=\"video-time\"]").first().text()); - } catch(Exception e) { - if(isLiveStream(li)) { - // -1 for no duration - return -1; - } else { - throw new ParsingException("Could not get Duration: " + getTitle(), e); - } - } - } - - @Override - public String getUploader() throws ParsingException { - return getChannelName(); - } - - @Override - public String getUploadDate() throws ParsingException { - try { - Element meta = li.select("div[class=\"yt-lockup-meta\"]").first(); - Element li = meta.select("li").first(); - if (li == null && meta != null) { - //this means we have a youtube red video - return ""; - }else { - return li.text(); - } - } catch(Exception e) { - throw new ParsingException("Could not get uplaod date", e); - } - } - - @Override - public long getViewCount() throws ParsingException { - String output; - String input; - try { - input = li.select("div[class=\"yt-lockup-meta\"]").first() - .select("li").get(1) - .text(); - } catch (IndexOutOfBoundsException e) { - return -1; - } - - output = Parser.matchGroup1("([0-9,\\. ]*)", input) - .replace(" ", "") - .replace(".", "") - .replace(",", ""); - - try { - return Long.parseLong(output); - } catch (NumberFormatException e) { - // if this happens the video probably has no views - if(!input.isEmpty()) { - return 0; - } else { - throw new ParsingException("Could not handle input: " + input, e); - } - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - String url; - Element te = li.select("span[class=\"yt-thumb-clip\"]").first() - .select("img").first(); - url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } - return url; - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - private boolean isLiveStream(Element item) { - Element bla = item.select("span[class*=\"yt-badge-live\"]").first(); - - if(bla == null) { - // sometimes livestreams dont have badges but sill are live streams - // if video time is not available we most likly have an offline livestream - if(item.select("span[class*=\"video-time\"]").first() == null) { - return true; - } - } - return bla != null; - } - }); - } - } - - return collector; - } - - @Override - public long getSubscriberCount() throws ParsingException { - Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]") - .first(); - if(el != null) { - subscriberCount = Long.parseLong(el.text().replaceAll("\\D+","")); - } else if(el == null && subscriberCount == -1) { - throw new ParsingException("Could not get subscriber count"); - } - return subscriberCount; - } - - @Override - public String getFeedUrl() throws ParsingException { - try { - if(userUrl.contains("channel")) { - //channels don't have feeds in youtube, only user can provide such - return ""; - } - if(!isAjaxPage) { - feedUrl = doc.select("link[title=\"RSS\"]").first().attr("abs:href"); - } - return feedUrl; - } catch(Exception e) { - throw new ParsingException("Could not get feed url", e); - } - } - - @Override - public boolean hasNextPage() throws ParsingException { - return !nextPageUrl.isEmpty(); - } - - private String getUserUrl(Document d) throws ParsingException { - return d.select("span[class=\"qualified-channel-title-text\"]").first() - .select("a").first().attr("abs:href"); - } - - private boolean isUserUrl(String url) throws ParsingException { - return url.contains("/user/"); - } - - private String getNextPageUrl(Document d) throws ParsingException { - try { - Element button = d.select("button[class*=\"yt-uix-load-more\"]").first(); - if(button != null) { - return button.attr("abs:data-uix-load-more-href"); - } else { - // sometimes channels are simply so small, they don't have a second/next4q page - return ""; - } - } catch(Exception e) { - throw new ParsingException("could not load next page url", e); - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java deleted file mode 100644 index 7dafb63e8..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.schabi.newpipe.extractor.Parser; -import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.jsoup.nodes.Element; - -/** - * Created by Christian Schabesberger on 12.02.17. - * - * Copyright (C) Christian Schabesberger 2017 - * YoutubeChannelInfoItemExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor { - private Element el; - - public YoutubeChannelInfoItemExtractor(Element el) { - this.el = el; - } - - public String getThumbnailUrl() throws ParsingException { - Element img = el.select("span[class*=\"yt-thumb-simple\"]").first() - .select("img").first(); - - String url = img.attr("abs:src"); - - if(url.contains("gif")) { - url = img.attr("abs:data-thumb"); - } - return url; - } - - public String getChannelName() throws ParsingException { - return el.select("a[class*=\"yt-uix-tile-link\"]").first() - .text(); - } - - public String getWebPageUrl() throws ParsingException { - return el.select("a[class*=\"yt-uix-tile-link\"]").first() - .attr("abs:href"); - } - - public long getSubscriberCount() throws ParsingException { - Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first(); - if(subsEl == null) { - return 0; - } else { - return Integer.parseInt(subsEl.text().replaceAll("\\D+","")); - } - } - - public int getVideoAmount() throws ParsingException { - Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first(); - if(metaEl == null) { - return 0; - } else { - return Integer.parseInt(metaEl.text().replaceAll("\\D+","")); - } - } - - public String getDescription() throws ParsingException { - Element desEl = el.select("div[class*=\"yt-lockup-description\"]").first(); - if(desEl == null) { - return ""; - } else { - return desEl.text(); - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java deleted file mode 100644 index 23960f608..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelUrlIdHandler.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.schabi.newpipe.extractor.Parser; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -/** - * Created by Christian Schabesberger on 25.07.16. - * - * Copyright (C) Christian Schabesberger 2016 - * YoutubeChannelUrlIdHandler.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeChannelUrlIdHandler implements UrlIdHandler { - - public String getUrl(String channelId) { - return "https://www.youtube.com/" + channelId; - } - - public String getId(String siteUrl) throws ParsingException { - return Parser.matchGroup1("/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)", siteUrl); - } - - public String cleanUrl(String siteUrl) throws ParsingException { - return getUrl(getId(siteUrl)); - } - - public boolean acceptUrl(String videoUrl) { - return (videoUrl.contains("youtube") || - videoUrl.contains("youtu.be")) && - ( videoUrl.contains("/user/") || - videoUrl.contains("/channel/")); - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java deleted file mode 100644 index 3b4ad42e1..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - - -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -/** - * Created by Christian Schabesberger on 02.03.16. - * - * Copyright (C) Christian Schabesberger 2016 - * YoutubeParsingHelper.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeParsingHelper { - - private YoutubeParsingHelper() { - } - - public static int parseDurationString(String input) - throws ParsingException, NumberFormatException { - String[] splitInput = input.split(":"); - String days = "0"; - String hours = "0"; - String minutes = "0"; - String seconds; - - switch(splitInput.length) { - case 4: - days = splitInput[0]; - hours = splitInput[1]; - minutes = splitInput[2]; - seconds = splitInput[3]; - break; - case 3: - hours = splitInput[0]; - minutes = splitInput[1]; - seconds = splitInput[2]; - break; - case 2: - minutes = splitInput[0]; - seconds = splitInput[1]; - break; - case 1: - seconds = splitInput[0]; - break; - default: - throw new ParsingException("Error duration string with unknown format: " + input); - } - return ((((Integer.parseInt(days) * 24) - + Integer.parseInt(hours) * 60) - + Integer.parseInt(minutes)) * 60) - + Integer.parseInt(seconds); - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java deleted file mode 100644 index d650535f7..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.search.InfoItemSearchCollector; -import org.schabi.newpipe.extractor.search.SearchEngine; - -import java.net.URLEncoder; -import java.io.IOException; -import java.util.EnumSet; - - -/** - * Created by Christian Schabesberger on 09.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * YoutubeSearchEngine.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeSearchEngine extends SearchEngine { - - private static final String TAG = YoutubeSearchEngine.class.toString(); - public static final String CHARSET_UTF_8 = "UTF-8"; - - public YoutubeSearchEngine(UrlIdHandler urlIdHandler, int serviceId) { - super(urlIdHandler, serviceId); - } - - @Override - public InfoItemSearchCollector search(String query, - int page, - String languageCode, - EnumSet filter) - throws IOException, ExtractionException { - InfoItemSearchCollector collector = getInfoItemSearchCollector(); - - - Downloader downloader = NewPipe.getDownloader(); - - String url = "https://www.youtube.com/results" - + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) - + "&page=" + Integer.toString(page + 1); - if(filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) { - url += "&sp=EgIQAQ%253D%253D"; - } else if(!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) { - url += "&sp=EgIQAg%253D%253D"; - } - - String site; - //String url = builder.build().toString(); - //if we've been passed a valid language code, append it to the URL - if(!languageCode.isEmpty()) { - //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); - site = downloader.download(url, languageCode); - } - else { - site = downloader.download(url); - } - - Document doc = Jsoup.parse(site, url); - Element list = doc.select("ol[class=\"item-section\"]").first(); - - for (Element item : list.children()) { - /* First we need to determine which kind of item we are working with. - Youtube depicts five different kinds of items on its search result page. These are - regular videos, playlists, channels, two types of video suggestions, and a "no video - found" item. Since we only want videos, we need to filter out all the others. - An example for this can be seen here: - https://www.youtube.com/results?search_query=asdf&page=1 - - We already applied a filter to the url, so we don't need to care about channels and - playlists now. - */ - - Element el; - - // both types of spell correction item - if ((el = item.select("div[class*=\"spell-correction\"]").first()) != null) { - collector.setSuggestion(el.select("a").first().text()); - if(list.children().size() == 1) { - throw new NothingFoundException("Did you mean: " + el.select("a").first().text()); - } - // search message item - } else if ((el = item.select("div[class*=\"search-message\"]").first()) != null) { - throw new NothingFoundException(el.text()); - - // video item type - } else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(el)); - } else if((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) { - collector.commit(new YoutubeChannelInfoItemExtractor(el)); - } else { - // noinspection ConstantConditions - // simply ignore not known items - // throw new ExtractionException("unexpected element found: \"" + item + "\""); - } - } - - return collector; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java deleted file mode 100644 index c8cc68fd5..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.channel.ChannelExtractor; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.search.SearchEngine; -import org.schabi.newpipe.extractor.SuggestionExtractor; -import org.schabi.newpipe.extractor.stream_info.StreamExtractor; - -import java.io.IOException; - - -/** - * Created by Christian Schabesberger on 23.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * YoutubeService.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeService extends StreamingService { - - public YoutubeService(int id) { - super(id); - } - - @Override - public ServiceInfo getServiceInfo() { - ServiceInfo serviceInfo = new ServiceInfo(); - serviceInfo.name = "Youtube"; - return serviceInfo; - } - @Override - public StreamExtractor getExtractorInstance(String url) - throws ExtractionException, IOException { - UrlIdHandler urlIdHandler = YoutubeStreamUrlIdHandler.getInstance(); - if(urlIdHandler.acceptUrl(url)) { - return new YoutubeStreamExtractor(urlIdHandler, url, getServiceId()); - } - else { - throw new IllegalArgumentException("supplied String is not a valid Youtube URL"); - } - } - @Override - public SearchEngine getSearchEngineInstance() { - return new YoutubeSearchEngine(getStreamUrlIdHandlerInstance(), getServiceId()); - } - - @Override - public UrlIdHandler getStreamUrlIdHandlerInstance() { - return YoutubeStreamUrlIdHandler.getInstance(); - } - - @Override - public UrlIdHandler getChannelUrlIdHandlerInstance() { - return new YoutubeChannelUrlIdHandler(); - } - - @Override - public ChannelExtractor getChannelExtractorInstance(String url, int page) - throws ExtractionException, IOException { - return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, page, getServiceId()); - } - - @Override - public SuggestionExtractor getSuggestionExtractorInstance() { - return new YoutubeSuggestionExtractor(getServiceId()); - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java deleted file mode 100644 index 2df361644..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ /dev/null @@ -1,878 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.json.JSONException; -import org.json.JSONObject; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.mozilla.javascript.Context; -import org.mozilla.javascript.Function; -import org.mozilla.javascript.ScriptableObject; -import org.schabi.newpipe.extractor.AbstractStreamInfo; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.stream_info.AudioStream; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.Parser; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.stream_info.StreamExtractor; -import org.schabi.newpipe.extractor.stream_info.StreamInfo; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemCollector; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor; -import org.schabi.newpipe.extractor.stream_info.VideoStream; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Vector; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Created by Christian Schabesberger on 06.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * YoutubeStreamExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeStreamExtractor extends StreamExtractor { - public static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map"; - public static final String HTTPS = "https:"; - public static final String CONTENT = "content"; - public static final String REGEX_INT = "[^\\d]"; - - // exceptions - - public class DecryptException extends ParsingException { - DecryptException(String message, Throwable cause) { - super(message, cause); - } - } - - // special content not available exceptions - - public class GemaException extends ContentNotAvailableException { - GemaException(String message) { - super(message); - } - } - - public class LiveStreamException extends ContentNotAvailableException { - LiveStreamException(String message) { - super(message); - } - } - - // ---------------- - - // Sometimes if the html page of youtube is already downloaded, youtube web page will internally - // download the /get_video_info page. Since a certain date dashmpd url is only available over - // this /get_video_info page, so we always need to download this one to. - // %%video_id%% will be replaced by the actual video id - // $$el_type$$ will be replaced by the actual el_type (se the declarations below) - private static final String GET_VIDEO_INFO_URL = - "https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en"; - // eltype is necessary for the url above - private static final String EL_INFO = "el=info"; - - public enum ItagType { - AUDIO, - VIDEO, - VIDEO_ONLY - } - - private static class ItagItem { - public ItagItem(int id, ItagType type, MediaFormat format, String res, int fps) { - this.id = id; - this.itagType = type; - this.mediaFormatId = format.id; - this.resolutionString = res; - this.fps = fps; - } - public ItagItem(int id, ItagType type, MediaFormat format, int samplingRate, int bandWidth) { - this.id = id; - this.itagType = type; - this.mediaFormatId = format.id; - this.samplingRate = samplingRate; - this.bandWidth = bandWidth; - } - public int id; - public ItagType itagType; - public int mediaFormatId; - public String resolutionString; - public int fps = -1; - public int samplingRate = -1; - public int bandWidth = -1; - } - - private static final ItagItem[] itagList = { - // video streams - // id, ItagType, MediaFormat, Resolution, fps - new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p", 12), - new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p", 24), - new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p", 24), - new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p", 24), - new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p", 24), - new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p", 24), - new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p", 24), - new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p", 24), - new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p", 24), - new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p", 24), - // audio streams - // id, ItagType, MediaFormat, samplingR, bandwidth - new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), // bandwith/samplingR 0 because not known - new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), - new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), - new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 0, 0), - new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), - // video only streams - new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p", 24), - new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p", 24), - new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p", 24), - new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p", 24), - new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p", 24), - new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p", 24), - }; - - /**These lists only contain itag formats that are supported by the common Android Video player. - However if you are looking for a list showing all itag formats, look at - https://github.com/rg3/youtube-dl/issues/1687 */ - - public static boolean itagIsSupported(int itag) { - for(ItagItem item : itagList) { - if(itag == item.id) { - return true; - } - } - return false; - } - - public static ItagItem getItagItem(int itag) throws ParsingException { - for(ItagItem item : itagList) { - if(itag == item.id) { - return item; - } - } - throw new ParsingException("itag=" + Integer.toString(itag) + " not supported"); - } - - private static final String TAG = YoutubeStreamExtractor.class.toString(); - private final Document doc; - private JSONObject playerArgs; - private boolean isAgeRestricted; - private Map videoInfoPage; - - // static values - private static final String DECRYPTION_FUNC_NAME="decrypt"; - - // cached values - private static volatile String decryptionCode = ""; - - UrlIdHandler urlidhandler = YoutubeStreamUrlIdHandler.getInstance(); - String pageUrl = ""; - - public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) - throws ExtractionException, IOException { - super(urlIdHandler, pageUrl, serviceId); - //most common videoInfo fields are now set in our superclass, for all services - this.pageUrl = pageUrl; - Downloader downloader = NewPipe.getDownloader(); - String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl)); - doc = Jsoup.parse(pageContent, pageUrl); - JSONObject ytPlayerConfig; - String playerUrl; - - // Check if the video is age restricted - if (pageContent.contains(" method - je.printStackTrace(); - System.err.println("failed to load title from JSON args; trying to extract it from HTML"); - try { // fall through to fall-back - return doc.select("meta[name=title]").attr(CONTENT); - } catch (Exception e) { - throw new ParsingException("failed permanently to load title.", e); - } - } - } - - @Override - public String getDescription() throws ParsingException { - try { - return doc.select("p[id=\"eow-description\"]").first().html(); - } catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know - throw new ParsingException("failed to load description.", e); - } - } - - @Override - public String getUploader() throws ParsingException { - try { - if (playerArgs == null) { - return videoInfoPage.get("author"); - } - //json player args method - return playerArgs.getString("author"); - } catch(JSONException je) { - je.printStackTrace(); - System.err.println( - "failed to load uploader name from JSON args; trying to extract it from HTML"); - } try {//fall through to fallback HTML method - return doc.select("div.yt-user-info").first().text(); - } catch (Exception e) { - throw new ParsingException("failed permanently to load uploader name.", e); - } - } - - @Override - public int getLength() throws ParsingException { - try { - if (playerArgs == null) { - return Integer.valueOf(videoInfoPage.get("length_seconds")); - } - return playerArgs.getInt("length_seconds"); - } catch (JSONException e) {//todo: find fallback method - throw new ParsingException("failed to load video duration from JSON args", e); - } - } - - @Override - public long getViewCount() throws ParsingException { - try { - String viewCountString = doc.select("meta[itemprop=interactionCount]").attr(CONTENT); - return Long.parseLong(viewCountString); - } catch (Exception e) {//todo: find fallback method - throw new ParsingException("failed to get number of views", e); - } - } - - @Override - public String getUploadDate() throws ParsingException { - try { - return doc.select("meta[itemprop=datePublished]").attr(CONTENT); - } catch (Exception e) {//todo: add fallback method - throw new ParsingException("failed to get upload date.", e); - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - //first attempt getting a small image version - //in the html extracting part we try to get a thumbnail with a higher resolution - // Try to get high resolution thumbnail if it fails use low res from the player instead - try { - return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); - } catch(Exception e) { - System.err.println("Could not find high res Thumbnail. Using low res instead"); - } - try { //fall through to fallback - return playerArgs.getString("thumbnail_url"); - } catch (JSONException je) { - throw new ParsingException( - "failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je); - } catch (NullPointerException ne) { - // Get from the video info page instead - return videoInfoPage.get("thumbnail_url"); - } - } - - @Override - public String getUploaderThumbnailUrl() throws ParsingException { - try { - return doc.select("a[class*=\"yt-user-photo\"]").first() - .select("img").first() - .attr("abs:data-thumb"); - } catch (Exception e) {//todo: add fallback method - throw new ParsingException("failed to get uploader thumbnail URL.", e); - } - } - - @Override - public String getDashMpdUrl() throws ParsingException { - try { - String dashManifestUrl = ""; - if(videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) { - dashManifestUrl = videoInfoPage.get("dashmpd"); - } else if (playerArgs.has("dashmpd")) { - dashManifestUrl = playerArgs.getString("dashmpd"); - } else { - return ""; - } - if(!dashManifestUrl.contains("/signature/")) { - String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl); - String decryptedSig; - - decryptedSig = decryptSignature(encryptedSig, decryptionCode); - dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); - } - return dashManifestUrl; - } catch (Exception e) { - throw new ParsingException( - "Could not get \"dashmpd\" maybe VideoInfoPage is broken.", e); - } - } - - - @Override - public List getAudioStreams() throws ParsingException { - Vector audioStreams = new Vector<>(); - try{ - String encodedUrlMap; - // playerArgs could be null if the video is age restricted - if (playerArgs == null) { - if(videoInfoPage.containsKey("adaptive_fmts")) { - encodedUrlMap = videoInfoPage.get("adaptive_fmts"); - } else { - return null; - } - } else { - if(playerArgs.has("adaptive_fmts")) { - encodedUrlMap = playerArgs.getString("adaptive_fmts"); - } else { - return null; - } - } - for(String url_data_str : encodedUrlMap.split(",")) { - // This loop iterates through multiple streams, therefor tags - // is related to one and the same stream at a time. - Map tags = Parser.compatParseMap( - org.jsoup.parser.Parser.unescapeEntities(url_data_str, true)); - - int itag = Integer.parseInt(tags.get("itag")); - - if (itagIsSupported(itag)) { - ItagItem itagItem = getItagItem(itag); - if (itagItem.itagType == ItagType.AUDIO) { - String streamUrl = tags.get("url"); - // if video has a signature: decrypt it and add it to the url - if (tags.get("s") != null) { - streamUrl = streamUrl + "&signature=" - + decryptSignature(tags.get("s"), decryptionCode); - } - - audioStreams.add(new AudioStream(streamUrl, - itagItem.mediaFormatId, - itagItem.bandWidth, - itagItem.samplingRate)); - } - } - } - } catch (Exception e) { - throw new ParsingException("Could not get audiostreams", e); - } - return audioStreams; - } - - @Override - public List getVideoStreams() throws ParsingException { - Vector videoStreams = new Vector<>(); - - try{ - String encodedUrlMap; - // playerArgs could be null if the video is age restricted - if (playerArgs == null) { - encodedUrlMap = videoInfoPage.get(URL_ENCODED_FMT_STREAM_MAP); - } else { - encodedUrlMap = playerArgs.getString(URL_ENCODED_FMT_STREAM_MAP); - } - for(String url_data_str : encodedUrlMap.split(",")) { - try { - // This loop iterates through multiple streams, therefor tags - // is related to one and the same stream at a time. - Map tags = Parser.compatParseMap( - org.jsoup.parser.Parser.unescapeEntities(url_data_str, true)); - - int itag = Integer.parseInt(tags.get("itag")); - - if (itagIsSupported(itag)) { - ItagItem itagItem = getItagItem(itag); - if(itagItem.itagType == ItagType.VIDEO) { - String streamUrl = tags.get("url"); - // if video has a signature: decrypt it and add it to the url - if (tags.get("s") != null) { - streamUrl = streamUrl + "&signature=" - + decryptSignature(tags.get("s"), decryptionCode); - } - videoStreams.add(new VideoStream( - streamUrl, - itagItem.mediaFormatId, - itagItem.resolutionString)); - } - } - } catch (Exception e) { - //todo: dont log throw an error - System.err.println("Could not get Video stream."); - e.printStackTrace(); - } - } - - } catch (Exception e) { - throw new ParsingException("Failed to get video streams", e); - } - - if(videoStreams.isEmpty()) { - throw new ParsingException("Failed to get any video stream"); - } - return videoStreams; - } - - @Override - public List getVideoOnlyStreams() throws ParsingException { - return null; - } - - /**Attempts to parse (and return) the offset to start playing the video from. - * @return the offset (in seconds), or 0 if no timestamp is found.*/ - @Override - public int getTimeStamp() throws ParsingException { - String timeStamp; - try { - timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); - } catch (Parser.RegexException e) { - // catch this instantly since an url does not necessarily have to have a time stamp - - // -2 because well the testing system will then know its the regex that failed :/ - // not good i know - return -2; - } - - if(!timeStamp.isEmpty()) { - try { - String secondsString = ""; - String minutesString = ""; - String hoursString = ""; - try { - secondsString = Parser.matchGroup1("(\\d{1,3})s", timeStamp); - minutesString = Parser.matchGroup1("(\\d{1,3})m", timeStamp); - hoursString = Parser.matchGroup1("(\\d{1,3})h", timeStamp); - } catch (Exception e) { - //it could be that time is given in another method - if (secondsString.isEmpty() //if nothing was got, - && minutesString.isEmpty()//treat as unlabelled seconds - && hoursString.isEmpty()) { - secondsString = Parser.matchGroup1("t=(\\d+)", timeStamp); - } - } - - int seconds = secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString); - int minutes = minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString); - int hours = hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString); - - //don't trust BODMAS! - return seconds + (60 * minutes) + (3600 * hours); - //Log.d(TAG, "derived timestamp value:"+ret); - //the ordering varies internationally - } catch (ParsingException e) { - throw new ParsingException("Could not get timestamp.", e); - } - } else { - return 0; - } - } - - @Override - public int getAgeLimit() throws ParsingException { - if (!isAgeRestricted) { - return 0; - } - try { - return Integer.valueOf(doc.head() - .getElementsByAttributeValue("property", "og:restrictions:age") - .attr(CONTENT).replace("+", "")); - } catch (Exception e) { - throw new ParsingException("Could not get age restriction"); - } - } - - @Override - public String getAverageRating() throws ParsingException { - try { - if (playerArgs == null) { - return videoInfoPage.get("avg_rating"); - } - return playerArgs.getString("avg_rating"); - } catch (JSONException e) { - throw new ParsingException("Could not get Average rating", e); - } - } - - @Override - public int getLikeCount() throws ParsingException { - String likesString = ""; - try { - - Element button = doc.select("button.like-button-renderer-like-button").first(); - try { - likesString = button.select("span.yt-uix-button-content").first().text(); - } catch (NullPointerException e) { - //if this ckicks in our button has no content and thefore likes/dislikes are disabled - return -1; - } - return Integer.parseInt(likesString.replaceAll(REGEX_INT, "")); - } catch (NumberFormatException nfe) { - throw new ParsingException( - "failed to parse likesString \"" + likesString + "\" as integers", nfe); - } catch (Exception e) { - throw new ParsingException("Could not get like count", e); - } - } - - @Override - public int getDislikeCount() throws ParsingException { - String dislikesString = ""; - try { - Element button = doc.select("button.like-button-renderer-dislike-button").first(); - try { - dislikesString = button.select("span.yt-uix-button-content").first().text(); - } catch (NullPointerException e) { - //if this kicks in our button has no content and therefore likes/dislikes are disabled - return -1; - } - return Integer.parseInt(dislikesString.replaceAll(REGEX_INT, "")); - } catch(NumberFormatException nfe) { - throw new ParsingException( - "failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe); - } catch(Exception e) { - throw new ParsingException("Could not get dislike count", e); - } - } - - @Override - public StreamInfoItemExtractor getNextVideo() throws ParsingException { - try { - return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() - .select("li").first()); - } catch(Exception e) { - throw new ParsingException("Could not get next video", e); - } - } - - @Override - public StreamInfoItemCollector getRelatedVideos() throws ParsingException { - try { - StreamInfoItemCollector collector = getStreamPreviewInfoCollector(); - Element ul = doc.select("ul[id=\"watch-related\"]").first(); - if(ul != null) { - for (Element li : ul.children()) { - // first check if we have a playlist. If so leave them out - if (li.select("a[class*=\"content-link\"]").first() != null) { - collector.commit(extractVideoPreviewInfo(li)); - } - } - } - return collector; - } catch(Exception e) { - throw new ParsingException("Could not get related videos", e); - } - } - - @Override - public String getPageUrl() { - return pageUrl; - } - - @Override - public String getChannelUrl() throws ParsingException { - try { - return doc.select("div[class=\"yt-user-info\"]").first().children() - .select("a").first().attr("abs:href"); - } catch(Exception e) { - throw new ParsingException("Could not get channel link", e); - } - } - - @Override - public StreamInfo.StreamType getStreamType() throws ParsingException { - //todo: if implementing livestream support this value should be generated dynamically - return StreamInfo.StreamType.VIDEO_STREAM; - } - - /**Provides information about links to other videos on the video page, such as related videos. - * This is encapsulated in a StreamInfoItem object, - * which is a subset of the fields in a full StreamInfo.*/ - private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) { - return new StreamInfoItemExtractor() { - @Override - public AbstractStreamInfo.StreamType getStreamType() throws ParsingException { - return AbstractStreamInfo.StreamType.VIDEO_STREAM; - } - - @Override - public boolean isAd() throws ParsingException { - if(!li.select("span[class*=\"icon-not-available\"]").isEmpty()) { - return true; - } else { - return false; - } - } - - @Override - public String getWebPageUrl() throws ParsingException { - return li.select("a.content-link").first().attr("abs:href"); - } - - @Override - public String getTitle() throws ParsingException { - //todo: check NullPointerException causing - return li.select("span.title").first().text(); - //this page causes the NullPointerException, after finding it by searching for "tjvg": - //https://www.youtube.com/watch?v=Uqg0aEhLFAg - } - - @Override - public int getDuration() throws ParsingException { - return YoutubeParsingHelper.parseDurationString( - li.select("span.video-time").first().text()); - } - - @Override - public String getUploader() throws ParsingException { - return li.select("span.g-hovercard").first().text(); - } - - @Override - public String getUploadDate() throws ParsingException { - return null; - } - - @Override - public long getViewCount() throws ParsingException { - //this line is unused - //String views = li.select("span.view-count").first().text(); - - //Log.i(TAG, "title:"+info.title); - //Log.i(TAG, "view count:"+views); - - try { - return Long.parseLong(li.select("span.view-count") - .first().text().replaceAll(REGEX_INT, "")); - } catch (Exception e) { - //related videos sometimes have no view count - return 0; - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - Element img = li.select("img").first(); - String thumbnailUrl = img.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we caught such an item. - if (thumbnailUrl.contains(".gif")) { - thumbnailUrl = img.attr("data-thumb"); - } - if (thumbnailUrl.startsWith("//")) { - thumbnailUrl = HTTPS + thumbnailUrl; - } - return thumbnailUrl; - } - }; - } - - - private String loadDecryptionCode(String playerUrl) throws DecryptException { - String decryptionFuncName; - String decryptionFunc; - String helperObjectName; - String helperObject; - String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}"; - String decryptionCode; - - try { - Downloader downloader = NewPipe.getDownloader(); - if(!playerUrl.contains("https://youtube.com")) { - //sometimes the https://youtube.com part does not get send with - //than we have to add it by hand - playerUrl = "https://youtube.com" + playerUrl; - } - String playerCode = downloader.download(playerUrl); - - decryptionFuncName = - Parser.matchGroup("([\"\\'])signature\\1\\s*,\\s*([a-zA-Z0-9$]+)\\(", playerCode, 2); - - String functionPattern = "(" - + decryptionFuncName.replace("$", "\\$") - + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})"; - decryptionFunc = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";"; - - helperObjectName = Parser - .matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc); - - String helperPattern = "(var " - + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; - helperObject = Parser.matchGroup1(helperPattern, playerCode); - - - callerFunc = callerFunc.replace("%%", decryptionFuncName); - decryptionCode = helperObject + decryptionFunc + callerFunc; - } catch(IOException ioe) { - throw new DecryptException("Could not load decrypt function", ioe); - } catch(Exception e) { - throw new DecryptException("Could not parse decrypt function ", e); - } - - return decryptionCode; - } - - private String decryptSignature(String encryptedSig, String decryptionCode) - throws DecryptException{ - Context context = Context.enter(); - context.setOptimizationLevel(-1); - Object result = null; - try { - ScriptableObject scope = context.initStandardObjects(); - context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null); - Function decryptionFunc = (Function) scope.get("decrypt", scope); - result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig}); - } catch (Exception e) { - throw new DecryptException("could not get decrypt signature", e); - } finally { - Context.exit(); - } - return result == null ? "" : result.toString(); - } - - private String findErrorReason(Document doc) { - String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); - if(errorMessage.contains("GEMA")) { - // Gema sometimes blocks youtube music content in germany: - // https://www.gema.de/en/ - // Detailed description: - // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29 - return "GEMA"; - } - return ""; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java deleted file mode 100644 index 44d606e2d..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java +++ /dev/null @@ -1,186 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.AbstractStreamInfo; -import org.schabi.newpipe.extractor.Parser; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.stream_info.StreamInfoItemExtractor; - -/** - * Copyright (C) Christian Schabesberger 2016 - * YoutubeStreamInfoItemExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { - - private final Element item; - - public YoutubeStreamInfoItemExtractor(Element item) throws FoundAdException { - this.item = item; - } - - @Override - public String getWebPageUrl() throws ParsingException { - try { - Element el = item.select("div[class*=\"yt-lockup-video\"").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.attr("abs:href"); - } catch (Exception e) { - throw new ParsingException("Could not get web page url for the video", e); - } - } - - @Override - public String getTitle() throws ParsingException { - try { - Element el = item.select("div[class*=\"yt-lockup-video\"").first(); - Element dl = el.select("h3").first().select("a").first(); - return dl.text(); - } catch (Exception e) { - throw new ParsingException("Could not get title", e); - } - } - - @Override - public int getDuration() throws ParsingException { - try { - return YoutubeParsingHelper.parseDurationString( - item.select("span[class=\"video-time\"]").first().text()); - } catch(Exception e) { - if(isLiveStream(item)) { - // -1 for no duration - return -1; - } else { - throw new ParsingException("Could not get Duration: " + getTitle(), e); - } - } - } - - @Override - public String getUploader() throws ParsingException { - try { - return item.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first() - .text(); - } catch (Exception e) { - throw new ParsingException("Could not get uploader", e); - } - } - - @Override - public String getUploadDate() throws ParsingException { - try { - Element div = item.select("div[class=\"yt-lockup-meta\"]").first(); - if(div == null) { - return null; - } else { - return div.select("li").first().text(); - } - } catch(Exception e) { - throw new ParsingException("Could not get uplaod date", e); - } - } - - @Override - public long getViewCount() throws ParsingException { - String output; - String input; - try { - Element div = item.select("div[class=\"yt-lockup-meta\"]").first(); - if(div == null) { - return -1; - } else { - input = div.select("li").get(1) - .text(); - } - } catch (IndexOutOfBoundsException e) { - if(isLiveStream(item)) { - // -1 for no view count - return -1; - } else { - throw new ParsingException( - "Could not parse yt-lockup-meta although available: " + getTitle(), e); - } - } - - output = Parser.matchGroup1("([0-9,\\. ]*)", input) - .replace(" ", "") - .replace(".", "") - .replace(",", ""); - - try { - return Long.parseLong(output); - } catch (NumberFormatException e) { - // if this happens the video probably has no views - if(!input.isEmpty()) { - return 0; - } else { - throw new ParsingException("Could not handle input: " + input, e); - } - } - } - - @Override - public String getThumbnailUrl() throws ParsingException { - try { - String url; - Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first() - .select("img").first(); - url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if (url.contains(".gif")) { - url = te.attr("abs:data-thumb"); - } - return url; - } catch (Exception e) { - throw new ParsingException("Could not get thumbnail url", e); - } - } - - @Override - public AbstractStreamInfo.StreamType getStreamType() { - if(isLiveStream(item)) { - return AbstractStreamInfo.StreamType.LIVE_STREAM; - } else { - return AbstractStreamInfo.StreamType.VIDEO_STREAM; - } - } - - @Override - public boolean isAd() throws ParsingException { - if(!item.select("span[class*=\"icon-not-available\"]").isEmpty()) { - return true; - } else { - return false; - } - } - - private boolean isLiveStream(Element item) { - Element bla = item.select("span[class*=\"yt-badge-live\"]").first(); - - if(bla == null) { - // sometimes livestreams dont have badges but sill are live streams - // if video time is not available we most likly have an offline livestream - if(item.select("span[class*=\"video-time\"]").first() == null) { - return true; - } - } - return bla != null; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java deleted file mode 100644 index 8c08a6101..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandler.java +++ /dev/null @@ -1,163 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.Parser; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URLDecoder; - -/** - * Created by Christian Schabesberger on 02.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * YoutubeStreamUrlIdHandler.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeStreamUrlIdHandler implements UrlIdHandler { - - private static final YoutubeStreamUrlIdHandler instance = new YoutubeStreamUrlIdHandler(); - private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})"; - - private YoutubeStreamUrlIdHandler() {} - - public static YoutubeStreamUrlIdHandler getInstance() { - return instance; - } - - @Override - public String getUrl(String videoId) { - return "https://www.youtube.com/watch?v=" + videoId; - } - - @Override - public String getId(String url) throws ParsingException, IllegalArgumentException { - if(url.isEmpty()) { - throw new IllegalArgumentException("The url parameter should not be empty"); - } - - String id; - String lowercaseUrl = url.toLowerCase(); - if(lowercaseUrl.contains("youtube")) { - if (url.contains("attribution_link")) { - try { - String escapedQuery = Parser.matchGroup1("u=(.[^&|$]*)", url); - String query = URLDecoder.decode(escapedQuery, "UTF-8"); - id = Parser.matchGroup1("v=" + ID_PATTERN, query); - } catch (UnsupportedEncodingException uee) { - throw new ParsingException("Could not parse attribution_link", uee); - } - } else if(lowercaseUrl.contains("youtube.com/shared?ci=")) { - return getRealIdFromSharedLink(url); - } else if (url.contains("vnd.youtube")) { - id = Parser.matchGroup1(ID_PATTERN, url); - } else if (url.contains("embed")) { - id = Parser.matchGroup1("embed/" + ID_PATTERN, url); - } else if(url.contains("googleads")) { - throw new FoundAdException("Error found add: " + url); - } else { - id = Parser.matchGroup1("[?&]v=" + ID_PATTERN, url); - } - } - else if(lowercaseUrl.contains("youtu.be")) { - if(url.contains("v=")) { - id = Parser.matchGroup1("v=" + ID_PATTERN, url); - } else { - id = Parser.matchGroup1("[Yy][Oo][Uu][Tt][Uu]\\.[Bb][Ee]/" + ID_PATTERN, url); - } - } - else { - throw new ParsingException("Error no suitable url: " + url); - } - - - if(!id.isEmpty()){ - return id; - } else { - throw new ParsingException("Error could not parse url: " + url); - } - } - - /** - * Get the real url from a shared uri. - * - * Shared URI's look like this: - * 
-     *     * https://www.youtube.com/shared?ci=PJICrTByb3E
-     *     * vnd.youtube://www.youtube.com/shared?ci=PJICrTByb3E&feature=twitter-deep-link
-     *
- * @param url The shared url - * @return the id of the stream - * @throws ParsingException - */ - private String getRealIdFromSharedLink(String url) throws ParsingException { - URI uri; - try { - uri = new URI(url); - } catch (URISyntaxException e) { - throw new ParsingException("Invalid shared link", e); - } - String sharedId = getSharedId(uri); - Downloader downloader = NewPipe.getDownloader(); - String content; - try { - content = downloader.download("https://www.youtube.com/shared?ci=" + sharedId); - } catch (IOException | ReCaptchaException e) { - throw new ParsingException("Unable to resolve shared link", e); - } - // is this bad? is this fragile?: - String realId = Parser.matchGroup1("rel=\"shortlink\" href=\"https://youtu.be/" + ID_PATTERN, content); - if(sharedId.equals(realId)) { - throw new ParsingException("Got same id for as shared id: " + sharedId); - } - return realId; - } - - private String getSharedId(URI uri) throws ParsingException { - if (!"/shared".equals(uri.getPath())) { - throw new ParsingException("Not a shared link: " + uri.toString() + " (path != " + uri.getPath() + ")"); - } - return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery()); - } - - public String cleanUrl(String complexUrl) throws ParsingException { - return getUrl(getId(complexUrl)); - } - - @Override - public boolean acceptUrl(String videoUrl) { - String lowercaseUrl = videoUrl.toLowerCase(); - if(lowercaseUrl.contains("youtube") || - lowercaseUrl.contains("youtu.be")) { - // bad programming I know - try { - getId(videoUrl); - return true; - } catch (Exception e) { - return false; - } - } else { - return false; - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java deleted file mode 100644 index 0535961e8..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.SuggestionExtractor; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.List; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - -/** - * Created by Christian Schabesberger on 28.09.16. - * - * Copyright (C) Christian Schabesberger 2015 - * YoutubeSuggestionExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeSuggestionExtractor extends SuggestionExtractor { - - public static final String CHARSET_UTF_8 = "UTF-8"; - - public YoutubeSuggestionExtractor(int serviceId) { - super(serviceId); - } - - @Override - public List suggestionList( - String query, String contentCountry) - throws ExtractionException, IOException { - List suggestions = new ArrayList<>(); - - Downloader dl = NewPipe.getDownloader(); - - String url = "https://suggestqueries.google.com/complete/search" - + "?client=" + "" - + "&output=" + "toolbar" - + "&ds=" + "yt" - + "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8) - + "&q=" + URLEncoder.encode(query, CHARSET_UTF_8); - - - String response = dl.download(url); - - //TODO: Parse xml data using Jsoup not done - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder; - org.w3c.dom.Document doc = null; - - try { - dBuilder = dbFactory.newDocumentBuilder(); - doc = dBuilder.parse(new InputSource( - new ByteArrayInputStream(response.getBytes(CHARSET_UTF_8)))); - doc.getDocumentElement().normalize(); - } catch (ParserConfigurationException | SAXException | IOException e) { - throw new ParsingException("Could not parse document."); - } - - try { - NodeList nList = doc.getElementsByTagName("CompleteSuggestion"); - for (int temp = 0; temp < nList.getLength(); temp++) { - - NodeList nList1 = doc.getElementsByTagName("suggestion"); - Node nNode1 = nList1.item(temp); - if (nNode1.getNodeType() == Node.ELEMENT_NODE) { - org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1; - suggestions.add(eElement.getAttribute("data")); - } - } - return suggestions; - } catch(Exception e) { - throw new ParsingException("Could not get suggestions form document.", e); - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/AudioStream.java b/app/src/main/java/org/schabi/newpipe/extractor/stream_info/AudioStream.java deleted file mode 100644 index 98eb17620..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/AudioStream.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.schabi.newpipe.extractor.stream_info; - -/** - * Created by Christian Schabesberger on 04.03.16. - * - * Copyright (C) Christian Schabesberger 2016 - * AudioStream.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class AudioStream { - public String url = ""; - public int format = -1; - public int bandwidth = -1; - public int sampling_rate = -1; - - public AudioStream(String url, int format, int bandwidth, int samplingRate) { - this.url = url; this.format = format; - this.bandwidth = bandwidth; this.sampling_rate = samplingRate; - } - - // reveals whether two streams are the same, but have different urls - public boolean equalStats(AudioStream cmp) { - return format == cmp.format - && bandwidth == cmp.bandwidth - && sampling_rate == cmp.sampling_rate; - } - - // reveals whether two streams are equal - public boolean equals(AudioStream cmp) { - return cmp != null && equalStats(cmp) - && url == cmp.url; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamExtractor.java deleted file mode 100644 index 1b7c1b5ab..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamExtractor.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.schabi.newpipe.extractor.stream_info; - -/** - * Created by Christian Schabesberger on 10.08.15. - * - * Copyright (C) Christian Schabesberger 2016 - * StreamExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import java.util.List; - -/**Scrapes information from a video streaming service (eg, YouTube).*/ - - -@SuppressWarnings("ALL") -public abstract class StreamExtractor { - - private int serviceId; - private String url; - private UrlIdHandler urlIdHandler; - private StreamInfoItemCollector previewInfoCollector; - - public class ExtractorInitException extends ExtractionException { - public ExtractorInitException(String message) { - super(message); - } - public ExtractorInitException(Throwable cause) { - super(cause); - } - public ExtractorInitException(String message, Throwable cause) { - super(message, cause); - } - } - - public class ContentNotAvailableException extends ParsingException { - public ContentNotAvailableException(String message) { - super(message); - } - public ContentNotAvailableException(String message, Throwable cause) { - super(message, cause); - } - } - - public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) { - this.serviceId = serviceId; - this.urlIdHandler = urlIdHandler; - previewInfoCollector = new StreamInfoItemCollector(urlIdHandler, serviceId); - } - - protected StreamInfoItemCollector getStreamPreviewInfoCollector() { - return previewInfoCollector; - } - - public String getUrl() { - return url; - } - - public UrlIdHandler getUrlIdHandler() { - return urlIdHandler; - } - - public abstract int getTimeStamp() throws ParsingException; - public abstract String getTitle() throws ParsingException; - public abstract String getDescription() throws ParsingException; - public abstract String getUploader() throws ParsingException; - public abstract String getChannelUrl() throws ParsingException; - public abstract int getLength() throws ParsingException; - public abstract long getViewCount() throws ParsingException; - public abstract String getUploadDate() throws ParsingException; - public abstract String getThumbnailUrl() throws ParsingException; - public abstract String getUploaderThumbnailUrl() throws ParsingException; - public abstract List getAudioStreams() throws ParsingException; - public abstract List getVideoStreams() throws ParsingException; - public abstract List getVideoOnlyStreams() throws ParsingException; - public abstract String getDashMpdUrl() throws ParsingException; - public abstract int getAgeLimit() throws ParsingException; - public abstract String getAverageRating() throws ParsingException; - public abstract int getLikeCount() throws ParsingException; - public abstract int getDislikeCount() throws ParsingException; - public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException; - public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException; - public abstract String getPageUrl(); - public abstract StreamInfo.StreamType getStreamType() throws ParsingException; - public int getServiceId() { - return serviceId; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfo.java b/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfo.java deleted file mode 100644 index cf3df49d8..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfo.java +++ /dev/null @@ -1,294 +0,0 @@ -package org.schabi.newpipe.extractor.stream_info; - -import org.schabi.newpipe.extractor.AbstractStreamInfo; -import org.schabi.newpipe.extractor.DashMpdParser; -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; - -import java.io.IOException; -import java.util.List; -import java.util.Vector; - -/** - * Created by Christian Schabesberger on 26.08.15. - * - * Copyright (C) Christian Schabesberger 2016 - * StreamInfo.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -/**Info object for opened videos, ie the video ready to play.*/ -@SuppressWarnings("ALL") -public class StreamInfo extends AbstractStreamInfo { - - public static class StreamExctractException extends ExtractionException { - StreamExctractException(String message) { - super(message); - } - } - - public StreamInfo() {} - - /**Creates a new StreamInfo object from an existing AbstractVideoInfo. - * All the shared properties are copied to the new StreamInfo.*/ - @SuppressWarnings("WeakerAccess") - public StreamInfo(AbstractStreamInfo avi) { - this.id = avi.id; - this.title = avi.title; - this.uploader = avi.uploader; - this.thumbnail_url = avi.thumbnail_url; - this.webpage_url = avi.webpage_url; - this.upload_date = avi.upload_date; - this.upload_date = avi.upload_date; - this.view_count = avi.view_count; - - //todo: better than this - if(avi instanceof StreamInfoItem) { - //shitty String to convert code - /* - String dur = ((StreamInfoItem)avi).duration; - int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":"))); - int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length())); - */ - this.duration = ((StreamInfoItem)avi).duration; - } - } - - public void addException(Exception e) { - errors.add(e); - } - - /**Fills out the video info fields which are common to all services. - * Probably needs to be overridden by subclasses*/ - public static StreamInfo getVideoInfo(StreamExtractor extractor) - throws ExtractionException, IOException { - StreamInfo streamInfo = new StreamInfo(); - - streamInfo = extractImportantData(streamInfo, extractor); - streamInfo = extractStreams(streamInfo, extractor); - streamInfo = extractOptionalData(streamInfo, extractor); - - return streamInfo; - } - - private static StreamInfo extractImportantData( - StreamInfo streamInfo, StreamExtractor extractor) - throws ExtractionException, IOException { - /* ---- important data, withoug the video can't be displayed goes here: ---- */ - // if one of these is not available an exception is meant to be thrown directly into the frontend. - - UrlIdHandler uiconv = extractor.getUrlIdHandler(); - - streamInfo.service_id = extractor.getServiceId(); - streamInfo.webpage_url = extractor.getPageUrl(); - streamInfo.stream_type = extractor.getStreamType(); - streamInfo.id = uiconv.getId(extractor.getPageUrl()); - streamInfo.title = extractor.getTitle(); - streamInfo.age_limit = extractor.getAgeLimit(); - - if((streamInfo.stream_type == StreamType.NONE) - || (streamInfo.webpage_url == null || streamInfo.webpage_url.isEmpty()) - || (streamInfo.id == null || streamInfo.id.isEmpty()) - || (streamInfo.title == null /* streamInfo.title can be empty of course */) - || (streamInfo.age_limit == -1)) { - throw new ExtractionException("Some importand stream information was not given."); - } - - return streamInfo; - } - - private static StreamInfo extractStreams( - StreamInfo streamInfo, StreamExtractor extractor) - throws ExtractionException, IOException { - /* ---- stream extraction goes here ---- */ - // At least one type of stream has to be available, - // otherwise an exception will be thrown directly into the frontend. - - try { - streamInfo.dashMpdUrl = extractor.getDashMpdUrl(); - } catch(Exception e) { - streamInfo.addException(new ExtractionException("Couldn't get Dash manifest", e)); - } - - /* Load and extract audio */ - try { - streamInfo.audio_streams = extractor.getAudioStreams(); - } catch(Exception e) { - streamInfo.addException(new ExtractionException("Couldn't get audio streams", e)); - } - // also try to get streams from the dashMpd - if(streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) { - if(streamInfo.audio_streams == null) { - streamInfo.audio_streams = new Vector<>(); - } - //todo: make this quick and dirty solution a real fallback - // same as the quick and dirty above - try { - streamInfo.audio_streams.addAll( - DashMpdParser.getAudioStreams(streamInfo.dashMpdUrl)); - } catch(Exception e) { - streamInfo.addException( - new ExtractionException("Couldn't get audio streams from dash mpd", e)); - } - } - /* Extract video stream url*/ - try { - streamInfo.video_streams = extractor.getVideoStreams(); - } catch (Exception e) { - streamInfo.addException( - new ExtractionException("Couldn't get video streams", e)); - } - /* Extract video only stream url*/ - try { - streamInfo.video_only_streams = extractor.getVideoOnlyStreams(); - } catch(Exception e) { - streamInfo.addException( - new ExtractionException("Couldn't get video only streams", e)); - } - - // either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream, - // and therefore failed. (Since video_only_streams are just optional they don't caunt). - if((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty()) - && (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty()) - && (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) { - throw new StreamExctractException( - "Could not get any stream. See error variable to get further details."); - } - - return streamInfo; - } - - private static StreamInfo extractOptionalData( - StreamInfo streamInfo, StreamExtractor extractor) { - /* ---- optional data goes here: ---- */ - // If one of these fails, the frontend needs to handle that they are not available. - // Exceptions are therefore not thrown into the frontend, but stored into the error List, - // so the frontend can afterwards check where errors happened. - - try { - streamInfo.thumbnail_url = extractor.getThumbnailUrl(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.duration = extractor.getLength(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.uploader = extractor.getUploader(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.channel_url = extractor.getChannelUrl(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.description = extractor.getDescription(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.view_count = extractor.getViewCount(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.upload_date = extractor.getUploadDate(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.uploader_thumbnail_url = extractor.getUploaderThumbnailUrl(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.start_position = extractor.getTimeStamp(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.average_rating = extractor.getAverageRating(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.like_count = extractor.getLikeCount(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - streamInfo.dislike_count = extractor.getDislikeCount(); - } catch(Exception e) { - streamInfo.addException(e); - } - try { - // get next video - if(streamInfo.next_video != null) - { - StreamInfoItemCollector c = new StreamInfoItemCollector( - extractor.getUrlIdHandler(), extractor.getServiceId()); - StreamInfoItemExtractor nextVideo = extractor.getNextVideo(); - c.commit(nextVideo); - if(c.getItemList().size() != 0) { - streamInfo.next_video = (StreamInfoItem) c.getItemList().get(0); - } - streamInfo.errors.addAll(c.getErrors()); - } - } - catch(Exception e) { - streamInfo.addException(e); - } - try { - // get related videos - StreamInfoItemCollector c = extractor.getRelatedVideos(); - streamInfo.related_streams = c.getItemList(); - streamInfo.errors.addAll(c.getErrors()); - } catch(Exception e) { - streamInfo.addException(e); - } - - return streamInfo; - } - - public String uploader_thumbnail_url = ""; - public String channel_url = ""; - public String description = ""; - - public List video_streams = null; - public List audio_streams = null; - public List video_only_streams = null; - // video streams provided by the dash mpd do not need to be provided as VideoStream. - // Later on this will also aplly to audio streams. Since dash mpd is standarized, - // crawling such a file is not service dependent. Therefore getting audio only streams by yust - // providing the dash mpd fille will be possible in the future. - public String dashMpdUrl = ""; - public int duration = -1; - - public int age_limit = -1; - public int like_count = -1; - public int dislike_count = -1; - public String average_rating = ""; - public StreamInfoItem next_video = null; - public List related_streams = null; - //in seconds. some metadata is not passed using a StreamInfo object! - public int start_position = 0; - - public List errors = new Vector<>(); -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItem.java b/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItem.java deleted file mode 100644 index 8a6db5bc0..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItem.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.schabi.newpipe.extractor.stream_info; - -/** - * Created by Christian Schabesberger on 26.08.15. - * - * Copyright (C) Christian Schabesberger 2016 - * StreamInfoItem.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -import org.schabi.newpipe.extractor.AbstractStreamInfo; -import org.schabi.newpipe.extractor.InfoItem; - -/**Info object for previews of unopened videos, eg search results, related videos*/ -public class StreamInfoItem extends AbstractStreamInfo implements InfoItem { - public int duration; - - public InfoType infoType() { - return InfoType.STREAM; - } - - public String getTitle() { - return title; - } - - public String getLink() { - return webpage_url; - } -} \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemCollector.java b/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemCollector.java deleted file mode 100644 index 03ff7bd83..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemCollector.java +++ /dev/null @@ -1,102 +0,0 @@ -package org.schabi.newpipe.extractor.stream_info; - -import org.schabi.newpipe.extractor.InfoItemCollector; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.UrlIdHandler; -import org.schabi.newpipe.extractor.exceptions.FoundAdException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import java.util.List; -import java.util.Vector; - -/** - * Created by Christian Schabesberger on 28.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * StreamInfoItemCollector.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class StreamInfoItemCollector extends InfoItemCollector { - - private UrlIdHandler urlIdHandler; - - public StreamInfoItemCollector(UrlIdHandler handler, int serviceId) { - super(serviceId); - urlIdHandler = handler; - } - - private UrlIdHandler getUrlIdHandler() { - return urlIdHandler; - } - - public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception { - if(extractor.isAd()) { - throw new FoundAdException("Found ad"); - } - - StreamInfoItem resultItem = new StreamInfoItem(); - // importand information - resultItem.service_id = getServiceId(); - resultItem.webpage_url = extractor.getWebPageUrl(); - if (getUrlIdHandler() == null) { - throw new ParsingException("Error: UrlIdHandler not set"); - } else if (!resultItem.webpage_url.isEmpty()) { - resultItem.id = NewPipe.getService(getServiceId()) - .getStreamUrlIdHandlerInstance() - .getId(resultItem.webpage_url); - } - resultItem.title = extractor.getTitle(); - resultItem.stream_type = extractor.getStreamType(); - - // optional information - try { - resultItem.duration = extractor.getDuration(); - } catch (Exception e) { - addError(e); - } - try { - resultItem.uploader = extractor.getUploader(); - } catch (Exception e) { - addError(e); - } - try { - resultItem.upload_date = extractor.getUploadDate(); - } catch (Exception e) { - addError(e); - } - try { - resultItem.view_count = extractor.getViewCount(); - } catch (Exception e) { - addError(e); - } - try { - resultItem.thumbnail_url = extractor.getThumbnailUrl(); - } catch (Exception e) { - addError(e); - } - return resultItem; - } - - public void commit(StreamInfoItemExtractor extractor) throws ParsingException { - try { - addItem(extract(extractor)); - } catch(FoundAdException ae) { - //System.out.println("AD_WARNING: " + ae.getMessage()); - } catch (Exception e) { - addError(e); - } - } -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemExtractor.java deleted file mode 100644 index b5432b42d..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/StreamInfoItemExtractor.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.schabi.newpipe.extractor.stream_info; - -import org.schabi.newpipe.extractor.AbstractStreamInfo; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -/** - * Created by Christian Schabesberger on 28.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * StreamInfoItemExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public interface StreamInfoItemExtractor { - AbstractStreamInfo.StreamType getStreamType() throws ParsingException; - String getWebPageUrl() throws ParsingException; - String getTitle() throws ParsingException; - int getDuration() throws ParsingException; - String getUploader() throws ParsingException; - String getUploadDate() throws ParsingException; - long getViewCount() throws ParsingException; - String getThumbnailUrl() throws ParsingException; - boolean isAd() throws ParsingException; -} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/VideoStream.java b/app/src/main/java/org/schabi/newpipe/extractor/stream_info/VideoStream.java deleted file mode 100644 index c3e12fdbd..000000000 --- a/app/src/main/java/org/schabi/newpipe/extractor/stream_info/VideoStream.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.schabi.newpipe.extractor.stream_info; - -/** - * Created by Christian Schabesberger on 04.03.16. - * - * Copyright (C) Christian Schabesberger 2016 - * VideoStream.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class VideoStream { - //url of the stream - public String url = ""; - public int format = -1; - public String resolution = ""; - - public VideoStream(String url, int format, String res) { - this.url = url; this.format = format; resolution = res; - } - - // reveals wether two streams are the same, but have diferent urls - public boolean equalStats(VideoStream cmp) { - return format == cmp.format - && resolution == cmp.resolution; - } - - // revelas wether two streams are equal - public boolean equals(VideoStream cmp) { - return cmp != null && equalStats(cmp) - && url == cmp.url; - } -} From 383857d110dbb2b058d634d131d91123b44403df Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Wed, 1 Mar 2017 19:08:12 +0100 Subject: [PATCH 4/5] add extractor submodule --- .gitmodules | 3 +++ app/src/main/java/org/schabi/newpipe/extractor | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 app/src/main/java/org/schabi/newpipe/extractor diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..bb31fd92e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "app/src/main/java/org/schabi/newpipe/extractor"] + path = app/src/main/java/org/schabi/newpipe/extractor + url = git@github.com:TeamNewPipe/NewPipeExtractor.git diff --git a/app/src/main/java/org/schabi/newpipe/extractor b/app/src/main/java/org/schabi/newpipe/extractor new file mode 160000 index 000000000..9005105e5 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/extractor @@ -0,0 +1 @@ +Subproject commit 9005105e5244b107e5a0c2a0653d2db37609cf1e From 6b9140c60d89c7dc41cd47c592069df5405a09d4 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Wed, 1 Mar 2017 19:25:19 +0100 Subject: [PATCH 5/5] make extractor sub use https --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index bb31fd92e..fc9d8f809 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "app/src/main/java/org/schabi/newpipe/extractor"] path = app/src/main/java/org/schabi/newpipe/extractor - url = git@github.com:TeamNewPipe/NewPipeExtractor.git + url = https://github.com/TeamNewPipe/NewPipeExtractor.git