2019-10-28 03:35:51 +01:00
|
|
|
package org.schabi.newpipe;
|
|
|
|
|
2020-03-15 22:53:29 +01:00
|
|
|
import android.content.Context;
|
2019-10-28 03:35:51 +01:00
|
|
|
|
2020-03-31 19:20:15 +02:00
|
|
|
import androidx.annotation.NonNull;
|
|
|
|
import androidx.annotation.Nullable;
|
2021-01-15 17:11:04 +01:00
|
|
|
import androidx.preference.PreferenceManager;
|
2020-03-31 19:20:15 +02:00
|
|
|
|
2020-12-09 12:42:01 +01:00
|
|
|
import org.schabi.newpipe.error.ReCaptchaActivity;
|
2019-10-28 03:35:51 +01:00
|
|
|
import org.schabi.newpipe.extractor.downloader.Downloader;
|
|
|
|
import org.schabi.newpipe.extractor.downloader.Request;
|
|
|
|
import org.schabi.newpipe.extractor.downloader.Response;
|
|
|
|
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
|
2020-03-15 22:53:29 +01:00
|
|
|
import org.schabi.newpipe.util.InfoCache;
|
2019-10-28 03:35:51 +01:00
|
|
|
|
|
|
|
import java.io.IOException;
|
2022-07-29 05:35:50 +02:00
|
|
|
import java.util.Arrays;
|
2020-03-15 22:53:29 +01:00
|
|
|
import java.util.HashMap;
|
2019-10-28 03:35:51 +01:00
|
|
|
import java.util.List;
|
|
|
|
import java.util.Map;
|
2022-07-29 05:35:50 +02:00
|
|
|
import java.util.Objects;
|
2019-10-28 03:35:51 +01:00
|
|
|
import java.util.concurrent.TimeUnit;
|
2022-07-29 05:35:50 +02:00
|
|
|
import java.util.stream.Collectors;
|
|
|
|
import java.util.stream.Stream;
|
2019-10-28 03:35:51 +01:00
|
|
|
|
|
|
|
import okhttp3.OkHttpClient;
|
|
|
|
import okhttp3.RequestBody;
|
|
|
|
import okhttp3.ResponseBody;
|
|
|
|
|
2020-03-31 19:20:15 +02:00
|
|
|
public final class DownloaderImpl extends Downloader {
|
2022-07-10 14:19:58 +02:00
|
|
|
public static final String USER_AGENT =
|
|
|
|
"Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0";
|
|
|
|
public static final String YOUTUBE_RESTRICTED_MODE_COOKIE_KEY =
|
|
|
|
"youtube_restricted_mode_key";
|
2020-04-12 00:26:16 +02:00
|
|
|
public static final String YOUTUBE_RESTRICTED_MODE_COOKIE = "PREF=f2=8000000";
|
2020-03-16 02:04:34 +01:00
|
|
|
public static final String YOUTUBE_DOMAIN = "youtube.com";
|
2020-03-15 22:53:29 +01:00
|
|
|
|
2019-10-28 03:35:51 +01:00
|
|
|
private static DownloaderImpl instance;
|
2020-11-18 23:50:00 +01:00
|
|
|
private final Map<String, String> mCookies;
|
|
|
|
private final OkHttpClient client;
|
2019-10-28 03:35:51 +01:00
|
|
|
|
2020-03-31 19:20:15 +02:00
|
|
|
private DownloaderImpl(final OkHttpClient.Builder builder) {
|
2019-10-28 03:35:51 +01:00
|
|
|
this.client = builder
|
|
|
|
.readTimeout(30, TimeUnit.SECONDS)
|
2020-03-31 19:20:15 +02:00
|
|
|
// .cache(new Cache(new File(context.getExternalCacheDir(), "okhttp"),
|
|
|
|
// 16 * 1024 * 1024))
|
2019-10-28 03:35:51 +01:00
|
|
|
.build();
|
2020-03-15 22:53:29 +01:00
|
|
|
this.mCookies = new HashMap<>();
|
2019-10-28 03:35:51 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* It's recommended to call exactly once in the entire lifetime of the application.
|
|
|
|
*
|
|
|
|
* @param builder if null, default builder will be used
|
2020-03-31 19:20:15 +02:00
|
|
|
* @return a new instance of {@link DownloaderImpl}
|
2019-10-28 03:35:51 +01:00
|
|
|
*/
|
2020-03-31 19:20:15 +02:00
|
|
|
public static DownloaderImpl init(@Nullable final OkHttpClient.Builder builder) {
|
|
|
|
instance = new DownloaderImpl(
|
|
|
|
builder != null ? builder : new OkHttpClient.Builder());
|
|
|
|
return instance;
|
2019-10-28 03:35:51 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
public static DownloaderImpl getInstance() {
|
|
|
|
return instance;
|
|
|
|
}
|
|
|
|
|
2020-03-16 02:04:34 +01:00
|
|
|
public String getCookies(final String url) {
|
2022-07-29 05:35:50 +02:00
|
|
|
final String youtubeCookie = url.contains(YOUTUBE_DOMAIN)
|
|
|
|
? getCookie(YOUTUBE_RESTRICTED_MODE_COOKIE_KEY) : null;
|
|
|
|
|
2020-03-16 02:04:34 +01:00
|
|
|
// Recaptcha cookie is always added TODO: not sure if this is necessary
|
2022-07-29 05:35:50 +02:00
|
|
|
return Stream.of(youtubeCookie, getCookie(ReCaptchaActivity.RECAPTCHA_COOKIES_KEY))
|
|
|
|
.filter(Objects::nonNull)
|
|
|
|
.flatMap(cookies -> Arrays.stream(cookies.split("; *")))
|
|
|
|
.distinct()
|
|
|
|
.collect(Collectors.joining("; "));
|
2020-03-15 22:53:29 +01:00
|
|
|
}
|
|
|
|
|
2020-04-11 23:21:52 +02:00
|
|
|
public String getCookie(final String key) {
|
2020-03-15 22:53:29 +01:00
|
|
|
return mCookies.get(key);
|
|
|
|
}
|
|
|
|
|
2020-04-11 23:21:52 +02:00
|
|
|
public void setCookie(final String key, final String cookie) {
|
2020-03-16 02:04:34 +01:00
|
|
|
mCookies.put(key, cookie);
|
2019-10-28 03:35:51 +01:00
|
|
|
}
|
|
|
|
|
2020-04-11 23:21:52 +02:00
|
|
|
public void removeCookie(final String key) {
|
2020-03-15 22:53:29 +01:00
|
|
|
mCookies.remove(key);
|
|
|
|
}
|
|
|
|
|
2020-04-12 22:13:04 +02:00
|
|
|
public void updateYoutubeRestrictedModeCookies(final Context context) {
|
2020-08-16 10:24:58 +02:00
|
|
|
final String restrictedModeEnabledKey =
|
2020-04-12 22:13:04 +02:00
|
|
|
context.getString(R.string.youtube_restricted_mode_enabled);
|
2020-08-16 10:24:58 +02:00
|
|
|
final boolean restrictedModeEnabled = PreferenceManager.getDefaultSharedPreferences(context)
|
2020-04-12 00:26:16 +02:00
|
|
|
.getBoolean(restrictedModeEnabledKey, false);
|
2020-04-12 22:13:04 +02:00
|
|
|
updateYoutubeRestrictedModeCookies(restrictedModeEnabled);
|
2020-03-15 22:53:29 +01:00
|
|
|
}
|
|
|
|
|
2020-04-12 22:13:04 +02:00
|
|
|
public void updateYoutubeRestrictedModeCookies(final boolean youtubeRestrictedModeEnabled) {
|
|
|
|
if (youtubeRestrictedModeEnabled) {
|
2020-04-12 00:26:16 +02:00
|
|
|
setCookie(YOUTUBE_RESTRICTED_MODE_COOKIE_KEY,
|
|
|
|
YOUTUBE_RESTRICTED_MODE_COOKIE);
|
2020-03-15 22:53:29 +01:00
|
|
|
} else {
|
2020-04-12 00:26:16 +02:00
|
|
|
removeCookie(YOUTUBE_RESTRICTED_MODE_COOKIE_KEY);
|
2020-03-15 22:53:29 +01:00
|
|
|
}
|
|
|
|
InfoCache.getInstance().clearCache();
|
2019-10-28 03:35:51 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get the size of the content that the url is pointing by firing a HEAD request.
|
|
|
|
*
|
|
|
|
* @param url an url pointing to the content
|
|
|
|
* @return the size of the content, in bytes
|
|
|
|
*/
|
2020-03-31 19:20:15 +02:00
|
|
|
public long getContentLength(final String url) throws IOException {
|
2019-10-28 03:35:51 +01:00
|
|
|
try {
|
|
|
|
final Response response = head(url);
|
|
|
|
return Long.parseLong(response.getHeader("Content-Length"));
|
2020-08-16 10:24:58 +02:00
|
|
|
} catch (final NumberFormatException e) {
|
2019-10-28 03:35:51 +01:00
|
|
|
throw new IOException("Invalid content length", e);
|
2020-08-16 10:24:58 +02:00
|
|
|
} catch (final ReCaptchaException e) {
|
2019-10-28 03:35:51 +01:00
|
|
|
throw new IOException(e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
2020-03-31 19:20:15 +02:00
|
|
|
public Response execute(@NonNull final Request request)
|
|
|
|
throws IOException, ReCaptchaException {
|
2019-10-28 03:35:51 +01:00
|
|
|
final String httpMethod = request.httpMethod();
|
|
|
|
final String url = request.url();
|
|
|
|
final Map<String, List<String>> headers = request.headers();
|
|
|
|
final byte[] dataToSend = request.dataToSend();
|
|
|
|
|
|
|
|
RequestBody requestBody = null;
|
|
|
|
if (dataToSend != null) {
|
2022-07-14 09:23:45 +02:00
|
|
|
requestBody = RequestBody.create(dataToSend);
|
2019-10-28 03:35:51 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
final okhttp3.Request.Builder requestBuilder = new okhttp3.Request.Builder()
|
|
|
|
.method(httpMethod, requestBody).url(url)
|
|
|
|
.addHeader("User-Agent", USER_AGENT);
|
|
|
|
|
2020-08-16 10:24:58 +02:00
|
|
|
final String cookies = getCookies(url);
|
2020-03-16 02:04:34 +01:00
|
|
|
if (!cookies.isEmpty()) {
|
|
|
|
requestBuilder.addHeader("Cookie", cookies);
|
2019-10-28 03:35:51 +01:00
|
|
|
}
|
|
|
|
|
2020-08-16 10:24:58 +02:00
|
|
|
for (final Map.Entry<String, List<String>> pair : headers.entrySet()) {
|
2019-10-28 03:35:51 +01:00
|
|
|
final String headerName = pair.getKey();
|
|
|
|
final List<String> headerValueList = pair.getValue();
|
|
|
|
|
|
|
|
if (headerValueList.size() > 1) {
|
|
|
|
requestBuilder.removeHeader(headerName);
|
2020-08-16 10:24:58 +02:00
|
|
|
for (final String headerValue : headerValueList) {
|
2019-10-28 03:35:51 +01:00
|
|
|
requestBuilder.addHeader(headerName, headerValue);
|
|
|
|
}
|
|
|
|
} else if (headerValueList.size() == 1) {
|
|
|
|
requestBuilder.header(headerName, headerValueList.get(0));
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
final okhttp3.Response response = client.newCall(requestBuilder.build()).execute();
|
|
|
|
|
|
|
|
if (response.code() == 429) {
|
|
|
|
response.close();
|
|
|
|
|
|
|
|
throw new ReCaptchaException("reCaptcha Challenge requested", url);
|
|
|
|
}
|
|
|
|
|
|
|
|
final ResponseBody body = response.body();
|
|
|
|
String responseBodyToReturn = null;
|
|
|
|
|
|
|
|
if (body != null) {
|
|
|
|
responseBodyToReturn = body.string();
|
|
|
|
}
|
|
|
|
|
2020-03-01 15:37:47 +01:00
|
|
|
final String latestUrl = response.request().url().toString();
|
2020-03-31 19:20:15 +02:00
|
|
|
return new Response(response.code(), response.message(), response.headers().toMultimap(),
|
|
|
|
responseBodyToReturn, latestUrl);
|
2019-12-08 17:09:16 +01:00
|
|
|
}
|
2019-10-28 03:35:51 +01:00
|
|
|
}
|