From 9e34fee58c1bd2ba9040a5457f73b9e7cc36a256 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Fri, 22 Mar 2019 22:54:07 -0300 Subject: [PATCH] New MP4 muxer + Queue changes + Storage fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main changes: * correctly check the available space (CircularFile.java) * misc cleanup (CircularFile.java) * use the "Error Reporter" for non-http errors * rewrite network state checking and add better support for API 21 (Lollipop) or higher * implement "metered networks" * add buttons in "Downloads" activity to start/pause all pending downloads, ignoring the queue flag or if the network is "metered" * add workaround for VPN connections and/or network switching. Example: switching WiFi to 3G * rewrite DataReader ¡Webm muxer is now 57% more faster! * rewrite CircularFile, use file buffers instead of memory buffers. Less troubles in low-end devices * fix missing offset for KaxCluster (WebMWriter.java), manifested as no thumbnails on file explorers Download queue: * remember queue status, unless the user pause the download (un-queue) * semi-automatic downloads, between networks. Effective if the user create a new download or the downloads activity is starts * allow enqueue failed downloads * new option, queue limit, enabled by default. Used to allow one or multiple downloads at same time Miscellaneous: * fix crash while selecting details/error menu (mistake on MissionFragment.java) * misc serialize changes (DownloadMission.java) * minor UI tweaks * allow overwrite paused downloads * fix wrong icons for grid/list button in downloads * add share option * implement #2006 * correct misspelled word in strings.xml (es) (cmn) * fix MissionAdapter crash during device shutdown New Mp4Muxer + required changes: * new mp4 muxer (from dash only) with this, muxing on Android 7 is possible now!!! * re-work in SharpStream * drop mp4 dash muxer * misc changes: add warning in SecondaryStreamHelper.java, * strip m4a DASH files to normal m4a format (youtube only) Fix storage issues: * warn to the user if is choosing a "read only" download directory (for external SD Cards), useless is rooted :) * "write proof" allow post-processing resuming only if the device ran out of space * implement "insufficient storage" error for downloads --- .../newpipe/download/DownloadDialog.java | 61 +- .../org/schabi/newpipe/report/UserAction.java | 4 +- .../settings/DownloadSettingsFragment.java | 14 +- .../schabi/newpipe/streams/DataReader.java | 237 ++++- .../schabi/newpipe/streams/Mp4DashReader.java | 444 +++++++--- .../schabi/newpipe/streams/Mp4DashWriter.java | 623 -------------- .../newpipe/streams/Mp4FromDashWriter.java | 810 ++++++++++++++++++ .../newpipe/streams/SubtitleConverter.java | 15 +- .../newpipe/streams/TrackDataChunk.java | 65 -- .../schabi/newpipe/streams/WebMReader.java | 16 +- .../schabi/newpipe/streams/WebMWriter.java | 130 ++- .../newpipe/streams/io/SharpStream.java | 22 +- .../org/schabi/newpipe/util/ListHelper.java | 14 +- .../newpipe/util/SecondaryStreamHelper.java | 2 +- .../giga/get/DownloadInitializer.java | 3 - .../us/shandian/giga/get/DownloadMission.java | 122 ++- .../java/us/shandian/giga/get/Mission.java | 5 + .../giga/postprocessing/M4aNoDash.java | 43 + ...p4DashMuxer.java => Mp4FromDashMuxer.java} | 58 +- .../giga/postprocessing/Mp4Muxer.java | 136 --- .../giga/postprocessing/Postprocessing.java | 118 ++- .../io/ChunkFileInputStream.java | 5 +- .../giga/postprocessing/io/CircularFile.java | 375 -------- .../postprocessing/io/CircularFileWriter.java | 459 ++++++++++ .../giga/postprocessing/io/FileStream.java | 126 --- .../postprocessing/io/SharpInputStream.java | 4 +- .../giga/service/DownloadManager.java | 227 +++-- .../giga/service/DownloadManagerService.java | 143 ++-- .../giga/ui/adapter/MissionAdapter.java | 255 ++++-- .../giga/ui/fragment/MissionsFragment.java | 33 +- .../res/drawable-hdpi/ic_pause_black_24dp.png | Bin 0 -> 135 bytes .../res/drawable-hdpi/ic_pause_white_24dp.png | Bin 0 -> 138 bytes .../res/drawable-mdpi/ic_pause_black_24dp.png | Bin 0 -> 109 bytes .../res/drawable-mdpi/ic_pause_white_24dp.png | Bin 0 -> 112 bytes .../drawable-xhdpi/ic_pause_black_24dp.png | Bin 0 -> 162 bytes .../drawable-xhdpi/ic_pause_white_24dp.png | Bin 0 -> 139 bytes .../drawable-xxhdpi/ic_pause_black_24dp.png | Bin 0 -> 196 bytes .../drawable-xxhdpi/ic_pause_white_24dp.png | Bin 0 -> 206 bytes .../drawable-xxxhdpi/ic_pause_black_24dp.png | Bin 0 -> 248 bytes .../drawable-xxxhdpi/ic_pause_white_24dp.png | Bin 0 -> 254 bytes app/src/main/res/menu/download_menu.xml | 12 + app/src/main/res/menu/mission.xml | 13 +- app/src/main/res/values-cmn/strings.xml | 2 +- app/src/main/res/values-es/strings.xml | 30 +- app/src/main/res/values/attrs.xml | 1 + app/src/main/res/values/settings_keys.xml | 1 + app/src/main/res/values/strings.xml | 14 +- app/src/main/res/values/styles.xml | 2 + app/src/main/res/xml/download_settings.xml | 7 + 49 files changed, 2715 insertions(+), 1936 deletions(-) delete mode 100644 app/src/main/java/org/schabi/newpipe/streams/Mp4DashWriter.java create mode 100644 app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java delete mode 100644 app/src/main/java/org/schabi/newpipe/streams/TrackDataChunk.java create mode 100644 app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java rename app/src/main/java/us/shandian/giga/postprocessing/{Mp4DashMuxer.java => Mp4FromDashMuxer.java} (60%) delete mode 100644 app/src/main/java/us/shandian/giga/postprocessing/Mp4Muxer.java delete mode 100644 app/src/main/java/us/shandian/giga/postprocessing/io/CircularFile.java create mode 100644 app/src/main/java/us/shandian/giga/postprocessing/io/CircularFileWriter.java delete mode 100644 app/src/main/java/us/shandian/giga/postprocessing/io/FileStream.java create mode 100644 app/src/main/res/drawable-hdpi/ic_pause_black_24dp.png create mode 100644 app/src/main/res/drawable-hdpi/ic_pause_white_24dp.png create mode 100644 app/src/main/res/drawable-mdpi/ic_pause_black_24dp.png create mode 100644 app/src/main/res/drawable-mdpi/ic_pause_white_24dp.png create mode 100644 app/src/main/res/drawable-xhdpi/ic_pause_black_24dp.png create mode 100644 app/src/main/res/drawable-xhdpi/ic_pause_white_24dp.png create mode 100644 app/src/main/res/drawable-xxhdpi/ic_pause_black_24dp.png create mode 100644 app/src/main/res/drawable-xxhdpi/ic_pause_white_24dp.png create mode 100644 app/src/main/res/drawable-xxxhdpi/ic_pause_black_24dp.png create mode 100644 app/src/main/res/drawable-xxxhdpi/ic_pause_white_24dp.png diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index ec6d42b29..0b4767133 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -7,6 +7,7 @@ import android.preference.PreferenceManager; import android.support.annotation.IdRes; import android.support.annotation.NonNull; import android.support.annotation.Nullable; +import android.support.annotation.StringRes; import android.support.v4.app.DialogFragment; import android.support.v7.app.AlertDialog; import android.support.v7.widget.Toolbar; @@ -52,6 +53,7 @@ import icepick.State; import io.reactivex.disposables.CompositeDisposable; import us.shandian.giga.postprocessing.Postprocessing; import us.shandian.giga.service.DownloadManagerService; +import us.shandian.giga.service.DownloadManagerService.MissionCheck; public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheckedChangeListener, AdapterView.OnItemSelectedListener { private static final String TAG = "DialogFragment"; @@ -263,7 +265,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck } @Override - public void onSaveInstanceState(Bundle outState) { + public void onSaveInstanceState(@NonNull Bundle outState) { super.onSaveInstanceState(outState); Icepick.saveInstanceState(this, outState); } @@ -476,23 +478,40 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck final String finalFileName = fileName; - DownloadManagerService.checkForRunningMission(context, location, fileName, (listed, finished) -> { - if (listed) { - AlertDialog.Builder builder = new AlertDialog.Builder(context); - builder.setTitle(R.string.download_dialog_title) - .setMessage(finished ? R.string.overwrite_warning : R.string.download_already_running) - .setPositiveButton( - finished ? R.string.overwrite : R.string.generate_unique_name, - (dialog, which) -> downloadSelected(context, stream, location, finalFileName, kind, threads) - ) - .setNegativeButton(android.R.string.cancel, (dialog, which) -> { - dialog.cancel(); - }) - .create() - .show(); - } else { - downloadSelected(context, stream, location, finalFileName, kind, threads); + DownloadManagerService.checkForRunningMission(context, location, fileName, (MissionCheck result) -> { + @StringRes int msgBtn; + @StringRes int msgBody; + + switch (result) { + case Finished: + msgBtn = R.string.overwrite; + msgBody = R.string.overwrite_warning; + break; + case Pending: + msgBtn = R.string.overwrite; + msgBody = R.string.download_already_pending; + break; + case PendingRunning: + msgBtn = R.string.generate_unique_name; + msgBody = R.string.download_already_running; + break; + default: + downloadSelected(context, stream, location, finalFileName, kind, threads); + return; } + + // overwrite or unique name actions are done by the download manager + + AlertDialog.Builder builder = new AlertDialog.Builder(context); + builder.setTitle(R.string.download_dialog_title) + .setMessage(msgBody) + .setPositiveButton( + msgBtn, + (dialog, which) -> downloadSelected(context, stream, location, finalFileName, kind, threads) + ) + .setNegativeButton(android.R.string.cancel, (dialog, which) -> dialog.cancel()) + .create() + .show(); }); } @@ -503,14 +522,18 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck String secondaryStreamUrl = null; long nearLength = 0; - if (selectedStream instanceof VideoStream) { + if (selectedStream instanceof AudioStream) { + if (selectedStream.getFormat() == MediaFormat.M4A) { + psName = Postprocessing.ALGORITHM_M4A_NO_DASH; + } + } else if (selectedStream instanceof VideoStream) { SecondaryStreamHelper secondaryStream = videoStreamsAdapter .getAllSecondary() .get(wrappedVideoStreams.getStreamsList().indexOf(selectedStream)); if (secondaryStream != null) { secondaryStreamUrl = secondaryStream.getStream().getUrl(); - psName = selectedStream.getFormat() == MediaFormat.MPEG_4 ? Postprocessing.ALGORITHM_MP4_MUXER : Postprocessing.ALGORITHM_WEBM_MUXER; + psName = selectedStream.getFormat() == MediaFormat.MPEG_4 ? Postprocessing.ALGORITHM_MP4_FROM_DASH_MUXER : Postprocessing.ALGORITHM_WEBM_MUXER; psArgs = null; long videoSize = wrappedVideoStreams.getSizeInBytes((VideoStream) selectedStream); diff --git a/app/src/main/java/org/schabi/newpipe/report/UserAction.java b/app/src/main/java/org/schabi/newpipe/report/UserAction.java index 2b2369ad3..2cca9305a 100644 --- a/app/src/main/java/org/schabi/newpipe/report/UserAction.java +++ b/app/src/main/java/org/schabi/newpipe/report/UserAction.java @@ -17,7 +17,9 @@ public enum UserAction { REQUESTED_KIOSK("requested kiosk"), REQUESTED_COMMENTS("requested comments"), DELETE_FROM_HISTORY("delete from history"), - PLAY_STREAM("Play stream"); + PLAY_STREAM("Play stream"), + DOWNLOAD_POSTPROCESSING("download post-processing"), + DOWNLOAD_FAILED("download failed"); private final String message; diff --git a/app/src/main/java/org/schabi/newpipe/settings/DownloadSettingsFragment.java b/app/src/main/java/org/schabi/newpipe/settings/DownloadSettingsFragment.java index 8214d7b4b..82c6853d5 100644 --- a/app/src/main/java/org/schabi/newpipe/settings/DownloadSettingsFragment.java +++ b/app/src/main/java/org/schabi/newpipe/settings/DownloadSettingsFragment.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.settings; import android.app.Activity; +import android.app.AlertDialog; import android.content.Intent; import android.os.Bundle; import android.support.annotation.Nullable; @@ -12,6 +13,8 @@ import com.nononsenseapps.filepicker.Utils; import org.schabi.newpipe.R; import org.schabi.newpipe.util.FilePickerActivityHelper; +import java.io.File; + public class DownloadSettingsFragment extends BasePreferenceFragment { private static final int REQUEST_DOWNLOAD_PATH = 0x1235; private static final int REQUEST_DOWNLOAD_AUDIO_PATH = 0x1236; @@ -45,7 +48,7 @@ public class DownloadSettingsFragment extends BasePreferenceFragment { @Override public boolean onPreferenceTreeClick(Preference preference) { if (DEBUG) { - Log.d(TAG, "onPreferenceTreeClick() called with: preference = [" + preference + "]"); + Log.d(TAG, "onPreferenceTreeClick() called with: preference = [" + preference + "]"); } if (preference.getKey().equals(DOWNLOAD_PATH_PREFERENCE) @@ -78,6 +81,15 @@ public class DownloadSettingsFragment extends BasePreferenceFragment { defaultPreferences.edit().putString(key, path).apply(); updatePreferencesSummary(); + + File target = new File(path); + if (!target.canWrite()) { + AlertDialog.Builder msg = new AlertDialog.Builder(getContext()); + msg.setTitle(R.string.download_to_sdcard_error_title); + msg.setMessage(R.string.download_to_sdcard_error_message); + msg.setPositiveButton(android.R.string.ok, (dialogInterface, i) -> { }); + msg.show(); + } } } } diff --git a/app/src/main/java/org/schabi/newpipe/streams/DataReader.java b/app/src/main/java/org/schabi/newpipe/streams/DataReader.java index d0e946eb7..567fa5229 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/DataReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/DataReader.java @@ -1,9 +1,10 @@ package org.schabi.newpipe.streams; +import org.schabi.newpipe.streams.io.SharpStream; + import java.io.EOFException; import java.io.IOException; - -import org.schabi.newpipe.streams.io.SharpStream; +import java.io.InputStream; /** * @author kapodamy @@ -15,89 +16,237 @@ public class DataReader { public final static int INTEGER_SIZE = 4; public final static int FLOAT_SIZE = 4; - private long pos; - public final SharpStream stream; - private final boolean rewind; + private long position = 0; + private final SharpStream stream; + + private InputStream view; + private int viewSize; public DataReader(SharpStream stream) { - this.rewind = stream.canRewind(); this.stream = stream; - this.pos = 0L; + this.readOffset = this.readBuffer.length; } public long position() { - return pos; + return position; } - public final int readInt() throws IOException { + public int read() throws IOException { + if (fillBuffer()) { + return -1; + } + + position++; + readCount--; + + return readBuffer[readOffset++] & 0xFF; + } + + public long skipBytes(long amount) throws IOException { + if (readCount < 0) { + return 0; + } else if (readCount == 0) { + amount = stream.skip(amount); + } else { + if (readCount > amount) { + readCount -= (int) amount; + readOffset += (int) amount; + } else { + amount = readCount + stream.skip(amount - readCount); + readCount = 0; + readOffset = readBuffer.length; + } + } + + position += amount; + return amount; + } + + public int readInt() throws IOException { primitiveRead(INTEGER_SIZE); return primitive[0] << 24 | primitive[1] << 16 | primitive[2] << 8 | primitive[3]; } - public final int read() throws IOException { - int value = stream.read(); - if (value == -1) { - throw new EOFException(); - } - - pos++; - return value; + public short readShort() throws IOException { + primitiveRead(SHORT_SIZE); + return (short) (primitive[0] << 8 | primitive[1]); } - public final long skipBytes(long amount) throws IOException { - amount = stream.skip(amount); - pos += amount; - return amount; - } - - public final long readLong() throws IOException { + public long readLong() throws IOException { primitiveRead(LONG_SIZE); long high = primitive[0] << 24 | primitive[1] << 16 | primitive[2] << 8 | primitive[3]; long low = primitive[4] << 24 | primitive[5] << 16 | primitive[6] << 8 | primitive[7]; return high << 32 | low; } - public final short readShort() throws IOException { - primitiveRead(SHORT_SIZE); - return (short) (primitive[0] << 8 | primitive[1]); - } - - public final int read(byte[] buffer) throws IOException { + public int read(byte[] buffer) throws IOException { return read(buffer, 0, buffer.length); } - public final int read(byte[] buffer, int offset, int count) throws IOException { - int res = stream.read(buffer, offset, count); - pos += res; + public int read(byte[] buffer, int offset, int count) throws IOException { + if (readCount < 0) { + return -1; + } + int total = 0; - return res; + if (count >= readBuffer.length) { + if (readCount > 0) { + System.arraycopy(readBuffer, readOffset, buffer, offset, readCount); + readOffset += readCount; + + offset += readCount; + count -= readCount; + + total = readCount; + readCount = 0; + } + total += Math.max(stream.read(buffer, offset, count), 0); + } else { + while (count > 0 && !fillBuffer()) { + int read = Math.min(readCount, count); + System.arraycopy(readBuffer, readOffset, buffer, offset, read); + + readOffset += read; + readCount -= read; + + offset += read; + count -= read; + + total += read; + } + } + + position += total; + return total; } - public final boolean available() { - return stream.available() > 0; + public boolean available() { + return readCount > 0 || stream.available() > 0; } public void rewind() throws IOException { stream.rewind(); - pos = 0; + + if ((position - viewSize) > 0) { + viewSize = 0;// drop view + } else { + viewSize += position; + } + + position = 0; + readOffset = readBuffer.length; } public boolean canRewind() { - return rewind; + return stream.canRewind(); } - private short[] primitive = new short[LONG_SIZE]; + /** + * Wraps this instance of {@code DataReader} into {@code InputStream} + * object. Note: Any read in the {@code DataReader} will not modify + * (decrease) the view size + * + * @param size the size of the view + * @return the view + */ + public InputStream getView(int size) { + if (view == null) { + view = new InputStream() { + @Override + public int read() throws IOException { + if (viewSize < 1) { + return -1; + } + int res = DataReader.this.read(); + if (res > 0) { + viewSize--; + } + return res; + } + + @Override + public int read(byte[] buffer) throws IOException { + return read(buffer, 0, buffer.length); + } + + @Override + public int read(byte[] buffer, int offset, int count) throws IOException { + if (viewSize < 1) { + return -1; + } + + int res = DataReader.this.read(buffer, offset, Math.min(viewSize, count)); + viewSize -= res; + + return res; + } + + @Override + public long skip(long amount) throws IOException { + if (viewSize < 1) { + return 0; + } + int res = (int) DataReader.this.skipBytes(Math.min(amount, viewSize)); + viewSize -= res; + + return res; + } + + @Override + public int available() { + return viewSize; + } + + @Override + public void close() { + viewSize = 0; + } + + @Override + public boolean markSupported() { + return false; + } + + }; + } + viewSize = size; + + return view; + } + + private final short[] primitive = new short[LONG_SIZE]; private void primitiveRead(int amount) throws IOException { byte[] buffer = new byte[amount]; - int read = stream.read(buffer, 0, amount); - pos += read; + int read = read(buffer, 0, amount); + if (read != amount) { - throw new EOFException("Truncated data, missing " + String.valueOf(amount - read) + " bytes"); + throw new EOFException("Truncated stream, missing " + String.valueOf(amount - read) + " bytes"); } - for (int i = 0; i < buffer.length; i++) { - primitive[i] = (short) (buffer[i] & 0xFF);// the "byte" datatype is signed and is very annoying + for (int i = 0; i < amount; i++) { + primitive[i] = (short) (buffer[i] & 0xFF);// the "byte" data type in java is signed and is very annoying } } + + private final byte[] readBuffer = new byte[8 * 1024]; + private int readOffset; + private int readCount; + + private boolean fillBuffer() throws IOException { + if (readCount < 0) { + return true; + } + if (readOffset >= readBuffer.length) { + readCount = stream.read(readBuffer); + if (readCount < 1) { + readCount = -1; + return true; + } + readOffset = 0; + } + + return readCount < 1; + } + } diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java index 271929d47..c52ebf3aa 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java @@ -1,17 +1,15 @@ package org.schabi.newpipe.streams; +import org.schabi.newpipe.streams.io.SharpStream; + import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; - import java.nio.ByteBuffer; - import java.util.ArrayList; import java.util.NoSuchElementException; -import org.schabi.newpipe.streams.io.SharpStream; - /** * @author kapodamy */ @@ -35,14 +33,29 @@ public class Mp4DashReader { private static final int ATOM_TREX = 0x74726578; private static final int ATOM_TKHD = 0x746B6864; private static final int ATOM_MFRA = 0x6D667261; - private static final int ATOM_TFRA = 0x74667261; private static final int ATOM_MDHD = 0x6D646864; + private static final int ATOM_EDTS = 0x65647473; + private static final int ATOM_ELST = 0x656C7374; + private static final int ATOM_HDLR = 0x68646C72; + private static final int ATOM_MINF = 0x6D696E66; + private static final int ATOM_DINF = 0x64696E66; + private static final int ATOM_STBL = 0x7374626C; + private static final int ATOM_STSD = 0x73747364; + private static final int ATOM_VMHD = 0x766D6864; + private static final int ATOM_SMHD = 0x736D6864; + private static final int BRAND_DASH = 0x64617368; + private static final int BRAND_ISO5 = 0x69736F35; + + private static final int HANDLER_VIDE = 0x76696465; + private static final int HANDLER_SOUN = 0x736F756E; + private static final int HANDLER_SUBT = 0x73756274; // private final DataReader stream; private Mp4Track[] tracks = null; + private int[] brands = null; private Box box; private Moof moof; @@ -50,9 +63,10 @@ public class Mp4DashReader { private boolean chunkZero = false; private int selectedTrack = -1; + private Box backupBox = null; public enum TrackKind { - Audio, Video, Other + Audio, Video, Subtitles, Other } public Mp4DashReader(SharpStream source) { @@ -65,8 +79,15 @@ public class Mp4DashReader { } box = readBox(ATOM_FTYP); - if (parse_ftyp() != BRAND_DASH) { - throw new NoSuchElementException("Main Brand is not dash"); + brands = parse_ftyp(box); + switch (brands[0]) { + case BRAND_DASH: + case BRAND_ISO5:// ¿why not? + break; + default: + throw new NoSuchElementException( + "Not a MPEG-4 DASH container, major brand is not 'dash' or 'iso5' is " + boxName(brands[0]) + ); } Moov moov = null; @@ -84,8 +105,6 @@ public class Mp4DashReader { break; case ATOM_MFRA: break; - case ATOM_MDAT: - throw new IOException("Expected moof, found mdat"); } } @@ -107,15 +126,26 @@ public class Mp4DashReader { } } - if (moov.trak[i].tkhd.bHeight == 0 && moov.trak[i].tkhd.bWidth == 0) { - tracks[i].kind = moov.trak[i].tkhd.bVolume == 0 ? TrackKind.Other : TrackKind.Audio; - } else { - tracks[i].kind = TrackKind.Video; + switch (moov.trak[i].mdia.hdlr.subType) { + case HANDLER_VIDE: + tracks[i].kind = TrackKind.Video; + break; + case HANDLER_SOUN: + tracks[i].kind = TrackKind.Audio; + break; + case HANDLER_SUBT: + tracks[i].kind = TrackKind.Subtitles; + break; + default: + tracks[i].kind = TrackKind.Other; + break; } } + + backupBox = box; } - public Mp4Track selectTrack(int index) { + Mp4Track selectTrack(int index) { selectedTrack = index; return tracks[index]; } @@ -126,7 +156,7 @@ public class Mp4DashReader { * @return list with a basic info * @throws IOException if the source stream is not seekeable */ - public int getFragmentsCount() throws IOException { + int getFragmentsCount() throws IOException { if (selectedTrack < 0) { throw new IllegalStateException("track no selected"); } @@ -136,7 +166,6 @@ public class Mp4DashReader { Box tmp; int count = 0; - long orig_offset = stream.position(); if (box.type == ATOM_MOOF) { tmp = box; @@ -162,17 +191,36 @@ public class Mp4DashReader { ensure(tmp); } while (stream.available() && (tmp = readBox()) != null); - stream.rewind(); - stream.skipBytes((int) orig_offset); + rewind(); return count; } + public int[] getBrands() { + if (brands == null) throw new IllegalStateException("Not parsed"); + return brands; + } + + public void rewind() throws IOException { + if (!stream.canRewind()) { + throw new IOException("The provided stream doesn't allow seek"); + } + if (box == null) { + return; + } + + box = backupBox; + chunkZero = false; + + stream.rewind(); + stream.skipBytes(backupBox.offset + (DataReader.INTEGER_SIZE * 2)); + } + public Mp4Track[] getAvailableTracks() { return tracks; } - public Mp4TrackChunk getNextChunk() throws IOException { + public Mp4DashChunk getNextChunk(boolean infoOnly) throws IOException { Mp4Track track = tracks[selectedTrack]; while (stream.available()) { @@ -208,7 +256,7 @@ public class Mp4DashReader { if (hasFlag(moof.traf.tfhd.bFlags, 0x10)) { moof.traf.trun.chunkSize = moof.traf.tfhd.defaultSampleSize * moof.traf.trun.entryCount; } else { - moof.traf.trun.chunkSize = box.size - 8; + moof.traf.trun.chunkSize = (int) (box.size - 8); } } if (!hasFlag(moof.traf.trun.bFlags, 0x900) && moof.traf.trun.chunkDuration == 0) { @@ -228,9 +276,12 @@ public class Mp4DashReader { continue;// find another chunk } - Mp4TrackChunk chunk = new Mp4TrackChunk(); + Mp4DashChunk chunk = new Mp4DashChunk(); chunk.moof = moof; - chunk.data = new TrackDataChunk(stream, moof.traf.trun.chunkSize); + if (!infoOnly) { + chunk.data = stream.getView(moof.traf.trun.chunkSize); + } + moof = null; stream.skipBytes(chunk.moof.traf.trun.dataOffset); @@ -269,6 +320,10 @@ public class Mp4DashReader { b.size = stream.readInt(); b.type = stream.readInt(); + if (b.size == 1) { + b.size = stream.readLong(); + } + return b; } @@ -280,6 +335,25 @@ public class Mp4DashReader { return b; } + private byte[] readFullBox(Box ref) throws IOException { + // full box reading is limited to 2 GiB, and should be enough + int size = (int) ref.size; + + ByteBuffer buffer = ByteBuffer.allocate(size); + buffer.putInt(size); + buffer.putInt(ref.type); + + int read = size - 8; + + if (stream.read(buffer.array(), 8, read) != read) { + throw new EOFException( + String.format("EOF reached in box: type=%s offset=%s size=%s", boxName(ref.type), ref.offset, ref.size) + ); + } + + return buffer.array(); + } + private void ensure(Box ref) throws IOException { long skip = ref.offset + ref.size - stream.position(); @@ -310,6 +384,14 @@ public class Mp4DashReader { return null; } + private Box untilAnyBox(Box ref) throws IOException { + if (stream.position() >= (ref.offset + ref.size)) { + return null; + } + + return readBox(); + } + // // @@ -329,7 +411,7 @@ public class Mp4DashReader { return obj; } } - + return obj; } @@ -397,14 +479,14 @@ public class Mp4DashReader { private long parse_tfdt() throws IOException { int version = stream.read(); - stream.skipBytes(3);// flags + stream.skipBytes(3);// flags return version == 0 ? readUint() : stream.readLong(); } private Trun parse_trun() throws IOException { Trun obj = new Trun(); obj.bFlags = stream.readInt(); - obj.entryCount = stream.readInt();// unsigned int + obj.entryCount = stream.readInt();// unsigned int obj.entries_rowSize = 0; if (hasFlag(obj.bFlags, 0x0100)) { @@ -448,11 +530,18 @@ public class Mp4DashReader { return obj; } - private int parse_ftyp() throws IOException { - int brand = stream.readInt(); + private int[] parse_ftyp(Box ref) throws IOException { + int i = 0; + int[] list = new int[(int) ((ref.offset + ref.size - stream.position() - 4) / 4)]; + + list[i++] = stream.readInt();// major brand + stream.skipBytes(4);// minor version - return brand; + for (; i < list.length; i++) + list[i] = stream.readInt();// compatible brands + + return list; } private Mvhd parse_mvhd() throws IOException { @@ -521,32 +610,66 @@ public class Mp4DashReader { trak.tkhd = parse_tkhd(); ensure(b); - b = untilBox(ref, ATOM_MDIA); - trak.mdia = new byte[b.size]; + while ((b = untilBox(ref, ATOM_MDIA, ATOM_EDTS)) != null) { + switch (b.type) { + case ATOM_MDIA: + trak.mdia = parse_mdia(b); + break; + case ATOM_EDTS: + trak.edst_elst = parse_edts(b); + break; + } - ByteBuffer buffer = ByteBuffer.wrap(trak.mdia); - buffer.putInt(b.size); - buffer.putInt(ATOM_MDIA); - stream.read(trak.mdia, 8, b.size - 8); - - trak.mdia_mdhd_timeScale = parse_mdia(buffer); + ensure(b); + } return trak; } - private int parse_mdia(ByteBuffer data) { - while (data.hasRemaining()) { - int end = data.position() + data.getInt(); - if (data.getInt() == ATOM_MDHD) { - byte version = data.get(); - data.position(data.position() + 3 + ((version == 0 ? 4 : 8) * 2)); - return data.getInt(); - } + private Mdia parse_mdia(Box ref) throws IOException { + Mdia obj = new Mdia(); - data.position(end); + Box b; + while ((b = untilBox(ref, ATOM_MDHD, ATOM_HDLR, ATOM_MINF)) != null) { + switch (b.type) { + case ATOM_MDHD: + obj.mdhd = readFullBox(b); + + // read time scale + ByteBuffer buffer = ByteBuffer.wrap(obj.mdhd); + byte version = buffer.get(8); + buffer.position(12 + ((version == 0 ? 4 : 8) * 2)); + obj.mdhd_timeScale = buffer.getInt(); + break; + case ATOM_HDLR: + obj.hdlr = parse_hdlr(b); + break; + case ATOM_MINF: + obj.minf = parse_minf(b); + break; + } + ensure(b); } - return 0;// this NEVER should happen + return obj; + } + + private Hdlr parse_hdlr(Box ref) throws IOException { + // version + // flags + stream.skipBytes(4); + + Hdlr obj = new Hdlr(); + obj.bReserved = new byte[12]; + + obj.type = stream.readInt(); + obj.subType = stream.readInt(); + stream.read(obj.bReserved); + + // component name (is a ansi/ascii string) + stream.skipBytes((ref.offset + ref.size) - stream.position()); + + return obj; } private Moov parse_moov(Box ref) throws IOException { @@ -570,7 +693,7 @@ public class Mp4DashReader { ensure(b); } - moov.trak = tmp.toArray(new Trak[tmp.size()]); + moov.trak = tmp.toArray(new Trak[0]); return moov; } @@ -584,7 +707,7 @@ public class Mp4DashReader { ensure(b); } - return tmp.toArray(new Trex[tmp.size()]); + return tmp.toArray(new Trex[0]); } private Trex parse_trex() throws IOException { @@ -602,74 +725,74 @@ public class Mp4DashReader { return obj; } - private Tfra parse_tfra() throws IOException { - int version = stream.read(); - - stream.skipBytes(3);// flags - - Tfra tfra = new Tfra(); - tfra.trackId = stream.readInt(); - - stream.skipBytes(3);// reserved - int bFlags = stream.read(); - int size_tts = ((bFlags >> 4) & 3) + ((bFlags >> 2) & 3) + (bFlags & 3); - - tfra.entries_time = new int[stream.readInt()]; - - for (int i = 0; i < tfra.entries_time.length; i++) { - tfra.entries_time[i] = version == 0 ? stream.readInt() : (int) stream.readLong(); - stream.skipBytes(size_tts + (version == 0 ? 4 : 8)); + private Elst parse_edts(Box ref) throws IOException { + Box b = untilBox(ref, ATOM_ELST); + if (b == null) { + return null; } - return tfra; - } - - private Sidx parse_sidx() throws IOException { - int version = stream.read(); + Elst obj = new Elst(); + boolean v1 = stream.read() == 1; stream.skipBytes(3);// flags - Sidx obj = new Sidx(); - obj.referenceId = stream.readInt(); - obj.timescale = stream.readInt(); + int entryCount = stream.readInt(); + if (entryCount < 1) { + obj.bMediaRate = 0x00010000;// default media rate (1.0) + return obj; + } - // earliest presentation entries_time - // first offset - // reserved - stream.skipBytes((2 * (version == 0 ? 4 : 8)) + 2); + if (v1) { + stream.skipBytes(DataReader.LONG_SIZE);// segment duration + obj.MediaTime = stream.readLong(); + // ignore all remain entries + stream.skipBytes((entryCount - 1) * (DataReader.LONG_SIZE * 2)); + } else { + stream.skipBytes(DataReader.INTEGER_SIZE);// segment duration + obj.MediaTime = stream.readInt(); + } - obj.entries_subsegmentDuration = new int[stream.readShort()]; + obj.bMediaRate = stream.readInt(); - for (int i = 0; i < obj.entries_subsegmentDuration.length; i++) { - // reference type - // referenced size - stream.skipBytes(4); - obj.entries_subsegmentDuration[i] = stream.readInt();// unsigned int + return obj; + } - // starts with SAP - // SAP type - // SAP delta entries_time - stream.skipBytes(4); + private Minf parse_minf(Box ref) throws IOException { + Minf obj = new Minf(); + + Box b; + while ((b = untilAnyBox(ref)) != null) { + + switch (b.type) { + case ATOM_DINF: + obj.dinf = readFullBox(b); + break; + case ATOM_STBL: + obj.stbl_stsd = parse_stbl(b); + break; + case ATOM_VMHD: + case ATOM_SMHD: + obj.$mhd = readFullBox(b); + break; + + } + ensure(b); } return obj; } - private Tfra[] parse_mfra(Box ref, int trackCount) throws IOException { - ArrayList tmp = new ArrayList<>(trackCount); - long limit = ref.offset + ref.size; + /** + * this only read the "stsd" box inside + */ + private byte[] parse_stbl(Box ref) throws IOException { + Box b = untilBox(ref, ATOM_STSD); - while (stream.position() < limit) { - box = readBox(); - - if (box.type == ATOM_TFRA) { - tmp.add(parse_tfra()); - } - - ensure(box); + if (b == null) { + return new byte[0];// this never should happens (missing codec startup data) } - return tmp.toArray(new Tfra[tmp.size()]); + return readFullBox(b); } // @@ -679,14 +802,7 @@ public class Mp4DashReader { int type; long offset; - int size; - } - - class Sidx { - - int timescale; - int referenceId; - int[] entries_subsegmentDuration; + long size; } public class Moof { @@ -711,12 +827,16 @@ public class Mp4DashReader { int defaultSampleFlags; } - public class TrunEntry { + class TrunEntry { + + int sampleDuration; + int sampleSize; + int sampleFlags; + int sampleCompositionTimeOffset; + + boolean hasCompositionTimeOffset; + boolean isKeyframe; - public int sampleDuration; - public int sampleSize; - public int sampleFlags; - public int sampleCompositionTimeOffset; } public class Trun { @@ -749,6 +869,31 @@ public class Mp4DashReader { entry.sampleCompositionTimeOffset = buffer.getInt(); } + entry.hasCompositionTimeOffset = hasFlag(bFlags, 0x0800); + entry.isKeyframe = !hasFlag(entry.sampleFlags, 0x10000); + + return entry; + } + + public TrunEntry getAbsoluteEntry(int i, Tfhd header) { + TrunEntry entry = getEntry(i); + + if (!hasFlag(bFlags, 0x0100) && hasFlag(header.bFlags, 0x20)) { + entry.sampleFlags = header.defaultSampleFlags; + } + + if (!hasFlag(bFlags, 0x0200) && hasFlag(header.bFlags, 0x10)) { + entry.sampleSize = header.defaultSampleSize; + } + + if (!hasFlag(bFlags, 0x0100) && hasFlag(header.bFlags, 0x08)) { + entry.sampleDuration = header.defaultSampleDuration; + } + + if (i == 0 && hasFlag(bFlags, 0x0004)) { + entry.sampleFlags = bFirstSampleFlags; + } + return entry; } } @@ -768,9 +913,9 @@ public class Mp4DashReader { public class Trak { public Tkhd tkhd; - public int mdia_mdhd_timeScale; + public Elst edst_elst; + public Mdia mdia; - byte[] mdia; } class Mvhd { @@ -786,12 +931,6 @@ public class Mp4DashReader { Trex[] mvex_trex; } - class Tfra { - - int trackId; - int[] entries_time; - } - public class Trex { private int trackId; @@ -801,6 +940,34 @@ public class Mp4DashReader { int defaultSampleFlags; } + public class Elst { + + public long MediaTime; + public int bMediaRate; + } + + public class Mdia { + + public int mdhd_timeScale; + public byte[] mdhd; + public Hdlr hdlr; + public Minf minf; + } + + public class Hdlr { + + public int type; + public int subType; + public byte[] bReserved; + } + + public class Minf { + + public byte[] dinf; + public byte[] stbl_stsd; + public byte[] $mhd; + } + public class Mp4Track { public TrackKind kind; @@ -808,10 +975,43 @@ public class Mp4DashReader { public Trex trex; } - public class Mp4TrackChunk { + public class Mp4DashChunk { public InputStream data; public Moof moof; + private int i = 0; + + public TrunEntry getNextSampleInfo() { + if (i >= moof.traf.trun.entryCount) { + return null; + } + return moof.traf.trun.getAbsoluteEntry(i++, moof.traf.tfhd); + } + + public Mp4DashSample getNextSample() throws IOException { + if (data == null) { + throw new IllegalStateException("This chunk has info only"); + } + if (i >= moof.traf.trun.entryCount) { + return null; + } + + Mp4DashSample sample = new Mp4DashSample(); + sample.info = moof.traf.trun.getAbsoluteEntry(i++, moof.traf.tfhd); + sample.data = new byte[sample.info.sampleSize]; + + if (data.read(sample.data) != sample.info.sampleSize) { + throw new EOFException("EOF reached while reading a sample"); + } + + return sample; + } + } + + public class Mp4DashSample { + + public TrunEntry info; + public byte[] data; } // } diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4DashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4DashWriter.java deleted file mode 100644 index babb2e24c..000000000 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4DashWriter.java +++ /dev/null @@ -1,623 +0,0 @@ -package org.schabi.newpipe.streams; - -import org.schabi.newpipe.streams.io.SharpStream; - -import org.schabi.newpipe.streams.Mp4DashReader.Mp4Track; -import org.schabi.newpipe.streams.Mp4DashReader.Mp4TrackChunk; -import org.schabi.newpipe.streams.Mp4DashReader.Trak; -import org.schabi.newpipe.streams.Mp4DashReader.Trex; - - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import static org.schabi.newpipe.streams.Mp4DashReader.hasFlag; - -/** - * - * @author kapodamy - */ -public class Mp4DashWriter { - - private final static byte DIMENSIONAL_FIVE = 5; - private final static byte DIMENSIONAL_TWO = 2; - private final static short DEFAULT_TIMESCALE = 1000; - private final static int BUFFER_SIZE = 8 * 1024; - private final static byte DEFAULT_TREX_SIZE = 32; - private final static byte[] TFRA_TTS_DEFAULT = new byte[]{0x01, 0x01, 0x01}; - private final static int EPOCH_OFFSET = 2082844800; - - private Mp4Track[] infoTracks; - private SharpStream[] sourceTracks; - - private Mp4DashReader[] readers; - private final long time; - - private boolean done = false; - private boolean parsed = false; - - private long written = 0; - private ArrayList> chunkTimes; - private ArrayList moofOffsets; - private ArrayList fragSizes; - - public Mp4DashWriter(SharpStream... source) { - sourceTracks = source; - readers = new Mp4DashReader[sourceTracks.length]; - infoTracks = new Mp4Track[sourceTracks.length]; - time = (System.currentTimeMillis() / 1000L) + EPOCH_OFFSET; - } - - public Mp4Track[] getTracksFromSource(int sourceIndex) throws IllegalStateException { - if (!parsed) { - throw new IllegalStateException("All sources must be parsed first"); - } - - return readers[sourceIndex].getAvailableTracks(); - } - - public void parseSources() throws IOException, IllegalStateException { - if (done) { - throw new IllegalStateException("already done"); - } - if (parsed) { - throw new IllegalStateException("already parsed"); - } - - try { - for (int i = 0; i < readers.length; i++) { - readers[i] = new Mp4DashReader(sourceTracks[i]); - readers[i].parse(); - } - - } finally { - parsed = true; - } - } - - public void selectTracks(int... trackIndex) throws IOException { - if (done) { - throw new IOException("already done"); - } - if (chunkTimes != null) { - throw new IOException("tracks already selected"); - } - - try { - chunkTimes = new ArrayList<>(readers.length); - moofOffsets = new ArrayList<>(32); - fragSizes = new ArrayList<>(32); - - for (int i = 0; i < readers.length; i++) { - infoTracks[i] = readers[i].selectTrack(trackIndex[i]); - - chunkTimes.add(new ArrayList(32)); - } - - } finally { - parsed = true; - } - } - - public long getBytesWritten() { - return written; - } - - public void build(SharpStream out) throws IOException, RuntimeException { - if (done) { - throw new RuntimeException("already done"); - } - if (!out.canWrite()) { - throw new IOException("the provided output is not writable"); - } - - long sidxOffsets = -1; - int maxFrags = 0; - - for (SharpStream stream : sourceTracks) { - if (!stream.canRewind()) { - sidxOffsets = -2;// sidx not available - } - } - - try { - dump(make_ftyp(), out); - dump(make_moov(), out); - - if (sidxOffsets == -1 && out.canRewind()) { - // - int reserved = 0; - for (Mp4DashReader reader : readers) { - int count = reader.getFragmentsCount(); - if (count > maxFrags) { - maxFrags = count; - } - reserved += 12 + calcSidxBodySize(count); - } - if (maxFrags > 0xFFFF) { - sidxOffsets = -3;// TODO: to many fragments, needs a multi-sidx implementation - } else { - sidxOffsets = written; - dump(make_free(reserved), out); - } - // - } - ArrayList chunks = new ArrayList<>(readers.length); - chunks.add(null); - - int read; - byte[] buffer = new byte[BUFFER_SIZE]; - int sequenceNumber = 1; - - while (true) { - chunks.clear(); - - for (int i = 0; i < readers.length; i++) { - Mp4TrackChunk chunk = readers[i].getNextChunk(); - if (chunk == null || chunk.moof.traf.trun.chunkSize < 1) { - continue; - } - chunk.moof.traf.tfhd.trackId = i + 1; - chunks.add(chunk); - - if (sequenceNumber == 1) { - if (chunk.moof.traf.trun.entryCount > 0 && hasFlag(chunk.moof.traf.trun.bFlags, 0x0800)) { - chunkTimes.get(i).add(chunk.moof.traf.trun.getEntry(0).sampleCompositionTimeOffset); - } else { - chunkTimes.get(i).add(0); - } - } - - chunkTimes.get(i).add(chunk.moof.traf.trun.chunkDuration); - } - - if (chunks.size() < 1) { - break; - } - - long offset = written; - moofOffsets.add(offset); - - dump(make_moof(sequenceNumber++, chunks, offset), out); - dump(make_mdat(chunks), out); - - for (Mp4TrackChunk chunk : chunks) { - while ((read = chunk.data.read(buffer)) > 0) { - out.write(buffer, 0, read); - written += read; - } - } - - fragSizes.add((int) (written - offset)); - } - - dump(make_mfra(), out); - - if (sidxOffsets > 0 && moofOffsets.size() == maxFrags) { - long len = written; - - out.rewind(); - out.skip(sidxOffsets); - - written = sidxOffsets; - sidxOffsets = moofOffsets.get(0); - - for (int i = 0; i < readers.length; i++) { - dump(make_sidx(i, sidxOffsets - written), out); - } - - written = len; - } - } finally { - done = true; - } - } - - public boolean isDone() { - return done; - } - - public boolean isParsed() { - return parsed; - } - - public void close() { - done = true; - parsed = true; - - for (SharpStream src : sourceTracks) { - src.dispose(); - } - - sourceTracks = null; - readers = null; - infoTracks = null; - moofOffsets = null; - chunkTimes = null; - } - - // - private void dump(byte[][] buffer, SharpStream stream) throws IOException { - for (byte[] buff : buffer) { - stream.write(buff); - written += buff.length; - } - } - - private byte[][] lengthFor(byte[][] buffer) { - int length = 0; - for (byte[] buff : buffer) { - length += buff.length; - } - - ByteBuffer.wrap(buffer[0]).putInt(length); - - return buffer; - } - - private int calcSidxBodySize(int entryCount) { - return 4 + 4 + 8 + 8 + 4 + (entryCount * 12); - } - // - - // - private byte[][] make_moof(int sequence, ArrayList chunks, long referenceOffset) { - int pos = 2; - TrunExtra[] extra = new TrunExtra[chunks.size()]; - - byte[][] buffer = new byte[pos + (extra.length * DIMENSIONAL_FIVE)][]; - buffer[0] = new byte[]{ - 0x00, 0x00, 0x00, 0x00, 0x6D, 0x6F, 0x6F, 0x66,// info header - 0x00, 0x00, 0x00, 0x10, 0x6D, 0x66, 0x68, 0x64, 0x00, 0x00, 0x00, 0x00//mfhd - }; - buffer[1] = new byte[4]; - ByteBuffer.wrap(buffer[1]).putInt(sequence); - - for (int i = 0; i < extra.length; i++) { - extra[i] = new TrunExtra(); - for (byte[] buff : make_traf(chunks.get(i), extra[i], referenceOffset)) { - buffer[pos++] = buff; - } - } - - lengthFor(buffer); - - int offset = 8 + ByteBuffer.wrap(buffer[0]).getInt(); - - for (int i = 0; i < extra.length; i++) { - extra[i].byteBuffer.putInt(offset); - offset += chunks.get(i).moof.traf.trun.chunkSize; - } - - return buffer; - } - - private byte[][] make_traf(Mp4TrackChunk chunk, TrunExtra extra, long moofOffset) { - byte[][] buffer = new byte[DIMENSIONAL_FIVE][]; - buffer[0] = new byte[]{ - 0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x61, 0x66, - 0x00, 0x00, 0x00, 0x00, 0x74, 0x66, 0x68, 0x64 - }; - - int flags = (chunk.moof.traf.tfhd.bFlags & 0x38) | 0x01; - byte tfhdBodySize = 8 + 8; - if (hasFlag(flags, 0x08)) { - tfhdBodySize += 4; - } - if (hasFlag(flags, 0x10)) { - tfhdBodySize += 4; - } - if (hasFlag(flags, 0x20)) { - tfhdBodySize += 4; - } - buffer[1] = new byte[tfhdBodySize]; - ByteBuffer set = ByteBuffer.wrap(buffer[1]); - set.position(4); - set.putInt(chunk.moof.traf.tfhd.trackId); - set.putLong(moofOffset); - if (hasFlag(flags, 0x08)) { - set.putInt(chunk.moof.traf.tfhd.defaultSampleDuration); - } - if (hasFlag(flags, 0x10)) { - set.putInt(chunk.moof.traf.tfhd.defaultSampleSize); - } - if (hasFlag(flags, 0x20)) { - set.putInt(chunk.moof.traf.tfhd.defaultSampleFlags); - } - set.putInt(0, flags); - ByteBuffer.wrap(buffer[0]).putInt(8, 8 + tfhdBodySize); - - buffer[2] = new byte[]{ - 0x00, 0x00, 0x00, 0x14, - 0x74, 0x66, 0x64, 0x74, - 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00 - }; - - ByteBuffer.wrap(buffer[2]).putLong(12, chunk.moof.traf.tfdt); - - buffer[3] = new byte[]{ - 0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x75, 0x6E, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00 - }; - - buffer[4] = chunk.moof.traf.trun.bEntries; - - lengthFor(buffer); - - set = ByteBuffer.wrap(buffer[3]); - set.putInt(buffer[3].length + buffer[4].length); - set.position(8); - set.putInt((chunk.moof.traf.trun.bFlags | 0x01) & 0x0F01); - set.putInt(chunk.moof.traf.trun.entryCount); - extra.byteBuffer = set; - - return buffer; - } - - private byte[][] make_mdat(ArrayList chunks) { - byte[][] buffer = new byte[][]{ - { - 0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x61, 0x74 - } - }; - - int length = 0; - - for (Mp4TrackChunk chunk : chunks) { - length += chunk.moof.traf.trun.chunkSize; - } - - ByteBuffer.wrap(buffer[0]).putInt(length + 8); - - return buffer; - } - - private byte[][] make_ftyp() { - return new byte[][]{ - { - 0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, 0x64, 0x61, 0x73, 0x68, 0x00, 0x00, 0x00, 0x00, - 0x6D, 0x70, 0x34, 0x31, 0x69, 0x73, 0x6F, 0x6D, 0x69, 0x73, 0x6F, 0x36, 0x69, 0x73, 0x6F, 0x32 - } - }; - } - - private byte[][] make_mvhd() { - byte[][] buffer = new byte[DIMENSIONAL_FIVE][]; - - buffer[0] = new byte[]{ - 0x00, 0x00, 0x00, 0x78, 0x6D, 0x76, 0x68, 0x64, 0x01, 0x00, 0x00, 0x00 - }; - buffer[1] = new byte[28]; - buffer[2] = new byte[]{ - 0x00, 0x01, 0x00, 0x00, 0x01, 0x00,// default volume and rate - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,// reserved values - // default matrix - 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x40, 0x00, 0x00, 0x00 - }; - buffer[3] = new byte[24];// predefined - buffer[4] = ByteBuffer.allocate(4).putInt(infoTracks.length + 1).array(); - - long longestTrack = 0; - - for (Mp4Track track : infoTracks) { - long tmp = (long) ((track.trak.tkhd.duration / (double) track.trak.mdia_mdhd_timeScale) * DEFAULT_TIMESCALE); - if (tmp > longestTrack) { - longestTrack = tmp; - } - } - - ByteBuffer.wrap(buffer[1]) - .putLong(time) - .putLong(time) - .putInt(DEFAULT_TIMESCALE) - .putLong(longestTrack); - - return buffer; - } - - private byte[][] make_trak(int trackId, Trak trak) throws RuntimeException { - if (trak.tkhd.matrix.length != 36) { - throw new RuntimeException("bad track matrix length (expected 36)"); - } - - byte[][] buffer = new byte[DIMENSIONAL_FIVE][]; - - buffer[0] = new byte[]{ - 0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x61, 0x6B,// trak header - 0x00, 0x00, 0x00, 0x68, 0x74, 0x6B, 0x68, 0x64, 0x01, 0x00, 0x00, 0x03 // tkhd header - }; - buffer[1] = new byte[48]; - buffer[2] = trak.tkhd.matrix; - buffer[3] = new byte[8]; - buffer[4] = trak.mdia; - - ByteBuffer set = ByteBuffer.wrap(buffer[1]); - set.putLong(time); - set.putLong(time); - set.putInt(trackId); - set.position(24); - set.putLong(trak.tkhd.duration); - set.position(40); - set.putShort(trak.tkhd.bLayer); - set.putShort(trak.tkhd.bAlternateGroup); - set.putShort(trak.tkhd.bVolume); - - ByteBuffer.wrap(buffer[3]) - .putInt(trak.tkhd.bWidth) - .putInt(trak.tkhd.bHeight); - - return lengthFor(buffer); - } - - private byte[][] make_moov() throws RuntimeException { - int pos = 1; - byte[][] buffer = new byte[2 + (DIMENSIONAL_TWO * infoTracks.length) + (DIMENSIONAL_FIVE * infoTracks.length) + DIMENSIONAL_FIVE + 1][]; - - buffer[0] = new byte[]{ - 0x00, 0x00, 0x00, 0x00, 0x6D, 0x6F, 0x6F, 0x76 - }; - - for (byte[] buff : make_mvhd()) { - buffer[pos++] = buff; - } - - for (int i = 0; i < infoTracks.length; i++) { - for (byte[] buff : make_trak(i + 1, infoTracks[i].trak)) { - buffer[pos++] = buff; - } - } - - buffer[pos] = new byte[]{ - 0x00, 0x00, 0x00, 0x00, 0x6D, 0x76, 0x65, 0x78 - }; - - ByteBuffer.wrap(buffer[pos++]).putInt((infoTracks.length * DEFAULT_TREX_SIZE) + 8); - - for (int i = 0; i < infoTracks.length; i++) { - for (byte[] buff : make_trex(i + 1, infoTracks[i].trex)) { - buffer[pos++] = buff; - } - } - - // default udta - buffer[pos] = new byte[]{ - 0x00, 0x00, 0x00, 0x5C, 0x75, 0x64, 0x74, 0x61, 0x00, 0x00, 0x00, 0x54, 0x6D, 0x65, 0x74, 0x61, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x68, 0x64, 0x6C, 0x72, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x69, 0x72, 0x61, 0x70, 0x70, 0x6C, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x69, 0x6C, 0x73, 0x74, 0x00, 0x00, 0x00, - 0x1F, (byte) 0xA9, 0x63, 0x6D, 0x74, 0x00, 0x00, 0x00, 0x17, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, - 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65// "NewPipe" binary string - }; - - return lengthFor(buffer); - } - - private byte[][] make_trex(int trackId, Trex trex) { - byte[][] buffer = new byte[][]{ - { - 0x00, 0x00, 0x00, 0x20, 0x74, 0x72, 0x65, 0x78, 0x00, 0x00, 0x00, 0x00 - }, - new byte[20] - }; - - ByteBuffer.wrap(buffer[1]) - .putInt(trackId) - .putInt(trex.defaultSampleDescriptionIndex) - .putInt(trex.defaultSampleDuration) - .putInt(trex.defaultSampleSize) - .putInt(trex.defaultSampleFlags); - - return buffer; - } - - private byte[][] make_tfra(int trackId, List times, List moofOffsets) { - int entryCount = times.size() - 1; - byte[][] buffer = new byte[DIMENSIONAL_TWO][]; - buffer[0] = new byte[]{ - 0x00, 0x00, 0x00, 0x00, 0x74, 0x66, 0x72, 0x61, 0x01, 0x00, 0x00, 0x00 - }; - buffer[1] = new byte[12 + ((16 + TFRA_TTS_DEFAULT.length) * entryCount)]; - - ByteBuffer set = ByteBuffer.wrap(buffer[1]); - set.putInt(trackId); - set.position(8); - set.putInt(entryCount); - - long decodeTime = 0; - - for (int i = 0; i < entryCount; i++) { - decodeTime += times.get(i); - set.putLong(decodeTime); - set.putLong(moofOffsets.get(i)); - set.put(TFRA_TTS_DEFAULT);// default values: traf number/trun number/sample number - } - - return lengthFor(buffer); - } - - private byte[][] make_mfra() { - byte[][] buffer = new byte[2 + (DIMENSIONAL_TWO * infoTracks.length)][]; - buffer[0] = new byte[]{ - 0x00, 0x00, 0x00, 0x00, 0x6D, 0x66, 0x72, 0x61 - }; - int pos = 1; - - for (int i = 0; i < infoTracks.length; i++) { - for (byte[] buff : make_tfra(i + 1, chunkTimes.get(i), moofOffsets)) { - buffer[pos++] = buff; - } - } - - buffer[pos] = new byte[]{// mfro - 0x00, 0x00, 0x00, 0x10, 0x6D, 0x66, 0x72, 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - }; - - lengthFor(buffer); - - ByteBuffer set = ByteBuffer.wrap(buffer[pos]); - set.position(12); - set.put(buffer[0], 0, 4); - - return buffer; - - } - - private byte[][] make_sidx(int internalTrackId, long firstOffset) { - List times = chunkTimes.get(internalTrackId); - int count = times.size() - 1;// the first item is ignored (composition time) - - if (count > 65535) { - throw new OutOfMemoryError("to many fragments. sidx limit is 65535, found " + String.valueOf(count)); - } - - byte[][] buffer = new byte[][]{ - new byte[]{ - 0x00, 0x00, 0x00, 0x00, 0x73, 0x69, 0x64, 0x78, 0x01, 0x00, 0x00, 0x00 - }, - new byte[calcSidxBodySize(count)] - }; - - lengthFor(buffer); - - ByteBuffer set = ByteBuffer.wrap(buffer[1]); - set.putInt(internalTrackId + 1); - set.putInt(infoTracks[internalTrackId].trak.mdia_mdhd_timeScale); - set.putLong(0); - set.putLong(firstOffset - ByteBuffer.wrap(buffer[0]).getInt()); - set.putInt(0xFFFF & count);// unsigned - - int i = 0; - while (i < count) { - set.putInt(fragSizes.get(i) & 0x7fffffff);// default reference type is 0 - set.putInt(times.get(i + 1)); - set.putInt(0x90000000);// default SAP settings - i++; - } - - return buffer; - } - - private byte[][] make_free(int totalSize) { - return lengthFor(new byte[][]{ - new byte[]{0x00, 0x00, 0x00, 0x00, 0x66, 0x72, 0x65, 0x65}, - new byte[totalSize - 8]// this is waste of RAM - }); - - } - -// - - class TrunExtra { - - ByteBuffer byteBuffer; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java new file mode 100644 index 000000000..5a4efbe32 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java @@ -0,0 +1,810 @@ +package org.schabi.newpipe.streams; + +import org.schabi.newpipe.streams.Mp4DashReader.Hdlr; +import org.schabi.newpipe.streams.Mp4DashReader.Mdia; +import org.schabi.newpipe.streams.Mp4DashReader.Mp4DashChunk; +import org.schabi.newpipe.streams.Mp4DashReader.Mp4DashSample; +import org.schabi.newpipe.streams.Mp4DashReader.Mp4Track; +import org.schabi.newpipe.streams.Mp4DashReader.TrunEntry; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * + * @author kapodamy + */ +public class Mp4FromDashWriter { + + private final static int EPOCH_OFFSET = 2082844800; + private final static short DEFAULT_TIMESCALE = 1000; + private final static byte SAMPLES_PER_CHUNK_INIT = 2; + private final static byte SAMPLES_PER_CHUNK = 6;// ffmpeg uses 2, basic uses 1 (with 60fps uses 21 or 22). NewPipe will use 6 + private final static long THRESHOLD_FOR_CO64 = 0xFFFEFFFFL;// near 3.999 GiB + private final static int THRESHOLD_MOOV_LENGTH = (256 * 1024) + (2048 * 1024); // 2.2 MiB enough for: 1080p 60fps 00h35m00s + + private final long time; + + private ByteBuffer auxBuffer; + private SharpStream outStream; + + private long lastWriteOffset = -1; + private long writeOffset; + + private boolean moovSimulation = true; + + private boolean done = false; + private boolean parsed = false; + + private Mp4Track[] tracks; + private SharpStream[] sourceTracks; + + private Mp4DashReader[] readers; + private Mp4DashChunk[] readersChunks; + + private int overrideMainBrand = 0x00; + + public Mp4FromDashWriter(SharpStream... sources) throws IOException { + for (SharpStream src : sources) { + if (!src.canRewind() && !src.canRead()) { + throw new IOException("All sources must be readable and allow rewind"); + } + } + + sourceTracks = sources; + readers = new Mp4DashReader[sourceTracks.length]; + readersChunks = new Mp4DashChunk[readers.length]; + time = (System.currentTimeMillis() / 1000L) + EPOCH_OFFSET; + } + + public Mp4Track[] getTracksFromSource(int sourceIndex) throws IllegalStateException { + if (!parsed) { + throw new IllegalStateException("All sources must be parsed first"); + } + + return readers[sourceIndex].getAvailableTracks(); + } + + public void parseSources() throws IOException, IllegalStateException { + if (done) { + throw new IllegalStateException("already done"); + } + if (parsed) { + throw new IllegalStateException("already parsed"); + } + + try { + for (int i = 0; i < readers.length; i++) { + readers[i] = new Mp4DashReader(sourceTracks[i]); + readers[i].parse(); + } + + } finally { + parsed = true; + } + } + + public void selectTracks(int... trackIndex) throws IOException { + if (done) { + throw new IOException("already done"); + } + if (tracks != null) { + throw new IOException("tracks already selected"); + } + + try { + tracks = new Mp4Track[readers.length]; + for (int i = 0; i < readers.length; i++) { + tracks[i] = readers[i].selectTrack(trackIndex[i]); + } + } finally { + parsed = true; + } + } + + public void setMainBrand(int brandId) { + overrideMainBrand = brandId; + } + + public boolean isDone() { + return done; + } + + public boolean isParsed() { + return parsed; + } + + public void close() throws IOException { + done = true; + parsed = true; + + for (SharpStream src : sourceTracks) { + src.dispose(); + } + + tracks = null; + sourceTracks = null; + + readers = null; + readersChunks = null; + + auxBuffer = null; + outStream = null; + } + + public void build(SharpStream output) throws IOException { + if (done) { + throw new RuntimeException("already done"); + } + if (!output.canWrite()) { + throw new IOException("the provided output is not writable"); + } + + // + // WARNING: the muxer requires at least 8 samples of every track + // not allowed for very short tracks (less than 0.5 seconds) + // + outStream = output; + int read = 8;// mdat box header size + long totalSampleSize = 0; + int[] sampleExtra = new int[readers.length]; + int[] defaultMediaTime = new int[readers.length]; + int[] defaultSampleDuration = new int[readers.length]; + int[] sampleCount = new int[readers.length]; + + TablesInfo[] tablesInfo = new TablesInfo[tracks.length]; + for (int i = 0; i < tablesInfo.length; i++) { + tablesInfo[i] = new TablesInfo(); + } + + // + for (int i = 0; i < readers.length; i++) { + int samplesSize = 0; + int sampleSizeChanges = 0; + int compositionOffsetLast = -1; + + Mp4DashChunk chunk; + while ((chunk = readers[i].getNextChunk(true)) != null) { + + if (defaultMediaTime[i] < 1 && chunk.moof.traf.tfhd.defaultSampleDuration > 0) { + defaultMediaTime[i] = chunk.moof.traf.tfhd.defaultSampleDuration; + } + + read += chunk.moof.traf.trun.chunkSize; + sampleExtra[i] += chunk.moof.traf.trun.chunkDuration;// calculate track duration + + TrunEntry info; + while ((info = chunk.getNextSampleInfo()) != null) { + if (info.isKeyframe) { + tablesInfo[i].stss++; + } + + if (info.sampleDuration > defaultSampleDuration[i]) { + defaultSampleDuration[i] = info.sampleDuration; + } + + tablesInfo[i].stsz++; + if (samplesSize != info.sampleSize) { + samplesSize = info.sampleSize; + sampleSizeChanges++; + } + + if (info.hasCompositionTimeOffset) { + if (info.sampleCompositionTimeOffset != compositionOffsetLast) { + tablesInfo[i].ctts++; + compositionOffsetLast = info.sampleCompositionTimeOffset; + } + } + + totalSampleSize += info.sampleSize; + } + } + + if (defaultMediaTime[i] < 1) { + defaultMediaTime[i] = defaultSampleDuration[i]; + } + + readers[i].rewind(); + + int tmp = tablesInfo[i].stsz - SAMPLES_PER_CHUNK_INIT; + tablesInfo[i].stco = (tmp / SAMPLES_PER_CHUNK) + 1;// +1 for samples in first chunk + + tmp = tmp % SAMPLES_PER_CHUNK; + if (tmp == 0) { + tablesInfo[i].stsc = 2;// first chunk (init) and succesive chunks + tablesInfo[i].stsc_bEntries = new int[]{ + 1, SAMPLES_PER_CHUNK_INIT, 1, + 2, SAMPLES_PER_CHUNK, 1 + }; + } else { + tablesInfo[i].stsc = 3;// first chunk (init) and succesive chunks and remain chunk + tablesInfo[i].stsc_bEntries = new int[]{ + 1, SAMPLES_PER_CHUNK_INIT, 1, + 2, SAMPLES_PER_CHUNK, 1, + tablesInfo[i].stco + 1, tmp, 1 + }; + tablesInfo[i].stco++; + } + + sampleCount[i] = tablesInfo[i].stsz; + + if (sampleSizeChanges == 1) { + tablesInfo[i].stsz = 0; + tablesInfo[i].stsz_default = samplesSize; + } else { + tablesInfo[i].stsz_default = 0; + } + + if (tablesInfo[i].stss == tablesInfo[i].stsz) { + tablesInfo[i].stss = -1;// for audio tracks (all samples are keyframes) + } + + // ensure track duration + if (tracks[i].trak.tkhd.duration < 1) { + tracks[i].trak.tkhd.duration = sampleExtra[i];// this never should happen + } + } + // + + boolean is64 = read > THRESHOLD_FOR_CO64; + + // calculate the moov size; + int auxSize = make_moov(defaultMediaTime, tablesInfo, is64); + + if (auxSize < THRESHOLD_MOOV_LENGTH) { + auxBuffer = ByteBuffer.allocate(auxSize);// cache moov in the memory + } + + moovSimulation = false; + writeOffset = 0; + + final int ftyp_size = make_ftyp(); + + // reserve moov space in the output stream + if (outStream.canSetLength()) { + long length = writeOffset + auxSize; + outStream.setLength(length); + outSeek(length); + } else { + // hard way + int length = auxSize; + byte[] buffer = new byte[8 * 1024];// 8 KiB + while (length > 0) { + int count = Math.min(length, buffer.length); + outWrite(buffer, 0, count); + length -= count; + } + } + if (auxBuffer == null) { + outSeek(ftyp_size); + } + + // tablesInfo contais row counts + // and after returning from make_moov() will contain table offsets + make_moov(defaultMediaTime, tablesInfo, is64); + + // write tables: stts stsc + // reset for ctts table: sampleCount sampleExtra + for (int i = 0; i < readers.length; i++) { + writeEntryArray(tablesInfo[i].stts, 2, sampleCount[i], defaultSampleDuration[i]); + writeEntryArray(tablesInfo[i].stsc, tablesInfo[i].stsc_bEntries.length, tablesInfo[i].stsc_bEntries); + tablesInfo[i].stsc_bEntries = null; + if (tablesInfo[i].ctts > 0) { + sampleCount[i] = 1;// index is not base zero + sampleExtra[i] = -1; + } + } + + if (auxBuffer == null) { + outRestore(); + } + + outWrite(make_mdat(totalSampleSize, is64)); + + int[] sampleIndex = new int[readers.length]; + int[] sizes = new int[SAMPLES_PER_CHUNK]; + int[] sync = new int[SAMPLES_PER_CHUNK]; + + int written = readers.length; + while (written > 0) { + written = 0; + + for (int i = 0; i < readers.length; i++) { + if (sampleIndex[i] < 0) { + continue;// track is done + } + + long chunkOffset = writeOffset; + int syncCount = 0; + int limit = sampleIndex[i] == 0 ? SAMPLES_PER_CHUNK_INIT : SAMPLES_PER_CHUNK; + + int j = 0; + for (; j < limit; j++) { + Mp4DashSample sample = getNextSample(i); + + if (sample == null) { + if (tablesInfo[i].ctts > 0 && sampleExtra[i] >= 0) { + writeEntryArray(tablesInfo[i].ctts, 1, sampleCount[i], sampleExtra[i]);// flush last entries + } + sampleIndex[i] = -1; + break; + } + + sampleIndex[i]++; + + if (tablesInfo[i].ctts > 0) { + if (sample.info.sampleCompositionTimeOffset == sampleExtra[i]) { + sampleCount[i]++; + } else { + if (sampleExtra[i] >= 0) { + tablesInfo[i].ctts = writeEntryArray(tablesInfo[i].ctts, 2, sampleCount[i], sampleExtra[i]); + outRestore(); + } + sampleCount[i] = 1; + sampleExtra[i] = sample.info.sampleCompositionTimeOffset; + } + } + + if (tablesInfo[i].stss > 0 && sample.info.isKeyframe) { + sync[syncCount++] = sampleIndex[i]; + } + + if (tablesInfo[i].stsz > 0) { + sizes[j] = sample.data.length; + } + + outWrite(sample.data, 0, sample.data.length); + } + + if (j > 0) { + written++; + + if (tablesInfo[i].stsz > 0) { + tablesInfo[i].stsz = writeEntryArray(tablesInfo[i].stsz, j, sizes); + } + + if (syncCount > 0) { + tablesInfo[i].stss = writeEntryArray(tablesInfo[i].stss, syncCount, sync); + } + + if (is64) { + tablesInfo[i].stco = writeEntry64(tablesInfo[i].stco, chunkOffset); + } else { + tablesInfo[i].stco = writeEntryArray(tablesInfo[i].stco, 1, (int) chunkOffset); + } + + outRestore(); + } + } + } + + if (auxBuffer != null) { + // dump moov + outSeek(ftyp_size); + outStream.write(auxBuffer.array(), 0, auxBuffer.capacity()); + auxBuffer = null; + } + } + + private Mp4DashSample getNextSample(int track) throws IOException { + if (readersChunks[track] == null) { + readersChunks[track] = readers[track].getNextChunk(false); + if (readersChunks[track] == null) { + return null;// EOF reached + } + } + + Mp4DashSample sample = readersChunks[track].getNextSample(); + if (sample == null) { + readersChunks[track] = null; + return getNextSample(track); + } else { + return sample; + } + } + + // + private int writeEntry64(int offset, long value) throws IOException { + outBackup(); + + auxSeek(offset); + auxWrite(ByteBuffer.allocate(8).putLong(value).array()); + + return offset + 8; + } + + private int writeEntryArray(int offset, int count, int... values) throws IOException { + outBackup(); + + auxSeek(offset); + + int size = count * 4; + ByteBuffer buffer = ByteBuffer.allocate(size); + + for (int i = 0; i < count; i++) { + buffer.putInt(values[i]); + } + + auxWrite(buffer.array()); + + return offset + size; + } + + private void outBackup() { + if (auxBuffer == null && lastWriteOffset < 0) { + lastWriteOffset = writeOffset; + } + } + + /** + * Restore to the previous position before the first call to writeEntry64() + * or writeEntryArray() methods. + */ + private void outRestore() throws IOException { + if (lastWriteOffset > 0) { + outSeek(lastWriteOffset); + lastWriteOffset = -1; + } + } + // + + // + private void outWrite(byte[] buffer) throws IOException { + outWrite(buffer, 0, buffer.length); + } + + private void outWrite(byte[] buffer, int offset, int count) throws IOException { + writeOffset += count; + outStream.write(buffer, offset, count); + } + + private void outSeek(long offset) throws IOException { + if (outStream.canSeek()) { + outStream.seek(offset); + writeOffset = offset; + } else if (outStream.canRewind()) { + outStream.rewind(); + writeOffset = 0; + outSkip(offset); + } else { + throw new IOException("cannot seek or rewind the output stream"); + } + } + + private void outSkip(long amount) throws IOException { + outStream.skip(amount); + writeOffset += amount; + } + + private int lengthFor(int offset) throws IOException { + int size = auxOffset() - offset; + + if (moovSimulation) { + return size; + } + + auxSeek(offset); + auxWrite(size); + auxSkip(size - 4); + + return size; + } + + private int make(int type, int extra, int columns, int rows) throws IOException { + final byte base = 16; + int size = columns * rows * 4; + int total = size + base; + int offset = auxOffset(); + + if (extra >= 0) { + total += 4; + } + + auxWrite(ByteBuffer.allocate(12) + .putInt(total) + .putInt(type) + .putInt(0x00)// default version & flags + .array() + ); + + if (extra >= 0) { + //size += 4;// commented for auxiliar buffer !!! + offset += 4; + auxWrite(extra); + } + + auxWrite(rows); + auxSkip(size); + + return offset + base; + } + + private void auxWrite(int value) throws IOException { + auxWrite(ByteBuffer.allocate(4) + .putInt(value) + .array() + ); + } + + private void auxWrite(byte[] buffer) throws IOException { + if (moovSimulation) { + writeOffset += buffer.length; + } else if (auxBuffer == null) { + outWrite(buffer, 0, buffer.length); + } else { + auxBuffer.put(buffer); + } + } + + private void auxSeek(int offset) throws IOException { + if (moovSimulation) { + writeOffset = offset; + } else if (auxBuffer == null) { + outSeek(offset); + } else { + auxBuffer.position(offset); + } + } + + private void auxSkip(int amount) throws IOException { + if (moovSimulation) { + writeOffset += amount; + } else if (auxBuffer == null) { + outSkip(amount); + } else { + auxBuffer.position(auxBuffer.position() + amount); + } + } + + private int auxOffset() { + return auxBuffer == null ? (int) writeOffset : auxBuffer.position(); + } + // + + // + private int make_ftyp() throws IOException { + byte[] buffer = new byte[]{ + 0x00, 0x00, 0x00, 0x1C, 0x66, 0x74, 0x79, 0x70,// ftyp + 0x6D, 0x70, 0x34, 0x32,// mayor brand (mp42) + 0x00, 0x00, 0x02, 0x00,// default minor version (512) + 0x6D, 0x70, 0x34, 0x31, 0x69, 0x73, 0x6F, 0x6D, 0x69, 0x73, 0x6F, 0x32// compatible brands: mp41 isom iso2 + }; + + if (overrideMainBrand != 0) + ByteBuffer.wrap(buffer).putInt(8, overrideMainBrand); + + outWrite(buffer); + + return buffer.length; + } + + private byte[] make_mdat(long refSize, boolean is64) { + if (is64) { + refSize += 16; + } else { + refSize += 8; + } + + ByteBuffer buffer = ByteBuffer.allocate(is64 ? 16 : 8) + .putInt(is64 ? 0x01 : (int) refSize) + .putInt(0x6D646174);// mdat + + if (is64) { + buffer.putLong(refSize); + } + + return buffer.array(); + } + + private void make_mvhd(long longestTrack) throws IOException { + auxWrite(new byte[]{ + 0x00, 0x00, 0x00, 0x78, 0x6D, 0x76, 0x68, 0x64, 0x01, 0x00, 0x00, 0x00 + }); + auxWrite(ByteBuffer.allocate(28) + .putLong(time) + .putLong(time) + .putInt(DEFAULT_TIMESCALE) + .putLong(longestTrack) + .array() + ); + + auxWrite(new byte[]{ + 0x00, 0x01, 0x00, 0x00, 0x01, 0x00,// default volume and rate + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,// reserved values + // default matrix + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00 + }); + auxWrite(new byte[24]);// predefined + auxWrite(ByteBuffer.allocate(4) + .putInt(tracks.length + 1) + .array() + ); + } + + private int make_moov(int[] defaultMediaTime, TablesInfo[] tablesInfo, boolean is64) throws RuntimeException, IOException { + int start = auxOffset(); + + auxWrite(new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x6D, 0x6F, 0x6F, 0x76 + }); + + long longestTrack = 0; + long[] durations = new long[tracks.length]; + + for (int i = 0; i < durations.length; i++) { + durations[i] = (long) Math.ceil( + ((double) tracks[i].trak.tkhd.duration / tracks[i].trak.mdia.mdhd_timeScale) * DEFAULT_TIMESCALE + ); + + if (durations[i] > longestTrack) { + longestTrack = durations[i]; + } + } + + make_mvhd(longestTrack); + + for (int i = 0; i < tracks.length; i++) { + if (tracks[i].trak.tkhd.matrix.length != 36) { + throw new RuntimeException("bad track matrix length (expected 36) in track n°" + i); + } + make_trak(i, durations[i], defaultMediaTime[i], tablesInfo[i], is64); + } + + // udta/meta/ilst/©too + auxWrite(new byte[]{ + 0x00, 0x00, 0x00, 0x5C, 0x75, 0x64, 0x74, 0x61, 0x00, 0x00, 0x00, 0x54, 0x6D, 0x65, 0x74, 0x61, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x68, 0x64, 0x6C, 0x72, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x69, 0x72, 0x61, 0x70, 0x70, 0x6C, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x69, 0x6C, 0x73, 0x74, 0x00, 0x00, 0x00, + 0x1F, (byte) 0xA9, 0x74, 0x6F, 0x6F, 0x00, 0x00, 0x00, 0x17, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65// "NewPipe" binary string + }); + + return lengthFor(start); + } + + private void make_trak(int index, long duration, int defaultMediaTime, TablesInfo tables, boolean is64) throws IOException { + int start = auxOffset(); + + auxWrite(new byte[]{ + 0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x61, 0x6B,// trak header + 0x00, 0x00, 0x00, 0x68, 0x74, 0x6B, 0x68, 0x64, 0x01, 0x00, 0x00, 0x03 // tkhd header + }); + + ByteBuffer buffer = ByteBuffer.allocate(48); + buffer.putLong(time); + buffer.putLong(time); + buffer.putInt(index + 1); + buffer.position(24); + buffer.putLong(duration); + buffer.position(40); + buffer.putShort(tracks[index].trak.tkhd.bLayer); + buffer.putShort(tracks[index].trak.tkhd.bAlternateGroup); + buffer.putShort(tracks[index].trak.tkhd.bVolume); + auxWrite(buffer.array()); + + auxWrite(tracks[index].trak.tkhd.matrix); + auxWrite(ByteBuffer.allocate(8) + .putInt(tracks[index].trak.tkhd.bWidth) + .putInt(tracks[index].trak.tkhd.bHeight) + .array() + ); + + auxWrite(new byte[]{ + 0x00, 0x00, 0x00, 0x24, 0x65, 0x64, 0x74, 0x73,// edts header + 0x00, 0x00, 0x00, 0x1C, 0x65, 0x6C, 0x73, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01// elst header + }); + + int bMediaRate; + int mediaTime; + + if (tracks[index].trak.edst_elst == null) { + // is a audio track ¿is edst/elst opcional for audio tracks? + mediaTime = 0x00;// ffmpeg set this value as zero, instead of defaultMediaTime + bMediaRate = 0x00010000; + } else { + mediaTime = (int) tracks[index].trak.edst_elst.MediaTime; + bMediaRate = tracks[index].trak.edst_elst.bMediaRate; + } + + auxWrite(ByteBuffer + .allocate(12) + .putInt((int) duration) + .putInt(mediaTime) + .putInt(bMediaRate) + .array() + ); + + make_mdia(tracks[index].trak.mdia, tables, is64); + + lengthFor(start); + } + + private void make_mdia(Mdia mdia, TablesInfo tablesInfo, boolean is64) throws IOException { + + int start_mdia = auxOffset(); + auxWrite(new byte[]{0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x69, 0x61});// mdia + auxWrite(mdia.mdhd); + auxWrite(make_hdlr(mdia.hdlr)); + + int start_minf = auxOffset(); + auxWrite(new byte[]{0x00, 0x00, 0x00, 0x00, 0x6D, 0x69, 0x6E, 0x66});// minf + auxWrite(mdia.minf.$mhd); + auxWrite(mdia.minf.dinf); + + int start_stbl = auxOffset(); + auxWrite(new byte[]{0x00, 0x00, 0x00, 0x00, 0x73, 0x74, 0x62, 0x6C});// stbl + auxWrite(mdia.minf.stbl_stsd); + + // + // In audio tracks the following tables is not required: ssts ctts + // And stsz can be empty if has a default sample size + // + if (moovSimulation) { + make(0x73747473, -1, 2, 1); + if (tablesInfo.stss > 0) { + make(0x73747373, -1, 1, tablesInfo.stss); + } + if (tablesInfo.ctts > 0) { + make(0x63747473, -1, 2, tablesInfo.ctts); + } + make(0x73747363, -1, 3, tablesInfo.stsc); + make(0x7374737A, tablesInfo.stsz_default, 1, tablesInfo.stsz); + make(is64 ? 0x636F3634 : 0x7374636F, -1, is64 ? 2 : 1, tablesInfo.stco); + } else { + tablesInfo.stts = make(0x73747473, -1, 2, 1); + if (tablesInfo.stss > 0) { + tablesInfo.stss = make(0x73747373, -1, 1, tablesInfo.stss); + } + if (tablesInfo.ctts > 0) { + tablesInfo.ctts = make(0x63747473, -1, 2, tablesInfo.ctts); + } + tablesInfo.stsc = make(0x73747363, -1, 3, tablesInfo.stsc); + tablesInfo.stsz = make(0x7374737A, tablesInfo.stsz_default, 1, tablesInfo.stsz); + tablesInfo.stco = make(is64 ? 0x636F3634 : 0x7374636F, -1, is64 ? 2 : 1, tablesInfo.stco); + } + + lengthFor(start_stbl); + lengthFor(start_minf); + lengthFor(start_mdia); + } + + private byte[] make_hdlr(Hdlr hdlr) { + ByteBuffer buffer = ByteBuffer.wrap(new byte[]{ + 0x00, 0x00, 0x00, 0x77, 0x68, 0x64, 0x6C, 0x72,// hdlr + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // binary string "ISO Media file created in NewPipe (A libre lightweight streaming frontend for Android)." + 0x49, 0x53, 0x4F, 0x20, 0x4D, 0x65, 0x64, 0x69, 0x61, 0x20, 0x66, 0x69, 0x6C, 0x65, 0x20, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x20, 0x69, 0x6E, 0x20, 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, + 0x65, 0x20, 0x28, 0x41, 0x20, 0x6C, 0x69, 0x62, 0x72, 0x65, 0x20, 0x6C, 0x69, 0x67, 0x68, 0x74, + 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x20, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x69, 0x6E, 0x67, + 0x20, 0x66, 0x72, 0x6F, 0x6E, 0x74, 0x65, 0x6E, 0x64, 0x20, 0x66, 0x6F, 0x72, 0x20, 0x41, 0x6E, + 0x64, 0x72, 0x6F, 0x69, 0x64, 0x29, 0x2E + }); + + buffer.position(12); + buffer.putInt(hdlr.type); + buffer.putInt(hdlr.subType); + buffer.put(hdlr.bReserved);// always is a zero array + + return buffer.array(); + } + // + + class TablesInfo { + + public int stts; + public int stsc; + public int[] stsc_bEntries; + public int ctts; + public int stsz; + public int stsz_default; + public int stss; + public int stco; + } +} diff --git a/app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java b/app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java index 26aaf49a5..c41db4373 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.streams; +import org.schabi.newpipe.streams.io.SharpStream; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -12,8 +13,6 @@ import java.nio.charset.Charset; import java.text.ParseException; import java.util.Locale; -import org.schabi.newpipe.streams.io.SharpStream; - import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -27,11 +26,11 @@ public class SubtitleConverter { public void dumpTTML(SharpStream in, final SharpStream out, final boolean ignoreEmptyFrames, final boolean detectYoutubeDuplicateLines ) throws IOException, ParseException, SAXException, ParserConfigurationException, XPathExpressionException { - + final FrameWriter callback = new FrameWriter() { int frameIndex = 0; final Charset charset = Charset.forName("utf-8"); - + @Override public void yield(SubtitleFrame frame) throws IOException { if (ignoreEmptyFrames && frame.isEmptyText()) { @@ -48,13 +47,13 @@ public class SubtitleConverter { out.write(NEW_LINE.getBytes(charset)); } }; - + read_xml_based(in, callback, detectYoutubeDuplicateLines, "tt", "xmlns", "http://www.w3.org/ns/ttml", new String[]{"timedtext", "head", "wp"}, new String[]{"body", "div", "p"}, "begin", "end", true - ); + ); } private void read_xml_based(SharpStream source, FrameWriter callback, boolean detectYoutubeDuplicateLines, @@ -70,7 +69,7 @@ public class SubtitleConverter { * Language parsing is not supported */ - byte[] buffer = new byte[source.available()]; + byte[] buffer = new byte[(int) source.available()]; source.read(buffer); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); @@ -206,7 +205,7 @@ public class SubtitleConverter { } } - private static NodeList selectNodes(Document xml, String[] path, String namespaceUri) throws XPathExpressionException { + private static NodeList selectNodes(Document xml, String[] path, String namespaceUri) { Element ref = xml.getDocumentElement(); for (int i = 0; i < path.length - 1; i++) { diff --git a/app/src/main/java/org/schabi/newpipe/streams/TrackDataChunk.java b/app/src/main/java/org/schabi/newpipe/streams/TrackDataChunk.java deleted file mode 100644 index 86eb5ff4f..000000000 --- a/app/src/main/java/org/schabi/newpipe/streams/TrackDataChunk.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.schabi.newpipe.streams; - -import java.io.InputStream; -import java.io.IOException; - -public class TrackDataChunk extends InputStream { - - private final DataReader base; - private int size; - - public TrackDataChunk(DataReader base, int size) { - this.base = base; - this.size = size; - } - - @Override - public int read() throws IOException { - if (size < 1) { - return -1; - } - - int res = base.read(); - - if (res >= 0) { - size--; - } - - return res; - } - - @Override - public int read(byte[] buffer) throws IOException { - return read(buffer, 0, buffer.length); - } - - @Override - public int read(byte[] buffer, int offset, int count) throws IOException { - count = Math.min(size, count); - int read = base.read(buffer, offset, count); - size -= count; - return read; - } - - @Override - public long skip(long amount) throws IOException { - long res = base.skipBytes(Math.min(amount, size)); - size -= res; - return res; - } - - @Override - public int available() { - return size; - } - - @Override - public void close() { - size = 0; - } - - @Override - public boolean markSupported() { - return false; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java index f61ef14c5..0c635ebe3 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java @@ -1,12 +1,13 @@ package org.schabi.newpipe.streams; +import org.schabi.newpipe.streams.io.SharpStream; + import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.NoSuchElementException; -import java.util.Objects; - -import org.schabi.newpipe.streams.io.SharpStream; /** * @@ -121,7 +122,7 @@ public class WebMReader { } private String readString(Element parent) throws IOException { - return new String(readBlob(parent), "utf-8"); + return new String(readBlob(parent), StandardCharsets.UTF_8);// or use "utf-8" } private byte[] readBlob(Element parent) throws IOException { @@ -193,6 +194,7 @@ public class WebMReader { return elem; } } + ensure(elem); } @@ -306,7 +308,7 @@ public class WebMReader { entry.trackNumber = readNumber(elem); break; case ID_TrackType: - entry.trackType = (int)readNumber(elem); + entry.trackType = (int) readNumber(elem); break; case ID_CodecID: entry.codecId = readString(elem); @@ -445,7 +447,7 @@ public class WebMReader { public class SimpleBlock { - public TrackDataChunk data; + public InputStream data; SimpleBlock(Element ref) { this.ref = ref; @@ -492,7 +494,7 @@ public class WebMReader { currentSimpleBlock = readSimpleBlock(elem); if (currentSimpleBlock.trackNumber == tracks[selectedTrack].trackNumber) { - currentSimpleBlock.data = new TrackDataChunk(stream, (int) currentSimpleBlock.dataSize); + currentSimpleBlock.data = stream.getView((int) currentSimpleBlock.dataSize); return currentSimpleBlock; } diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java index ea038c607..eba2bbb87 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java @@ -1,20 +1,20 @@ package org.schabi.newpipe.streams; +import android.support.annotation.NonNull; + import org.schabi.newpipe.streams.WebMReader.Cluster; import org.schabi.newpipe.streams.WebMReader.Segment; import org.schabi.newpipe.streams.WebMReader.SimpleBlock; import org.schabi.newpipe.streams.WebMReader.WebMTrack; +import org.schabi.newpipe.streams.io.SharpStream; import java.io.IOException; import java.io.InputStream; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import org.schabi.newpipe.streams.io.SharpStream; - /** - * * @author kapodamy */ public class WebMWriter { @@ -94,10 +94,6 @@ public class WebMWriter { } } - public long getBytesWritten() { - return written; - } - public boolean isDone() { return done; } @@ -138,42 +134,42 @@ public class WebMWriter { /* segment */ listBuffer.add(new byte[]{ - 0x18, 0x53, (byte) 0x80, 0x67, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00// segment content size + 0x18, 0x53, (byte) 0x80, 0x67, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00// segment content size }); long baseSegmentOffset = written + listBuffer.get(0).length; /* seek head */ listBuffer.add(new byte[]{ - 0x11, 0x4d, (byte) 0x9b, 0x74, (byte) 0xbe, - 0x4d, (byte) 0xbb, (byte) 0x8b, - 0x53, (byte) 0xab, (byte) 0x84, 0x15, 0x49, (byte) 0xa9, 0x66, 0x53, - (byte) 0xac, (byte) 0x81, /*info offset*/ 0x43, - 0x4d, (byte) 0xbb, (byte) 0x8b, 0x53, (byte) 0xab, - (byte) 0x84, 0x16, 0x54, (byte) 0xae, 0x6b, 0x53, (byte) 0xac, (byte) 0x81, - /*tracks offset*/ 0x6a, - 0x4d, (byte) 0xbb, (byte) 0x8e, 0x53, (byte) 0xab, (byte) 0x84, 0x1f, - 0x43, (byte) 0xb6, 0x75, 0x53, (byte) 0xac, (byte) 0x84, /*cluster offset [2]*/ 0x00, 0x00, 0x00, 0x00, - 0x4d, (byte) 0xbb, (byte) 0x8e, 0x53, (byte) 0xab, (byte) 0x84, 0x1c, 0x53, - (byte) 0xbb, 0x6b, 0x53, (byte) 0xac, (byte) 0x84, /*cues offset [7]*/ 0x00, 0x00, 0x00, 0x00 + 0x11, 0x4d, (byte) 0x9b, 0x74, (byte) 0xbe, + 0x4d, (byte) 0xbb, (byte) 0x8b, + 0x53, (byte) 0xab, (byte) 0x84, 0x15, 0x49, (byte) 0xa9, 0x66, 0x53, + (byte) 0xac, (byte) 0x81, /*info offset*/ 0x43, + 0x4d, (byte) 0xbb, (byte) 0x8b, 0x53, (byte) 0xab, + (byte) 0x84, 0x16, 0x54, (byte) 0xae, 0x6b, 0x53, (byte) 0xac, (byte) 0x81, + /*tracks offset*/ 0x6a, + 0x4d, (byte) 0xbb, (byte) 0x8e, 0x53, (byte) 0xab, (byte) 0x84, 0x1f, + 0x43, (byte) 0xb6, 0x75, 0x53, (byte) 0xac, (byte) 0x84, /*cluster offset [2]*/ 0x00, 0x00, 0x00, 0x00, + 0x4d, (byte) 0xbb, (byte) 0x8e, 0x53, (byte) 0xab, (byte) 0x84, 0x1c, 0x53, + (byte) 0xbb, 0x6b, 0x53, (byte) 0xac, (byte) 0x84, /*cues offset [7]*/ 0x00, 0x00, 0x00, 0x00 }); /* info */ listBuffer.add(new byte[]{ - 0x15, 0x49, (byte) 0xa9, 0x66, (byte) 0xa2, 0x2a, (byte) 0xd7, (byte) 0xb1 + 0x15, 0x49, (byte) 0xa9, 0x66, (byte) 0xa2, 0x2a, (byte) 0xd7, (byte) 0xb1 }); listBuffer.add(encode(DEFAULT_TIMECODE_SCALE, true));// this value MUST NOT exceed 4 bytes listBuffer.add(new byte[]{0x44, (byte) 0x89, (byte) 0x84, - 0x00, 0x00, 0x00, 0x00,// info.duration - - /* MuxingApp */ - 0x4d, (byte) 0x80, (byte) 0x87, 0x4E, - 0x65, 0x77, 0x50, 0x69, 0x70, 0x65, // "NewPipe" binary string - - /* WritingApp */ - 0x57, 0x41, (byte) 0x87, 0x4E, - 0x65, 0x77, 0x50, 0x69, 0x70, 0x65// "NewPipe" binary string + 0x00, 0x00, 0x00, 0x00,// info.duration + + /* MuxingApp */ + 0x4d, (byte) 0x80, (byte) 0x87, 0x4E, + 0x65, 0x77, 0x50, 0x69, 0x70, 0x65, // "NewPipe" binary string + + /* WritingApp */ + 0x57, 0x41, (byte) 0x87, 0x4E, + 0x65, 0x77, 0x50, 0x69, 0x70, 0x65// "NewPipe" binary string }); /* tracks */ @@ -200,7 +196,6 @@ public class WebMWriter { long nextCueTime = infoTracks[cuesForTrackId].trackType == 1 ? -1 : 0; ArrayList keyFrames = new ArrayList<>(32); - //ArrayList chunks = new ArrayList<>(readers.length); ArrayList clusterOffsets = new ArrayList<>(32); ArrayList clusterSizes = new ArrayList<>(32); @@ -283,24 +278,21 @@ public class WebMWriter { long segmentSize = written - offsetSegmentSizeSet - 7; - // final step write offsets and sizes - out.rewind(); - written = 0; - - skipTo(out, offsetSegmentSizeSet); + /* ---- final step write offsets and sizes ---- */ + seekTo(out, offsetSegmentSizeSet); writeLong(out, segmentSize); if (predefinedDurations[durationFromTrackId] > -1) { duration += predefinedDurations[durationFromTrackId];// this value is full-filled in makeTrackEntry() method } - skipTo(out, offsetInfoDurationSet); + seekTo(out, offsetInfoDurationSet); writeFloat(out, duration); firstClusterOffset -= baseSegmentOffset; - skipTo(out, offsetClusterSet); + seekTo(out, offsetClusterSet); writeInt(out, firstClusterOffset); - skipTo(out, cueReservedOffset); + seekTo(out, cueReservedOffset); /* Cue */ dump(new byte[]{0x1c, 0x53, (byte) 0xbb, 0x6b, 0x20, 0x00, 0x00}, out); @@ -321,17 +313,14 @@ public class WebMWriter { voidBuffer.putShort((short) (firstClusterOffset - written - 4)); dump(voidBuffer.array(), out); - out.rewind(); - written = 0; - - skipTo(out, offsetCuesSet); + seekTo(out, offsetCuesSet); writeInt(out, (int) (cueReservedOffset - baseSegmentOffset)); - skipTo(out, cueReservedOffset + 5); + seekTo(out, cueReservedOffset + 5); writeShort(out, cueSize); for (int i = 0; i < clusterSizes.size(); i++) { - skipTo(out, clusterOffsets.get(i)); + seekTo(out, clusterOffsets.get(i)); byte[] size = ByteBuffer.allocate(4).putInt(clusterSizes.get(i) | 0x200000).array(); out.write(size, 1, 3); written += 3; @@ -365,20 +354,29 @@ public class WebMWriter { bloq.dataSize = (int) res.dataSize; bloq.trackNumber = internalTrackId; bloq.flags = res.flags; - bloq.absoluteTimecode = convertTimecode(res.relativeTimeCode, readersSegment[internalTrackId].info.timecodeScale, DEFAULT_TIMECODE_SCALE); + bloq.absoluteTimecode = convertTimecode(res.relativeTimeCode, readersSegment[internalTrackId].info.timecodeScale); bloq.absoluteTimecode += readersCluter[internalTrackId].timecode; return bloq; } - private short convertTimecode(int time, long oldTimeScale, int newTimeScale) { - return (short) (time * (newTimeScale / oldTimeScale)); + private short convertTimecode(int time, long oldTimeScale) { + return (short) (time * (DEFAULT_TIMECODE_SCALE / oldTimeScale)); } - private void skipTo(SharpStream stream, long absoluteOffset) throws IOException { - absoluteOffset -= written; - written += absoluteOffset; - stream.skip(absoluteOffset); + private void seekTo(SharpStream stream, long offset) throws IOException { + if (stream.canSeek()) { + stream.seek(offset); + } else { + if (offset > written) { + stream.skip(offset - written); + } else { + stream.rewind(); + stream.skip(offset); + } + } + + written = offset; } private void writeLong(SharpStream stream, long number) throws IOException { @@ -468,12 +466,12 @@ public class WebMWriter { private void makeEBML(SharpStream stream) throws IOException { // deafult values dump(new byte[]{ - 0x1A, 0x45, (byte) 0xDF, (byte) 0xA3, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x1F, 0x42, (byte) 0x86, (byte) 0x81, 0x01, - 0x42, (byte) 0xF7, (byte) 0x81, 0x01, 0x42, (byte) 0xF2, (byte) 0x81, 0x04, - 0x42, (byte) 0xF3, (byte) 0x81, 0x08, 0x42, (byte) 0x82, (byte) 0x84, 0x77, - 0x65, 0x62, 0x6D, 0x42, (byte) 0x87, (byte) 0x81, 0x02, - 0x42, (byte) 0x85, (byte) 0x81, 0x02 + 0x1A, 0x45, (byte) 0xDF, (byte) 0xA3, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x1F, 0x42, (byte) 0x86, (byte) 0x81, 0x01, + 0x42, (byte) 0xF7, (byte) 0x81, 0x01, 0x42, (byte) 0xF2, (byte) 0x81, 0x04, + 0x42, (byte) 0xF3, (byte) 0x81, 0x08, 0x42, (byte) 0x82, (byte) 0x84, 0x77, + 0x65, 0x62, 0x6D, 0x42, (byte) 0x87, (byte) 0x81, 0x02, + 0x42, (byte) 0x85, (byte) 0x81, 0x02 }, stream); } @@ -618,9 +616,10 @@ public class WebMWriter { int offset = withLength ? 1 : 0; byte[] buffer = new byte[offset + length]; - long marker = (long) Math.floor((length - 1) / 8); + long marker = (long) Math.floor((length - 1f) / 8f); - for (int i = length - 1, mul = 1; i >= 0; i--, mul *= 0x100) { + float mul = 1; + for (int i = length - 1; i >= 0; i--, mul *= 0x100) { long b = (long) Math.floor(number / mul); if (!withLength && i == marker) { b = b | (0x80 >> (length - 1)); @@ -637,11 +636,7 @@ public class WebMWriter { private ArrayList encode(String value) { byte[] str; - try { - str = value.getBytes("utf-8"); - } catch (UnsupportedEncodingException err) { - str = value.getBytes(); - } + str = value.getBytes(StandardCharsets.UTF_8);// or use "utf-8" ArrayList buffer = new ArrayList<>(2); buffer.add(encode(str.length, false)); @@ -720,9 +715,10 @@ public class WebMWriter { return (flags & 0x80) == 0x80; } + @NonNull @Override public String toString() { - return String.format("trackNumber=%s isKeyFrame=%S absoluteTimecode=%s", trackNumber, (flags & 0x80) == 0x80, absoluteTimecode); + return String.format("trackNumber=%s isKeyFrame=%S absoluteTimecode=%s", trackNumber, isKeyframe(), absoluteTimecode); } } } diff --git a/app/src/main/java/org/schabi/newpipe/streams/io/SharpStream.java b/app/src/main/java/org/schabi/newpipe/streams/io/SharpStream.java index 48bea06f6..ea2f60837 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/io/SharpStream.java +++ b/app/src/main/java/org/schabi/newpipe/streams/io/SharpStream.java @@ -3,7 +3,7 @@ package org.schabi.newpipe.streams.io; import java.io.IOException; /** - * based c# + * based on c# */ public abstract class SharpStream { @@ -15,23 +15,27 @@ public abstract class SharpStream { public abstract long skip(long amount) throws IOException; - - public abstract int available(); + public abstract long available(); public abstract void rewind() throws IOException; - public abstract void dispose(); public abstract boolean isDisposed(); - public abstract boolean canRewind(); public abstract boolean canRead(); public abstract boolean canWrite(); + public boolean canSetLength() { + return false; + } + + public boolean canSeek() { + return false; + } public abstract void write(byte value) throws IOException; @@ -39,9 +43,15 @@ public abstract class SharpStream { public abstract void write(byte[] buffer, int offset, int count) throws IOException; - public abstract void flush() throws IOException; + public void flush() throws IOException { + // STUB + } public void setLength(long length) throws IOException { throw new IOException("Not implemented"); } + + public void seek(long offset) throws IOException { + throw new IOException("Not implemented"); + } } diff --git a/app/src/main/java/org/schabi/newpipe/util/ListHelper.java b/app/src/main/java/org/schabi/newpipe/util/ListHelper.java index 8fc423837..fa5530f12 100644 --- a/app/src/main/java/org/schabi/newpipe/util/ListHelper.java +++ b/app/src/main/java/org/schabi/newpipe/util/ListHelper.java @@ -430,24 +430,26 @@ public final class ListHelper { */ private static String getResolutionLimit(Context context) { String resolutionLimit = null; - if (!isWifiActive(context)) { + if (isMeteredNetwork(context)) { SharedPreferences preferences = PreferenceManager.getDefaultSharedPreferences(context); String defValue = context.getString(R.string.limit_data_usage_none_key); String value = preferences.getString( context.getString(R.string.limit_mobile_data_usage_key), defValue); - resolutionLimit = value.equals(defValue) ? null : value; + resolutionLimit = defValue.equals(value) ? null : value; } return resolutionLimit; } /** - * Are we connected to wifi? + * The current network is metered (like mobile data)? * @param context App context - * @return {@code true} if connected to wifi + * @return {@code true} if connected to a metered network */ - private static boolean isWifiActive(Context context) + private static boolean isMeteredNetwork(Context context) { ConnectivityManager manager = (ConnectivityManager)context.getSystemService(Context.CONNECTIVITY_SERVICE); - return manager != null && manager.getActiveNetworkInfo() != null && manager.getActiveNetworkInfo().getType() == ConnectivityManager.TYPE_WIFI; + if (manager == null || manager.getActiveNetworkInfo() == null) return false; + + return manager.isActiveNetworkMetered(); } } diff --git a/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java b/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java index b3522aea0..7febfa053 100644 --- a/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java +++ b/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java @@ -38,7 +38,7 @@ public class SecondaryStreamHelper { public static AudioStream getAudioStreamFor(@NonNull List audioStreams, @NonNull VideoStream videoStream) { switch (videoStream.getFormat()) { case WEBM: - case MPEG_4: + case MPEG_4:// ¿is mpeg-4 DASH? break; default: return null; diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index b864cf4fb..abc934878 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -164,9 +164,6 @@ public class DownloadInitializer extends Thread { } } - // hide marquee in the progress bar - mMission.done++; - mMission.start(); } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 243a8585a..b8849482a 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -40,6 +40,9 @@ public class DownloadMission extends Mission { public static final int ERROR_UNKNOWN_HOST = 1005; public static final int ERROR_CONNECT_HOST = 1006; public static final int ERROR_POSTPROCESSING = 1007; + public static final int ERROR_POSTPROCESSING_STOPPED = 1008; + public static final int ERROR_POSTPROCESSING_HOLD = 1009; + public static final int ERROR_INSUFFICIENT_STORAGE = 1010; public static final int ERROR_HTTP_NO_CONTENT = 204; public static final int ERROR_HTTP_UNSUPPORTED_RANGE = 206; @@ -83,8 +86,9 @@ public class DownloadMission extends Mission { * 0: ready * 1: running * 2: completed + * 3: hold */ - public int postprocessingState; + public volatile int postprocessingState; /** * Indicate if the post-processing algorithm works on the same file @@ -92,19 +96,19 @@ public class DownloadMission extends Mission { public boolean postprocessingThis; /** - * The current resource to download {@code urls[current]} + * The current resource to download, see {@code urls[current]} and {@code offsets[current]} */ public int current; /** * Metadata where the mission state is saved */ - public File metadata; + public transient File metadata; /** * maximum attempts */ - public int maxRetry; + public transient int maxRetry; /** * Approximated final length, this represent the sum of all resources sizes @@ -115,11 +119,11 @@ public class DownloadMission extends Mission { boolean fallback; private int finishCount; public transient boolean running; - public transient boolean enqueued = true; + public boolean enqueued; public int errCode = ERROR_NOTHING; - public transient Exception errObject = null; + public Exception errObject = null; public transient boolean recovered; public transient Handler mHandler; private transient boolean mWritingToFile; @@ -131,7 +135,7 @@ public class DownloadMission extends Mission { private transient boolean deleted; int currentThreadCount; - private transient Thread[] threads = new Thread[0]; + public transient volatile Thread[] threads = new Thread[0]; private transient Thread init = null; @@ -155,6 +159,8 @@ public class DownloadMission extends Mission { this.location = location; this.kind = kind; this.offsets = new long[urls.length]; + this.enqueued = true; + this.maxRetry = 3; if (postprocessingName != null) { Postprocessing algorithm = Postprocessing.getAlgorithm(postprocessingName, null); @@ -183,6 +189,7 @@ public class DownloadMission extends Mission { */ boolean isBlockPreserved(long block) { checkBlock(block); + //noinspection ConstantConditions return blockState.containsKey(block) ? blockState.get(block) : false; } @@ -247,6 +254,12 @@ public class DownloadMission extends Mission { HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setInstanceFollowRedirects(true); + // BUG workaround: switching between networks can freeze the download forever + + //conn.setRequestProperty("Connection", "close"); + conn.setConnectTimeout(30000); + conn.setReadTimeout(10000); + if (rangeStart >= 0) { String req = "bytes=" + rangeStart + "-"; if (rangeEnd > 0) req += rangeEnd; @@ -342,11 +355,32 @@ public class DownloadMission extends Mission { } } - synchronized void notifyError(int code, Exception err) { + public synchronized void notifyError(int code, Exception err) { Log.e(TAG, "notifyError() code = " + code, err); + if (err instanceof IOException) { + if (err.getMessage().contains("Permission denied")) { + code = ERROR_PERMISSION_DENIED; + err = null; + } else if (err.getMessage().contains("write failed: ENOSPC")) { + code = ERROR_INSUFFICIENT_STORAGE; + err = null; + } else { + try { + File storage = new File(location); + if (storage.canWrite() && storage.getUsableSpace() < (getLength() - done)) { + code = ERROR_INSUFFICIENT_STORAGE; + err = null; + } + } catch (SecurityException e) { + // is a permission error + } + } + } + errCode = code; errObject = err; + enqueued = false; pause(); @@ -378,6 +412,7 @@ public class DownloadMission extends Mission { if (!doPostprocessing()) return; + enqueued = false; running = false; deleteThisFromFile(); @@ -386,22 +421,20 @@ public class DownloadMission extends Mission { } private void notifyPostProcessing(int state) { - if (DEBUG) { - String action; - switch (state) { - case 1: - action = "Running"; - break; - case 2: - action = "Completed"; - break; - default: - action = "Failed"; - } - - Log.d(TAG, action + " postprocessing on " + location + File.separator + name); + String action; + switch (state) { + case 1: + action = "Running"; + break; + case 2: + action = "Completed"; + break; + default: + action = "Failed"; } + Log.d(TAG, action + " postprocessing on " + location + File.separator + name); + synchronized (blockState) { // don't return without fully write the current state postprocessingState = state; @@ -420,7 +453,6 @@ public class DownloadMission extends Mission { if (threads != null) for (Thread thread : threads) joinForThread(thread); - enqueued = false; running = true; errCode = ERROR_NOTHING; @@ -463,7 +495,7 @@ public class DownloadMission extends Mission { } /** - * Pause the mission, does not affect the blocks that are being downloaded. + * Pause the mission */ public synchronized void pause() { if (!running) return; @@ -477,7 +509,6 @@ public class DownloadMission extends Mission { running = false; recovered = true; - enqueued = false; if (init != null && Thread.currentThread() != init && init.isAlive()) { init.interrupt(); @@ -514,7 +545,7 @@ public class DownloadMission extends Mission { } /** - * Removes the file and the meta file + * Removes the downloaded file and the meta file */ @Override public boolean delete() { @@ -580,12 +611,21 @@ public class DownloadMission extends Mission { * @return true, otherwise, false */ public boolean isPsRunning() { - return postprocessingName != null && postprocessingState == 1; + return postprocessingName != null && (postprocessingState == 1 || postprocessingState == 3); + } + + /** + * Indicated if the mission is ready + * + * @return true, otherwise, false + */ + public boolean isInitialized() { + return blocks >= 0; // DownloadMissionInitializer was executed } public long getLength() { long calculated; - if (postprocessingState == 1) { + if (postprocessingState == 1 || postprocessingState == 3) { calculated = length; } else { calculated = offsets[current < offsets.length ? current : (offsets.length - 1)] + length; @@ -596,13 +636,37 @@ public class DownloadMission extends Mission { return calculated > nearLength ? calculated : nearLength; } + /** + * set this mission state on the queue + * + * @param queue true to add to the queue, otherwise, false + */ + public void setEnqueued(boolean queue) { + enqueued = queue; + runAsync(-2, this::writeThisToFile); + } + + /** + * Attempts to continue a blocked post-processing + * + * @param recover {@code true} to retry, otherwise, {@code false} to cancel + */ + public void psContinue(boolean recover) { + postprocessingState = 1; + errCode = recover ? ERROR_NOTHING : ERROR_POSTPROCESSING; + threads[0].interrupt(); + } + private boolean doPostprocessing() { if (postprocessingName == null || postprocessingState == 2) return true; notifyPostProcessing(1); notifyProgress(0); - Thread.currentThread().setName("[" + TAG + "] post-processing = " + postprocessingName + " filename = " + name); + if (DEBUG) + Thread.currentThread().setName("[" + TAG + "] post-processing = " + postprocessingName + " filename = " + name); + + threads = new Thread[]{Thread.currentThread()}; Exception exception = null; diff --git a/app/src/main/java/us/shandian/giga/get/Mission.java b/app/src/main/java/us/shandian/giga/get/Mission.java index ec2ddaa26..53c81b08b 100644 --- a/app/src/main/java/us/shandian/giga/get/Mission.java +++ b/app/src/main/java/us/shandian/giga/get/Mission.java @@ -47,6 +47,11 @@ public abstract class Mission implements Serializable { return new File(location, name); } + /** + * Delete the downloaded file + * + * @return {@code true] if and only if the file is successfully deleted, otherwise, {@code false} + */ public boolean delete() { deleted = true; return getDownloadedFile().delete(); diff --git a/app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java b/app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java new file mode 100644 index 000000000..fa0c2c7ae --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/M4aNoDash.java @@ -0,0 +1,43 @@ +package us.shandian.giga.postprocessing; + +import org.schabi.newpipe.streams.Mp4DashReader; +import org.schabi.newpipe.streams.Mp4FromDashWriter; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; + +import us.shandian.giga.get.DownloadMission; + +public class M4aNoDash extends Postprocessing { + + M4aNoDash(DownloadMission mission) { + super(mission, 0, true); + } + + @Override + boolean test(SharpStream... sources) throws IOException { + // check if the mp4 file is DASH (youtube) + + Mp4DashReader reader = new Mp4DashReader(sources[0]); + reader.parse(); + + switch (reader.getBrands()[0]) { + case 0x64617368:// DASH + case 0x69736F35:// ISO5 + return true; + default: + return false; + } + } + + @Override + int process(SharpStream out, SharpStream... sources) throws IOException { + Mp4FromDashWriter muxer = new Mp4FromDashWriter(sources[0]); + muxer.setMainBrand(0x4D344120);// binary string "M4A " + muxer.parseSources(); + muxer.selectTracks(0); + muxer.build(out); + + return OK_RESULT; + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java similarity index 60% rename from app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java rename to app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java index 45c06dd4b..09f5d9661 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java @@ -1,29 +1,29 @@ -package us.shandian.giga.postprocessing; - -import org.schabi.newpipe.streams.Mp4DashWriter; -import org.schabi.newpipe.streams.io.SharpStream; - -import java.io.IOException; - -import us.shandian.giga.get.DownloadMission; - -/** - * @author kapodamy - */ -class Mp4DashMuxer extends Postprocessing { - - Mp4DashMuxer(DownloadMission mission) { - super(mission, 15360 * 1024/* 15 MiB */, true); - } - - @Override - int process(SharpStream out, SharpStream... sources) throws IOException { - Mp4DashWriter muxer = new Mp4DashWriter(sources); - muxer.parseSources(); - muxer.selectTracks(0, 0); - muxer.build(out); - - return OK_RESULT; - } - -} +package us.shandian.giga.postprocessing; + +import org.schabi.newpipe.streams.Mp4FromDashWriter; +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.IOException; + +import us.shandian.giga.get.DownloadMission; + +/** + * @author kapodamy + */ +class Mp4FromDashMuxer extends Postprocessing { + + Mp4FromDashMuxer(DownloadMission mission) { + super(mission, 2 * 1024 * 1024/* 2 MiB */, true); + } + + @Override + int process(SharpStream out, SharpStream... sources) throws IOException { + Mp4FromDashWriter muxer = new Mp4FromDashWriter(sources); + muxer.parseSources(); + muxer.selectTracks(0, 0); + muxer.build(out); + + return OK_RESULT; + } + +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Mp4Muxer.java b/app/src/main/java/us/shandian/giga/postprocessing/Mp4Muxer.java deleted file mode 100644 index bf932d5c1..000000000 --- a/app/src/main/java/us/shandian/giga/postprocessing/Mp4Muxer.java +++ /dev/null @@ -1,136 +0,0 @@ -package us.shandian.giga.postprocessing; - -import android.media.MediaCodec.BufferInfo; -import android.media.MediaExtractor; -import android.media.MediaMuxer; -import android.media.MediaMuxer.OutputFormat; -import android.util.Log; - -import static org.schabi.newpipe.BuildConfig.DEBUG; - -import org.schabi.newpipe.streams.io.SharpStream; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -import us.shandian.giga.get.DownloadMission; - - -class Mp4Muxer extends Postprocessing { - private static final String TAG = "Mp4Muxer"; - private static final int NOTIFY_BYTES_INTERVAL = 128 * 1024;// 128 KiB - - Mp4Muxer(DownloadMission mission) { - super(mission, 0, false); - } - - @Override - int process(SharpStream out, SharpStream... sources) throws IOException { - File dlFile = mission.getDownloadedFile(); - File tmpFile = new File(mission.location, mission.name.concat(".tmp")); - - if (tmpFile.exists()) - if (!tmpFile.delete()) return DownloadMission.ERROR_FILE_CREATION; - - if (!tmpFile.createNewFile()) return DownloadMission.ERROR_FILE_CREATION; - - FileInputStream source = null; - MediaMuxer muxer = null; - - //noinspection TryFinallyCanBeTryWithResources - try { - source = new FileInputStream(dlFile); - MediaExtractor tracks[] = { - getMediaExtractor(source, mission.offsets[0], mission.offsets[1] - mission.offsets[0]), - getMediaExtractor(source, mission.offsets[1], mission.length - mission.offsets[1]) - }; - - muxer = new MediaMuxer(tmpFile.getAbsolutePath(), OutputFormat.MUXER_OUTPUT_MPEG_4); - - int tracksIndex[] = { - muxer.addTrack(tracks[0].getTrackFormat(0)), - muxer.addTrack(tracks[1].getTrackFormat(0)) - }; - - ByteBuffer buffer = ByteBuffer.allocate(512 * 1024);// 512 KiB - BufferInfo info = new BufferInfo(); - - long written = 0; - long nextReport = NOTIFY_BYTES_INTERVAL; - - muxer.start(); - - while (true) { - int done = 0; - - for (int i = 0; i < tracks.length; i++) { - if (tracksIndex[i] < 0) continue; - - info.set(0, - tracks[i].readSampleData(buffer, 0), - tracks[i].getSampleTime(), - tracks[i].getSampleFlags() - ); - - if (info.size >= 0) { - muxer.writeSampleData(tracksIndex[i], buffer, info); - written += info.size; - done++; - } - if (!tracks[i].advance()) { - // EOF reached - tracks[i].release(); - tracksIndex[i] = -1; - } - - if (written > nextReport) { - nextReport = written + NOTIFY_BYTES_INTERVAL; - super.progressReport(written); - } - } - - if (done < 1) break; - } - - // this part should not fail - if (!dlFile.delete()) return DownloadMission.ERROR_FILE_CREATION; - if (!tmpFile.renameTo(dlFile)) return DownloadMission.ERROR_FILE_CREATION; - - return OK_RESULT; - } finally { - try { - if (muxer != null) { - muxer.stop(); - muxer.release(); - } - } catch (Exception err) { - if (DEBUG) - Log.e(TAG, "muxer stop/release failed", err); - } - - if (source != null) { - try { - source.close(); - } catch (IOException e) { - // nothing to do - } - } - - // if the operation fails, delete the temporal file - if (tmpFile.exists()) { - //noinspection ResultOfMethodCallIgnored - tmpFile.delete(); - } - } - } - - private MediaExtractor getMediaExtractor(FileInputStream source, long offset, long length) throws IOException { - MediaExtractor extractor = new MediaExtractor(); - extractor.setDataSource(source.getFD(), offset, length); - extractor.selectTrack(0); - - return extractor; - } -} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java index 635140bd3..df8549010 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java @@ -1,6 +1,7 @@ package us.shandian.giga.postprocessing; import android.os.Message; +import android.util.Log; import org.schabi.newpipe.streams.io.SharpStream; @@ -9,17 +10,22 @@ import java.io.IOException; import us.shandian.giga.get.DownloadMission; import us.shandian.giga.postprocessing.io.ChunkFileInputStream; -import us.shandian.giga.postprocessing.io.CircularFile; +import us.shandian.giga.postprocessing.io.CircularFileWriter; +import us.shandian.giga.postprocessing.io.CircularFileWriter.OffsetChecker; import us.shandian.giga.service.DownloadManagerService; +import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; +import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; +import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; + public abstract class Postprocessing { - static final byte OK_RESULT = DownloadMission.ERROR_NOTHING; + static final byte OK_RESULT = ERROR_NOTHING; public static final String ALGORITHM_TTML_CONVERTER = "ttml"; - public static final String ALGORITHM_MP4_DASH_MUXER = "mp4D"; - public static final String ALGORITHM_MP4_MUXER = "mp4"; public static final String ALGORITHM_WEBM_MUXER = "webm"; + public static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4"; + public static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a"; public static Postprocessing getAlgorithm(String algorithmName, DownloadMission mission) { if (null == algorithmName) { @@ -27,14 +33,14 @@ public abstract class Postprocessing { } else switch (algorithmName) { case ALGORITHM_TTML_CONVERTER: return new TtmlConverter(mission); - case ALGORITHM_MP4_DASH_MUXER: - return new Mp4DashMuxer(mission); - case ALGORITHM_MP4_MUXER: - return new Mp4Muxer(mission); case ALGORITHM_WEBM_MUXER: return new WebMMuxer(mission); + case ALGORITHM_MP4_FROM_DASH_MUXER: + return new Mp4FromDashMuxer(mission); + case ALGORITHM_M4A_NO_DASH: + return new M4aNoDash(mission); /*case "example-algorithm": - return new ExampleAlgorithm(mission);*/ + return new ExampleAlgorithm(mission);*/ default: throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName); } @@ -65,7 +71,8 @@ public abstract class Postprocessing { public void run() throws IOException { File file = mission.getDownloadedFile(); - CircularFile out = null; + File temp = null; + CircularFileWriter out = null; int result; long finalLength = -1; @@ -81,29 +88,54 @@ public abstract class Postprocessing { } sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.getDownloadedFile().length(), "rw"); - int[] idx = {0}; - CircularFile.OffsetChecker checker = () -> { - while (idx[0] < sources.length) { - /* - * WARNING: never use rewind() in any chunk after any writing (especially on first chunks) - * or the CircularFile can lead to unexpected results - */ - if (sources[idx[0]].isDisposed() || sources[idx[0]].available() < 1) { - idx[0]++; - continue;// the selected source is not used anymore + if (test(sources)) { + for (SharpStream source : sources) source.rewind(); + + OffsetChecker checker = () -> { + for (ChunkFileInputStream source : sources) { + /* + * WARNING: never use rewind() in any chunk after any writing (especially on first chunks) + * or the CircularFileWriter can lead to unexpected results + */ + if (source.isDisposed() || source.available() < 1) { + continue;// the selected source is not used anymore + } + + return source.getFilePointer() - 1; } - return sources[idx[0]].getFilePointer() - 1; - } + return -1; + }; - return -1; - }; - out = new CircularFile(file, 0, this::progressReport, checker); + temp = new File(mission.location, mission.name + ".tmp"); - result = process(out, sources); + out = new CircularFileWriter(file, temp, checker); + out.onProgress = this::progressReport; - if (result == OK_RESULT) - finalLength = out.finalizeFile(); + out.onWriteError = (err) -> { + mission.postprocessingState = 3; + mission.notifyError(ERROR_POSTPROCESSING_HOLD, err); + + try { + synchronized (this) { + while (mission.postprocessingState == 3) + wait(); + } + } catch (InterruptedException e) { + // nothing to do + Log.e(this.getClass().getSimpleName(), "got InterruptedException"); + } + + return mission.errCode == ERROR_NOTHING; + }; + + result = process(out, sources); + + if (result == OK_RESULT) + finalLength = out.finalizeFile(); + } else { + result = OK_RESULT; + } } finally { for (SharpStream source : sources) { if (source != null && !source.isDisposed()) { @@ -113,17 +145,22 @@ public abstract class Postprocessing { if (out != null) { out.dispose(); } + if (temp != null) { + //noinspection ResultOfMethodCallIgnored + temp.delete(); + } } } else { - result = process(null); + result = test() ? process(null) : OK_RESULT; } if (result == OK_RESULT) { - if (finalLength < 0) finalLength = file.length(); - mission.done = finalLength; - mission.length = finalLength; + if (finalLength != -1) { + mission.done = finalLength; + mission.length = finalLength; + } } else { - mission.errCode = DownloadMission.ERROR_UNKNOWN_EXCEPTION; + mission.errCode = ERROR_UNKNOWN_EXCEPTION; mission.errObject = new RuntimeException("post-processing algorithm returned " + result); } @@ -134,7 +171,18 @@ public abstract class Postprocessing { } /** - * Abstract method to execute the pos-processing algorithm + * Test if the post-processing algorithm can be skipped + * + * @param sources files to be processed + * @return {@code true} if the post-processing is required, otherwise, {@code false} + * @throws IOException if an I/O error occurs. + */ + boolean test(SharpStream... sources) throws IOException { + return true; + } + + /** + * Abstract method to execute the post-processing algorithm * * @param out output stream * @param sources files to be processed @@ -151,7 +199,7 @@ public abstract class Postprocessing { return mission.postprocessingArgs[index]; } - void progressReport(long done) { + private void progressReport(long done) { mission.done = done; if (mission.length < mission.done) mission.length = mission.done; diff --git a/app/src/main/java/us/shandian/giga/postprocessing/io/ChunkFileInputStream.java b/app/src/main/java/us/shandian/giga/postprocessing/io/ChunkFileInputStream.java index cd62c5d22..ee2fcddd5 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/io/ChunkFileInputStream.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/io/ChunkFileInputStream.java @@ -94,7 +94,7 @@ public class ChunkFileInputStream extends SharpStream { } @Override - public int available() { + public long available() { return (int) (length - position); } @@ -147,7 +147,4 @@ public class ChunkFileInputStream extends SharpStream { public void write(byte[] buffer, int offset, int count) { } - @Override - public void flush() { - } } diff --git a/app/src/main/java/us/shandian/giga/postprocessing/io/CircularFile.java b/app/src/main/java/us/shandian/giga/postprocessing/io/CircularFile.java deleted file mode 100644 index d2fc82d33..000000000 --- a/app/src/main/java/us/shandian/giga/postprocessing/io/CircularFile.java +++ /dev/null @@ -1,375 +0,0 @@ -package us.shandian.giga.postprocessing.io; - -import org.schabi.newpipe.streams.io.SharpStream; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.ArrayList; - -public class CircularFile extends SharpStream { - - private final static int AUX_BUFFER_SIZE = 1024 * 1024;// 1 MiB - private final static int AUX_BUFFER_SIZE2 = 512 * 1024;// 512 KiB - private final static int NOTIFY_BYTES_INTERVAL = 64 * 1024;// 64 KiB - private final static int QUEUE_BUFFER_SIZE = 8 * 1024;// 8 KiB - private final static boolean IMMEDIATE_AUX_BUFFER_FLUSH = false; - - private RandomAccessFile out; - private long position; - private long maxLengthKnown = -1; - - private ArrayList auxiliaryBuffers; - private OffsetChecker callback; - private ManagedBuffer queue; - private long startOffset; - private ProgressReport onProgress; - private long reportPosition; - - public CircularFile(File file, long offset, ProgressReport progressReport, OffsetChecker checker) throws IOException { - if (checker == null) { - throw new NullPointerException("checker is null"); - } - - try { - queue = new ManagedBuffer(QUEUE_BUFFER_SIZE); - out = new RandomAccessFile(file, "rw"); - out.seek(offset); - position = offset; - } catch (IOException err) { - try { - if (out != null) { - out.close(); - } - } catch (IOException e) { - // nothing to do - } - throw err; - } - - auxiliaryBuffers = new ArrayList<>(15); - callback = checker; - startOffset = offset; - reportPosition = offset; - onProgress = progressReport; - - } - - /** - * Close the file without flushing any buffer - */ - @Override - public void dispose() { - try { - auxiliaryBuffers = null; - if (out != null) { - out.close(); - out = null; - } - } catch (IOException err) { - // nothing to do - } - } - - /** - * Flush any buffer and close the output file. Use this method if the - * operation is successful - * - * @return the final length of the file - * @throws IOException if an I/O error occurs - */ - public long finalizeFile() throws IOException { - flushEverything(); - - if (maxLengthKnown > -1) { - position = maxLengthKnown; - } - if (position < out.length()) { - out.setLength(position); - } - - dispose(); - - return position; - } - - @Override - public void write(byte b) throws IOException { - write(new byte[]{b}, 0, 1); - } - - @Override - public void write(byte b[]) throws IOException { - write(b, 0, b.length); - } - - @Override - public void write(byte b[], int off, int len) throws IOException { - if (len == 0) { - return; - } - - long end = callback.check(); - long available; - - if (end == -1) { - available = Long.MAX_VALUE; - } else { - if (end < startOffset) { - throw new IOException("The reported offset is invalid. reported offset is " + String.valueOf(end)); - } - available = end - position; - } - - // Check if possible flush one or more auxiliary buffer - if (auxiliaryBuffers.size() > 0) { - ManagedBuffer aux = auxiliaryBuffers.get(0); - - // check if there is enough space to flush it completely - while (available >= (aux.size + queue.size)) { - available -= aux.size; - writeQueue(aux.buffer, 0, aux.size); - aux.dereference(); - auxiliaryBuffers.remove(0); - - if (auxiliaryBuffers.size() < 1) { - aux = null; - break; - } - aux = auxiliaryBuffers.get(0); - } - - if (IMMEDIATE_AUX_BUFFER_FLUSH) { - // try partial flush to avoid allocate another auxiliary buffer - if (aux != null && aux.available() < len && available > queue.size) { - int size = Math.min(aux.size, (int) available - queue.size); - - writeQueue(aux.buffer, 0, size); - aux.dereference(size); - - available -= size; - } - } - } - - if (auxiliaryBuffers.size() < 1 && available > (len + queue.size)) { - writeQueue(b, off, len); - } else { - int i = auxiliaryBuffers.size() - 1; - while (len > 0) { - if (i < 0) { - // allocate a new auxiliary buffer - auxiliaryBuffers.add(new ManagedBuffer(AUX_BUFFER_SIZE)); - i++; - } - - ManagedBuffer aux = auxiliaryBuffers.get(i); - available = aux.available(); - - if (available < 1) { - // secondary auxiliary buffer - available = len; - aux = new ManagedBuffer(Math.max(len, AUX_BUFFER_SIZE2)); - auxiliaryBuffers.add(aux); - i++; - } else { - available = Math.min(len, available); - } - - aux.write(b, off, (int) available); - - len -= available; - if (len > 0) off += available; - } - } - } - - private void writeOutside(byte buffer[], int offset, int length) throws IOException { - out.write(buffer, offset, length); - position += length; - - if (onProgress != null && position > reportPosition) { - reportPosition = position + NOTIFY_BYTES_INTERVAL; - onProgress.report(position); - } - } - - private void writeQueue(byte[] buffer, int offset, int length) throws IOException { - while (length > 0) { - if (queue.available() < length) { - flushQueue(); - - if (length >= queue.buffer.length) { - writeOutside(buffer, offset, length); - return; - } - } - - int size = Math.min(queue.available(), length); - queue.write(buffer, offset, size); - - offset += size; - length -= size; - } - - if (queue.size >= queue.buffer.length) { - flushQueue(); - } - } - - private void flushQueue() throws IOException { - writeOutside(queue.buffer, 0, queue.size); - queue.size = 0; - } - - private void flushEverything() throws IOException { - flushQueue(); - - if (auxiliaryBuffers.size() > 0) { - for (ManagedBuffer aux : auxiliaryBuffers) { - writeOutside(aux.buffer, 0, aux.size); - aux.dereference(); - } - auxiliaryBuffers.clear(); - } - } - - /** - * Flush any buffer directly to the file. Warning: use this method ONLY if - * all read dependencies are disposed - * - * @throws IOException if the dependencies are not disposed - */ - @Override - public void flush() throws IOException { - if (callback.check() != -1) { - throw new IOException("All read dependencies of this file must be disposed first"); - } - flushEverything(); - - // Save the current file length in case the method {@code rewind()} is called - if (position > maxLengthKnown) { - maxLengthKnown = position; - } - } - - @Override - public void rewind() throws IOException { - flush(); - out.seek(startOffset); - - if (onProgress != null) { - onProgress.report(-position); - } - - position = startOffset; - reportPosition = startOffset; - - } - - @Override - public long skip(long amount) throws IOException { - flush(); - position += amount; - - out.seek(position); - - return amount; - } - - @Override - public boolean isDisposed() { - return out == null; - } - - @Override - public boolean canRewind() { - return true; - } - - @Override - public boolean canWrite() { - return true; - } - - // - @Override - public boolean canRead() { - return false; - } - - @Override - public int read() { - throw new UnsupportedOperationException("write-only"); - } - - @Override - public int read(byte[] buffer) { - throw new UnsupportedOperationException("write-only"); - } - - @Override - public int read(byte[] buffer, int offset, int count) { - throw new UnsupportedOperationException("write-only"); - } - - @Override - public int available() { - throw new UnsupportedOperationException("write-only"); - } -// - - public interface OffsetChecker { - - /** - * Checks the amount of available space ahead - * - * @return absolute offset in the file where no more data SHOULD NOT be - * written. If the value is -1 the whole file will be used - */ - long check(); - } - - public interface ProgressReport { - - void report(long progress); - } - - class ManagedBuffer { - - byte[] buffer; - int size; - - ManagedBuffer(int length) { - buffer = new byte[length]; - } - - void dereference() { - buffer = null; - size = 0; - } - - void dereference(int amount) { - if (amount > size) { - throw new IndexOutOfBoundsException("Invalid dereference amount (" + amount + ">=" + size + ")"); - } - size -= amount; - System.arraycopy(buffer, amount, buffer, 0, size); - } - - protected int available() { - return buffer.length - size; - } - - private void write(byte[] b, int off, int len) { - System.arraycopy(b, off, buffer, size, len); - size += len; - } - - @Override - public String toString() { - return "holding: " + String.valueOf(size) + " length: " + String.valueOf(buffer.length) + " available: " + String.valueOf(available()); - } - - } -} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/io/CircularFileWriter.java b/app/src/main/java/us/shandian/giga/postprocessing/io/CircularFileWriter.java new file mode 100644 index 000000000..4c4160fa3 --- /dev/null +++ b/app/src/main/java/us/shandian/giga/postprocessing/io/CircularFileWriter.java @@ -0,0 +1,459 @@ +package us.shandian.giga.postprocessing.io; + +import android.support.annotation.NonNull; + +import org.schabi.newpipe.streams.io.SharpStream; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; + +public class CircularFileWriter extends SharpStream { + + private final static int QUEUE_BUFFER_SIZE = 8 * 1024;// 8 KiB + private final static int NOTIFY_BYTES_INTERVAL = 64 * 1024;// 64 KiB + private final static int THRESHOLD_AUX_LENGTH = 3 * 1024 * 1024;// 3 MiB + + private OffsetChecker callback; + + public ProgressReport onProgress; + public WriteErrorHandle onWriteError; + + private long reportPosition; + private long maxLengthKnown = -1; + + private BufferedFile out; + private BufferedFile aux; + + public CircularFileWriter(File source, File temp, OffsetChecker checker) throws IOException { + if (checker == null) { + throw new NullPointerException("checker is null"); + } + + if (!temp.exists()) { + if (!temp.createNewFile()) { + throw new IOException("Cannot create a temporal file"); + } + } + + aux = new BufferedFile(temp); + out = new BufferedFile(source); + + callback = checker; + + reportPosition = NOTIFY_BYTES_INTERVAL; + } + + private void flushAuxiliar() throws IOException { + if (aux.length < 1) { + return; + } + + boolean underflow = out.getOffset() >= out.length; + + out.flush(); + aux.flush(); + + aux.target.seek(0); + out.target.seek(out.length); + + long length = aux.length; + out.length += aux.length; + + while (length > 0) { + int read = (int) Math.min(length, Integer.MAX_VALUE); + read = aux.target.read(aux.queue, 0, Math.min(read, aux.queue.length)); + + out.writeProof(aux.queue, read); + length -= read; + } + + if (underflow) { + out.offset += aux.offset; + out.target.seek(out.offset); + } else { + out.offset = out.length; + } + + if (out.length > maxLengthKnown) { + maxLengthKnown = out.length; + } + + if (aux.length > THRESHOLD_AUX_LENGTH) { + aux.target.setLength(THRESHOLD_AUX_LENGTH);// or setLength(0); + } + + aux.reset(); + } + + /** + * Flush any buffer and close the output file. Use this method if the + * operation is successful + * + * @return the final length of the file + * @throws IOException if an I/O error occurs + */ + public long finalizeFile() throws IOException { + flushAuxiliar(); + + out.flush(); + + // change file length (if required) + long length = Math.max(maxLengthKnown, out.length); + if (length != out.target.length()) { + out.target.setLength(length); + } + + dispose(); + + return length; + } + + /** + * Close the file without flushing any buffer + */ + @Override + public void dispose() { + if (out != null) { + out.dispose(); + out = null; + } + if (aux != null) { + aux.dispose(); + aux = null; + } + } + + @Override + public void write(byte b) throws IOException { + write(new byte[]{b}, 0, 1); + } + + @Override + public void write(byte b[]) throws IOException { + write(b, 0, b.length); + } + + @Override + public void write(byte b[], int off, int len) throws IOException { + if (len == 0) { + return; + } + + long available; + long offsetOut = out.getOffset(); + long offsetAux = aux.getOffset(); + long end = callback.check(); + + if (end == -1) { + available = Integer.MAX_VALUE; + } else if (end < offsetOut) { + throw new IOException("The reported offset is invalid: " + String.valueOf(offsetOut)); + } else { + available = end - offsetOut; + } + + boolean usingAux = aux.length > 0 && offsetOut >= out.length; + boolean underflow = offsetAux < aux.length || offsetOut < out.length; + + if (usingAux) { + // before continue calculate the final length of aux + long length = offsetAux + len; + if (underflow) { + if (aux.length > length) { + length = aux.length;// the length is not changed + } + } else { + length = aux.length + len; + } + + if (length > available || length < THRESHOLD_AUX_LENGTH) { + aux.write(b, off, len); + } else { + if (underflow) { + aux.write(b, off, len); + flushAuxiliar(); + } else { + flushAuxiliar(); + out.write(b, off, len);// write directly on the output + } + } + } else { + if (underflow) { + available = out.length - offsetOut; + } + + int length = Math.min(len, (int) available); + out.write(b, off, length); + + len -= length; + off += length; + + if (len > 0) { + aux.write(b, off, len); + } + } + + if (onProgress != null) { + long absoluteOffset = out.getOffset() + aux.getOffset(); + if (absoluteOffset > reportPosition) { + reportPosition = absoluteOffset + NOTIFY_BYTES_INTERVAL; + onProgress.report(absoluteOffset); + } + } + } + + @Override + public void flush() throws IOException { + aux.flush(); + out.flush(); + + long total = out.length + aux.length; + if (total > maxLengthKnown) { + maxLengthKnown = total;// save the current file length in case the method {@code rewind()} is called + } + } + + @Override + public long skip(long amount) throws IOException { + seek(out.getOffset() + aux.getOffset() + amount); + return amount; + } + + @Override + public void rewind() throws IOException { + if (onProgress != null) { + onProgress.report(-out.length - aux.length);// rollback the whole progress + } + + seek(0); + + reportPosition = NOTIFY_BYTES_INTERVAL; + } + + @Override + public void seek(long offset) throws IOException { + long total = out.length + aux.length; + if (offset == total) { + return;// nothing to do + } + + // flush everything, avoid any underflow + flush(); + + if (offset < 0 || offset > total) { + throw new IOException("desired offset is outside of range=0-" + total + " offset=" + offset); + } + + if (offset > out.length) { + out.seek(out.length); + aux.seek(offset - out.length); + } else { + out.seek(offset); + aux.seek(0); + } + } + + @Override + public boolean isDisposed() { + return out == null; + } + + @Override + public boolean canRewind() { + return true; + } + + @Override + public boolean canWrite() { + return true; + } + + @Override + public boolean canSeek() { + return true; + } + + // + @Override + public boolean canRead() { + return false; + } + + @Override + public int read() { + throw new UnsupportedOperationException("write-only"); + } + + @Override + public int read(byte[] buffer + ) { + throw new UnsupportedOperationException("write-only"); + } + + @Override + public int read(byte[] buffer, int offset, int count + ) { + throw new UnsupportedOperationException("write-only"); + } + + @Override + public long available() { + throw new UnsupportedOperationException("write-only"); + } + // + + public interface OffsetChecker { + + /** + * Checks the amount of available space ahead + * + * @return absolute offset in the file where no more data SHOULD NOT be + * written. If the value is -1 the whole file will be used + */ + long check(); + } + + public interface ProgressReport { + + /** + * Report the size of the new file + * + * @param progress the new size + */ + void report(long progress); + } + + public interface WriteErrorHandle { + + /** + * Attempts to handle a I/O exception + * + * @param err the cause + * @return {@code true} to retry and continue, otherwise, {@code false} + * and throw the exception + */ + boolean handle(Exception err); + } + + class BufferedFile { + + protected final RandomAccessFile target; + + private long offset; + protected long length; + + private byte[] queue; + private int queueSize; + + BufferedFile(File file) throws FileNotFoundException { + queue = new byte[QUEUE_BUFFER_SIZE]; + target = new RandomAccessFile(file, "rw"); + } + + protected long getOffset() { + return offset + queueSize;// absolute offset in the file + } + + protected void dispose() { + try { + queue = null; + target.close(); + } catch (IOException e) { + // nothing to do + } + } + + protected void write(byte b[], int off, int len) throws IOException { + while (len > 0) { + // if the queue is full, the method available() will flush the queue + int read = Math.min(available(), len); + + // enqueue incoming buffer + System.arraycopy(b, off, queue, queueSize, read); + queueSize += read; + + len -= read; + off += read; + } + + long total = offset + queueSize; + if (total > length) { + length = total;// save length + } + } + + protected void flush() throws IOException { + writeProof(queue, queueSize); + offset += queueSize; + queueSize = 0; + } + + protected void rewind() throws IOException { + offset = 0; + target.seek(0); + } + + protected int available() throws IOException { + if (queueSize >= queue.length) { + flush(); + return queue.length; + } + + return queue.length - queueSize; + } + + protected void reset() throws IOException { + offset = 0; + length = 0; + target.seek(0); + } + + protected void seek(long absoluteOffset) throws IOException { + offset = absoluteOffset; + target.seek(absoluteOffset); + } + + protected void writeProof(byte[] buffer, int length) throws IOException { + if (onWriteError == null) { + target.write(buffer, 0, length); + return; + } + + while (true) { + try { + target.write(buffer, 0, length); + return; + } catch (Exception e) { + if (!onWriteError.handle(e)) { + throw e;// give up + } + } + } + } + + @NonNull + @Override + public String toString() { + String absOffset; + String absLength; + + try { + absOffset = Long.toString(target.getFilePointer()); + } catch (IOException e) { + absOffset = "[" + e.getLocalizedMessage() + "]"; + } + try { + absLength = Long.toString(target.length()); + } catch (IOException e) { + absLength = "[" + e.getLocalizedMessage() + "]"; + } + + return String.format( + "offset=%s length=%s queue=%s absOffset=%s absLength=%s", + offset, length, queueSize, absOffset, absLength + ); + } + } +} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/io/FileStream.java b/app/src/main/java/us/shandian/giga/postprocessing/io/FileStream.java deleted file mode 100644 index c1b675eef..000000000 --- a/app/src/main/java/us/shandian/giga/postprocessing/io/FileStream.java +++ /dev/null @@ -1,126 +0,0 @@ -package us.shandian.giga.postprocessing.io; - -import org.schabi.newpipe.streams.io.SharpStream; - -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; - -/** - * @author kapodamy - */ -public class FileStream extends SharpStream { - - public enum Mode { - Read, - ReadWrite - } - - public RandomAccessFile source; - private final Mode mode; - - public FileStream(String path, Mode mode) throws IOException { - String flags; - - if (mode == Mode.Read) { - flags = "r"; - } else { - flags = "rw"; - } - - this.mode = mode; - source = new RandomAccessFile(path, flags); - } - - @Override - public int read() throws IOException { - return source.read(); - } - - @Override - public int read(byte b[]) throws IOException { - return read(b, 0, b.length); - } - - @Override - public int read(byte b[], int off, int len) throws IOException { - return source.read(b, off, len); - } - - @Override - public long skip(long pos) throws IOException { - FileChannel fc = source.getChannel(); - fc.position(fc.position() + pos); - return pos; - } - - @Override - public int available() { - try { - return (int) (source.length() - source.getFilePointer()); - } catch (IOException ex) { - return 0; - } - } - - @SuppressWarnings("EmptyCatchBlock") - @Override - public void dispose() { - try { - source.close(); - } catch (IOException err) { - - } finally { - source = null; - } - } - - @Override - public boolean isDisposed() { - return source == null; - } - - @Override - public void rewind() throws IOException { - source.getChannel().position(0); - } - - @Override - public boolean canRewind() { - return true; - } - - @Override - public boolean canRead() { - return mode == Mode.Read || mode == Mode.ReadWrite; - } - - @Override - public boolean canWrite() { - return mode == Mode.ReadWrite; - } - - @Override - public void write(byte value) throws IOException { - source.write(value); - } - - @Override - public void write(byte[] buffer) throws IOException { - source.write(buffer); - } - - @Override - public void write(byte[] buffer, int offset, int count) throws IOException { - source.write(buffer, offset, count); - } - - @Override - public void flush() { - } - - @Override - public void setLength(long length) throws IOException { - source.setLength(length); - } -} diff --git a/app/src/main/java/us/shandian/giga/postprocessing/io/SharpInputStream.java b/app/src/main/java/us/shandian/giga/postprocessing/io/SharpInputStream.java index 52e0775da..586456d98 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/io/SharpInputStream.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/io/SharpInputStream.java @@ -14,6 +14,7 @@ import java.io.InputStream; /** * Wrapper for the classic {@link java.io.InputStream} + * * @author kapodamy */ public class SharpInputStream extends InputStream { @@ -49,7 +50,8 @@ public class SharpInputStream extends InputStream { @Override public int available() { - return base.available(); + long res = base.available(); + return res > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) res; } @Override diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index 883c26850..58246beb1 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -21,6 +21,8 @@ import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.FinishedMission; import us.shandian.giga.get.Mission; import us.shandian.giga.get.sqlite.DownloadDataSource; +import us.shandian.giga.service.DownloadManagerService.DMChecker; +import us.shandian.giga.service.DownloadManagerService.MissionCheck; import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; @@ -28,7 +30,7 @@ import static org.schabi.newpipe.BuildConfig.DEBUG; public class DownloadManager { private static final String TAG = DownloadManager.class.getSimpleName(); - enum NetworkState {Unavailable, WifiOperating, MobileOperating, OtherOperating} + enum NetworkState {Unavailable, Operating, MeteredOperating} public final static int SPECIAL_NOTHING = 0; public final static int SPECIAL_PENDING = 1; @@ -45,7 +47,9 @@ public class DownloadManager { private NetworkState mLastNetworkStatus = NetworkState.Unavailable; int mPrefMaxRetry; - boolean mPrefCrossNetwork; + boolean mPrefMeteredDownloads; + boolean mPrefQueueLimit; + private boolean mSelfMissionsControl; /** * Create a new instance @@ -152,8 +156,8 @@ public class DownloadManager { } mis.postprocessingState = 0; - mis.errCode = DownloadMission.ERROR_POSTPROCESSING; - mis.errObject = new RuntimeException("stopped unexpectedly"); + mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED; + mis.errObject = null; } else if (exists && !dl.isFile()) { // probably a folder, this should never happens if (!sub.delete()) { @@ -162,20 +166,21 @@ public class DownloadManager { continue; } - if (!exists) { + if (!exists && mis.isInitialized()) { // downloaded file deleted, reset mission state DownloadMission m = new DownloadMission(mis.urls, mis.name, mis.location, mis.kind, mis.postprocessingName, mis.postprocessingArgs); m.timestamp = mis.timestamp; m.threadCount = mis.threadCount; m.source = mis.source; - m.maxRetry = mis.maxRetry; m.nearLength = mis.nearLength; + m.setEnqueued(mis.enqueued); mis = m; } mis.running = false; mis.recovered = exists; mis.metadata = sub; + mis.maxRetry = mPrefMaxRetry; mis.mHandler = mHandler; mMissionsPending.add(mis); @@ -205,17 +210,25 @@ public class DownloadManager { synchronized (this) { // check for existing pending download DownloadMission pendingMission = getPendingMission(location, name); + if (pendingMission != null) { - // generate unique filename (?) - try { - name = generateUniqueName(location, name); - } catch (Exception e) { - Log.e(TAG, "Unable to generate unique name", e); - name = System.currentTimeMillis() + name; - Log.i(TAG, "Using " + name); + if (pendingMission.running) { + // generate unique filename (?) + try { + name = generateUniqueName(location, name); + } catch (Exception e) { + Log.e(TAG, "Unable to generate unique name", e); + name = System.currentTimeMillis() + name; + Log.i(TAG, "Using " + name); + } + } else { + // dispose the mission + mMissionsPending.remove(pendingMission); + mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_DELETED); + pendingMission.delete(); } } else { - // check for existing finished download + // check for existing finished download and dispose (if exists) int index = getFinishedMissionIndex(location, name); if (index >= 0) mDownloadDataSource.deleteMission(mMissionsFinished.remove(index)); } @@ -242,14 +255,17 @@ public class DownloadManager { mission.timestamp = System.currentTimeMillis(); } + mSelfMissionsControl = true; mMissionsPending.add(mission); - // Before starting, save the state in case the internet connection is not available + // Before continue, save the metadata in case the internet connection is not available Utility.writeToFile(mission.metadata, mission); - if (canDownloadInCurrentNetwork() && (getRunningMissionsCount() < 1)) { + boolean start = !mPrefQueueLimit || getRunningMissionsCount() < 1; + + if (canDownloadInCurrentNetwork() && start) { + mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); mission.start(); - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_RUNNING); } } } @@ -257,13 +273,14 @@ public class DownloadManager { public void resumeMission(DownloadMission mission) { if (!mission.running) { + mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); mission.start(); - mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_RUNNING); } } public void pauseMission(DownloadMission mission) { if (mission.running) { + mission.setEnqueued(false); mission.pause(); mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); } @@ -335,7 +352,7 @@ public class DownloadManager { int count = 0; synchronized (this) { for (DownloadMission mission : mMissionsPending) { - if (mission.running && !mission.isFinished() && !mission.isPsFailed()) + if (mission.running && !mission.isPsFailed() && !mission.isFinished()) count++; } } @@ -343,10 +360,36 @@ public class DownloadManager { return count; } - void pauseAllMissions() { + public void pauseAllMissions(boolean force) { + boolean flag = false; + synchronized (this) { - for (DownloadMission mission : mMissionsPending) mission.pause(); + for (DownloadMission mission : mMissionsPending) { + if (!mission.running || mission.isPsRunning() || mission.isFinished()) continue; + + if (force) mission.threads = null;// avoid waiting for threads + + mission.pause(); + flag = true; + } } + + if (flag) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); + } + + public void startAllMissions() { + boolean flag = false; + + synchronized (this) { + for (DownloadMission mission : mMissionsPending) { + if (mission.running || mission.isPsFailed() || mission.isFinished()) continue; + + flag = true; + mission.start(); + } + } + + if (flag) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); } @@ -415,31 +458,35 @@ public class DownloadManager { } /** - * runs another mission in queue if possible + * runs one or multiple missions in from queue if possible * - * @return true if exits pending missions running or a mission was started, otherwise, false + * @return true if one or multiple missions are running, otherwise, false */ - boolean runAnotherMission() { + boolean runMissions() { synchronized (this) { if (mMissionsPending.size() < 1) return false; - - int i = getRunningMissionsCount(); - if (i > 0) return true; - if (!canDownloadInCurrentNetwork()) return false; - for (DownloadMission mission : mMissionsPending) { - if (!mission.running && mission.errCode == DownloadMission.ERROR_NOTHING && mission.enqueued) { - resumeMission(mission); - return true; - } + if (mPrefQueueLimit) { + for (DownloadMission mission : mMissionsPending) + if (!mission.isFinished() && mission.running) return true; } - return false; + boolean flag = false; + for (DownloadMission mission : mMissionsPending) { + if (mission.running || !mission.enqueued || mission.isFinished()) continue; + + resumeMission(mission); + if (mPrefQueueLimit) return true; + flag = true; + } + + return flag; } } public MissionIterator getIterator() { + mSelfMissionsControl = true; return new MissionIterator(); } @@ -457,31 +504,43 @@ public class DownloadManager { private boolean canDownloadInCurrentNetwork() { if (mLastNetworkStatus == NetworkState.Unavailable) return false; - return !(mPrefCrossNetwork && mLastNetworkStatus == NetworkState.MobileOperating); + return !(mPrefMeteredDownloads && mLastNetworkStatus == NetworkState.MeteredOperating); } - void handleConnectivityChange(NetworkState currentStatus) { + void handleConnectivityState(NetworkState currentStatus, boolean updateOnly) { if (currentStatus == mLastNetworkStatus) return; mLastNetworkStatus = currentStatus; + if (currentStatus == NetworkState.Unavailable) return; - if (currentStatus == NetworkState.Unavailable) { - return; - } else if (currentStatus != NetworkState.MobileOperating || !mPrefCrossNetwork) { - return; + if (!mSelfMissionsControl || updateOnly) { + return;// don't touch anything without the user interaction } - boolean flag = false; + boolean isMetered = mPrefMeteredDownloads && mLastNetworkStatus == NetworkState.MeteredOperating; + + int running = 0; + int paused = 0; synchronized (this) { for (DownloadMission mission : mMissionsPending) { - if (mission.running && !mission.isFinished() && !mission.isPsRunning()) { - flag = true; + if (mission.isFinished() || mission.isPsRunning()) continue; + + if (mission.running && isMetered) { + paused++; mission.pause(); + } else if (!mission.running && !isMetered && mission.enqueued) { + running++; + mission.start(); + if (mPrefQueueLimit) break; } } } - if (flag) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); + if (running > 0) { + mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PROGRESS); + return; + } + if (paused > 0) mHandler.sendEmptyMessage(DownloadManagerService.MESSAGE_PAUSED); } void updateMaximumAttempts() { @@ -506,21 +565,24 @@ public class DownloadManager { ), Toast.LENGTH_LONG).show(); } - void checkForRunningMission(String location, String name, DownloadManagerService.DMChecker check) { - boolean listed; - boolean finished = false; + void checkForRunningMission(String location, String name, DMChecker check) { + MissionCheck result = MissionCheck.None; synchronized (this) { - DownloadMission mission = getPendingMission(location, name); - if (mission != null) { - listed = true; + DownloadMission pending = getPendingMission(location, name); + + if (pending == null) { + if (getFinishedMissionIndex(location, name) >= 0) result = MissionCheck.Finished; } else { - listed = getFinishedMissionIndex(location, name) >= 0; - finished = listed; + if (pending.isFinished()) { + result = MissionCheck.Finished;// this never should happen (race-condition) + } else { + result = pending.running ? MissionCheck.PendingRunning : MissionCheck.Pending; + } } } - check.callback(listed, finished); + check.callback(result); } public class MissionIterator extends DiffUtil.Callback { @@ -592,39 +654,6 @@ public class DownloadManager { return SPECIAL_NOTHING; } - public MissionItem getItemUnsafe(int position) { - synchronized (DownloadManager.this) { - int count = mMissionsPending.size(); - int count2 = mMissionsFinished.size(); - - if (count > 0) { - position--; - if (position == -1) - return new MissionItem(SPECIAL_PENDING); - else if (position < count) - return new MissionItem(SPECIAL_NOTHING, mMissionsPending.get(position)); - else if (position == count && count2 > 0) - return new MissionItem(SPECIAL_FINISHED); - else - position -= count; - } else { - if (count2 > 0 && position == 0) { - return new MissionItem(SPECIAL_FINISHED); - } - } - - position--; - - if (count2 < 1) { - throw new RuntimeException( - String.format("Out of range. pending_count=%s finished_count=%s position=%s", count, count2, position) - ); - } - - return new MissionItem(SPECIAL_NOTHING, mMissionsFinished.get(position)); - } - } - public void start() { current = getSpecialItems(); @@ -647,6 +676,32 @@ public class DownloadManager { return hasFinished; } + /** + * Check if exists missions running and paused. Corrupted and hidden missions are not counted + * + * @return two-dimensional array contains the current missions state. + * 1° entry: true if has at least one mission running + * 2° entry: true if has at least one mission paused + */ + public boolean[] hasValidPendingMissions() { + boolean running = false; + boolean paused = false; + + synchronized (DownloadManager.this) { + for (DownloadMission mission : mMissionsPending) { + if (hidden.contains(mission) || mission.isPsFailed() || mission.isFinished()) + continue; + + if (mission.running) + paused = true; + else + running = true; + } + } + + return new boolean[]{running, paused}; + } + @Override public int getOldListSize() { diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index a57fe1734..be1e20dd6 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -15,7 +15,9 @@ import android.content.SharedPreferences; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.net.ConnectivityManager; +import android.net.Network; import android.net.NetworkInfo; +import android.net.NetworkRequest; import android.net.Uri; import android.os.Binder; import android.os.Build; @@ -24,6 +26,7 @@ import android.os.IBinder; import android.os.Looper; import android.os.Message; import android.preference.PreferenceManager; +import android.support.annotation.NonNull; import android.support.v4.app.NotificationCompat; import android.support.v4.app.NotificationCompat.Builder; import android.support.v4.content.PermissionChecker; @@ -48,7 +51,6 @@ public class DownloadManagerService extends Service { private static final String TAG = "DownloadManagerService"; - public static final int MESSAGE_RUNNING = 0; public static final int MESSAGE_PAUSED = 1; public static final int MESSAGE_FINISHED = 2; public static final int MESSAGE_PROGRESS = 3; @@ -76,7 +78,7 @@ public class DownloadManagerService extends Service { private Notification mNotification; private Handler mHandler; private boolean mForeground = false; - private NotificationManager notificationManager = null; + private NotificationManager mNotificationManager = null; private boolean mDownloadNotificationEnable = true; private int downloadDoneCount = 0; @@ -85,7 +87,9 @@ public class DownloadManagerService extends Service { private final ArrayList mEchoObservers = new ArrayList<>(1); - private BroadcastReceiver mNetworkStateListener; + private ConnectivityManager mConnectivityManager; + private BroadcastReceiver mNetworkStateListener = null; + private ConnectivityManager.NetworkCallback mNetworkStateListenerL = null; private SharedPreferences mPrefs = null; private final SharedPreferences.OnSharedPreferenceChangeListener mPrefChangeListener = this::handlePreferenceChange; @@ -147,25 +151,39 @@ public class DownloadManagerService extends Service { .setContentText(getString(R.string.msg_running_detail)); mNotification = builder.build(); - notificationManager = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE); - mNetworkStateListener = new BroadcastReceiver() { - @Override - public void onReceive(Context context, Intent intent) { - if (intent.getBooleanExtra(ConnectivityManager.EXTRA_NO_CONNECTIVITY, false)) { - handleConnectivityChange(null); - return; + mNotificationManager = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE); + mConnectivityManager = (ConnectivityManager) getSystemService(Context.CONNECTIVITY_SERVICE); + + if (android.os.Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + mNetworkStateListenerL = new ConnectivityManager.NetworkCallback() { + @Override + public void onAvailable(Network network) { + handleConnectivityState(false); } - handleConnectivityChange(intent.getParcelableExtra(ConnectivityManager.EXTRA_NETWORK_INFO)); - } - }; - registerReceiver(mNetworkStateListener, new IntentFilter(ConnectivityManager.CONNECTIVITY_ACTION)); + + @Override + public void onLost(Network network) { + handleConnectivityState(false); + } + }; + mConnectivityManager.registerNetworkCallback(new NetworkRequest.Builder().build(), mNetworkStateListenerL); + } else { + mNetworkStateListener = new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + handleConnectivityState(false); + } + }; + registerReceiver(mNetworkStateListener, new IntentFilter(ConnectivityManager.CONNECTIVITY_ACTION)); + } mPrefs = PreferenceManager.getDefaultSharedPreferences(this); mPrefs.registerOnSharedPreferenceChangeListener(mPrefChangeListener); handlePreferenceChange(mPrefs, getString(R.string.downloads_cross_network)); handlePreferenceChange(mPrefs, getString(R.string.downloads_maximum_retry)); + handlePreferenceChange(mPrefs, getString(R.string.downloads_queue_limit)); mLock = new LockManager(this); } @@ -173,12 +191,11 @@ public class DownloadManagerService extends Service { @Override public int onStartCommand(Intent intent, int flags, int startId) { if (DEBUG) { - if (intent == null) { - Log.d(TAG, "Restarting"); - return START_NOT_STICKY; - } - Log.d(TAG, "Starting"); + Log.d(TAG, intent == null ? "Restarting" : "Starting"); } + + if (intent == null) return START_NOT_STICKY; + Log.i(TAG, "Got intent: " + intent); String action = intent.getAction(); if (action != null) { @@ -193,6 +210,8 @@ public class DownloadManagerService extends Service { String source = intent.getStringExtra(EXTRA_SOURCE); long nearLength = intent.getLongExtra(EXTRA_NEAR_LENGTH, 0); + handleConnectivityState(true);// first check the actual network status + mHandler.post(() -> mManager.startMission(urls, location, name, kind, threads, source, psName, psArgs, nearLength)); } else if (downloadDoneNotification != null) { @@ -221,21 +240,25 @@ public class DownloadManagerService extends Service { stopForeground(true); - if (notificationManager != null && downloadDoneNotification != null) { + if (mNotificationManager != null && downloadDoneNotification != null) { downloadDoneNotification.setDeleteIntent(null);// prevent NewPipe running when is killed, cleared from recent, etc - notificationManager.notify(DOWNLOADS_NOTIFICATION_ID, downloadDoneNotification.build()); + mNotificationManager.notify(DOWNLOADS_NOTIFICATION_ID, downloadDoneNotification.build()); } - mManager.pauseAllMissions(); - manageLock(false); - unregisterReceiver(mNetworkStateListener); + if (android.os.Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) + mConnectivityManager.unregisterNetworkCallback(mNetworkStateListenerL); + else + unregisterReceiver(mNetworkStateListener); + mPrefs.unregisterOnSharedPreferenceChangeListener(mPrefChangeListener); if (icDownloadDone != null) icDownloadDone.recycle(); if (icDownloadFailed != null) icDownloadFailed.recycle(); if (icLauncher != null) icLauncher.recycle(); + + mManager.pauseAllMissions(true); } @Override @@ -264,15 +287,16 @@ public class DownloadManagerService extends Service { notifyMediaScanner(mission.getDownloadedFile()); notifyFinishedDownload(mission.name); mManager.setFinished(mission); - updateForegroundState(mManager.runAnotherMission()); + handleConnectivityState(false); + updateForegroundState(mManager.runMissions()); break; - case MESSAGE_RUNNING: case MESSAGE_PROGRESS: updateForegroundState(true); break; case MESSAGE_ERROR: notifyFailedDownload(mission); - updateForegroundState(mManager.runAnotherMission()); + handleConnectivityState(false); + updateForegroundState(mManager.runMissions()); break; case MESSAGE_PAUSED: updateForegroundState(mManager.getRunningMissionsCount() > 0); @@ -293,36 +317,30 @@ public class DownloadManagerService extends Service { } } - private void handleConnectivityChange(NetworkInfo info) { + private void handleConnectivityState(boolean updateOnly) { + NetworkInfo info = mConnectivityManager.getActiveNetworkInfo(); NetworkState status; if (info == null) { status = NetworkState.Unavailable; - Log.i(TAG, "actual connectivity status is unavailable"); - } else if (!info.isAvailable() || !info.isConnected()) { - status = NetworkState.Unavailable; - Log.i(TAG, "actual connectivity status is not available and not connected"); + Log.i(TAG, "Active network [connectivity is unavailable]"); } else { - int type = info.getType(); - if (type == ConnectivityManager.TYPE_MOBILE || type == ConnectivityManager.TYPE_MOBILE_DUN) { - status = NetworkState.MobileOperating; - } else if (type == ConnectivityManager.TYPE_WIFI) { - status = NetworkState.WifiOperating; - } else if (type == ConnectivityManager.TYPE_WIMAX || - type == ConnectivityManager.TYPE_ETHERNET || - type == ConnectivityManager.TYPE_BLUETOOTH) { - status = NetworkState.OtherOperating; - } else { + boolean connected = info.isConnected(); + boolean metered = mConnectivityManager.isActiveNetworkMetered(); + + if (connected) + status = metered ? NetworkState.MeteredOperating : NetworkState.Operating; + else status = NetworkState.Unavailable; - } - Log.i(TAG, "actual connectivity status is " + status.name()); + + Log.i(TAG, "Active network [connected=" + connected + " metered=" + metered + "] " + info.toString()); } if (mManager == null) return;// avoid race-conditions while the service is starting - mManager.handleConnectivityChange(status); + mManager.handleConnectivityState(status, updateOnly); } - private void handlePreferenceChange(SharedPreferences prefs, String key) { + private void handlePreferenceChange(SharedPreferences prefs, @NonNull String key) { if (key.equals(getString(R.string.downloads_maximum_retry))) { try { String value = prefs.getString(key, getString(R.string.downloads_maximum_retry_default)); @@ -332,7 +350,9 @@ public class DownloadManagerService extends Service { } mManager.updateMaximumAttempts(); } else if (key.equals(getString(R.string.downloads_cross_network))) { - mManager.mPrefCrossNetwork = prefs.getBoolean(key, false); + mManager.mPrefMeteredDownloads = prefs.getBoolean(key, false); + } else if (key.equals(getString(R.string.downloads_queue_limit))) { + mManager.mPrefQueueLimit = prefs.getBoolean(key, true); } } @@ -366,19 +386,20 @@ public class DownloadManagerService extends Service { context.startService(intent); } - public static void checkForRunningMission(Context context, String location, String name, DMChecker check) { + public static void checkForRunningMission(Context context, String location, String name, DMChecker checker) { Intent intent = new Intent(); intent.setClass(context, DownloadManagerService.class); + context.startService(intent); + context.bindService(intent, new ServiceConnection() { @Override public void onServiceConnected(ComponentName cname, IBinder service) { try { - ((DMBinder) service).getDownloadManager().checkForRunningMission(location, name, check); + ((DMBinder) service).getDownloadManager().checkForRunningMission(location, name, checker); } catch (Exception err) { Log.w(TAG, "checkForRunningMission() callback is defective", err); } - // TODO: find a efficient way to unbind the service. This destroy the service due idle, but is started again when the user start a download. context.unbindService(this); } @@ -389,7 +410,7 @@ public class DownloadManagerService extends Service { } public void notifyFinishedDownload(String name) { - if (!mDownloadNotificationEnable || notificationManager == null) { + if (!mDownloadNotificationEnable || mNotificationManager == null) { return; } @@ -428,7 +449,7 @@ public class DownloadManagerService extends Service { downloadDoneNotification.setContentText(downloadDoneList); } - notificationManager.notify(DOWNLOADS_NOTIFICATION_ID, downloadDoneNotification.build()); + mNotificationManager.notify(DOWNLOADS_NOTIFICATION_ID, downloadDoneNotification.build()); downloadDoneCount++; } @@ -458,7 +479,7 @@ public class DownloadManagerService extends Service { .bigText(mission.name)); } - notificationManager.notify(id, downloadFailedNotification.build()); + mNotificationManager.notify(id, downloadFailedNotification.build()); } private PendingIntent makePendingIntent(String action) { @@ -487,7 +508,11 @@ public class DownloadManagerService extends Service { mLockAcquired = acquire; } - // Wrapper of DownloadManager + + //////////////////////////////////////////////////////////////////////////////////////////////// + // Wrappers for DownloadManager + //////////////////////////////////////////////////////////////////////////////////////////////// + public class DMBinder extends Binder { public DownloadManager getDownloadManager() { return mManager; @@ -502,15 +527,15 @@ public class DownloadManagerService extends Service { } public void clearDownloadNotifications() { - if (notificationManager == null) return; + if (mNotificationManager == null) return; if (downloadDoneNotification != null) { - notificationManager.cancel(DOWNLOADS_NOTIFICATION_ID); + mNotificationManager.cancel(DOWNLOADS_NOTIFICATION_ID); downloadDoneList.setLength(0); downloadDoneCount = 0; } if (downloadFailedNotification != null) { for (; downloadFailedNotificationID > DOWNLOADS_NOTIFICATION_ID; downloadFailedNotificationID--) { - notificationManager.cancel(downloadFailedNotificationID); + mNotificationManager.cancel(downloadFailedNotificationID); } mFailedDownloads.clear(); downloadFailedNotificationID++; @@ -524,7 +549,9 @@ public class DownloadManagerService extends Service { } public interface DMChecker { - void callback(boolean listed, boolean finished); + void callback(MissionCheck result); } + public enum MissionCheck {None, Pending, PendingRunning, Finished} + } diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 4a35aa166..cada3aeb8 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -14,15 +14,17 @@ import android.os.Looper; import android.os.Message; import android.support.annotation.NonNull; import android.support.annotation.Nullable; +import android.support.annotation.StringRes; import android.support.v4.content.FileProvider; import android.support.v4.view.ViewCompat; import android.support.v7.app.AlertDialog; import android.support.v7.util.DiffUtil; import android.support.v7.widget.RecyclerView; -import android.support.v7.widget.RecyclerView.ViewHolder; import android.support.v7.widget.RecyclerView.Adapter; +import android.support.v7.widget.RecyclerView.ViewHolder; import android.util.Log; import android.util.SparseArray; +import android.view.HapticFeedbackConstants; import android.view.LayoutInflater; import android.view.Menu; import android.view.MenuItem; @@ -36,14 +38,17 @@ import android.widget.Toast; import org.schabi.newpipe.BuildConfig; import org.schabi.newpipe.R; +import org.schabi.newpipe.report.ErrorActivity; +import org.schabi.newpipe.report.UserAction; import org.schabi.newpipe.util.NavigationHelper; -import java.io.File; import java.lang.ref.WeakReference; import java.util.ArrayList; +import java.util.Collections; import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.FinishedMission; +import us.shandian.giga.get.Mission; import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; import us.shandian.giga.ui.common.Deleter; @@ -57,10 +62,13 @@ import static us.shandian.giga.get.DownloadMission.ERROR_CONNECT_HOST; import static us.shandian.giga.get.DownloadMission.ERROR_FILE_CREATION; import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_NO_CONTENT; import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_UNSUPPORTED_RANGE; +import static us.shandian.giga.get.DownloadMission.ERROR_INSUFFICIENT_STORAGE; import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; import static us.shandian.giga.get.DownloadMission.ERROR_PATH_CREATION; import static us.shandian.giga.get.DownloadMission.ERROR_PERMISSION_DENIED; import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING; +import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; +import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_STOPPED; import static us.shandian.giga.get.DownloadMission.ERROR_SSL_EXCEPTION; import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_HOST; @@ -69,6 +77,7 @@ public class MissionAdapter extends Adapter { private static final SparseArray ALGORITHMS = new SparseArray<>(); private static final String TAG = "MissionAdapter"; private static final String UNDEFINED_PROGRESS = "--.-%"; + private static final String DEFAULT_MIME_TYPE = "*/*"; static { @@ -85,9 +94,11 @@ public class MissionAdapter extends Adapter { private ArrayList mPendingDownloadsItems = new ArrayList<>(); private Handler mHandler; private MenuItem mClear; + private MenuItem mStartButton; + private MenuItem mPauseButton; private View mEmptyMessage; - public MissionAdapter(Context context, DownloadManager downloadManager, MenuItem clearButton, View emptyMessage) { + public MissionAdapter(Context context, DownloadManager downloadManager, View emptyMessage) { mContext = context; mDownloadManager = downloadManager; mDeleter = null; @@ -105,10 +116,18 @@ public class MissionAdapter extends Adapter { onServiceMessage(msg); break; } + + if (mStartButton != null && mPauseButton != null) switch (msg.what) { + case DownloadManagerService.MESSAGE_DELETED: + case DownloadManagerService.MESSAGE_ERROR: + case DownloadManagerService.MESSAGE_FINISHED: + case DownloadManagerService.MESSAGE_PAUSED: + checkMasterButtonsVisibility(); + break; + } } }; - mClear = clearButton; mEmptyMessage = emptyMessage; mIterator = downloadManager.getIterator(); @@ -225,8 +244,10 @@ public class MissionAdapter extends Adapter { long deltaDone = mission.done - h.lastDone; boolean hasError = mission.errCode != ERROR_NOTHING; - // on error hide marquee or show if condition (mission.done < 1 || mission.unknownLength) is true - h.progress.setMarquee(!hasError && (mission.done < 1 || mission.unknownLength)); + // hide on error + // show if current resource length is not fetched + // show if length is unknown + h.progress.setMarquee(!hasError && (!mission.isInitialized() || mission.unknownLength)); float progress; if (mission.unknownLength) { @@ -305,36 +326,64 @@ public class MissionAdapter extends Adapter { } } - private boolean viewWithFileProvider(@NonNull File file) { - if (!file.exists()) return true; + private void viewWithFileProvider(Mission mission) { + if (checkInvalidFile(mission)) return; - String ext = Utility.getFileExt(file.getName()); - if (ext == null) return false; + String mimeType = resolveMimeType(mission); - String mimeType = MimeTypeMap.getSingleton().getMimeTypeFromExtension(ext.substring(1)); - Log.v(TAG, "Mime: " + mimeType + " package: " + BuildConfig.APPLICATION_ID + ".provider"); + if (BuildConfig.DEBUG) + Log.v(TAG, "Mime: " + mimeType + " package: " + BuildConfig.APPLICATION_ID + ".provider"); - Uri uri = FileProvider.getUriForFile(mContext, BuildConfig.APPLICATION_ID + ".provider", file); + Uri uri = FileProvider.getUriForFile( + mContext, + BuildConfig.APPLICATION_ID + ".provider", + mission.getDownloadedFile() + ); Intent intent = new Intent(); intent.setAction(Intent.ACTION_VIEW); intent.setDataAndType(uri, mimeType); intent.addFlags(FLAG_GRANT_READ_URI_PERMISSION); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { intent.addFlags(FLAG_GRANT_PREFIX_URI_PERMISSION); } - if(Build.VERSION.SDK_INT <= Build.VERSION_CODES.M) { + if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.M) { intent.addFlags(FLAG_ACTIVITY_NEW_TASK); } + //mContext.grantUriPermission(packageName, uri, Intent.FLAG_GRANT_READ_URI_PERMISSION); - Log.v(TAG, "Starting intent: " + intent); + if (intent.resolveActivity(mContext.getPackageManager()) != null) { mContext.startActivity(intent); } else { - Toast noPlayerToast = Toast.makeText(mContext, R.string.toast_no_player, Toast.LENGTH_LONG); - noPlayerToast.show(); + Toast.makeText(mContext, R.string.toast_no_player, Toast.LENGTH_LONG).show(); } + } + private void shareFile(Mission mission) { + if (checkInvalidFile(mission)) return; + + Intent intent = new Intent(Intent.ACTION_SEND); + intent.setType(resolveMimeType(mission)); + intent.putExtra(Intent.EXTRA_STREAM, mission.getDownloadedFile().toURI()); + + mContext.startActivity(Intent.createChooser(intent, null)); + } + + private static String resolveMimeType(@NonNull Mission mission) { + String ext = Utility.getFileExt(mission.getDownloadedFile().getName()); + if (ext == null) return DEFAULT_MIME_TYPE; + + String mimeType = MimeTypeMap.getSingleton().getMimeTypeFromExtension(ext.substring(1)); + + return mimeType == null ? DEFAULT_MIME_TYPE : mimeType; + } + + private boolean checkInvalidFile(@NonNull Mission mission) { + if (mission.getDownloadedFile().exists()) return false; + + Toast.makeText(mContext, R.string.missing_file, Toast.LENGTH_SHORT).show(); return true; } @@ -343,15 +392,9 @@ public class MissionAdapter extends Adapter { } private void onServiceMessage(@NonNull Message msg) { - switch (msg.what) { - case DownloadManagerService.MESSAGE_PROGRESS: - setAutoRefresh(true); - return; - case DownloadManagerService.MESSAGE_ERROR: - case DownloadManagerService.MESSAGE_FINISHED: - break; - default: - return; + if (msg.what == DownloadManagerService.MESSAGE_PROGRESS) { + setAutoRefresh(true); + return; } for (int i = 0; i < mPendingDownloadsItems.size(); i++) { @@ -370,74 +413,98 @@ public class MissionAdapter extends Adapter { } private void showError(@NonNull DownloadMission mission) { - StringBuilder str = new StringBuilder(); - str.append(mContext.getString(R.string.label_code)); - str.append(": "); - str.append(mission.errCode); - str.append('\n'); + @StringRes int msg = R.string.general_error; + String msgEx = null; switch (mission.errCode) { case 416: - str.append(mContext.getString(R.string.error_http_requested_range_not_satisfiable)); + msg = R.string.error_http_requested_range_not_satisfiable; break; case 404: - str.append(mContext.getString(R.string.error_http_not_found)); + msg = R.string.error_http_not_found; break; case ERROR_NOTHING: - str.append("¿?"); - break; + return;// this never should happen case ERROR_FILE_CREATION: - str.append(mContext.getString(R.string.error_file_creation)); + msg = R.string.error_file_creation; break; case ERROR_HTTP_NO_CONTENT: - str.append(mContext.getString(R.string.error_http_no_content)); + msg = R.string.error_http_no_content; break; case ERROR_HTTP_UNSUPPORTED_RANGE: - str.append(mContext.getString(R.string.error_http_unsupported_range)); + msg = R.string.error_http_unsupported_range; break; case ERROR_PATH_CREATION: - str.append(mContext.getString(R.string.error_path_creation)); + msg = R.string.error_path_creation; break; case ERROR_PERMISSION_DENIED: - str.append(mContext.getString(R.string.permission_denied)); + msg = R.string.permission_denied; break; case ERROR_SSL_EXCEPTION: - str.append(mContext.getString(R.string.error_ssl_exception)); + msg = R.string.error_ssl_exception; break; case ERROR_UNKNOWN_HOST: - str.append(mContext.getString(R.string.error_unknown_host)); + msg = R.string.error_unknown_host; break; case ERROR_CONNECT_HOST: - str.append(mContext.getString(R.string.error_connect_host)); + msg = R.string.error_connect_host; + break; + case ERROR_POSTPROCESSING_STOPPED: + msg = R.string.error_postprocessing_stopped; break; case ERROR_POSTPROCESSING: - str.append(mContext.getString(R.string.error_postprocessing_failed)); - case ERROR_UNKNOWN_EXCEPTION: + case ERROR_POSTPROCESSING_HOLD: + showError(mission.errObject, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed); + return; + case ERROR_INSUFFICIENT_STORAGE: + msg = R.string.error_insufficient_storage; break; + case ERROR_UNKNOWN_EXCEPTION: + showError(mission.errObject, UserAction.DOWNLOAD_FAILED, R.string.general_error); + return; default: if (mission.errCode >= 100 && mission.errCode < 600) { - str = new StringBuilder(8); - str.append("HTTP "); - str.append(mission.errCode); + msgEx = "HTTP " + mission.errCode; } else if (mission.errObject == null) { - str.append("(not_decelerated_error_code)"); + msgEx = "(not_decelerated_error_code)"; + } else { + showError(mission.errObject, UserAction.DOWNLOAD_FAILED, msg); + return; } break; } - if (mission.errObject != null) { - str.append("\n\n"); - str.append(mission.errObject.toString()); + AlertDialog.Builder builder = new AlertDialog.Builder(mContext); + + if (msgEx != null) + builder.setMessage(msgEx); + else + builder.setMessage(msg); + + // add report button for non-HTTP errors (range 100-599) + if (mission.errObject != null && (mission.errCode < 100 || mission.errCode >= 600)) { + @StringRes final int mMsg = msg; + builder.setPositiveButton(R.string.error_report_title, (dialog, which) -> + showError(mission.errObject, UserAction.DOWNLOAD_FAILED, mMsg) + ); } - AlertDialog.Builder builder = new AlertDialog.Builder(mContext); - builder.setTitle(mission.name) - .setMessage(str) - .setNegativeButton(android.R.string.ok, (dialog, which) -> dialog.cancel()) + builder.setNegativeButton(android.R.string.ok, (dialog, which) -> dialog.cancel()) + .setTitle(mission.name) .create() .show(); } + private void showError(Exception exception, UserAction action, @StringRes int reason) { + ErrorActivity.reportError( + mContext, + Collections.singletonList(exception), + null, + null, + ErrorActivity.ErrorInfo.make(action, "-", "-", reason) + ); + } + public void clearFinishedDownloads() { mDownloadManager.forgetFinishedDownloads(); applyChanges(); @@ -466,16 +533,24 @@ public class MissionAdapter extends Adapter { showError(mission); return true; case R.id.queue: - h.queue.setChecked(!h.queue.isChecked()); - mission.enqueued = h.queue.isChecked(); + boolean flag = !h.queue.isChecked(); + h.queue.setChecked(flag); + mission.setEnqueued(flag); updateProgress(h); return true; + case R.id.retry: + mission.psContinue(true); + return true; + case R.id.cancel: + mission.psContinue(false); + return false; } } switch (id) { - case R.id.open: - return viewWithFileProvider(h.item.mission.getDownloadedFile()); + case R.id.menu_item_share: + shareFile(h.item.mission); + return true; case R.id.delete: if (mDeleter == null) { mDownloadManager.deleteMission(h.item.mission); @@ -529,15 +604,42 @@ public class MissionAdapter extends Adapter { } public void setClearButton(MenuItem clearButton) { - if (mClear == null) clearButton.setVisible(mIterator.hasFinishedMissions()); + if (mClear == null) + clearButton.setVisible(mIterator.hasFinishedMissions()); + mClear = clearButton; } + public void setMasterButtons(MenuItem startButton, MenuItem pauseButton) { + boolean init = mStartButton == null || mPauseButton == null; + + mStartButton = startButton; + mPauseButton = pauseButton; + + if (init) checkMasterButtonsVisibility(); + } + private void checkEmptyMessageVisibility() { int flag = mIterator.getOldListSize() > 0 ? View.GONE : View.VISIBLE; if (mEmptyMessage.getVisibility() != flag) mEmptyMessage.setVisibility(flag); } + private void checkMasterButtonsVisibility() { + boolean[] state = mIterator.hasValidPendingMissions(); + + mStartButton.setVisible(state[0]); + mPauseButton.setVisible(state[1]); + } + + public void ensurePausedMissions() { + for (ViewHolderItem h : mPendingDownloadsItems) { + if (((DownloadMission) h.item.mission).running) continue; + updateProgress(h); + h.lastTimeStamp = -1; + h.lastDone = -1; + } + } + public void deleterDispose(Bundle bundle) { if (mDeleter != null) mDeleter.dispose(bundle); @@ -604,6 +706,8 @@ public class MissionAdapter extends Adapter { ProgressDrawable progress; PopupMenu popupMenu; + MenuItem retry; + MenuItem cancel; MenuItem start; MenuItem pause; MenuItem open; @@ -636,22 +740,34 @@ public class MissionAdapter extends Adapter { button.setOnClickListener(v -> showPopupMenu()); Menu menu = popupMenu.getMenu(); + retry = menu.findItem(R.id.retry); + cancel = menu.findItem(R.id.cancel); start = menu.findItem(R.id.start); pause = menu.findItem(R.id.pause); - open = menu.findItem(R.id.open); + open = menu.findItem(R.id.menu_item_share); queue = menu.findItem(R.id.queue); showError = menu.findItem(R.id.error_message_view); delete = menu.findItem(R.id.delete); source = menu.findItem(R.id.source); checksum = menu.findItem(R.id.checksum); - itemView.setOnClickListener((v) -> { + itemView.setHapticFeedbackEnabled(true); + + itemView.setOnClickListener(v -> { if (item.mission instanceof FinishedMission) - viewWithFileProvider(item.mission.getDownloadedFile()); + viewWithFileProvider(item.mission); + }); + + itemView.setOnLongClickListener(v -> { + v.performHapticFeedback(HapticFeedbackConstants.LONG_PRESS); + showPopupMenu(); + return true; }); } private void showPopupMenu() { + retry.setVisible(false); + cancel.setVisible(false); start.setVisible(false); pause.setVisible(false); open.setVisible(false); @@ -664,7 +780,16 @@ public class MissionAdapter extends Adapter { DownloadMission mission = item.mission instanceof DownloadMission ? (DownloadMission) item.mission : null; if (mission != null) { - if (!mission.isPsRunning()) { + if (mission.isPsRunning()) { + switch (mission.errCode) { + case ERROR_INSUFFICIENT_STORAGE: + case ERROR_POSTPROCESSING_HOLD: + retry.setVisible(true); + cancel.setVisible(true); + showError.setVisible(true); + break; + } + } else { if (mission.running) { pause.setVisible(true); } else { diff --git a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java index c4fd3b5fd..a3786a5e6 100644 --- a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java +++ b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java @@ -35,6 +35,8 @@ public class MissionsFragment extends Fragment { private boolean mLinear; private MenuItem mSwitch; private MenuItem mClear = null; + private MenuItem mStart = null; + private MenuItem mPause = null; private RecyclerView mList; private View mEmpty; @@ -54,9 +56,11 @@ public class MissionsFragment extends Fragment { mBinder = (DownloadManagerService.DMBinder) binder; mBinder.clearDownloadNotifications(); - mAdapter = new MissionAdapter(mContext, mBinder.getDownloadManager(), mClear, mEmpty); + mAdapter = new MissionAdapter(mContext, mBinder.getDownloadManager(), mEmpty); mAdapter.deleterLoad(mBundle, getView()); + setAdapterButtons(); + mBundle = null; mBinder.addMissionEventListener(mAdapter.getMessenger()); @@ -132,7 +136,7 @@ public class MissionsFragment extends Fragment { public void onAttach(Activity activity) { super.onAttach(activity); - mContext = activity.getApplicationContext(); + mContext = activity; } @@ -154,7 +158,11 @@ public class MissionsFragment extends Fragment { public void onPrepareOptionsMenu(Menu menu) { mSwitch = menu.findItem(R.id.switch_mode); mClear = menu.findItem(R.id.clear_list); - if (mAdapter != null) mAdapter.setClearButton(mClear); + mStart = menu.findItem(R.id.start_downloads); + mPause = menu.findItem(R.id.pause_downloads); + + if (mAdapter != null) setAdapterButtons(); + super.onPrepareOptionsMenu(menu); } @@ -168,6 +176,14 @@ public class MissionsFragment extends Fragment { case R.id.clear_list: mAdapter.clearFinishedDownloads(); return true; + case R.id.start_downloads: + item.setVisible(false); + mBinder.getDownloadManager().startAllMissions(); + return true; + case R.id.pause_downloads: + item.setVisible(false); + mBinder.getDownloadManager().pauseAllMissions(false); + mAdapter.ensurePausedMissions();// update items view default: return super.onOptionsItemSelected(item); } @@ -193,9 +209,9 @@ public class MissionsFragment extends Fragment { int icon; if (mLinear) - icon = isLight ? R.drawable.ic_list_black_24dp : R.drawable.ic_list_white_24dp; - else icon = isLight ? R.drawable.ic_grid_black_24dp : R.drawable.ic_grid_white_24dp; + else + icon = isLight ? R.drawable.ic_list_black_24dp : R.drawable.ic_list_white_24dp; mSwitch.setIcon(icon); mSwitch.setTitle(mLinear ? R.string.grid : R.string.list); @@ -203,6 +219,13 @@ public class MissionsFragment extends Fragment { } } + private void setAdapterButtons() { + if (mClear == null || mStart == null || mPause == null) return; + + mAdapter.setClearButton(mClear); + mAdapter.setMasterButtons(mStart, mPause); + } + @Override public void onSaveInstanceState(Bundle outState) { super.onSaveInstanceState(outState); diff --git a/app/src/main/res/drawable-hdpi/ic_pause_black_24dp.png b/app/src/main/res/drawable-hdpi/ic_pause_black_24dp.png new file mode 100644 index 0000000000000000000000000000000000000000..3770b9124c41ecdf986c9f15ebc9d33f4d308449 GIT binary patch literal 135 zcmeAS@N?(olHy`uVBq!ia0vp^Dj>|k1|%Oc%$NbB{5)M8Ln2z=PBY|UFyLUW-FBh< z^_6eTubwCrvELNhVW`Bwps3y|k1|%Oc%$NbB0zF+ELn2z=UfamYV8Fq8u}3$h zU~LlHiQ}i74$0oo+;eGbN5?&*;_i(}_pkV>zv%dwbpNWax`*h|x)sb04GfG-EF1y~ m4)V>%_HNibuSV(DN^Z6)zQ%0|Gu8tQXYh3Ob6Mw<&;$UCzc1YY literal 0 HcmV?d00001 diff --git a/app/src/main/res/drawable-mdpi/ic_pause_black_24dp.png b/app/src/main/res/drawable-mdpi/ic_pause_black_24dp.png new file mode 100644 index 0000000000000000000000000000000000000000..6e81d3ad4f2029f7a521349688dff470b5858019 GIT binary patch literal 109 zcmeAS@N?(olHy`uVBq!ia0vp^5+KaM1|%Pp+x`GjCY~;iAs(H{2@4 literal 0 HcmV?d00001 diff --git a/app/src/main/res/drawable-mdpi/ic_pause_white_24dp.png b/app/src/main/res/drawable-mdpi/ic_pause_white_24dp.png new file mode 100644 index 0000000000000000000000000000000000000000..f5236e8aa89c24a1f42dbd298dbb08b871a05722 GIT binary patch literal 112 zcmeAS@N?(olHy`uVBq!ia0vp^5+KaM1|%Pp+x`Gj=AJH&As(H{2@lo1`?|x>U_dm*KlFIf!UpaSA_<7IjIM7lCPgg&ebxsLQ02LECbpQYW literal 0 HcmV?d00001 diff --git a/app/src/main/res/drawable-xhdpi/ic_pause_white_24dp.png b/app/src/main/res/drawable-xhdpi/ic_pause_white_24dp.png new file mode 100644 index 0000000000000000000000000000000000000000..b94b2ae40da8a50607f4dbb92efc57ea7fac9e4d GIT binary patch literal 139 zcmeAS@N?(olHy`uVBq!ia0vp^1|ZDA1|-9oezpUtAWs*^kc`H+Hx#)T6a-uy-`C&l zJ+?I>mRE`6^MMtAR&G=CoMfkETwVC+L0w4x9)^%y_A?7l12v+9F2?_Lx?Yo(bn_}W Vew)4B;Nvfl3Qt!*mvv4FO#oAVDPsTt literal 0 HcmV?d00001 diff --git a/app/src/main/res/drawable-xxhdpi/ic_pause_black_24dp.png b/app/src/main/res/drawable-xxhdpi/ic_pause_black_24dp.png new file mode 100644 index 0000000000000000000000000000000000000000..2691adeb30e1ada4ed84340315848782199c6ceb GIT binary patch literal 196 zcmeAS@N?(olHy`uVBq!ia0vp^9w5xY1|&n@ZgvM!-JULvAr*7p-rC4}K!Jzh;G6yG z+#4K>9KRTe9-aF{>~#ER8=xwNKR22q+25~!*!Ox*v!!JMe|e1i0)+qp4Gs|&E+*E- jCI?6Ix&2Q-Hhllhz@PfGi~DL_K8Wk->gTe~DWM4fwX--o literal 0 HcmV?d00001 diff --git a/app/src/main/res/drawable-xxhdpi/ic_pause_white_24dp.png b/app/src/main/res/drawable-xxhdpi/ic_pause_white_24dp.png new file mode 100644 index 0000000000000000000000000000000000000000..15cb0b51ce27438dd5cfa97a2a8f84c456647ec5 GIT binary patch literal 206 zcmeAS@N?(olHy`uVBq!ia0vp^9w5xY1|&n@ZgvM!Q$1ZALn`LHy=BOC*g%BU@&5mL zyBzdBT;9R6M)dI^1LiE%d)KV^fC?GvX3ji+EVBO2{<3=&Nso0mr&k}a{If~j?4DA9 wfCh&M3l|e>W0Qm90tK?Ui8U(hK#MAx_0^?TpQBfB_G~#&`Cq zW+yn_@)bVN@OZG@_uc&Gj6hXD&`?}$@cPg{(>)Kr-#d7xqQMr(VCmo#5LQrg@M)OA vV91!nbcDtNH=dk6QJVN)>K?>N3=HmeHaFkJH)U`C2NLmg^>bP0l+XkKR3S{3 literal 0 HcmV?d00001 diff --git a/app/src/main/res/drawable-xxxhdpi/ic_pause_white_24dp.png b/app/src/main/res/drawable-xxxhdpi/ic_pause_white_24dp.png new file mode 100644 index 0000000000000000000000000000000000000000..12a49bc12b8a00c5acdbcac690d1a3cc266c466f GIT binary patch literal 254 zcmeAS@N?(olHy`uVBq!ia0vp^2_VeD1|%QND7OGoM?GB}Ln`LHy>pQBh=Bl$<8%3? z#swVJ^6n2rP1r*hY(MfP{yHmA9SF#uJ+s;3d*ko-XQlakF3g(ymErw_`Eze5s5 + + + + + + + + + @@ -13,8 +22,8 @@ android:checkable="true"/> + android:id="@+id/menu_item_share" + android:title="@string/share" /> 系统拒绝该行动 下载失败 下载完成 - %已下载完毕 + %s已下载完毕 生成独特的名字 覆写 同名的已下载文件已经存在 diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index e75d905df..818efc74c 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -159,7 +159,7 @@ abrir en modo popup Si tienes ideas de; traducción, cambios de diseño, limpieza de código o cambios de código realmente fuertes—la ayuda siempre es bienvenida. Cuanto más se hace, mejor se pone! Leer licencia Contribuir - Suscribirse +Suscribirse Suscrito Canal no suscrito No se pudo cambiar la suscripción @@ -211,8 +211,8 @@ abrir en modo popup Vídeos Elemento eliminado - ¿Desea eliminar este elemento del historial de búsqueda? - Contenido de la página principal +¿Desea eliminar este elemento del historial de búsqueda? +Contenido de la página principal Página en blanco Página del kiosco Página de suscripción @@ -224,7 +224,7 @@ abrir en modo popup Kiosco Tendencias Top 50 - Mostrar sugerencia cuando se presiona el botón de segundo plano o popup en la página de detalles del vídeo +Mostrar sugerencia cuando se presiona el botón de segundo plano o popup en la página de detalles del vídeo En cola en el reproductor de fondo En cola en el reproductor popup Reproducir todo @@ -242,7 +242,7 @@ abrir en modo popup Comenzar a reproducir aquí Comenzar aquí en segundo plano Comenzar aquí en popup - Mostrar consejo \"Mantener para poner en la cola\" +Mostrar consejo \"Mantener para poner en la cola\" Nuevo y popular Mantener para poner en la cola Donar @@ -270,7 +270,7 @@ abrir en modo popup Reproductor de popup Obteniendo información… Cargando contenido solicitado - Importar base de datos +Importar base de datos Exportar base de datos Reemplazará su historial actual y sus suscripciones Exportar historial, suscripciones y listas de reproducción @@ -325,6 +325,7 @@ abrir en modo popup DIRECTO SINCRONIZAR Archivo + Archivo movido o eliminado No existe el directorio No existe la fuente del archivo/contenido El archivo no existe o insuficientes permisos para leerlo o escribir en él @@ -419,6 +420,8 @@ abrir en modo popup Sobrescribir Ya existe un archivo descargado con este nombre Hay una descarga en curso con este nombre + Hay una descarga pendiente con este nombre + Mostrar como grilla Mostrar como lista Limpiar descargas finalizadas @@ -426,8 +429,14 @@ abrir en modo popup Detener Intentos máximos Cantidad máxima de intentos antes de cancelar la descarga - Pausar al cambiar a datos moviles - Las descargas que no se pueden pausar serán reiniciadas + Interrumpir en redes medidas + Útil al cambiar a Datos Móviles, solo algunas descargas no se pueden suspender + Limitar cola de descarga + Solo se permitirá una descarga a la vez + Iniciar descargas + Pausar descargas + + Mostrar error Codigo @@ -439,9 +448,12 @@ abrir en modo popup No se puede conectar con el servidor El servidor no devolvio datos El servidor no acepta descargas multi-hilos, intente de nuevo con @string/msg_threads = 1 - El rango solicitado no se puede satisfacer + No se logro obtener el rango solicitado No encontrado Fallo el post-procesado + NewPipe se cerro mientras se trabajaba en el archivo + No hay suficiente espacio disponible en el dispositivo + Desuscribirse Nueva pestaña Elige la pestaña diff --git a/app/src/main/res/values/attrs.xml b/app/src/main/res/values/attrs.xml index 865b68c24..37bc9eec6 100644 --- a/app/src/main/res/values/attrs.xml +++ b/app/src/main/res/values/attrs.xml @@ -25,6 +25,7 @@ + diff --git a/app/src/main/res/values/settings_keys.xml b/app/src/main/res/values/settings_keys.xml index 3861f53d5..214a074c4 100644 --- a/app/src/main/res/values/settings_keys.xml +++ b/app/src/main/res/values/settings_keys.xml @@ -192,6 +192,7 @@ cross_network_downloads + downloads_queue_limit default_download_threads diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 154b8e0c4..afc6afeb3 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -196,6 +196,7 @@ Invalid URL No video streams found No audio streams found + File moved or deleted No such folder No such file/content source The file doesn\'t exist or permission to read or write to it is lacking @@ -513,6 +514,8 @@ Overwrite A downloaded file with this name already exists There is a download in progress with this name + There is a pending download with this name + Show error Code @@ -527,13 +530,20 @@ Requested range not satisfiable Not found Post-processing failed + NewPipe was closed while working on the file + No space left on device + Clear finished downloads Continue your %s pending transfers from Downloads Stop Maximum retries Maximum number of attempts before canceling the download - Pause on switching to mobile data - Downloads that can not be paused will be restarted + Interrupt on metered networks + Useful when switching to mobile data, although some downloads cannot be suspended Close + Limit download queue + One download will run at the same time + Start downloads + Pause downloads \ No newline at end of file diff --git a/app/src/main/res/values/styles.xml b/app/src/main/res/values/styles.xml index a7686dedc..51043718a 100644 --- a/app/src/main/res/values/styles.xml +++ b/app/src/main/res/values/styles.xml @@ -41,6 +41,7 @@ @drawable/ic_arrow_top_left_black_24dp @drawable/ic_more_vert_black_24dp @drawable/ic_play_arrow_black_24dp + @drawable/ic_pause_black_24dp @drawable/ic_settings_black_24dp @drawable/ic_whatshot_black_24dp @drawable/ic_channel_black_24dp @@ -119,6 +120,7 @@ @drawable/ic_list_white_24dp @drawable/ic_grid_white_24dp @drawable/ic_delete_white_24dp + @drawable/ic_pause_white_24dp @drawable/ic_settings_update_white @color/dark_separator_color diff --git a/app/src/main/res/xml/download_settings.xml b/app/src/main/res/xml/download_settings.xml index be015018a..9f32e7f2f 100644 --- a/app/src/main/res/xml/download_settings.xml +++ b/app/src/main/res/xml/download_settings.xml @@ -50,4 +50,11 @@ android:summary="@string/pause_downloads_on_mobile_desc" android:title="@string/pause_downloads_on_mobile" /> + +