package us.shandian.giga.service; import android.content.Context; import android.os.Handler; import android.util.Log; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.recyclerview.widget.DiffUtil; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.FinishedMission; import us.shandian.giga.get.Mission; import us.shandian.giga.get.sqlite.FinishedMissionStore; import org.schabi.newpipe.streams.io.StoredDirectoryHelper; import org.schabi.newpipe.streams.io.StoredFileHelper; import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; public class DownloadManager { private static final String TAG = DownloadManager.class.getSimpleName(); enum NetworkState {Unavailable, Operating, MeteredOperating} public final static int SPECIAL_NOTHING = 0; public final static int SPECIAL_PENDING = 1; public final static int SPECIAL_FINISHED = 2; public static final String TAG_AUDIO = "audio"; public static final String TAG_VIDEO = "video"; private static final String DOWNLOADS_METADATA_FOLDER = "pending_downloads"; private final FinishedMissionStore mFinishedMissionStore; private final ArrayList mMissionsPending = new ArrayList<>(); private final ArrayList mMissionsFinished; private final Handler mHandler; private final File mPendingMissionsDir; private NetworkState mLastNetworkStatus = NetworkState.Unavailable; int mPrefMaxRetry; boolean mPrefMeteredDownloads; boolean mPrefQueueLimit; private boolean mSelfMissionsControl; StoredDirectoryHelper mMainStorageAudio; StoredDirectoryHelper mMainStorageVideo; /** * Create a new instance * * @param context Context for the data source for finished downloads * @param handler Thread required for Messaging */ DownloadManager(@NonNull Context context, Handler handler, StoredDirectoryHelper storageVideo, StoredDirectoryHelper storageAudio) { if (DEBUG) { Log.d(TAG, "new DownloadManager instance. 0x" + Integer.toHexString(this.hashCode())); } mFinishedMissionStore = new FinishedMissionStore(context); mHandler = handler; mMainStorageAudio = storageAudio; mMainStorageVideo = storageVideo; mMissionsFinished = loadFinishedMissions(); mPendingMissionsDir = getPendingDir(context); loadPendingMissions(context); } private static File getPendingDir(@NonNull Context context) { File dir = context.getExternalFilesDir(DOWNLOADS_METADATA_FOLDER); if (testDir(dir)) return dir; dir = new File(context.getFilesDir(), DOWNLOADS_METADATA_FOLDER); if (testDir(dir)) return dir; throw new RuntimeException("path to pending downloads are not accessible"); } private static boolean testDir(@Nullable File dir) { if (dir == null) return false; try { if (!Utility.mkdir(dir, false)) { Log.e(TAG, "testDir() cannot create the directory in path: " + dir.getAbsolutePath()); return false; } File tmp = new File(dir, ".tmp"); if (!tmp.createNewFile()) return false; return tmp.delete();// if the file was created, SHOULD BE deleted too } catch (Exception e) { Log.e(TAG, "testDir() failed: " + dir.getAbsolutePath(), e); return false; } } /** * Loads finished missions from the data source and forgets finished missions whose file does * not exist anymore. */ private ArrayList loadFinishedMissions() { ArrayList finishedMissions = mFinishedMissionStore.loadFinishedMissions(); // check if the files exists, otherwise, forget the download for (int i = finishedMissions.size() - 1; i >= 0; i--) { FinishedMission mission = finishedMissions.get(i); if (!mission.storage.existsAsFile()) { if (DEBUG) Log.d(TAG, "downloaded file removed: " + mission.storage.getName()); mFinishedMissionStore.deleteMission(mission); finishedMissions.remove(i); } } return finishedMissions; } private void loadPendingMissions(Context ctx) { File[] subs = mPendingMissionsDir.listFiles(); if (subs == null) { Log.e(TAG, "listFiles() returned null"); return; } if (subs.length < 1) { return; } if (DEBUG) { Log.d(TAG, "Loading pending downloads from directory: " + mPendingMissionsDir.getAbsolutePath()); } File tempDir = pickAvailableTemporalDir(ctx); Log.i(TAG, "using '" + tempDir + "' as temporal directory"); for (File sub : subs) { if (!sub.isFile()) continue; if (sub.getName().equals(".tmp")) continue; DownloadMission mis = Utility.readFromFile(sub); if (mis == null || mis.isFinished() || mis.hasInvalidStorage()) { //noinspection ResultOfMethodCallIgnored sub.delete(); continue; } mis.threads = new Thread[0]; boolean exists; try { mis.storage = StoredFileHelper.deserialize(mis.storage, ctx); exists = !mis.storage.isInvalid() && mis.storage.existsAsFile(); } catch (Exception ex) { Log.e(TAG, "Failed to load the file source of " + mis.storage.toString(), ex); mis.storage.invalidate(); exists = false; } if (mis.isPsRunning()) { if (mis.psAlgorithm.worksOnSameFile) { // Incomplete post-processing results in a corrupted download file // because the selected algorithm works on the same file to save space. // the file will be deleted if the storage API // is Java IO (avoid showing the "Save as..." dialog) if (exists && mis.storage.isDirect() && !mis.storage.delete()) Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath()); } mis.psState = 0; mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED; } else if (!exists) { tryRecover(mis); // the progress is lost, reset mission state if (mis.isInitialized()) mis.resetState(true, true, DownloadMission.ERROR_PROGRESS_LOST); } if (mis.psAlgorithm != null) { mis.psAlgorithm.cleanupTemporalDir(); mis.psAlgorithm.setTemporalDir(tempDir); } mis.metadata = sub; mis.maxRetry = mPrefMaxRetry; mis.mHandler = mHandler; mMissionsPending.add(mis); } if (mMissionsPending.size() > 1) Collections.sort(mMissionsPending, Comparator.comparingLong(Mission::getTimestamp)); } /** * Start a new download mission * * @param mission the new download mission to add and run (if possible) */ void startMission(DownloadMission mission) { synchronized (this) { mission.timestamp = System.currentTimeMillis(); mission.mHandler = mHandler; mission.maxRetry = mPrefMaxRetry; // create metadata file while (true) { mission.metadata = new File(mPendingMissionsDir, String.valueOf(mission.timestamp)); if (!mission.metadata.isFile() && !mission.metadata.exists()) { try { if (!mission.metadata.createNewFile()) throw new RuntimeException("Cant create download metadata file"); } catch (IOException e) { throw new RuntimeException(e); } break; } mission.timestamp = System.currentTimeMillis(); } mSelfMissionsControl = true; mMissionsPending.add(mission); // Before continue, save the metadata in case the internet connection is not available Utility.writeToFile(mission.metadata, mission); if (mission.storage == null) { // noting to do here mission.errCode = DownloadMission.ERROR_FILE_CREATION; if (mission.errObject != null) mission.errObject = new IOException("DownloadMission.storage == NULL"); return; } boolean start = !mPrefQueueLimit || getRunningMissionsCount() < 1; if (canDownloadInCurrentNetwork() && start) { mission.start(); } } } public void resumeMission(DownloadMission mission) { if (!mission.running) { mission.start(); } } public void pauseMission(DownloadMission mission) { if (mission.running) { mission.setEnqueued(false); mission.pause(); } } public void deleteMission(Mission mission) { synchronized (this) { if (mission instanceof DownloadMission) { mMissionsPending.remove(mission); } else if (mission instanceof FinishedMission) { mMissionsFinished.remove(mission); mFinishedMissionStore.deleteMission(mission); } mission.delete(); } } public void forgetMission(StoredFileHelper storage) { synchronized (this) { Mission mission = getAnyMission(storage); if (mission == null) return; if (mission instanceof DownloadMission) { mMissionsPending.remove(mission); } else if (mission instanceof FinishedMission) { mMissionsFinished.remove(mission); mFinishedMissionStore.deleteMission(mission); } mission.storage = null; mission.delete(); } } public void tryRecover(DownloadMission mission) { StoredDirectoryHelper mainStorage = getMainStorage(mission.storage.getTag()); if (!mission.storage.isInvalid() && mission.storage.create()) return; // using javaIO cannot recreate the file // using SAF in older devices (no tree available) // // force the user to pick again the save path mission.storage.invalidate(); if (mainStorage == null) return; // if the user has changed the save path before this download, the original save path will be lost StoredFileHelper newStorage = mainStorage.createFile(mission.storage.getName(), mission.storage.getType()); if (newStorage != null) mission.storage = newStorage; } /** * Get a pending mission by its path * * @param storage where the file possible is stored * @return the mission or null if no such mission exists */ @Nullable private DownloadMission getPendingMission(StoredFileHelper storage) { for (DownloadMission mission : mMissionsPending) { if (mission.storage.equals(storage)) { return mission; } } return null; } /** * Get the index into {@link #mMissionsFinished} of a finished mission by its path, return * {@code -1} if there is no such mission. This function also checks if the matched mission's * file exists, and, if it does not, the related mission is forgotten about (like in {@link * #loadFinishedMissions()}) and {@code -1} is returned. * * @param storage where the file would be stored * @return the mission index or -1 if no such mission exists */ private int getFinishedMissionIndex(StoredFileHelper storage) { for (int i = 0; i < mMissionsFinished.size(); i++) { if (mMissionsFinished.get(i).storage.equals(storage)) { // If the file does not exist the mission is not valid anymore. Also checking if // length == 0 since the file picker may create an empty file before yielding it, // but that does not mean the file really belonged to a previous mission. if (!storage.existsAsFile() || storage.length() == 0) { if (DEBUG) { Log.d(TAG, "matched downloaded file removed: " + storage.getName()); } mFinishedMissionStore.deleteMission(mMissionsFinished.get(i)); mMissionsFinished.remove(i); return -1; // finished mission whose associated file was removed } return i; } } return -1; } private Mission getAnyMission(StoredFileHelper storage) { synchronized (this) { Mission mission = getPendingMission(storage); if (mission != null) return mission; int idx = getFinishedMissionIndex(storage); if (idx >= 0) return mMissionsFinished.get(idx); } return null; } int getRunningMissionsCount() { int count = 0; synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (mission.running && !mission.isPsFailed() && !mission.isFinished()) count++; } } return count; } public void pauseAllMissions(boolean force) { synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (!mission.running || mission.isPsRunning() || mission.isFinished()) continue; if (force) { // avoid waiting for threads mission.init = null; mission.threads = new Thread[0]; } mission.pause(); } } } public void startAllMissions() { synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (mission.running || mission.isCorrupt()) continue; mission.start(); } } } /** * Set a pending download as finished * * @param mission the desired mission */ void setFinished(DownloadMission mission) { synchronized (this) { mMissionsPending.remove(mission); mMissionsFinished.add(0, new FinishedMission(mission)); mFinishedMissionStore.addFinishedMission(mission); } } /** * runs one or multiple missions in from queue if possible * * @return true if one or multiple missions are running, otherwise, false */ boolean runMissions() { synchronized (this) { if (mMissionsPending.size() < 1) return false; if (!canDownloadInCurrentNetwork()) return false; if (mPrefQueueLimit) { for (DownloadMission mission : mMissionsPending) if (!mission.isFinished() && mission.running) return true; } boolean flag = false; for (DownloadMission mission : mMissionsPending) { if (mission.running || !mission.enqueued || mission.isFinished()) continue; resumeMission(mission); if (mission.errCode != DownloadMission.ERROR_NOTHING) continue; if (mPrefQueueLimit) return true; flag = true; } return flag; } } public MissionIterator getIterator() { mSelfMissionsControl = true; return new MissionIterator(); } /** * Forget all finished downloads, but, doesn't delete any file */ public void forgetFinishedDownloads() { synchronized (this) { for (FinishedMission mission : mMissionsFinished) { mFinishedMissionStore.deleteMission(mission); } mMissionsFinished.clear(); } } private boolean canDownloadInCurrentNetwork() { if (mLastNetworkStatus == NetworkState.Unavailable) return false; return !(mPrefMeteredDownloads && mLastNetworkStatus == NetworkState.MeteredOperating); } void handleConnectivityState(NetworkState currentStatus, boolean updateOnly) { if (currentStatus == mLastNetworkStatus) return; mLastNetworkStatus = currentStatus; if (currentStatus == NetworkState.Unavailable) return; if (!mSelfMissionsControl || updateOnly) { return;// don't touch anything without the user interaction } boolean isMetered = mPrefMeteredDownloads && mLastNetworkStatus == NetworkState.MeteredOperating; synchronized (this) { for (DownloadMission mission : mMissionsPending) { if (mission.isCorrupt() || mission.isPsRunning()) continue; if (mission.running && isMetered) { mission.pause(); } else if (!mission.running && !isMetered && mission.enqueued) { mission.start(); if (mPrefQueueLimit) break; } } } } void updateMaximumAttempts() { synchronized (this) { for (DownloadMission mission : mMissionsPending) mission.maxRetry = mPrefMaxRetry; } } public MissionState checkForExistingMission(StoredFileHelper storage) { synchronized (this) { DownloadMission pending = getPendingMission(storage); if (pending == null) { if (getFinishedMissionIndex(storage) >= 0) return MissionState.Finished; } else { if (pending.isFinished()) { return MissionState.Finished;// this never should happen (race-condition) } else { return pending.running ? MissionState.PendingRunning : MissionState.Pending; } } } return MissionState.None; } private static boolean isDirectoryAvailable(File directory) { return directory != null && directory.canWrite() && directory.exists(); } static File pickAvailableTemporalDir(@NonNull Context ctx) { File dir = ctx.getExternalFilesDir(null); if (isDirectoryAvailable(dir)) return dir; dir = ctx.getFilesDir(); if (isDirectoryAvailable(dir)) return dir; // this never should happen dir = ctx.getDir("muxing_tmp", Context.MODE_PRIVATE); if (isDirectoryAvailable(dir)) return dir; // fallback to cache dir dir = ctx.getCacheDir(); if (isDirectoryAvailable(dir)) return dir; throw new RuntimeException("Not temporal directories are available"); } @Nullable private StoredDirectoryHelper getMainStorage(@NonNull String tag) { if (tag.equals(TAG_AUDIO)) return mMainStorageAudio; if (tag.equals(TAG_VIDEO)) return mMainStorageVideo; Log.w(TAG, "Unknown download category, not [audio video]: " + tag); return null;// this never should happen } public class MissionIterator extends DiffUtil.Callback { final Object FINISHED = new Object(); final Object PENDING = new Object(); ArrayList snapshot; ArrayList current; ArrayList hidden; boolean hasFinished = false; private MissionIterator() { hidden = new ArrayList<>(2); current = null; snapshot = getSpecialItems(); } private ArrayList getSpecialItems() { synchronized (DownloadManager.this) { ArrayList pending = new ArrayList<>(mMissionsPending); ArrayList finished = new ArrayList<>(mMissionsFinished); List remove = new ArrayList<>(hidden); // hide missions (if required) remove.removeIf(mission -> pending.remove(mission) || finished.remove(mission)); int fakeTotal = pending.size(); if (fakeTotal > 0) fakeTotal++; fakeTotal += finished.size(); if (finished.size() > 0) fakeTotal++; ArrayList list = new ArrayList<>(fakeTotal); if (pending.size() > 0) { list.add(PENDING); list.addAll(pending); } if (finished.size() > 0) { list.add(FINISHED); list.addAll(finished); } hasFinished = finished.size() > 0; return list; } } public MissionItem getItem(int position) { Object object = snapshot.get(position); if (object == PENDING) return new MissionItem(SPECIAL_PENDING); if (object == FINISHED) return new MissionItem(SPECIAL_FINISHED); return new MissionItem(SPECIAL_NOTHING, (Mission) object); } public int getSpecialAtItem(int position) { Object object = snapshot.get(position); if (object == PENDING) return SPECIAL_PENDING; if (object == FINISHED) return SPECIAL_FINISHED; return SPECIAL_NOTHING; } public void start() { current = getSpecialItems(); } public void end() { snapshot = current; current = null; } public void hide(Mission mission) { hidden.add(mission); } public void unHide(Mission mission) { hidden.remove(mission); } public boolean hasFinishedMissions() { return hasFinished; } /** * Check if exists missions running and paused. Corrupted and hidden missions are not counted * * @return two-dimensional array contains the current missions state. * 1° entry: true if has at least one mission running * 2° entry: true if has at least one mission paused */ public boolean[] hasValidPendingMissions() { boolean running = false; boolean paused = false; synchronized (DownloadManager.this) { for (DownloadMission mission : mMissionsPending) { if (hidden.contains(mission) || mission.isCorrupt()) continue; if (mission.running) running = true; else paused = true; } } return new boolean[]{running, paused}; } @Override public int getOldListSize() { return snapshot.size(); } @Override public int getNewListSize() { return current.size(); } @Override public boolean areItemsTheSame(int oldItemPosition, int newItemPosition) { return snapshot.get(oldItemPosition) == current.get(newItemPosition); } @Override public boolean areContentsTheSame(int oldItemPosition, int newItemPosition) { Object x = snapshot.get(oldItemPosition); Object y = current.get(newItemPosition); if (x instanceof Mission && y instanceof Mission) { return ((Mission) x).storage.equals(((Mission) y).storage); } return false; } } public static class MissionItem { public int special; public Mission mission; MissionItem(int s, Mission m) { special = s; mission = m; } MissionItem(int s) { this(s, null); } } }