diff --git a/app/schemas/org.schabi.newpipe.database.AppDatabase/3.json b/app/schemas/org.schabi.newpipe.database.AppDatabase/3.json index 86852c85e..dcedb5a7f 100644 --- a/app/schemas/org.schabi.newpipe.database.AppDatabase/3.json +++ b/app/schemas/org.schabi.newpipe.database.AppDatabase/3.json @@ -2,7 +2,7 @@ "formatVersion": 1, "database": { "version": 3, - "identityHash": "ecffbb2ea251aeb38a8f508acf2aa404", + "identityHash": "83d5d68663102d5fa28d63caaffb396d", "entities": [ { "tableName": "subscriptions", @@ -119,7 +119,7 @@ }, { "tableName": "streams", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `service_id` INTEGER NOT NULL, `url` TEXT NOT NULL, `title` TEXT NOT NULL, `stream_type` TEXT NOT NULL, `duration` INTEGER NOT NULL, `uploader` TEXT NOT NULL, `thumbnail_url` TEXT, `view_count` INTEGER, `textual_upload_date` TEXT, `upload_date` INTEGER)", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `service_id` INTEGER NOT NULL, `url` TEXT NOT NULL, `title` TEXT NOT NULL, `stream_type` TEXT NOT NULL, `duration` INTEGER NOT NULL, `uploader` TEXT NOT NULL, `thumbnail_url` TEXT, `view_count` INTEGER, `textual_upload_date` TEXT, `upload_date` INTEGER, `is_upload_date_approximation` INTEGER)", "fields": [ { "fieldPath": "uid", @@ -186,6 +186,12 @@ "columnName": "upload_date", "affinity": "INTEGER", "notNull": false + }, + { + "fieldPath": "isUploadDateApproximation", + "columnName": "is_upload_date_approximation", + "affinity": "INTEGER", + "notNull": false } ], "primaryKey": { @@ -637,11 +643,50 @@ ] } ] + }, + { + "tableName": "feed_last_updated", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`subscription_id` INTEGER NOT NULL, `last_updated` INTEGER, PRIMARY KEY(`subscription_id`), FOREIGN KEY(`subscription_id`) REFERENCES `subscriptions`(`uid`) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)", + "fields": [ + { + "fieldPath": "subscriptionId", + "columnName": "subscription_id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastUpdated", + "columnName": "last_updated", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "subscription_id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [ + { + "table": "subscriptions", + "onDelete": "CASCADE", + "onUpdate": "CASCADE", + "columns": [ + "subscription_id" + ], + "referencedColumns": [ + "uid" + ] + } + ] } ], + "views": [], "setupQueries": [ "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", - "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, \"ecffbb2ea251aeb38a8f508acf2aa404\")" + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '83d5d68663102d5fa28d63caaffb396d')" ] } } \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/database/AppDatabase.java b/app/src/main/java/org/schabi/newpipe/database/AppDatabase.java index 49acd41a0..d3cd6eb80 100644 --- a/app/src/main/java/org/schabi/newpipe/database/AppDatabase.java +++ b/app/src/main/java/org/schabi/newpipe/database/AppDatabase.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.database.feed.dao.FeedGroupDAO; import org.schabi.newpipe.database.feed.model.FeedEntity; import org.schabi.newpipe.database.feed.model.FeedGroupEntity; import org.schabi.newpipe.database.feed.model.FeedGroupSubscriptionEntity; +import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity; import org.schabi.newpipe.database.history.dao.SearchHistoryDAO; import org.schabi.newpipe.database.history.dao.StreamHistoryDAO; import org.schabi.newpipe.database.history.model.SearchHistoryEntry; @@ -34,7 +35,8 @@ import static org.schabi.newpipe.database.Migrations.DB_VER_3; SubscriptionEntity.class, SearchHistoryEntry.class, StreamEntity.class, StreamHistoryEntity.class, StreamStateEntity.class, PlaylistEntity.class, PlaylistStreamEntity.class, PlaylistRemoteEntity.class, - FeedEntity.class, FeedGroupEntity.class, FeedGroupSubscriptionEntity.class + FeedEntity.class, FeedGroupEntity.class, FeedGroupSubscriptionEntity.class, + FeedLastUpdatedEntity.class }, version = DB_VER_3 ) diff --git a/app/src/main/java/org/schabi/newpipe/database/Migrations.java b/app/src/main/java/org/schabi/newpipe/database/Migrations.java index 4724bf21d..b489900a0 100644 --- a/app/src/main/java/org/schabi/newpipe/database/Migrations.java +++ b/app/src/main/java/org/schabi/newpipe/database/Migrations.java @@ -78,10 +78,11 @@ public class Migrations { // Add NOT NULLs and new fields database.execSQL("CREATE TABLE IF NOT EXISTS streams_new " + "(uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, service_id INTEGER NOT NULL, url TEXT NOT NULL, title TEXT NOT NULL, stream_type TEXT NOT NULL," + - " duration INTEGER NOT NULL, uploader TEXT NOT NULL, thumbnail_url TEXT, view_count INTEGER, textual_upload_date TEXT, upload_date INTEGER)"); + " duration INTEGER NOT NULL, uploader TEXT NOT NULL, thumbnail_url TEXT, view_count INTEGER, textual_upload_date TEXT, upload_date INTEGER," + + " is_upload_date_approximation INTEGER)"); - database.execSQL("INSERT INTO streams_new (uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, view_count, textual_upload_date, upload_date)"+ - " SELECT uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, NULL, NULL, NULL FROM streams"); + database.execSQL("INSERT INTO streams_new (uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, view_count, textual_upload_date, upload_date, is_upload_date_approximation)"+ + " SELECT uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, NULL, NULL, NULL, NULL FROM streams"); database.execSQL("DROP TABLE streams"); database.execSQL("ALTER TABLE streams_new RENAME TO streams"); @@ -93,6 +94,7 @@ public class Migrations { database.execSQL("CREATE TABLE IF NOT EXISTS feed_group (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, icon_id INTEGER NOT NULL)"); database.execSQL("CREATE TABLE IF NOT EXISTS feed_group_subscription_join (group_id INTEGER NOT NULL, subscription_id INTEGER NOT NULL, PRIMARY KEY(group_id, subscription_id), FOREIGN KEY(group_id) REFERENCES feed_group(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)"); database.execSQL("CREATE INDEX index_feed_group_subscription_join_subscription_id ON feed_group_subscription_join (subscription_id)"); + database.execSQL("CREATE TABLE IF NOT EXISTS feed_last_updated (subscription_id INTEGER NOT NULL, last_updated INTEGER, PRIMARY KEY(subscription_id), FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)"); } }; diff --git a/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt b/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt index bc7e9feb6..cba834942 100644 --- a/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt +++ b/app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt @@ -1,12 +1,11 @@ package org.schabi.newpipe.database.feed.dao -import androidx.room.Dao -import androidx.room.Insert -import androidx.room.OnConflictStrategy -import androidx.room.Query +import androidx.room.* import io.reactivex.Flowable import org.schabi.newpipe.database.feed.model.FeedEntity +import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity import org.schabi.newpipe.database.stream.model.StreamEntity +import org.schabi.newpipe.database.subscription.SubscriptionEntity import java.util.* @Dao @@ -80,4 +79,69 @@ abstract class FeedDAO { @Insert(onConflict = OnConflictStrategy.IGNORE) abstract fun insertAll(entities: List): List + + @Insert(onConflict = OnConflictStrategy.IGNORE) + internal abstract fun insertLastUpdated(lastUpdatedEntity: FeedLastUpdatedEntity): Long + + @Update(onConflict = OnConflictStrategy.IGNORE) + internal abstract fun updateLastUpdated(lastUpdatedEntity: FeedLastUpdatedEntity) + + @Transaction + open fun setLastUpdatedForSubscription(lastUpdatedEntity: FeedLastUpdatedEntity) { + val id = insertLastUpdated(lastUpdatedEntity) + + if (id == -1L) { + updateLastUpdated(lastUpdatedEntity) + } + } + + @Query(""" + SELECT MIN(lu.last_updated) FROM feed_last_updated lu + + INNER JOIN feed_group_subscription_join fgs + ON fgs.subscription_id = lu.subscription_id AND fgs.group_id = :groupId + """) + abstract fun oldestSubscriptionUpdate(groupId: Long): Flowable> + + @Query("SELECT MIN(last_updated) FROM feed_last_updated") + abstract fun oldestSubscriptionUpdateFromAll(): Flowable> + + @Query("SELECT COUNT(*) FROM feed_last_updated WHERE last_updated IS NULL") + abstract fun notLoadedCount(): Flowable + + @Query(""" + SELECT COUNT(*) FROM subscriptions s + + INNER JOIN feed_group_subscription_join fgs + ON s.uid = fgs.subscription_id AND fgs.group_id = :groupId + + LEFT JOIN feed_last_updated lu + ON s.uid = lu.subscription_id + + WHERE lu.last_updated IS NULL + """) + abstract fun notLoadedCountForGroup(groupId: Long): Flowable + + @Query(""" + SELECT s.* FROM subscriptions s + + LEFT JOIN feed_last_updated lu + ON s.uid = lu.subscription_id + + WHERE lu.last_updated IS NULL OR lu.last_updated < :outdatedThreshold + """) + abstract fun getAllOutdated(outdatedThreshold: Date): Flowable> + + @Query(""" + SELECT s.* FROM subscriptions s + + INNER JOIN feed_group_subscription_join fgs + ON s.uid = fgs.subscription_id AND fgs.group_id = :groupId + + LEFT JOIN feed_last_updated lu + ON s.uid = lu.subscription_id + + WHERE lu.last_updated IS NULL OR lu.last_updated < :outdatedThreshold + """) + abstract fun getAllOutdatedForGroup(groupId: Long, outdatedThreshold: Date): Flowable> } diff --git a/app/src/main/java/org/schabi/newpipe/database/feed/model/FeedLastUpdatedEntity.kt b/app/src/main/java/org/schabi/newpipe/database/feed/model/FeedLastUpdatedEntity.kt new file mode 100644 index 000000000..d6d7e7dec --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/database/feed/model/FeedLastUpdatedEntity.kt @@ -0,0 +1,37 @@ +package org.schabi.newpipe.database.feed.model + +import androidx.room.ColumnInfo +import androidx.room.Entity +import androidx.room.ForeignKey +import androidx.room.PrimaryKey +import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity.Companion.FEED_LAST_UPDATED_TABLE +import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity.Companion.SUBSCRIPTION_ID +import org.schabi.newpipe.database.subscription.SubscriptionEntity +import java.util.* + +@Entity( + tableName = FEED_LAST_UPDATED_TABLE, + foreignKeys = [ + ForeignKey( + entity = SubscriptionEntity::class, + parentColumns = [SubscriptionEntity.SUBSCRIPTION_UID], + childColumns = [SUBSCRIPTION_ID], + onDelete = ForeignKey.CASCADE, onUpdate = ForeignKey.CASCADE, deferred = true) + ] +) +data class FeedLastUpdatedEntity( + @PrimaryKey + @ColumnInfo(name = SUBSCRIPTION_ID) + var subscriptionId: Long, + + @ColumnInfo(name = LAST_UPDATED) + var lastUpdated: Date? = null +) { + + companion object { + const val FEED_LAST_UPDATED_TABLE = "feed_last_updated" + + const val SUBSCRIPTION_ID = "subscription_id" + const val LAST_UPDATED = "last_updated" + } +} \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/database/stream/dao/StreamDAO.kt b/app/src/main/java/org/schabi/newpipe/database/stream/dao/StreamDAO.kt index ed99d7e81..43793becb 100644 --- a/app/src/main/java/org/schabi/newpipe/database/stream/dao/StreamDAO.kt +++ b/app/src/main/java/org/schabi/newpipe/database/stream/dao/StreamDAO.kt @@ -6,7 +6,8 @@ import org.schabi.newpipe.database.BasicDAO import org.schabi.newpipe.database.stream.model.StreamEntity import org.schabi.newpipe.database.stream.model.StreamEntity.Companion.STREAM_ID import org.schabi.newpipe.extractor.stream.StreamType -import org.schabi.newpipe.extractor.stream.StreamType.* +import org.schabi.newpipe.extractor.stream.StreamType.AUDIO_LIVE_STREAM +import org.schabi.newpipe.extractor.stream.StreamType.LIVE_STREAM import java.util.* import kotlin.collections.ArrayList @@ -31,8 +32,8 @@ abstract class StreamDAO : BasicDAO { internal abstract fun silentInsertAllInternal(streams: List): List @Query(""" - SELECT uid, stream_type, textual_upload_date, upload_date FROM streams - WHERE url = :url AND service_id = :serviceId + SELECT uid, stream_type, textual_upload_date, upload_date, is_upload_date_approximation, duration + FROM streams WHERE url = :url AND service_id = :serviceId """) internal abstract fun getMinimalStreamForCompare(serviceId: Int, url: String): StreamCompareFeed? @@ -79,8 +80,16 @@ abstract class StreamDAO : BasicDAO { val isNewerStreamLive = newerStream.streamType == AUDIO_LIVE_STREAM || newerStream.streamType == LIVE_STREAM if (!isNewerStreamLive) { - if (existentMinimalStream.uploadDate != null) newerStream.uploadDate = existentMinimalStream.uploadDate - if (existentMinimalStream.textualUploadDate != null) newerStream.textualUploadDate = existentMinimalStream.textualUploadDate + if (existentMinimalStream.uploadDate != null && existentMinimalStream.isUploadDateApproximation != true) { + newerStream.uploadDate = existentMinimalStream.uploadDate + newerStream.textualUploadDate = existentMinimalStream.textualUploadDate + newerStream.isUploadDateApproximation = existentMinimalStream.isUploadDateApproximation + } + + if (existentMinimalStream.duration > 0 && newerStream.duration < 0) { + newerStream.duration = existentMinimalStream.duration + } + } } @@ -105,12 +114,18 @@ abstract class StreamDAO : BasicDAO { @ColumnInfo(name = STREAM_ID) var uid: Long = 0, - @field:ColumnInfo(name = StreamEntity.STREAM_TYPE) + @ColumnInfo(name = StreamEntity.STREAM_TYPE) var streamType: StreamType, - @field:ColumnInfo(name = StreamEntity.STREAM_TEXTUAL_UPLOAD_DATE) + @ColumnInfo(name = StreamEntity.STREAM_TEXTUAL_UPLOAD_DATE) var textualUploadDate: String? = null, - @field:ColumnInfo(name = StreamEntity.STREAM_UPLOAD_DATE) - var uploadDate: Date? = null) + @ColumnInfo(name = StreamEntity.STREAM_UPLOAD_DATE) + var uploadDate: Date? = null, + + @ColumnInfo(name = StreamEntity.STREAM_IS_UPLOAD_DATE_APPROXIMATION) + var isUploadDateApproximation: Boolean? = null, + + @ColumnInfo(name = StreamEntity.STREAM_DURATION) + var duration: Long) } diff --git a/app/src/main/java/org/schabi/newpipe/database/stream/model/StreamEntity.kt b/app/src/main/java/org/schabi/newpipe/database/stream/model/StreamEntity.kt index 5e9464df9..ed9dc6b42 100644 --- a/app/src/main/java/org/schabi/newpipe/database/stream/model/StreamEntity.kt +++ b/app/src/main/java/org/schabi/newpipe/database/stream/model/StreamEntity.kt @@ -50,7 +50,10 @@ data class StreamEntity( var textualUploadDate: String? = null, @ColumnInfo(name = STREAM_UPLOAD_DATE) - var uploadDate: Date? = null + var uploadDate: Date? = null, + + @ColumnInfo(name = STREAM_IS_UPLOAD_DATE_APPROXIMATION) + var isUploadDateApproximation: Boolean? = null ) : Serializable { @Ignore @@ -58,7 +61,8 @@ data class StreamEntity( serviceId = item.serviceId, url = item.url, title = item.name, streamType = item.streamType, duration = item.duration, uploader = item.uploaderName, thumbnailUrl = item.thumbnailUrl, viewCount = item.viewCount, - textualUploadDate = item.textualUploadDate, uploadDate = item.uploadDate?.date()?.time + textualUploadDate = item.textualUploadDate, uploadDate = item.uploadDate?.date()?.time, + isUploadDateApproximation = item.uploadDate?.isApproximation ) @Ignore @@ -66,7 +70,8 @@ data class StreamEntity( serviceId = info.serviceId, url = info.url, title = info.name, streamType = info.streamType, duration = info.duration, uploader = info.uploaderName, thumbnailUrl = info.thumbnailUrl, viewCount = info.viewCount, - textualUploadDate = info.textualUploadDate, uploadDate = info.uploadDate?.date()?.time + textualUploadDate = info.textualUploadDate, uploadDate = info.uploadDate?.date()?.time, + isUploadDateApproximation = info.uploadDate?.isApproximation ) @Ignore @@ -84,7 +89,9 @@ data class StreamEntity( if (viewCount != null) item.viewCount = viewCount as Long item.textualUploadDate = textualUploadDate - item.uploadDate = uploadDate?.let { DateWrapper(Calendar.getInstance().apply { time = it }) } + item.uploadDate = uploadDate?.let { + DateWrapper(Calendar.getInstance().apply { time = it }, isUploadDateApproximation ?: false) + } return item } @@ -103,5 +110,6 @@ data class StreamEntity( const val STREAM_VIEWS = "view_count" const val STREAM_TEXTUAL_UPLOAD_DATE = "textual_upload_date" const val STREAM_UPLOAD_DATE = "upload_date" + const val STREAM_IS_UPLOAD_DATE_APPROXIMATION = "is_upload_date_approximation" } } diff --git a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java index b28c71d72..86198650c 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java @@ -1246,12 +1246,22 @@ public class VideoDetailFragment final boolean playbackResumeEnabled = prefs.getBoolean(activity.getString(R.string.enable_watch_history_key), true) && prefs.getBoolean(activity.getString(R.string.enable_playback_resume_key), true); + if (!playbackResumeEnabled || info.getDuration() <= 0) { positionView.setVisibility(View.INVISIBLE); detailPositionView.setVisibility(View.GONE); - return; + + // TODO: Remove this check when separation of concerns is done. + // (live streams weren't getting updated because they are mixed) + if (!info.getStreamType().equals(StreamType.LIVE_STREAM) && + !info.getStreamType().equals(StreamType.AUDIO_LIVE_STREAM)) { + return; + } } final HistoryRecordManager recordManager = new HistoryRecordManager(requireContext()); + + // TODO: Separate concerns when updating database data. + // (move the updating part to when the loading happens) positionSubscriber = recordManager.loadStreamState(info) .subscribeOn(Schedulers.io()) .onErrorComplete() diff --git a/app/src/main/java/org/schabi/newpipe/local/feed/FeedDatabaseManager.kt b/app/src/main/java/org/schabi/newpipe/local/feed/FeedDatabaseManager.kt index 281a790b7..253e1c0fe 100644 --- a/app/src/main/java/org/schabi/newpipe/local/feed/FeedDatabaseManager.kt +++ b/app/src/main/java/org/schabi/newpipe/local/feed/FeedDatabaseManager.kt @@ -1,7 +1,6 @@ package org.schabi.newpipe.local.feed import android.content.Context -import android.preference.PreferenceManager import android.util.Log import io.reactivex.Completable import io.reactivex.Flowable @@ -10,9 +9,9 @@ import io.reactivex.android.schedulers.AndroidSchedulers import io.reactivex.schedulers.Schedulers import org.schabi.newpipe.MainActivity.DEBUG import org.schabi.newpipe.NewPipeDatabase -import org.schabi.newpipe.R import org.schabi.newpipe.database.feed.model.FeedEntity import org.schabi.newpipe.database.feed.model.FeedGroupEntity +import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity import org.schabi.newpipe.database.stream.model.StreamEntity import org.schabi.newpipe.extractor.stream.StreamInfoItem import org.schabi.newpipe.extractor.stream.StreamType @@ -55,6 +54,22 @@ class FeedDatabaseManager(context: Context) { } } + fun outdatedSubscriptions(outdatedThreshold: Date) = feedTable.getAllOutdated(outdatedThreshold) + + fun notLoadedCount(groupId: Long = -1): Flowable { + return if (groupId != -1L) { + feedTable.notLoadedCountForGroup(groupId) + } else { + feedTable.notLoadedCount() + } + } + + fun outdatedSubscriptionsForGroup(groupId: Long = -1, outdatedThreshold: Date) = + feedTable.getAllOutdatedForGroup(groupId, outdatedThreshold) + + fun markAsOutdated(subscriptionId: Long) = feedTable + .setLastUpdatedForSubscription(FeedLastUpdatedEntity(subscriptionId, null)) + fun upsertAll(subscriptionId: Long, items: List, oldestAllowedDate: Date = FEED_OLDEST_ALLOWED_DATE.time) { val itemsToInsert = ArrayList() @@ -77,24 +92,8 @@ class FeedDatabaseManager(context: Context) { feedTable.insertAll(feedEntities) } - } - fun getLastUpdated(context: Context): Calendar? { - val lastUpdatedMillis = PreferenceManager.getDefaultSharedPreferences(context) - .getLong(context.getString(R.string.feed_last_updated_key), -1) - - val calendar = Calendar.getInstance() - if (lastUpdatedMillis > 0) { - calendar.timeInMillis = lastUpdatedMillis - return calendar - } - - return null - } - - fun setLastUpdated(context: Context, lastUpdated: Calendar?) { - PreferenceManager.getDefaultSharedPreferences(context).edit() - .putLong(context.getString(R.string.feed_last_updated_key), lastUpdated?.timeInMillis ?: -1).apply() + feedTable.setLastUpdatedForSubscription(FeedLastUpdatedEntity(subscriptionId, Calendar.getInstance().time)) } fun removeOrphansOrOlderStreams(oldestAllowedDate: Date = FEED_OLDEST_ALLOWED_DATE.time) { @@ -147,4 +146,13 @@ class FeedDatabaseManager(context: Context) { .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) } + + fun oldestSubscriptionUpdate(groupId: Long): Flowable> { + return if (groupId == -1L) { + feedTable.oldestSubscriptionUpdateFromAll() + } else { + feedTable.oldestSubscriptionUpdate(groupId) + } + + } } diff --git a/app/src/main/java/org/schabi/newpipe/local/feed/FeedFragment.kt b/app/src/main/java/org/schabi/newpipe/local/feed/FeedFragment.kt index 1ae249194..3abe01850 100644 --- a/app/src/main/java/org/schabi/newpipe/local/feed/FeedFragment.kt +++ b/app/src/main/java/org/schabi/newpipe/local/feed/FeedFragment.kt @@ -35,14 +35,15 @@ import org.schabi.newpipe.local.feed.service.FeedLoadService import org.schabi.newpipe.report.UserAction import org.schabi.newpipe.util.AnimationUtils.animateView import org.schabi.newpipe.util.Localization +import java.util.* class FeedFragment : BaseListFragment() { private lateinit var viewModel: FeedViewModel - private lateinit var feedDatabaseManager: FeedDatabaseManager @State @JvmField var listState: Parcelable? = null private var groupId = -1L private var groupName = "" + private var oldestSubscriptionUpdate: Calendar? = null init { setHasOptionsMenu(true) @@ -54,11 +55,6 @@ class FeedFragment : BaseListFragment() { groupId = arguments?.getLong(KEY_GROUP_ID, -1) ?: -1 groupName = arguments?.getString(KEY_GROUP_NAME) ?: "" - - feedDatabaseManager = FeedDatabaseManager(requireContext()) - if (feedDatabaseManager.getLastUpdated(requireContext()) == null) { - triggerUpdate() - } } override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View? { @@ -193,11 +189,7 @@ class FeedFragment : BaseListFragment() { loading_progress_bar.isIndeterminate = isIndeterminate || (progressState.maxProgress > 0 && progressState.currentProgress == 0) - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - loading_progress_bar?.setProgress(progressState.currentProgress, true) - } else { - loading_progress_bar.progress = progressState.currentProgress - } + loading_progress_bar.progress = progressState.currentProgress loading_progress_bar.max = progressState.maxProgress } @@ -209,9 +201,18 @@ class FeedFragment : BaseListFragment() { listState = null } - if (!loadedState.itemsErrors.isEmpty()) { + oldestSubscriptionUpdate = loadedState.oldestUpdate + + if (loadedState.notLoadedCount > 0) { + refresh_subtitle_text.visibility = View.VISIBLE + refresh_subtitle_text.text = getString(R.string.feed_subscription_not_loaded_count, loadedState.notLoadedCount) + } else { + refresh_subtitle_text.visibility = View.GONE + } + + if (loadedState.itemsErrors.isNotEmpty()) { showSnackBarError(loadedState.itemsErrors, UserAction.REQUESTED_FEED, - "none", "Loading feed", R.string.general_error); + "none", "Loading feed", R.string.general_error) } if (loadedState.items.isEmpty()) { @@ -237,13 +238,12 @@ class FeedFragment : BaseListFragment() { } private fun updateRefreshViewState() { - val lastUpdated = feedDatabaseManager.getLastUpdated(requireContext()) - val updatedAt = when { - lastUpdated != null -> Localization.relativeTime(lastUpdated) + val oldestSubscriptionUpdateText = when { + oldestSubscriptionUpdate != null -> Localization.relativeTime(oldestSubscriptionUpdate!!) else -> "—" } - refresh_text?.text = getString(R.string.feed_last_updated, updatedAt) + refresh_text?.text = getString(R.string.feed_oldest_subscription_update, oldestSubscriptionUpdateText) } /////////////////////////////////////////////////////////////////////////// @@ -256,7 +256,9 @@ class FeedFragment : BaseListFragment() { override fun hasMoreItems() = false private fun triggerUpdate() { - getActivity()?.startService(Intent(requireContext(), FeedLoadService::class.java)) + getActivity()?.startService(Intent(requireContext(), FeedLoadService::class.java).apply { + putExtra(FeedLoadService.EXTRA_GROUP_ID, groupId) + }) listState = null } diff --git a/app/src/main/java/org/schabi/newpipe/local/feed/FeedState.kt b/app/src/main/java/org/schabi/newpipe/local/feed/FeedState.kt index 1329d1ea4..c37d6a0b3 100644 --- a/app/src/main/java/org/schabi/newpipe/local/feed/FeedState.kt +++ b/app/src/main/java/org/schabi/newpipe/local/feed/FeedState.kt @@ -5,7 +5,20 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem import java.util.* sealed class FeedState { - data class ProgressState(val currentProgress: Int = -1, val maxProgress: Int = -1, @StringRes val progressMessage: Int = 0) : FeedState() - data class LoadedState(val lastUpdated: Calendar? = null, val items: List, var itemsErrors: List = emptyList()) : FeedState() - data class ErrorState(val error: Throwable? = null) : FeedState() + data class ProgressState( + val currentProgress: Int = -1, + val maxProgress: Int = -1, + @StringRes val progressMessage: Int = 0 + ) : FeedState() + + data class LoadedState( + val items: List, + val oldestUpdate: Calendar? = null, + val notLoadedCount: Long, + val itemsErrors: List = emptyList() + ) : FeedState() + + data class ErrorState( + val error: Throwable? = null + ) : FeedState() } \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/local/feed/FeedViewModel.kt b/app/src/main/java/org/schabi/newpipe/local/feed/FeedViewModel.kt index fa6e6bcfe..0e887f581 100644 --- a/app/src/main/java/org/schabi/newpipe/local/feed/FeedViewModel.kt +++ b/app/src/main/java/org/schabi/newpipe/local/feed/FeedViewModel.kt @@ -6,12 +6,13 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.ViewModelProvider import io.reactivex.Flowable import io.reactivex.android.schedulers.AndroidSchedulers -import io.reactivex.functions.Function3 +import io.reactivex.functions.Function4 import io.reactivex.schedulers.Schedulers import org.schabi.newpipe.extractor.stream.StreamInfoItem import org.schabi.newpipe.local.feed.service.FeedEventManager -import org.schabi.newpipe.local.subscription.SubscriptionManager +import org.schabi.newpipe.local.feed.service.FeedEventManager.Event.* import org.schabi.newpipe.util.DEFAULT_THROTTLE_TIMEOUT +import java.util.* import java.util.concurrent.TimeUnit class FeedViewModel(applicationContext: Context, val groupId: Long = -1) : ViewModel() { @@ -23,7 +24,6 @@ class FeedViewModel(applicationContext: Context, val groupId: Long = -1) : ViewM } private var feedDatabaseManager: FeedDatabaseManager = FeedDatabaseManager(applicationContext) - private var subscriptionManager: SubscriptionManager = SubscriptionManager(applicationContext) val stateLiveData = MutableLiveData() @@ -31,30 +31,30 @@ class FeedViewModel(applicationContext: Context, val groupId: Long = -1) : ViewM .combineLatest( FeedEventManager.events(), feedDatabaseManager.asStreamItems(groupId), - subscriptionManager.subscriptionTable().rowCount(), + feedDatabaseManager.notLoadedCount(groupId), + feedDatabaseManager.oldestSubscriptionUpdate(groupId), - Function3 { t1: FeedEventManager.Event, t2: List, t3: Long -> return@Function3 Triple(first = t1, second = t2, third = t3) } + Function4 { t1: FeedEventManager.Event, t2: List, t3: Long, t4: List -> + return@Function4 CombineResultHolder(t1, t2, t3, t4.firstOrNull()) + } ) .throttleLatest(DEFAULT_THROTTLE_TIMEOUT, TimeUnit.MILLISECONDS) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .subscribe { - val (event, listFromDB, subsCount) = it + val (event, listFromDB, notLoadedCount, oldestUpdate) = it - var lastUpdated = feedDatabaseManager.getLastUpdated(applicationContext) - if (subsCount == 0L && lastUpdated != null) { - feedDatabaseManager.setLastUpdated(applicationContext, null) - lastUpdated = null - } + val oldestUpdateCalendar = + oldestUpdate?.let { Calendar.getInstance().apply { time = it } } stateLiveData.postValue(when (event) { - is FeedEventManager.Event.IdleEvent -> FeedState.LoadedState(lastUpdated, listFromDB) - is FeedEventManager.Event.ProgressEvent -> FeedState.ProgressState(event.currentProgress, event.maxProgress, event.progressMessage) - is FeedEventManager.Event.SuccessResultEvent -> FeedState.LoadedState(lastUpdated, listFromDB, event.itemsErrors) - is FeedEventManager.Event.ErrorResultEvent -> throw event.error + is IdleEvent -> FeedState.LoadedState(listFromDB, oldestUpdateCalendar, notLoadedCount) + is ProgressEvent -> FeedState.ProgressState(event.currentProgress, event.maxProgress, event.progressMessage) + is SuccessResultEvent -> FeedState.LoadedState(listFromDB, oldestUpdateCalendar, notLoadedCount, event.itemsErrors) + is ErrorResultEvent -> FeedState.ErrorState(event.error) }) - if (event is FeedEventManager.Event.ErrorResultEvent || event is FeedEventManager.Event.SuccessResultEvent) { + if (event is ErrorResultEvent || event is SuccessResultEvent) { FeedEventManager.reset() } } @@ -63,4 +63,6 @@ class FeedViewModel(applicationContext: Context, val groupId: Long = -1) : ViewM super.onCleared() combineDisposable.dispose() } + + private data class CombineResultHolder(val t1: FeedEventManager.Event, val t2: List, val t3: Long, val t4: Date?) } \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/local/feed/service/FeedLoadService.kt b/app/src/main/java/org/schabi/newpipe/local/feed/service/FeedLoadService.kt index 8f5e551da..26a23342a 100644 --- a/app/src/main/java/org/schabi/newpipe/local/feed/service/FeedLoadService.kt +++ b/app/src/main/java/org/schabi/newpipe/local/feed/service/FeedLoadService.kt @@ -23,6 +23,7 @@ import android.app.Service import android.content.Intent import android.os.Build import android.os.IBinder +import android.preference.PreferenceManager import android.util.Log import androidx.core.app.NotificationCompat import androidx.core.app.NotificationManagerCompat @@ -71,6 +72,8 @@ class FeedLoadService : Service() { * Number of items to buffer to mass-insert in the database. */ private const val BUFFER_COUNT_BEFORE_INSERT = 20 + + const val EXTRA_GROUP_ID: String = "FeedLoadService.EXTRA_GROUP_ID" } private var loadingSubscription: Subscription? = null @@ -103,7 +106,15 @@ class FeedLoadService : Service() { } setupNotification() - startLoading() + val defaultSharedPreferences = PreferenceManager.getDefaultSharedPreferences(this) + + val groupId = intent.getLongExtra(EXTRA_GROUP_ID, -1) + val thresholdOutdatedMinutesString = defaultSharedPreferences + .getString(getString(R.string.feed_update_threshold_key), getString(R.string.feed_update_threshold_default_value)) + val thresholdOutdatedMinutes = thresholdOutdatedMinutesString!!.toInt() + + startLoading(groupId, thresholdOutdatedMinutes) + return START_NOT_STICKY } @@ -129,23 +140,31 @@ class FeedLoadService : Service() { // Loading & Handling /////////////////////////////////////////////////////////////////////////// - private class RequestException(message: String, cause: Throwable) : Exception(message, cause) { + private class RequestException(val subscriptionId: Long, message: String, cause: Throwable) : Exception(message, cause) { companion object { - fun wrapList(info: ChannelInfo): List { + fun wrapList(subscriptionId: Long, info: ChannelInfo): List { val toReturn = ArrayList(info.errors.size) for (error in info.errors) { - toReturn.add(RequestException(info.serviceId.toString() + ":" + info.url, error)) + toReturn.add(RequestException(subscriptionId, info.serviceId.toString() + ":" + info.url, error)) } return toReturn } } } - private fun startLoading() { + private fun startLoading(groupId: Long = -1, thresholdOutdatedMinutes: Int) { feedResultsHolder = ResultsHolder() - subscriptionManager - .subscriptions() + val outdatedThreshold = Calendar.getInstance().apply { + add(Calendar.MINUTE, -thresholdOutdatedMinutes) + }.time + + val subscriptions = when (groupId) { + -1L -> feedDatabaseManager.outdatedSubscriptions(outdatedThreshold) + else -> feedDatabaseManager.outdatedSubscriptionsForGroup(groupId, outdatedThreshold) + } + + subscriptions .limit(1) .doOnNext { @@ -174,7 +193,7 @@ class FeedLoadService : Service() { return@map Notification.createOnNext(Pair(subscriptionEntity.uid, channelInfo)) } catch (e: Throwable) { val request = "${subscriptionEntity.serviceId}:${subscriptionEntity.url}" - val wrapper = RequestException(request, e) + val wrapper = RequestException(subscriptionEntity.uid, request, e) return@map Notification.createOnError>(wrapper) } } @@ -235,7 +254,6 @@ class FeedLoadService : Service() { postEvent(ProgressEvent(R.string.feed_processing_message)) feedDatabaseManager.removeOrphansOrOlderStreams() - feedDatabaseManager.setLastUpdated(this@FeedLoadService, feedResultsHolder.lastUpdated) postEvent(SuccessResultEvent(feedResultsHolder.itemsErrors)) true @@ -266,11 +284,17 @@ class FeedLoadService : Service() { subscriptionManager.updateFromInfo(subscriptionId, info) if (info.errors.isNotEmpty()) { - feedResultsHolder.addErrors(RequestException.wrapList(info)) + feedResultsHolder.addErrors(RequestException.wrapList(subscriptionId, info)) + feedDatabaseManager.markAsOutdated(subscriptionId) } } else if (notification.isOnError) { - feedResultsHolder.addError(notification.error!!) + val error = notification.error!! + feedResultsHolder.addError(error) + + if (error is RequestException) { + feedDatabaseManager.markAsOutdated(error.subscriptionId) + } } } } @@ -371,11 +395,6 @@ class FeedLoadService : Service() { /////////////////////////////////////////////////////////////////////////// class ResultsHolder { - /** - * The time the items have been loaded. - */ - internal lateinit var lastUpdated: Calendar - /** * List of errors that may have happen during loading. */ @@ -393,7 +412,6 @@ class FeedLoadService : Service() { fun ready() { itemsErrors = itemsErrorsHolder.toList() - lastUpdated = Calendar.getInstance() } } } \ No newline at end of file diff --git a/app/src/main/res/layout/fragment_feed.xml b/app/src/main/res/layout/fragment_feed.xml index c81d0ee00..7d166a3f5 100644 --- a/app/src/main/res/layout/fragment_feed.xml +++ b/app/src/main/res/layout/fragment_feed.xml @@ -1,6 +1,5 @@ - - + android:gravity="center_vertical" + android:orientation="vertical"> + + + + + + tools:ignore="ContentDescription" /> + android:layout_marginRight="8dp" + android:background="?attr/separator_color" /> + tools:listitem="@layout/list_stream_item" + tools:visibility="visible" /> + tools:visibility="visible" /> + tools:visibility="visible" /> @@ -101,7 +120,7 @@ android:layout_height="wrap_content" android:layout_centerInParent="true" android:visibility="gone" - tools:visibility="visible"/> + tools:visibility="visible" /> + tools:visibility="visible" /> + android:background="?attr/toolbar_shadow_drawable" /> \ No newline at end of file diff --git a/app/src/main/res/layout/list_stream_item.xml b/app/src/main/res/layout/list_stream_item.xml index 02e8f1531..d2000381d 100644 --- a/app/src/main/res/layout/list_stream_item.xml +++ b/app/src/main/res/layout/list_stream_item.xml @@ -75,6 +75,7 @@ android:layout_alignParentBottom="true" android:layout_toRightOf="@+id/itemThumbnailView" android:layout_toEndOf="@+id/itemThumbnailView" + android:ellipsize="end" android:lines="1" android:textAppearance="?android:attr/textAppearanceSmall" android:textSize="@dimen/video_item_search_upload_date_text_size" diff --git a/app/src/main/res/layout/list_stream_playlist_item.xml b/app/src/main/res/layout/list_stream_playlist_item.xml index 2747038f6..00b431cc6 100644 --- a/app/src/main/res/layout/list_stream_playlist_item.xml +++ b/app/src/main/res/layout/list_stream_playlist_item.xml @@ -78,6 +78,7 @@ android:layout_toStartOf="@id/itemHandle" android:layout_toRightOf="@+id/itemThumbnailView" android:layout_toEndOf="@+id/itemThumbnailView" + android:ellipsize="end" android:lines="1" android:textAppearance="?android:attr/textAppearanceSmall" android:textSize="@dimen/video_item_search_uploader_text_size" diff --git a/app/src/main/res/values/settings_keys.xml b/app/src/main/res/values/settings_keys.xml index 1ffb84db4..8a391e983 100644 --- a/app/src/main/res/values/settings_keys.xml +++ b/app/src/main/res/values/settings_keys.xml @@ -181,6 +181,28 @@ app_language_key enable_lock_screen_video_thumbnail + feed_update_threshold_key + 5 + + + @string/feed_update_threshold_option_always_update + 5 minutes + 15 minutes + 1 hour + 6 hours + 12 hours + 1 day + + + 0 + 5 + 15 + 60 + 360 + 720 + 1440 + + import_data export_data diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index bcf7b18b1..1f9b4e0e8 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -601,7 +601,8 @@ What\'s New Feed groups - Last updated: %s + Oldest subscription update: %s + Not loaded: %d Loading feed… Processing feed… Select subscriptions @@ -610,4 +611,9 @@ Empty group name Name New + + Feed + Feed update threshold + Time after last update before a subscription is considered outdated — %s + Always update \ No newline at end of file diff --git a/app/src/main/res/xml/content_settings.xml b/app/src/main/res/xml/content_settings.xml index fd87de9ef..bd418b776 100644 --- a/app/src/main/res/xml/content_settings.xml +++ b/app/src/main/res/xml/content_settings.xml @@ -89,4 +89,18 @@ android:title="@string/export_data_title" android:key="@string/export_data" android:summary="@string/export_data_summary"/> + + + + + diff --git a/assets/db.dia b/assets/db.dia index adcf8fb9b..ebd6d6219 100644 Binary files a/assets/db.dia and b/assets/db.dia differ