Add new tables for better FreshRSS synchronization speed

build.gradle

@@ -18,6 +18,7 @@ allprojects {
         google()
         jcenter()
         mavenCentral()
+        maven { url 'https://jitpack.io' }
     }
     afterEvaluate {
         tasks.withType(JavaCompile.class) {

db/build.gradle

@@ -67,6 +67,9 @@ dependencies {
     implementation 'androidx.room:room-rxjava2:2.2.5'
     androidTestImplementation "androidx.room:room-testing:2.2.5"
 
+    implementation 'com.github.MatrixDev.Roomigrant:RoomigrantLib:0.2.0'
+    kapt 'com.github.MatrixDev.Roomigrant:RoomigrantCompiler:0.2.0'
+
     api 'androidx.paging:paging-runtime:2.1.2'
     api 'androidx.paging:paging-common:2.1.2'
 

db/schemas/com.readrops.db.Database/3.json

@@ -2,7 +2,7 @@
   "formatVersion": 1,
   "database": {
     "version": 3,
-    "identityHash": "474b2c463c0a4bc9f0a8b69feb2feeec",
+    "identityHash": "c5754bffea62bb91c7d79c091c61aca8",
     "entities": [
       {
         "tableName": "Feed",
@@ -440,12 +440,264 @@
         },
         "indices": [],
         "foreignKeys": []
+      },
+      {
+        "tableName": "UnreadItemsIds",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `remote_id` TEXT NOT NULL, `account_id` INTEGER NOT NULL, FOREIGN KEY(`account_id`) REFERENCES `Account`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+        "fields": [
+          {
+            "fieldPath": "id",
+            "columnName": "id",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "remoteId",
+            "columnName": "remote_id",
+            "affinity": "TEXT",
+            "notNull": true
+          },
+          {
+            "fieldPath": "accountId",
+            "columnName": "account_id",
+            "affinity": "INTEGER",
+            "notNull": true
+          }
+        ],
+        "primaryKey": {
+          "columnNames": [
+            "id"
+          ],
+          "autoGenerate": true
+        },
+        "indices": [
+          {
+            "name": "index_UnreadItemsIds_remote_id",
+            "unique": false,
+            "columnNames": [
+              "remote_id"
+            ],
+            "createSql": "CREATE INDEX IF NOT EXISTS `index_UnreadItemsIds_remote_id` ON `${TABLE_NAME}` (`remote_id`)"
+          },
+          {
+            "name": "index_UnreadItemsIds_account_id",
+            "unique": false,
+            "columnNames": [
+              "account_id"
+            ],
+            "createSql": "CREATE INDEX IF NOT EXISTS `index_UnreadItemsIds_account_id` ON `${TABLE_NAME}` (`account_id`)"
+          }
+        ],
+        "foreignKeys": [
+          {
+            "table": "Account",
+            "onDelete": "CASCADE",
+            "onUpdate": "NO ACTION",
+            "columns": [
+              "account_id"
+            ],
+            "referencedColumns": [
+              "id"
+            ]
+          }
+        ]
+      },
+      {
+        "tableName": "ReadStarStateChange",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `read_change` INTEGER NOT NULL, `star_change` INTEGER NOT NULL, PRIMARY KEY(`id`))",
+        "fields": [
+          {
+            "fieldPath": "id",
+            "columnName": "id",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "readChange",
+            "columnName": "read_change",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "starChange",
+            "columnName": "star_change",
+            "affinity": "INTEGER",
+            "notNull": true
+          }
+        ],
+        "primaryKey": {
+          "columnNames": [
+            "id"
+          ],
+          "autoGenerate": false
+        },
+        "indices": [],
+        "foreignKeys": []
+      },
+      {
+        "tableName": "StarredItem",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `description` TEXT, `clean_description` TEXT, `link` TEXT, `image_link` TEXT, `author` TEXT, `pub_date` INTEGER, `content` TEXT, `feed_id` INTEGER NOT NULL, `guid` TEXT, `read_time` REAL NOT NULL, `read` INTEGER NOT NULL, `read_changed` INTEGER NOT NULL, `starred` INTEGER NOT NULL, `starred_changed` INTEGER NOT NULL, `read_it_later` INTEGER NOT NULL, `remoteId` TEXT, FOREIGN KEY(`feed_id`) REFERENCES `Feed`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
+        "fields": [
+          {
+            "fieldPath": "id",
+            "columnName": "id",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "title",
+            "columnName": "title",
+            "affinity": "TEXT",
+            "notNull": false
+          },
+          {
+            "fieldPath": "description",
+            "columnName": "description",
+            "affinity": "TEXT",
+            "notNull": false
+          },
+          {
+            "fieldPath": "cleanDescription",
+            "columnName": "clean_description",
+            "affinity": "TEXT",
+            "notNull": false
+          },
+          {
+            "fieldPath": "link",
+            "columnName": "link",
+            "affinity": "TEXT",
+            "notNull": false
+          },
+          {
+            "fieldPath": "imageLink",
+            "columnName": "image_link",
+            "affinity": "TEXT",
+            "notNull": false
+          },
+          {
+            "fieldPath": "author",
+            "columnName": "author",
+            "affinity": "TEXT",
+            "notNull": false
+          },
+          {
+            "fieldPath": "pubDate",
+            "columnName": "pub_date",
+            "affinity": "INTEGER",
+            "notNull": false
+          },
+          {
+            "fieldPath": "content",
+            "columnName": "content",
+            "affinity": "TEXT",
+            "notNull": false
+          },
+          {
+            "fieldPath": "feedId",
+            "columnName": "feed_id",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "guid",
+            "columnName": "guid",
+            "affinity": "TEXT",
+            "notNull": false
+          },
+          {
+            "fieldPath": "readTime",
+            "columnName": "read_time",
+            "affinity": "REAL",
+            "notNull": true
+          },
+          {
+            "fieldPath": "read",
+            "columnName": "read",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "readChanged",
+            "columnName": "read_changed",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "starred",
+            "columnName": "starred",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "starredChanged",
+            "columnName": "starred_changed",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "readItLater",
+            "columnName": "read_it_later",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "remoteId",
+            "columnName": "remoteId",
+            "affinity": "TEXT",
+            "notNull": false
+          }
+        ],
+        "primaryKey": {
+          "columnNames": [
+            "id"
+          ],
+          "autoGenerate": true
+        },
+        "indices": [
+          {
+            "name": "index_StarredItem_feed_id",
+            "unique": false,
+            "columnNames": [
+              "feed_id"
+            ],
+            "createSql": "CREATE INDEX IF NOT EXISTS `index_StarredItem_feed_id` ON `${TABLE_NAME}` (`feed_id`)"
+          },
+          {
+            "name": "index_StarredItem_guid",
+            "unique": false,
+            "columnNames": [
+              "guid"
+            ],
+            "createSql": "CREATE INDEX IF NOT EXISTS `index_StarredItem_guid` ON `${TABLE_NAME}` (`guid`)"
+          },
+          {
+            "name": "index_StarredItem_starred_changed",
+            "unique": false,
+            "columnNames": [
+              "starred_changed"
+            ],
+            "createSql": "CREATE INDEX IF NOT EXISTS `index_StarredItem_starred_changed` ON `${TABLE_NAME}` (`starred_changed`)"
+          }
+        ],
+        "foreignKeys": [
+          {
+            "table": "Feed",
+            "onDelete": "CASCADE",
+            "onUpdate": "NO ACTION",
+            "columns": [
+              "feed_id"
+            ],
+            "referencedColumns": [
+              "id"
+            ]
+          }
+        ]
       }
     ],
     "views": [],
     "setupQueries": [
       "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
-      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '474b2c463c0a4bc9f0a8b69feb2feeec')"
+      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'c5754bffea62bb91c7d79c091c61aca8')"
     ]
   }
 }

db/src/main/java/com/readrops/db/Database.java

@@ -1,49 +0,0 @@
-package com.readrops.db;
-
-import androidx.annotation.NonNull;
-import androidx.room.RoomDatabase;
-import androidx.room.TypeConverters;
-import androidx.room.migration.Migration;
-import androidx.sqlite.db.SupportSQLiteDatabase;
-
-import com.readrops.db.dao.AccountDao;
-import com.readrops.db.dao.FeedDao;
-import com.readrops.db.dao.FolderDao;
-import com.readrops.db.dao.ItemDao;
-import com.readrops.db.entities.Feed;
-import com.readrops.db.entities.Folder;
-import com.readrops.db.entities.Item;
-import com.readrops.db.entities.account.Account;
-
-
-@androidx.room.Database(entities = {Feed.class, Item.class, Folder.class, Account.class}, version = 3)
-@TypeConverters({Converters.class})
-public abstract class Database extends RoomDatabase {
-
-    public abstract FeedDao feedDao();
-
-    public abstract ItemDao itemDao();
-
-    public abstract FolderDao folderDao();
-
-    public abstract AccountDao accountDao();
-    
-    public static final Migration MIGRATION_1_2 = new Migration(1, 2) {
-        @Override
-        public void migrate(@NonNull SupportSQLiteDatabase database) {
-            database.execSQL("Alter Table Account Add Column notifications_enabled INTEGER Not Null Default 0");
-
-            database.execSQL("Alter Table Feed Add Column notification_enabled INTEGER Not Null Default 1");
-        }
-    };
-
-    public static final Migration MIGRATION_2_3 = new Migration(2, 3) {
-        @Override
-        public void migrate(@NonNull SupportSQLiteDatabase database) {
-            database.execSQL("Alter Table Item Add Column starred INTEGER Not Null Default 0");
-            database.execSQL("Alter Table Item Add Column starred_changed INTEGER Not Null Default 0");
-
-            database.execSQL("CREATE INDEX `index_Item_starred_changed` ON `Item` (`starred_changed`);");
-        }
-    };
-}

db/src/main/java/com/readrops/db/Database.kt

@@ -0,0 +1,26 @@
+package com.readrops.db
+
+import androidx.room.Database
+import androidx.room.RoomDatabase
+import androidx.room.TypeConverters
+import com.readrops.db.dao.*
+import com.readrops.db.entities.*
+import com.readrops.db.entities.account.Account
+import dev.matrix.roomigrant.GenerateRoomMigrations
+
+@Database(entities = [Feed::class, Item::class, Folder::class, Account::class, UnreadItemsIds::class, ReadStarStateChange::class, StarredItem::class], version = 3)
+@TypeConverters(Converters::class)
+@GenerateRoomMigrations
+abstract class Database : RoomDatabase() {
+    abstract fun feedDao(): FeedDao
+
+    abstract fun itemDao(): ItemDao
+
+    abstract fun folderDao(): FolderDao
+
+    abstract fun accountDao(): AccountDao
+
+    abstract fun itemsIdsDao(): ItemsIdsDao
+
+    abstract fun starredItemDao(): StarredItemDao
+}

db/src/main/java/com/readrops/db/DbModule.kt

@@ -7,8 +7,7 @@ val dbModule = module {
 
     single(createdAtStart = true) {
         Room.databaseBuilder(get(), Database::class.java, "readrops-db")
-                .addMigrations(Database.MIGRATION_1_2)
+                .addMigrations(*Database_Migrations.build())
-                .addMigrations(Database.MIGRATION_2_3)
                 .build()
     }
 }

db/src/main/java/com/readrops/db/dao/ItemDao.java

@@ -7,6 +7,7 @@ import androidx.room.Dao;
 import androidx.room.Query;
 import androidx.room.RawQuery;
 import androidx.room.RoomWarnings;
+import androidx.room.Transaction;
 import androidx.sqlite.db.SupportSQLiteQuery;
 
 import com.readrops.db.entities.Feed;
@@ -95,9 +96,11 @@ public interface ItemDao extends BaseDao<Item> {
     @Query("Update Item set starred = :starred, starred_changed = :starredChanged Where id = :itemId")
     Completable setStarState(int itemId, boolean starred, boolean starredChanged);
 
-    @Query("Update Item set starred = 1 Where remoteId In (:ids) And feed_id In (Select id From Feed Where account_id = :accountId)")
+    @Transaction
-    void starItems(List<String> ids, int accountId);
+    @Query("Update Item set read = 0 Where Item.remoteId In (Select  remote_id From UnreadItemsIds) And feed_id In (Select id From Feed Where account_id = :accountId)")
+    void updateUnreadState(int accountId);
 
-    @Query("Update Item set starred = 0 Where remoteId Not In (:ids) And feed_id In (Select id From Feed Where account_id = :accountId)")
+    @Transaction
-    void unstarItems(List<String> ids, int accountId);
+    @Query("Update Item set read = 1 Where Item.remoteId Not In (Select remote_id From UnreadItemsIds) And feed_id In (Select id From Feed Where account_id = :accountId)")
+    void updateReadState(int accountId);
 }

db/src/main/java/com/readrops/db/dao/ItemsIdsDao.kt

@@ -0,0 +1,25 @@
+package com.readrops.db.dao
+
+import androidx.room.Dao
+import androidx.room.Insert
+import androidx.room.Query
+import com.readrops.db.entities.ReadStarStateChange
+import com.readrops.db.entities.UnreadItemsIds
+
+@Dao
+interface ItemsIdsDao {
+
+    @Insert
+    fun insertUnreadItemsIds(unreadItemsIds: List<UnreadItemsIds>)
+
+    @Insert
+    fun insertReadStarStateChange(readStarStateChange: ReadStarStateChange)
+
+    @Query("Delete From UnreadItemsIds Where account_id = :accountId")
+    fun deleteUnreadItemsIds(accountId: Int)
+
+    @Query("Delete From ReadStarStateChange")
+    fun deleteReadStarStateChanges()
+
+
+}

db/src/main/java/com/readrops/db/dao/StarredItemDao.kt

@@ -0,0 +1,23 @@
+package com.readrops.db.dao
+
+import androidx.paging.DataSource
+import androidx.room.Dao
+import androidx.room.Query
+import androidx.room.RawQuery
+import androidx.sqlite.db.SupportSQLiteQuery
+import com.readrops.db.entities.Feed
+import com.readrops.db.entities.Folder
+import com.readrops.db.entities.Item
+import com.readrops.db.entities.StarredItem
+import com.readrops.db.pojo.ItemWithFeed
+
+@Dao
+interface StarredItemDao : BaseDao<StarredItem> {
+
+    @Query("Delete From StarredItem Where feed_id In (Select feed_id From Feed Where account_id = :accountId)")
+    fun deleteStarredItems(accountId: Int)
+
+    @RawQuery(observedEntities = [StarredItem::class, Folder::class, Feed::class])
+    fun selectAll(query: SupportSQLiteQuery?): DataSource.Factory<Int?, ItemWithFeed?>?
+
+}

db/src/main/java/com/readrops/db/entities/ItemsIds.kt

@@ -0,0 +1,17 @@
+package com.readrops.db.entities
+
+import androidx.room.ColumnInfo
+import androidx.room.Entity
+import androidx.room.ForeignKey
+import androidx.room.PrimaryKey
+import com.readrops.db.entities.account.Account
+
+@Entity(foreignKeys = [ForeignKey(entity = Account::class, parentColumns = ["id"],
+        childColumns = ["account_id"], onDelete = ForeignKey.CASCADE)])
+data class UnreadItemsIds(@PrimaryKey(autoGenerate = true) val id: Int = 0,
+                          @ColumnInfo(name = "remote_id", index = true) val remoteId: String,
+                          @ColumnInfo(name = "account_id", index = true) val accountId: Int)
+@Entity
+data class ReadStarStateChange(@PrimaryKey val id: Int = 0,
+                               @ColumnInfo(name = "read_change") val readChange: Boolean = false,
+                               @ColumnInfo(name = "star_change") val starChange: Boolean = false)

db/src/main/java/com/readrops/db/entities/StarredItem.kt

@@ -0,0 +1,35 @@
+package com.readrops.db.entities
+
+import androidx.room.Entity
+import androidx.room.ForeignKey
+import androidx.room.Ignore
+
+@Entity(foreignKeys = [ForeignKey(entity = Feed::class, parentColumns = ["id"], childColumns = ["feed_id"],
+        onDelete = ForeignKey.CASCADE)], inheritSuperIndices = true)
+class StarredItem() : Item() {
+
+    // TODO really hacky, should be replaced by something better
+    @Ignore
+    constructor(item: Item) : this() {
+        id = item.id
+        title = item.title
+        description = item.description
+        cleanDescription = item.cleanDescription
+        link = item.link
+        imageLink = item.imageLink
+        author = item.author
+        pubDate = item.pubDate
+        content = item.content
+        feedId = item.feedId
+        guid = item.guid
+        readTime = item.readTime
+        isRead = item.isRead
+        isReadChanged = item.isReadChanged
+        isStarred = true // important here for the items query compatibility
+        isStarredChanged = item.isStarredChanged
+        isReadItLater = item.isReadItLater
+        remoteId = item.remoteId
+        feedRemoteId = item.feedRemoteId
+    }
+
+}