From 110be62694d9e707357e7f8560c981065ca894d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=B5=A9=E8=BF=9C?= Date: Tue, 1 Sep 2020 11:29:08 +0800 Subject: [PATCH] db performance and schema changes --- src/bridges/settings.ts | 4 +- src/components/feeds/feed.tsx | 2 +- src/components/log-menu.tsx | 2 +- src/components/page.tsx | 2 +- src/containers/article-container.tsx | 2 +- src/containers/log-menu-container.tsx | 2 +- src/main/settings.ts | 4 +- src/main/update-scripts.ts | 2 +- src/scripts/db.ts | 15 +++-- src/scripts/models/app.ts | 6 +- src/scripts/models/feed.ts | 4 +- src/scripts/models/item.ts | 30 +++++----- src/scripts/models/page.ts | 4 +- src/scripts/models/service.ts | 66 ++++++++++++---------- src/scripts/models/services/feedbin.ts | 26 ++++----- src/scripts/models/services/fever.ts | 20 +++---- src/scripts/models/source.ts | 65 +++++++++++++-------- src/scripts/settings.ts | 78 ++++++++++++++------------ src/scripts/utils.ts | 19 +++++-- 19 files changed, 198 insertions(+), 155 deletions(-) diff --git a/src/bridges/settings.ts b/src/bridges/settings.ts index e05f239..3cc1640 100644 --- a/src/bridges/settings.ts +++ b/src/bridges/settings.ts @@ -106,8 +106,8 @@ const settingsBridge = { getNeDBStatus: (): boolean => { return ipcRenderer.sendSync("get-nedb-status") }, - setNeDBStatus: () => { - ipcRenderer.invoke("set-nedb-status") + setNeDBStatus: (flag: boolean) => { + ipcRenderer.invoke("set-nedb-status", flag) }, getAll: () => { diff --git a/src/components/feeds/feed.tsx b/src/components/feeds/feed.tsx index af9c963..b189bdd 100644 --- a/src/components/feeds/feed.tsx +++ b/src/components/feeds/feed.tsx @@ -11,7 +11,7 @@ export type FeedProps = FeedReduxProps & { viewType: ViewType viewConfigs?: ViewConfigs items: RSSItem[] - currentItem: string + currentItem: number sourceMap: Object filter: FeedFilter shortcuts: (item: RSSItem, e: KeyboardEvent) => void diff --git a/src/components/log-menu.tsx b/src/components/log-menu.tsx index dba2862..7e7a80e 100644 --- a/src/components/log-menu.tsx +++ b/src/components/log-menu.tsx @@ -8,7 +8,7 @@ type LogMenuProps = { display: boolean logs: AppLog[] close: () => void - showItem: (iid: string) => void + showItem: (iid: number) => void } function getLogIcon(log: AppLog) { diff --git a/src/components/page.tsx b/src/components/page.tsx index 9b1db59..0d427d6 100644 --- a/src/components/page.tsx +++ b/src/components/page.tsx @@ -10,7 +10,7 @@ type PageProps = { contextOn: boolean settingsOn: boolean feeds: string[] - itemId: string + itemId: number itemFromFeed: boolean viewType: ViewType dismissItem: () => void diff --git a/src/containers/article-container.tsx b/src/containers/article-container.tsx index 5e3140d..15820c4 100644 --- a/src/containers/article-container.tsx +++ b/src/containers/article-container.tsx @@ -8,7 +8,7 @@ import Article from "../components/article" import { openTextMenu, closeContextMenu, openImageMenu } from "../scripts/models/app" type ArticleContainerProps = { - itemId: string + itemId: number } const getItem = (state: RootState, props: ArticleContainerProps) => state.items[props.itemId] diff --git a/src/containers/log-menu-container.tsx b/src/containers/log-menu-container.tsx index e4c5e64..5b26a4f 100644 --- a/src/containers/log-menu-container.tsx +++ b/src/containers/log-menu-container.tsx @@ -12,7 +12,7 @@ const mapStateToProps = createSelector(getLogs, logs => logs) const mapDispatchToProps = dispatch => { return { close: () => dispatch(toggleLogMenu()), - showItem: (iid: string) => dispatch(showItemFromId(iid)) + showItem: (iid: number) => dispatch(showItemFromId(iid)) } } diff --git a/src/main/settings.ts b/src/main/settings.ts index dddc480..50ab326 100644 --- a/src/main/settings.ts +++ b/src/main/settings.ts @@ -177,6 +177,6 @@ const NEDB_STATUS_STORE_KEY = "useNeDB" ipcMain.on("get-nedb-status", (event) => { event.returnValue = store.get(NEDB_STATUS_STORE_KEY, true) }) -ipcMain.handle("set-nedb-status", () => { - store.set(NEDB_STATUS_STORE_KEY, false) +ipcMain.handle("set-nedb-status", (_, flag: boolean) => { + store.set(NEDB_STATUS_STORE_KEY, flag) }) diff --git a/src/main/update-scripts.ts b/src/main/update-scripts.ts index eac80af..5a273bf 100644 --- a/src/main/update-scripts.ts +++ b/src/main/update-scripts.ts @@ -7,7 +7,7 @@ export default function performUpdate(store: Store) { let useNeDB = store.get("useNeDB", undefined) let currentVersion = app.getVersion() - if (useNeDB === undefined) { + if (useNeDB === undefined && version !== null) { const revs = version.split(".").map(s => parseInt(s)) if ((revs[0] === 0 && revs[1] < 8) || !app.isPackaged) { store.set("useNeDB", true) diff --git a/src/scripts/db.ts b/src/scripts/db.ts index 9aeb112..b12f021 100644 --- a/src/scripts/db.ts +++ b/src/scripts/db.ts @@ -11,7 +11,7 @@ sdbSchema.createTable("sources"). addColumn("name", lf.Type.STRING). addColumn("openTarget", lf.Type.NUMBER). addColumn("lastFetched", lf.Type.DATE_TIME). - addColumn("serviceRef", lf.Type.NUMBER). + addColumn("serviceRef", lf.Type.STRING). addColumn("fetchFrequency", lf.Type.NUMBER). addColumn("rules", lf.Type.OBJECT). addNullable(["iconurl", "serviceRef", "rules"]). @@ -33,9 +33,10 @@ idbSchema.createTable("items"). addColumn("starred", lf.Type.BOOLEAN). addColumn("hidden", lf.Type.BOOLEAN). addColumn("notify", lf.Type.BOOLEAN). - addColumn("serviceRef", lf.Type.NUMBER). + addColumn("serviceRef", lf.Type.STRING). addNullable(["thumb", "creator", "serviceRef"]). - addIndex("idxDate", ["date"], false, lf.Order.DESC) + addIndex("idxDate", ["date"], false, lf.Order.DESC). + addIndex("idxService", ["serviceRef"], false) export let sourcesDB: lf.Database export let sources: lf.schema.Table @@ -73,14 +74,14 @@ export async function init() { }) }) const sRows = sourceDocs.map(doc => { - //doc.serviceRef = String(doc.serviceRef) + if (doc.serviceRef !== undefined) doc.serviceRef = String(doc.serviceRef) // @ts-ignore delete doc._id if (!doc.fetchFrequency) doc.fetchFrequency = 0 return sources.createRow(doc) }) const iRows = itemDocs.map(doc => { - //doc.serviceRef = String(doc.serviceRef) + if (doc.serviceRef !== undefined) doc.serviceRef = String(doc.serviceRef) delete doc._id doc.starred = Boolean(doc.starred) doc.hidden = Boolean(doc.hidden) @@ -91,6 +92,8 @@ export async function init() { sourcesDB.insert().into(sources).values(sRows).exec(), itemsDB.insert().into(items).values(iRows).exec() ]) - window.settings.setNeDBStatus() + window.settings.setNeDBStatus(false) + sdb.remove({}, { multi: true }, () => { sdb.persistence.compactDatafile() }) + idb.remove({}, { multi: true }, () => { idb.persistence.compactDatafile() }) } } diff --git a/src/scripts/models/app.ts b/src/scripts/models/app.ts index c6b95e0..3e79fa8 100644 --- a/src/scripts/models/app.ts +++ b/src/scripts/models/app.ts @@ -22,10 +22,10 @@ export class AppLog { type: AppLogType title: string details?: string - iid?: string + iid?: number time: Date - constructor(type: AppLogType, title: string, details: string=null, iid: string = null) { + constructor(type: AppLogType, title: string, details: string=null, iid: number = null) { this.type = type this.title = title this.details = details @@ -121,7 +121,7 @@ interface ToggleLogMenuAction { type: typeof TOGGLE_LOGS } interface PushNotificationAction { type: typeof PUSH_NOTIFICATION - iid: string + iid: number title: string source: string } diff --git a/src/scripts/models/feed.ts b/src/scripts/models/feed.ts index 6b797c6..d72b43e 100644 --- a/src/scripts/models/feed.ts +++ b/src/scripts/models/feed.ts @@ -84,7 +84,7 @@ export class RSSFeed { loading: boolean allLoaded: boolean sids: number[] - iids: string[] + iids: number[] filter: FeedFilter constructor (id: string = null, sids=[], filter=null) { @@ -240,7 +240,7 @@ export function feedReducer( switch (action.status) { case ActionStatus.Success: return { ...state, - [ALL]: new RSSFeed(ALL, action.sources.map(s => s.sid)) + [ALL]: new RSSFeed(ALL, Object.values(action.sources).map(s => s.sid)) } default: return state } diff --git a/src/scripts/models/item.ts b/src/scripts/models/item.ts index affe0d4..5e54cda 100644 --- a/src/scripts/models/item.ts +++ b/src/scripts/models/item.ts @@ -9,7 +9,7 @@ import { pushNotification, setupAutoFetch } from "./app" import { getServiceHooks, syncWithService, ServiceActionTypes, SYNC_LOCAL_ITEMS } from "./service" export class RSSItem { - _id: string + _id: number source: number title: string link: string @@ -23,7 +23,7 @@ export class RSSItem { starred: boolean hidden: boolean notify: boolean - serviceRef?: string | number + serviceRef?: string constructor (item: Parser.Item, source: RSSSource) { for (let field of ["title", "link", "creator"]) { @@ -79,7 +79,7 @@ export class RSSItem { } export type ItemState = { - [_id: string]: RSSItem + [_id: number]: RSSItem } export const FETCH_ITEMS = "FETCH_ITEMS" @@ -306,7 +306,7 @@ const toggleStarredDone = (item: RSSItem): ItemActionTypes => ({ export function toggleStarred(item: RSSItem): AppThunk { return (dispatch) => { - if (item.starred === true) { + if (item.starred) { db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.starred, false).exec() } else { db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.starred, true).exec() @@ -327,10 +327,10 @@ const toggleHiddenDone = (item: RSSItem): ItemActionTypes => ({ export function toggleHidden(item: RSSItem): AppThunk { return (dispatch) => { - if (item.hidden === true) { - db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.hidden, true).exec() - } else { + if (item.hidden) { db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.hidden, false).exec() + } else { + db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.hidden, true).exec() } dispatch(toggleHiddenDone(item)) } @@ -371,7 +371,7 @@ export function applyItemReduction(item: RSSItem, type: string) { break } case TOGGLE_HIDDEN: { - item.hidden = !item.hidden + nextItem.hidden = !item.hidden break } } @@ -406,13 +406,13 @@ export function itemReducer( case MARK_ALL_READ: { let nextState = { ...state } let sids = new Set(action.sids) - for (let [id, item] of Object.entries(state)) { + for (let item of Object.values(state)) { if (sids.has(item.source) && !item.hasRead) { if (!action.time || (action.before ? item.date.getTime() <= action.time : item.date.getTime() >= action.time) ) { - nextState[id] = { + nextState[item._id] = { ...item, hasRead: true } @@ -435,15 +435,13 @@ export function itemReducer( } } case SYNC_LOCAL_ITEMS: { - const unreadSet = new Set(action.unreadIds) - const starredSet = new Set(action.starredIds) let nextState = { ...state } - for (let [id, item] of Object.entries(state)) { + for (let item of Object.values(state)) { if (item.hasOwnProperty("serviceRef")) { const nextItem = { ...item } - nextItem.hasRead = !unreadSet.has(nextItem.serviceRef as number) - nextItem.starred = starredSet.has(item.serviceRef as number) - nextState[id] = nextItem + nextItem.hasRead = !action.unreadIds.has(item.serviceRef) + nextItem.starred = action.starredIds.has(item.serviceRef) + nextState[item._id] = nextItem } } return nextState diff --git a/src/scripts/models/page.ts b/src/scripts/models/page.ts index 1c8781b..c6dbdd0 100644 --- a/src/scripts/models/page.ts +++ b/src/scripts/models/page.ts @@ -114,7 +114,7 @@ export function showItem(feedId: string, item: RSSItem): AppThunk { } } } -export function showItemFromId(iid: string): AppThunk { +export function showItemFromId(iid: number): AppThunk { return (dispatch, getState) => { const state = getState() const item = state.items[iid] @@ -237,7 +237,7 @@ export class PageState { viewConfigs = window.settings.getViewConfigs(window.settings.getDefaultView()) filter = new FeedFilter() feedId = ALL - itemId = null as string + itemId = null as number itemFromFeed = true searchOn = false } diff --git a/src/scripts/models/service.ts b/src/scripts/models/service.ts index 0209a94..90f8f2b 100644 --- a/src/scripts/models/service.ts +++ b/src/scripts/models/service.ts @@ -14,9 +14,9 @@ import { feedbinServiceHooks } from "./services/feedbin" export interface ServiceHooks { authenticate?: (configs: ServiceConfigs) => Promise - updateSources?: () => AppThunk]>> + updateSources?: () => AppThunk]>> fetchItems?: () => AppThunk> - syncItems?: () => AppThunk> + syncItems?: () => AppThunk, Set]>> markRead?: (item: RSSItem) => AppThunk markUnread?: (item: RSSItem) => AppThunk markAllRead?: (sids?: number[], date?: Date, before?: boolean) => AppThunk> @@ -71,7 +71,7 @@ export function syncWithService(background = false): AppThunk> { function updateSources(hook: ServiceHooks["updateSources"]): AppThunk> { return async (dispatch, getState) => { const [sources, groupsMap] = await dispatch(hook()) - const existing = new Map() + const existing = new Map() for (let source of Object.values(getState().sources)) { if (source.serviceRef) { existing.set(source.serviceRef, source) @@ -138,29 +138,37 @@ function syncItems(hook: ServiceHooks["syncItems"]): AppThunk> { return async (dispatch, getState) => { const state = getState() const [unreadRefs, starredRefs] = await dispatch(hook()) - const promises = [ - db.itemsDB.update(db.items).set(db.items.hasRead, false).where(lf.op.and( - db.items.serviceRef.in(unreadRefs), - db.items.hasRead.eq(true) - )).exec(), - db.itemsDB.update(db.items).set(db.items.hasRead, true).where(lf.op.and( - lf.op.not(db.items.serviceRef.in(unreadRefs)), - db.items.hasRead.eq(false) - )).exec(), - db.itemsDB.update(db.items).set(db.items.starred, true).where(lf.op.and( - db.items.serviceRef.in(starredRefs), - db.items.starred.eq(false) - )).exec(), - db.itemsDB.update(db.items).set(db.items.starred, false).where(lf.op.and( - lf.op.not(db.items.serviceRef.in(starredRefs)), - db.items.hasRead.eq(true) - )).exec(), - ] - await Promise.all(promises) - await dispatch(updateUnreadCounts()) - dispatch(syncLocalItems(unreadRefs, starredRefs)) - if (!(state.page.filter.type & FilterType.ShowRead) || !(state.page.filter.type & FilterType.ShowNotStarred)) { - dispatch(initFeeds(true)) + const unreadCopy = new Set(unreadRefs) + const starredCopy = new Set(starredRefs) + const rows = await db.itemsDB.select( + db.items.serviceRef, db.items.hasRead, db.items.starred + ).from(db.items).where(lf.op.and( + db.items.serviceRef.isNotNull(), + lf.op.or(db.items.hasRead.eq(false), db.items.starred.eq(true)) + )).exec() + const updates = new Array() + for (let row of rows) { + const serviceRef = row["serviceRef"] + if (row["hasRead"] === false && !unreadRefs.delete(serviceRef)) { + updates.push(db.itemsDB.update(db.items).set(db.items.hasRead, true).where(db.items.serviceRef.eq(serviceRef))) + } + if (row["starred"] === true && !starredRefs.delete(serviceRef)) { + updates.push(db.itemsDB.update(db.items).set(db.items.starred, false).where(db.items.serviceRef.eq(serviceRef))) + } + } + for (let unread of unreadRefs) { + updates.push(db.itemsDB.update(db.items).set(db.items.hasRead, false).where(db.items.serviceRef.eq(unread))) + } + for (let starred of starredRefs) { + updates.push(db.itemsDB.update(db.items).set(db.items.starred, true).where(db.items.serviceRef.eq(starred))) + } + if (updates.length > 0) { + await db.itemsDB.createTransaction().exec(updates) + await dispatch(updateUnreadCounts()) + dispatch(syncLocalItems(unreadCopy, starredCopy)) + if (!(state.page.filter.type & FilterType.ShowRead) || !(state.page.filter.type & FilterType.ShowNotStarred)) { + dispatch(initFeeds(true)) + } } } } @@ -224,8 +232,8 @@ interface SyncWithServiceAction { interface SyncLocalItemsAction { type: typeof SYNC_LOCAL_ITEMS - unreadIds: (string | number)[] - starredIds: (string | number)[] + unreadIds: Set + starredIds: Set } export type ServiceActionTypes = SaveServiceConfigsAction | SyncWithServiceAction | SyncLocalItemsAction @@ -240,7 +248,7 @@ export function saveServiceConfigs(configs: ServiceConfigs): AppThunk { } } -function syncLocalItems(unread: (string | number)[], starred: (string | number)[]): ServiceActionTypes { +function syncLocalItems(unread: Set, starred: Set): ServiceActionTypes { return { type: SYNC_LOCAL_ITEMS, unreadIds: unread, diff --git a/src/scripts/models/services/feedbin.ts b/src/scripts/models/services/feedbin.ts index 439b550..0be037a 100644 --- a/src/scripts/models/services/feedbin.ts +++ b/src/scripts/models/services/feedbin.ts @@ -62,7 +62,7 @@ export const feedbinServiceHooks: ServiceHooks = { const response = await fetchAPI(configs, "subscriptions.json") if (response.status !== 200) throw APIError() const subscriptions: any[] = await response.json() - let groupsMap: Map + let groupsMap: Map if (configs.importGroups) { const tagsResponse = await fetchAPI(configs, "taggings.json") if (tagsResponse.status !== 200) throw APIError() @@ -75,12 +75,12 @@ export const feedbinServiceHooks: ServiceHooks = { tagsSet.add(title) dispatch(createSourceGroup(title)) } - groupsMap.set(tag.feed_id, title) + groupsMap.set(String(tag.feed_id), title) } } const sources = subscriptions.map(s => { const source = new RSSSource(s.feed_url, s.title) - source.serviceRef = s.feed_id + source.serviceRef = String(s.feed_id) return source }) return [sources, groupsMap] @@ -95,7 +95,7 @@ export const feedbinServiceHooks: ServiceHooks = { if (unreadResponse.status !== 200 || starredResponse.status !== 200) throw APIError() const unread = await unreadResponse.json() const starred = await starredResponse.json() - return [unread, starred] + return [new Set(unread.map(i => String(i))), new Set(starred.map(i => String(i)))] }, fetchItems: () => async (_, getState) => { @@ -120,10 +120,10 @@ export const feedbinServiceHooks: ServiceHooks = { ) configs.lastId = items.reduce((m, n) => Math.max(m, n.id), configs.lastId) if (items.length > 0) { - const fidMap = new Map() + const fidMap = new Map() for (let source of Object.values(state.sources)) { if (source.serviceRef) { - fidMap.set(source.serviceRef as number, source) + fidMap.set(source.serviceRef, source) } } const [unreadResponse, starredResponse] = await Promise.all([ @@ -134,7 +134,7 @@ export const feedbinServiceHooks: ServiceHooks = { const unread: Set = new Set(await unreadResponse.json()) const starred: Set = new Set(await starredResponse.json()) const parsedItems = items.map(i => { - const source = fidMap.get(i.feed_id) + const source = fidMap.get(String(i.feed_id)) const dom = domParser.parseFromString(i.content, "text/html") const item = { source: source.sid, @@ -149,7 +149,7 @@ export const feedbinServiceHooks: ServiceHooks = { starred: starred.has(i.id), hidden: false, notify: false, - serviceRef: i.id, + serviceRef: String(i.id), } as RSSItem if (i.images && i.images.original_url) { item.thumb = i.images.original_url @@ -186,23 +186,23 @@ export const feedbinServiceHooks: ServiceHooks = { predicates.push(before ? db.items.date.lte(date) : db.items.date.gte(date)) } const rows = await db.itemsDB.select(db.items.serviceRef).where(lf.op.and.apply(null, predicates)).exec() - const refs = rows.map(row => row["serviceRef"]) as number[] + const refs = rows.map(row => parseInt(row["serviceRef"])) markItems(configs, "unread", "DELETE", refs) }, markRead: (item: RSSItem) => async (_, getState) => { - await markItems(getState().service as FeedbinConfigs, "unread", "DELETE", [item.serviceRef as number]) + await markItems(getState().service as FeedbinConfigs, "unread", "DELETE", [parseInt(item.serviceRef)]) }, markUnread: (item: RSSItem) => async (_, getState) => { - await markItems(getState().service as FeedbinConfigs, "unread", "POST", [item.serviceRef as number]) + await markItems(getState().service as FeedbinConfigs, "unread", "POST", [parseInt(item.serviceRef)]) }, star: (item: RSSItem) => async (_, getState) => { - await markItems(getState().service as FeedbinConfigs, "starred", "POST", [item.serviceRef as number]) + await markItems(getState().service as FeedbinConfigs, "starred", "POST", [parseInt(item.serviceRef)]) }, unstar: (item: RSSItem) => async (_, getState) => { - await markItems(getState().service as FeedbinConfigs, "starred", "DELETE", [item.serviceRef as number]) + await markItems(getState().service as FeedbinConfigs, "starred", "DELETE", [parseInt(item.serviceRef)]) }, } \ No newline at end of file diff --git a/src/scripts/models/services/fever.ts b/src/scripts/models/services/fever.ts index 3b6dc1e..db54ab1 100644 --- a/src/scripts/models/services/fever.ts +++ b/src/scripts/models/services/fever.ts @@ -53,7 +53,7 @@ export const feverServiceHooks: ServiceHooks = { const feeds: any[] = response.feeds const feedGroups: any[] = response.feeds_groups if (feeds === undefined) throw APIError() - let groupsMap: Map + let groupsMap: Map if (configs.importGroups) { // Import groups on the first sync const groups: any[] = (await fetchAPI(configs, "&groups")).groups @@ -66,14 +66,14 @@ export const feverServiceHooks: ServiceHooks = { } groupsMap = new Map() for (let group of feedGroups) { - for (let fid of group.feed_ids.split(",").map(s => parseInt(s))) { + for (let fid of group.feed_ids.split(",")) { groupsMap.set(fid, groupsIdMap.get(group.group_id)) } } } const sources = feeds.map(f => { const source = new RSSSource(f.url, f.title) - source.serviceRef = f.id + source.serviceRef = String(f.id) return source }) return [sources, groupsMap] @@ -104,14 +104,14 @@ export const feverServiceHooks: ServiceHooks = { ) configs.lastId = items.reduce((m, n) => Math.max(m, n.id), configs.lastId) if (items.length > 0) { - const fidMap = new Map() + const fidMap = new Map() for (let source of Object.values(state.sources)) { if (source.serviceRef) { - fidMap.set(source.serviceRef as number, source) + fidMap.set(source.serviceRef, source) } } const parsedItems = items.map(i => { - const source = fidMap.get(i.feed_id) + const source = fidMap.get(String(i.feed_id)) const item = { source: source.sid, title: i.title, @@ -125,7 +125,7 @@ export const feverServiceHooks: ServiceHooks = { starred: Boolean(i.is_saved), hidden: false, notify: false, - serviceRef: typeof i.id === "string" ? parseInt(i.id) : i.id, + serviceRef: String(i.id), } as RSSItem // Try to get the thumbnail of the item let dom = domParser.parseFromString(item.content, "text/html") @@ -163,9 +163,9 @@ export const feverServiceHooks: ServiceHooks = { if (typeof unreadResponse.unread_item_ids !== "string" || typeof starredResponse.saved_item_ids !== "string") { throw APIError() } - const unreadFids: number[] = unreadResponse.unread_item_ids.split(",").map(s => parseInt(s)) - const starredFids: number[] = starredResponse.saved_item_ids.split(",").map(s => parseInt(s)) - return [unreadFids, starredFids] + const unreadFids: string[] = unreadResponse.unread_item_ids.split(",") + const starredFids: string[] = starredResponse.saved_item_ids.split(",") + return [new Set(unreadFids), new Set(starredFids)] }, markAllRead: (sids, date, before) => async (_, getState) => { diff --git a/src/scripts/models/source.ts b/src/scripts/models/source.ts index 33a6aea..88f5103 100644 --- a/src/scripts/models/source.ts +++ b/src/scripts/models/source.ts @@ -19,7 +19,7 @@ export class RSSSource { openTarget: SourceOpenTarget unreadCount: number lastFetched: Date - serviceRef?: string | number + serviceRef?: string fetchFrequency: number // in minutes rules?: SourceRule[] @@ -81,12 +81,13 @@ export type SourceState = { export const INIT_SOURCES = "INIT_SOURCES" export const ADD_SOURCE = "ADD_SOURCE" export const UPDATE_SOURCE = "UPDATE_SOURCE" +export const UPDATE_UNREAD_COUNTS = "UPDATE_UNREAD_COUNTS" export const DELETE_SOURCE = "DELETE_SOURCE" interface InitSourcesAction { type: typeof INIT_SOURCES status: ActionStatus - sources?: RSSSource[] + sources?: SourceState err? } @@ -103,12 +104,18 @@ interface UpdateSourceAction { source: RSSSource } +interface UpdateUnreadCountsAction { + type: typeof UPDATE_UNREAD_COUNTS + sources: SourceState +} + interface DeleteSourceAction { type: typeof DELETE_SOURCE, source: RSSSource } -export type SourceActionTypes = InitSourcesAction | AddSourceAction | UpdateSourceAction | DeleteSourceAction +export type SourceActionTypes = InitSourcesAction | AddSourceAction | UpdateSourceAction + | UpdateUnreadCountsAction | DeleteSourceAction export function initSourcesRequest(): SourceActionTypes { return { @@ -117,7 +124,7 @@ export function initSourcesRequest(): SourceActionTypes { } } -export function initSourcesSuccess(sources: RSSSource[]): SourceActionTypes { +export function initSourcesSuccess(sources: SourceState): SourceActionTypes { return { type: INIT_SOURCES, status: ActionStatus.Success, @@ -133,21 +140,34 @@ export function initSourcesFailure(err): SourceActionTypes { } } -async function unreadCount(source: RSSSource): Promise { - source.unreadCount = (await db.itemsDB.select( +async function unreadCount(sources: SourceState): Promise { + const rows = await db.itemsDB.select( + db.items.source, lf.fn.count(db.items._id) - ).from(db.items).where(lf.op.and( - db.items.source.eq(source.sid), + ).from(db.items).where( db.items.hasRead.eq(false) - )).exec())[0]["COUNT(_id)"] - return source + ).groupBy( + db.items.source + ).exec() + for (let row of rows) { + sources[row["source"]].unreadCount = row["COUNT(_id)"] + } + return sources } export function updateUnreadCounts(): AppThunk> { return async (dispatch, getState) => { - await Promise.all(Object.values(getState().sources).map(async s => { - dispatch(updateSourceDone(await unreadCount(s))) - })) + const sources: SourceState = {} + for (let source of Object.values(getState().sources)) { + sources[source.sid] = { + ...source, + unreadCount: 0 + } + } + dispatch({ + type: UPDATE_UNREAD_COUNTS, + sources: await unreadCount(sources), + }) } } @@ -156,9 +176,13 @@ export function initSources(): AppThunk> { dispatch(initSourcesRequest()) await db.init() const sources = (await db.sourcesDB.select().from(db.sources).exec()) as RSSSource[] - const promises = sources.map(s => unreadCount(s)) - const counted = await Promise.all(promises) - dispatch(initSourcesSuccess(counted)) + const state: SourceState = {} + for (let source of sources) { + source.unreadCount = 0 + state[source.sid] = source + } + await unreadCount(sources) + dispatch(initSourcesSuccess(sources)) } } @@ -313,15 +337,10 @@ export function sourceReducer( switch (action.type) { case INIT_SOURCES: switch (action.status) { - case ActionStatus.Success: { - let newState: SourceState = {} - for (let source of action.sources) { - newState[source.sid] = source - } - return newState - } + case ActionStatus.Success: return action.sources default: return state } + case UPDATE_UNREAD_COUNTS: return action.sources case ADD_SOURCE: switch (action.status) { case ActionStatus.Success: return { diff --git a/src/scripts/settings.ts b/src/scripts/settings.ts index cbcc18e..9e74b36 100644 --- a/src/scripts/settings.ts +++ b/src/scripts/settings.ts @@ -1,3 +1,4 @@ +import * as db from "./db" import { IPartialTheme, loadTheme } from "@fluentui/react" import locales from "./i18n/_locales" import { ThemeSettings } from "../schema-types" @@ -57,29 +58,17 @@ export function getCurrentLocale() { return (locale in locales) ? locale : "en-US" } -export function exportAll() { +export async function exportAll() { const filters = [{ name: intl.get("app.frData"), extensions: ["frdata"] }] - window.utils.showSaveDialog(filters, "*/Fluent_Reader_Backup.frdata").then(write => { - if (write) { - let output = window.settings.getAll() - output["nedb"] = {} - let openRequest = window.indexedDB.open("NeDB") - openRequest.onsuccess = () => { - let db = openRequest.result - let objectStore = db.transaction("nedbdata").objectStore("nedbdata") - let cursorRequest = objectStore.openCursor() - cursorRequest.onsuccess = () => { - let cursor = cursorRequest.result - if (cursor) { - output["nedb"][cursor.key] = cursor.value - cursor.continue() - } else { - write(JSON.stringify(output), intl.get("settings.writeError")) - } - } - } + const write = await window.utils.showSaveDialog(filters, "*/Fluent_Reader_Backup.frdata") + if (write) { + let output = window.settings.getAll() + output["lovefield"] = { + sources: await db.sourcesDB.select().from(db.sources).exec(), + items: await db.itemsDB.select().from(db.items).exec(), } - }) + write(JSON.stringify(output), intl.get("settings.writeError")) + } } export async function importAll() { @@ -94,21 +83,40 @@ export async function importAll() { ) if (!confirmed) return true let configs = JSON.parse(data) - let openRequest = window.indexedDB.open("NeDB") - openRequest.onsuccess = () => { - let db = openRequest.result - let objectStore = db.transaction("nedbdata", "readwrite").objectStore("nedbdata") - let requests = Object.entries(configs.nedb).map(([key, value]) => { - return objectStore.put(value, key) + await db.sourcesDB.delete().from(db.sources).exec() + await db.itemsDB.delete().from(db.items).exec() + if (configs.nedb) { + let openRequest = window.indexedDB.open("NeDB") + configs.useNeDB = true + openRequest.onsuccess = () => { + let db = openRequest.result + let objectStore = db.transaction("nedbdata", "readwrite").objectStore("nedbdata") + let requests = Object.entries(configs.nedb).map(([key, value]) => { + return objectStore.put(value, key) + }) + let promises = requests.map(req => new Promise((resolve, reject) => { + req.onsuccess = () => resolve() + req.onerror = () => reject() + })) + Promise.all(promises).then(() => { + delete configs.nedb + window.settings.setAll(configs) + }) + } + } else { + const sRows = configs.lovefield.sources.map(s => { + s.lastFetched = new Date(s.lastFetched) + return db.sources.createRow(s) }) - let promises = requests.map(req => new Promise((resolve, reject) => { - req.onsuccess = () => resolve() - req.onerror = () => reject() - })) - Promise.all(promises).then(() => { - delete configs.nedb - window.settings.setAll(configs) + const iRows = configs.lovefield.items.map(i => { + i.date = new Date(i.date) + i.fetchedDate = new Date(i.fetchedDate) + return db.items.createRow(i) }) - } + await db.sourcesDB.insert().into(db.sources).values(sRows).exec() + await db.itemsDB.insert().into(db.items).values(iRows).exec() + delete configs.lovefield + window.settings.setAll(configs) + } return false } diff --git a/src/scripts/utils.ts b/src/scripts/utils.ts index 854b94a..91400e8 100644 --- a/src/scripts/utils.ts +++ b/src/scripts/utils.ts @@ -189,15 +189,22 @@ function byteLength(str: string) { export function calculateItemSize(): Promise { return new Promise((resolve, reject) => { - let openRequest = window.indexedDB.open("NeDB") + let result = 0 + let openRequest = window.indexedDB.open("itemsDB") openRequest.onsuccess = () => { let db = openRequest.result - let objectStore = db.transaction("nedbdata").objectStore("nedbdata") - let getRequest = objectStore.get("items") - getRequest.onsuccess = () => { - resolve(byteLength(getRequest.result)) + let objectStore = db.transaction("items").objectStore("items") + let cursorRequest = objectStore.openCursor() + cursorRequest.onsuccess = () => { + let cursor = cursorRequest.result + if (cursor) { + result += byteLength(JSON.stringify(cursor.value)) + cursor.continue() + } else { + resolve(result) + } } - getRequest.onerror = () => reject() + cursorRequest.onerror = () => reject() } openRequest.onerror = () => reject() })