diff --git a/package-lock.json b/package-lock.json index 8d711e1..4c988df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -166,6 +166,12 @@ "integrity": "sha512-iYCgjm1dGPRuo12+BStjd1HiVQqhlRhWDOQigNxn023HcjnhsiFz9pc6CzJj4HwDCSQca9bxTL4PxJDbkdm3PA==", "dev": true }, + "@types/lovefield": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@types/lovefield/-/lovefield-2.1.3.tgz", + "integrity": "sha512-LaMHENSO3MOcAgYD+iYsbuvEjDBQ0a5GmYyjca99MlHli0A5tcOTEy03UHiRk5cfubdPhVYCfCTXYN7/CLvnWA==", + "dev": true + }, "@types/nedb": { "version": "1.8.9", "resolved": "https://registry.npmjs.org/@types/nedb/-/nedb-1.8.9.tgz", @@ -571,6 +577,12 @@ } } }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true + }, "acorn": { "version": "6.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", @@ -4244,6 +4256,50 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, + "lovefield": { + "version": "2.1.12", + "resolved": "https://registry.npmjs.org/lovefield/-/lovefield-2.1.12.tgz", + "integrity": "sha1-B/ufHXhAV47HhHBsrQ++feTJP1I=", + "dev": true, + "requires": { + "js-yaml": "~3.1.0", + "nopt": "~2.2.1" + }, + "dependencies": { + "argparse": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-0.1.16.tgz", + "integrity": "sha1-z9AeD7uj1srtBJ+9dY1A9lGW9Xw=", + "dev": true, + "requires": { + "underscore": "~1.7.0", + "underscore.string": "~2.4.0" + } + }, + "esprima": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz", + "integrity": "sha1-n1V+CPw7TSbs6d00+Pv0drYlha0=", + "dev": true + }, + "js-yaml": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.1.0.tgz", + "integrity": "sha1-NroC5hjFB0jnct01JCiQTLutz0Q=", + "dev": true, + "requires": { + "argparse": "~ 0.1.11", + "esprima": "~ 1.0.2" + } + }, + "underscore": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", + "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=", + "dev": true + } + } + }, "lower-case": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.1.tgz", @@ -4605,6 +4661,15 @@ "integrity": "sha512-UdS4swXs85fCGWWf6t6DMGgpN/vnlKeSGEQ7hJcrs7PBFoxoKLmibc3QRb7fwiYsjdL7PX8iI/TMSlZ90dgHhQ==", "dev": true }, + "nopt": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-2.2.1.tgz", + "integrity": "sha1-KqCbfRdoSHs7ianFqlIzW/8Lrqc=", + "dev": true, + "requires": { + "abbrev": "1" + } + }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -6401,6 +6466,12 @@ "integrity": "sha1-YaajIBBiKvoHljvzJSA88SI51gQ=", "dev": true }, + "underscore.string": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.4.0.tgz", + "integrity": "sha1-jN2PusTi0uoefi6Al8QvRCKA+Fs=", + "dev": true + }, "union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", diff --git a/package.json b/package.json index 47ebbea..861299f 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ }, "devDependencies": { "@fluentui/react": "^7.126.2", + "@types/lovefield": "^2.1.3", "@types/nedb": "^1.8.9", "@types/react": "^16.9.35", "@types/react-dom": "^16.9.8", @@ -99,6 +100,7 @@ "hard-source-webpack-plugin": "^0.13.1", "html-webpack-plugin": "^4.3.0", "js-md5": "^0.7.3", + "lovefield": "^2.1.12", "nedb": "^1.8.0", "qrcode.react": "^1.0.0", "react": "^16.13.1", diff --git a/src/bridges/settings.ts b/src/bridges/settings.ts index 2527837..e05f239 100644 --- a/src/bridges/settings.ts +++ b/src/bridges/settings.ts @@ -103,6 +103,13 @@ const settingsBridge = { ipcRenderer.invoke("set-view-configs", view, configs) }, + getNeDBStatus: (): boolean => { + return ipcRenderer.sendSync("get-nedb-status") + }, + setNeDBStatus: () => { + ipcRenderer.invoke("set-nedb-status") + }, + getAll: () => { return ipcRenderer.sendSync("get-all-settings") as Object }, diff --git a/src/main/settings.ts b/src/main/settings.ts index 53f0b50..dddc480 100644 --- a/src/main/settings.ts +++ b/src/main/settings.ts @@ -172,3 +172,11 @@ ipcMain.handle("set-view-configs", (_, view: ViewType, configs: ViewConfigs) => break } }) + +const NEDB_STATUS_STORE_KEY = "useNeDB" +ipcMain.on("get-nedb-status", (event) => { + event.returnValue = store.get(NEDB_STATUS_STORE_KEY, true) +}) +ipcMain.handle("set-nedb-status", () => { + store.set(NEDB_STATUS_STORE_KEY, false) +}) diff --git a/src/main/update-scripts.ts b/src/main/update-scripts.ts index 6cf8cbf..eac80af 100644 --- a/src/main/update-scripts.ts +++ b/src/main/update-scripts.ts @@ -4,8 +4,17 @@ import { SchemaTypes } from "../schema-types" export default function performUpdate(store: Store) { let version = store.get("version", null) + let useNeDB = store.get("useNeDB", undefined) let currentVersion = app.getVersion() + if (useNeDB === undefined) { + const revs = version.split(".").map(s => parseInt(s)) + if ((revs[0] === 0 && revs[1] < 8) || !app.isPackaged) { + store.set("useNeDB", true) + } else { + store.set("useNeDB", false) + } + } if (version != currentVersion) { store.set("version", currentVersion) } diff --git a/src/schema-types.ts b/src/schema-types.ts index 2036df0..5fb2e92 100644 --- a/src/schema-types.ts +++ b/src/schema-types.ts @@ -76,4 +76,5 @@ export type SchemaTypes = { serviceConfigs: ServiceConfigs filterType: number listViewConfigs: ViewConfigs + useNeDB: boolean } diff --git a/src/scripts/db.ts b/src/scripts/db.ts index 2e5fbee..4c4c175 100644 --- a/src/scripts/db.ts +++ b/src/scripts/db.ts @@ -1,17 +1,41 @@ import Datastore from "nedb" +import lf from "lovefield" import { RSSSource } from "./models/source" import { RSSItem } from "./models/item" -export const sdb = new Datastore({ - filename: "sources", - autoload: true, - onload: (err) => { - if (err) window.console.log(err) - } -}) -sdb.ensureIndex({ fieldName: "sid", unique: true }) -sdb.ensureIndex({ fieldName: "url", unique: true }) -//sdb.remove({}, { multi: true }) +const sdbSchema = lf.schema.create("sourcesDB", 1) +sdbSchema.createTable("sources"). + addColumn("sid", lf.Type.INTEGER).addPrimaryKey(["sid"], false). + addColumn("url", lf.Type.STRING). + addColumn("iconurl", lf.Type.STRING). + addColumn("name", lf.Type.STRING). + addColumn("openTarget", lf.Type.NUMBER). + addColumn("lastFetched", lf.Type.DATE_TIME). + addColumn("serviceRef", lf.Type.NUMBER). + addColumn("fetchFrequency", lf.Type.NUMBER). + addColumn("rules", lf.Type.OBJECT). + addNullable(["iconurl", "serviceRef", "rules"]). + addIndex("idxURL", ["url"], true) + +const idbSchema = lf.schema.create("itemsDB", 1) +idbSchema.createTable("items"). + addColumn("_id", lf.Type.INTEGER).addPrimaryKey(["_id"], true). + addColumn("source", lf.Type.INTEGER). + addColumn("title", lf.Type.STRING). + addColumn("link", lf.Type.STRING). + addColumn("date", lf.Type.DATE_TIME). + addColumn("fetchedDate", lf.Type.DATE_TIME). + addColumn("thumb", lf.Type.STRING). + addColumn("content", lf.Type.STRING). + addColumn("snippet", lf.Type.STRING). + addColumn("creator", lf.Type.STRING). + addColumn("hasRead", lf.Type.BOOLEAN). + addColumn("starred", lf.Type.BOOLEAN). + addColumn("hidden", lf.Type.BOOLEAN). + addColumn("notify", lf.Type.BOOLEAN). + addColumn("serviceRef", lf.Type.NUMBER). + addNullable(["thumb", "creator", "serviceRef"]). + addIndex("idxDate", ["date"], false, lf.Order.DESC) export const idb = new Datastore({ filename: "items", @@ -23,4 +47,54 @@ export const idb = new Datastore({ idb.ensureIndex({ fieldName: "source" }) //idb.removeIndex("id") //idb.update({}, {$unset: {id: true}}, {multi: true}) -//idb.remove({}, { multi: true }) \ No newline at end of file +//idb.remove({}, { multi: true }) +export let sourcesDB: lf.Database +export let sources: lf.schema.Table +export let itemsDB: lf.Database +export let items: lf.schema.Table + +export async function init() { + sourcesDB = await sdbSchema.connect() + sources = sourcesDB.getSchema().table("sources") + itemsDB = await idbSchema.connect() + items = itemsDB.getSchema().table("items") + if (window.settings.getNeDBStatus()) { + const sdb = new Datastore({ + filename: "sources", + autoload: true, + onload: (err) => { + if (err) window.console.log(err) + } + }) + const sourceDocs = await new Promise(resolve => { + sdb.find({}, (_, docs) => { + resolve(docs) + }) + }) + const itemDocs = await new Promise(resolve => { + idb.find({}, (_, docs) => { + resolve(docs) + }) + }) + const sRows = sourceDocs.map(doc => { + //doc.serviceRef = String(doc.serviceRef) + // @ts-ignore + delete doc._id + if (!doc.fetchFrequency) doc.fetchFrequency = 0 + return sources.createRow(doc) + }) + const iRows = itemDocs.map(doc => { + //doc.serviceRef = String(doc.serviceRef) + delete doc._id + doc.starred = Boolean(doc.starred) + doc.hidden = Boolean(doc.hidden) + doc.notify = Boolean(doc.notify) + return items.createRow(doc) + }) + await Promise.all([ + sourcesDB.insert().into(sources).values(sRows).exec(), + itemsDB.insert().into(items).values(iRows).exec() + ]) + window.settings.setNeDBStatus() + } +} diff --git a/src/scripts/models/app.ts b/src/scripts/models/app.ts index e09799b..9d28a58 100644 --- a/src/scripts/models/app.ts +++ b/src/scripts/models/app.ts @@ -301,7 +301,6 @@ export function initApp(): AppThunk { dispatch(fixBrokenGroups()) await dispatch(fetchItems()) }).then(() => { - db.sdb.persistence.compactDatafile() db.idb.persistence.compactDatafile() dispatch(updateFavicon()) }) diff --git a/src/scripts/models/item.ts b/src/scripts/models/item.ts index 48ad7ba..8dbba0f 100644 --- a/src/scripts/models/item.ts +++ b/src/scripts/models/item.ts @@ -1,7 +1,7 @@ import * as db from "../db" import intl from "react-intl-universal" import { domParser, htmlDecode, ActionStatus, AppThunk, platformCtrl } from "../utils" -import { RSSSource } from "./source" +import { RSSSource, updateSource } from "./source" import { FeedActionTypes, INIT_FEED, LOAD_MORE, FilterType, initFeeds } from "./feed" import Parser from "@yang991178/rss-parser" import { pushNotification, setupAutoFetch } from "./app" @@ -19,9 +19,9 @@ export class RSSItem { snippet: string creator?: string hasRead: boolean - starred?: true - hidden?: true - notify?: true + starred?: boolean + hidden?: boolean + notify?: boolean serviceRef?: string | number constructor (item: Parser.Item, source: RSSSource) { @@ -189,6 +189,7 @@ export function fetchItems(background = false, sids: number[] = null): AppThunk< : sids.map(sid => sourcesState[sid]).filter(s => !s.serviceRef) for (let source of sources) { let promise = RSSSource.fetchItems(source) + promise.then(() => dispatch(updateSource({ ...source, lastFetched: new Date() }))) promise.finally(() => dispatch(fetchItemsIntermediate())) promises.push(promise) } diff --git a/src/scripts/models/service.ts b/src/scripts/models/service.ts index cd5fffe..37ec15e 100644 --- a/src/scripts/models/service.ts +++ b/src/scripts/models/service.ts @@ -79,46 +79,42 @@ function updateSources(hook: ServiceHooks["updateSources"]): AppThunk { if (!(getState().app.settings.saving)) dispatch(saveSettings()) } - let promises = sources.map(s => new Promise((resolve, reject) => { + let promises = sources.map(async (s) => { if (existing.has(s.serviceRef)) { const doc = existing.get(s.serviceRef) existing.delete(s.serviceRef) - resolve(doc) + return doc } else { - db.sdb.findOne({ url: s.url }, (err, doc) => { - if (err) { - reject(err) - } else if (doc === null) { - // Create a new source - forceSettings() - dispatch(insertSource(s)) - .then((inserted) => { - inserted.unreadCount = 0 - resolve(inserted) - dispatch(addSourceSuccess(inserted, true)) - window.settings.saveGroups(getState().groups) - dispatch(updateFavicon([inserted.sid])) - }) - .catch((err) => { - reject(err) - }) - } else if (doc.serviceRef !== s.serviceRef) { - // Mark an existing source as remote and remove all items - forceSettings() - doc.serviceRef = s.serviceRef - doc.unreadCount = 0 - dispatch(updateSource(doc)).finally(() => { - db.idb.remove({ source: doc.sid }, { multi: true }, (err) => { - if (err) reject(err) - else resolve(doc) - }) + const docs = (await db.sourcesDB.select().from(db.sources).where( + db.sources.url.eq(s.url) + ).exec()) as RSSSource[] + if (docs.length === 0) { + // Create a new source + forceSettings() + const inserted = await dispatch(insertSource(s)) + inserted.unreadCount = 0 + dispatch(addSourceSuccess(inserted, true)) + window.settings.saveGroups(getState().groups) + dispatch(updateFavicon([inserted.sid])) + return inserted + } else if (docs[0].serviceRef !== s.serviceRef) { + // Mark an existing source as remote and remove all items + const doc = docs[0] + forceSettings() + doc.serviceRef = s.serviceRef + doc.unreadCount = 0 + await dispatch(updateSource(doc)) + await new Promise((resolve, reject) => { + db.idb.remove({ source: doc.sid }, { multi: true }, (err) => { + if (err) reject(err) + else resolve(doc) }) - } else { - resolve(doc) - } - }) + }) + } else { + return docs[0] + } } - })) + }) for (let [_, source] of existing) { // Delete sources removed from the service side forceSettings() diff --git a/src/scripts/models/source.ts b/src/scripts/models/source.ts index 0519e6a..29e2b0d 100644 --- a/src/scripts/models/source.ts +++ b/src/scripts/models/source.ts @@ -19,7 +19,7 @@ export class RSSSource { unreadCount: number lastFetched: Date serviceRef?: string | number - fetchFrequency?: number // in minutes + fetchFrequency: number // in minutes rules?: SourceRule[] constructor(url: string, name: string = null) { @@ -27,6 +27,7 @@ export class RSSSource { this.name = name this.openTarget = SourceOpenTarget.Local this.lastFetched = new Date() + this.fetchFrequency = 0 } static async fetchMetaData(source: RSSSource) { @@ -74,7 +75,6 @@ export class RSSSource { static async fetchItems(source: RSSSource) { let feed = await parseRSS(source.url) - db.sdb.update({ sid: source.sid }, { $set: { lastFetched: new Date() } }) return await this.checkItems(source, feed.items) } } @@ -160,24 +160,13 @@ export function updateUnreadCounts(): AppThunk> { } export function initSources(): AppThunk> { - return (dispatch) => { + return async (dispatch) => { dispatch(initSourcesRequest()) - return new Promise((resolve, reject) => { - db.sdb.find({}).sort({ sid: 1 }).exec((err, sources) => { - if (err) { - dispatch(initSourcesFailure(err)) - reject(err) - } else { - let p = sources.map(s => unreadCount(s)) - Promise.all(p) - .then(values => { - dispatch(initSourcesSuccess(values)) - resolve() - }) - .catch(err => reject(err)) - } - }) - }) + await db.init() + const sources = (await db.sourcesDB.select().from(db.sources).exec()) as RSSSource[] + const promises = sources.map(s => unreadCount(s)) + const counted = await Promise.all(promises) + dispatch(initSourcesSuccess(counted)) } } @@ -211,22 +200,17 @@ let insertPromises = Promise.resolve() export function insertSource(source: RSSSource): AppThunk> { return (_, getState) => { return new Promise((resolve, reject) => { - insertPromises = insertPromises.then(() => new Promise(innerResolve => { + insertPromises = insertPromises.then(async () => { let sids = Object.values(getState().sources).map(s => s.sid) source.sid = Math.max(...sids, -1) + 1 - db.sdb.insert(source, (err, inserted) => { - if (err) { - if ((new RegExp(`^Can't insert key ${source.url},`)).test(err.message)) { - reject(intl.get("sources.exist")) - } else { - reject(err) - } - } else { - resolve(inserted) - } - innerResolve() - }) - })) + const row = db.sources.createRow(source) + try { + const inserted = (await db.sourcesDB.insert().into(db.sources).values([row]).exec()) as RSSSource[] + resolve(inserted[0]) + } catch { + reject(intl.get("sources.exist")) + } + }) }) } } @@ -267,17 +251,13 @@ export function updateSourceDone(source: RSSSource): SourceActionTypes { } export function updateSource(source: RSSSource): AppThunk> { - return (dispatch) => new Promise((resolve) => { + return async (dispatch) => { let sourceCopy = { ...source } - delete sourceCopy.sid delete sourceCopy.unreadCount - db.sdb.update({ sid: source.sid }, { $set: { ...sourceCopy }}, {}, err => { - if (!err) { - dispatch(updateSourceDone(source)) - } - resolve() - }) - }) + const row = db.sources.createRow(sourceCopy) + await db.sourcesDB.insertOrReplace().into(db.sources).values([row]).exec() + dispatch(updateSourceDone(source)) + } } export function deleteSourceDone(source: RSSSource): SourceActionTypes { @@ -297,17 +277,17 @@ export function deleteSource(source: RSSSource, batch = false): AppThunk { - if (err) { - console.log(err) - if (!batch) dispatch(saveSettings()) - resolve() - } else { - dispatch(deleteSourceDone(source)) - window.settings.saveGroups(getState().groups) - if (!batch) dispatch(saveSettings()) - resolve() - } + db.sourcesDB.delete().from(db.sources).where( + db.sources.sid.eq(source.sid) + ).exec().then(() => { + dispatch(deleteSourceDone(source)) + window.settings.saveGroups(getState().groups) + if (!batch) dispatch(saveSettings()) + resolve() + }).catch(err => { + console.log(err) + if (!batch) dispatch(saveSettings()) + resolve() }) } })