FreshRSS
/
FluentReader
mirror of https://github.com/yang991178/fluent-reader.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use lovefield for sources

This commit is contained in:
刘浩远 2020-08-31 14:35:00 +08:00
parent 237c584363
commit 8251bb25ac
11 changed files with 251 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
package-lock.json generated
View File

@ -166,6 +166,12 @@
"integrity": "sha512-iYCgjm1dGPRuo12+BStjd1HiVQqhlRhWDOQigNxn023HcjnhsiFz9pc6CzJj4HwDCSQca9bxTL4PxJDbkdm3PA==",
"dev": true
},
"@types/lovefield": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/@types/lovefield/-/lovefield-2.1.3.tgz",
"integrity": "sha512-LaMHENSO3MOcAgYD+iYsbuvEjDBQ0a5GmYyjca99MlHli0A5tcOTEy03UHiRk5cfubdPhVYCfCTXYN7/CLvnWA==",
"dev": true
},
"@types/nedb": {
"version": "1.8.9",
"resolved": "https://registry.npmjs.org/@types/nedb/-/nedb-1.8.9.tgz",
@ -571,6 +577,12 @@
}
}
},
"abbrev": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
"dev": true
},
"acorn": {
"version": "6.4.1",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz",
@ -4244,6 +4256,50 @@
"js-tokens": "^3.0.0 || ^4.0.0"
}
},
"lovefield": {
"version": "2.1.12",
"resolved": "https://registry.npmjs.org/lovefield/-/lovefield-2.1.12.tgz",
"integrity": "sha1-B/ufHXhAV47HhHBsrQ++feTJP1I=",
"dev": true,
"requires": {
"js-yaml": "~3.1.0",
"nopt": "~2.2.1"
},
"dependencies": {
"argparse": {
"version": "0.1.16",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-0.1.16.tgz",
"integrity": "sha1-z9AeD7uj1srtBJ+9dY1A9lGW9Xw=",
"dev": true,
"requires": {
"underscore": "~1.7.0",
"underscore.string": "~2.4.0"
}
},
"esprima": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz",
"integrity": "sha1-n1V+CPw7TSbs6d00+Pv0drYlha0=",
"dev": true
},
"js-yaml": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.1.0.tgz",
"integrity": "sha1-NroC5hjFB0jnct01JCiQTLutz0Q=",
"dev": true,
"requires": {
"argparse": "~ 0.1.11",
"esprima": "~ 1.0.2"
}
},
"underscore": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz",
"integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=",
"dev": true
}
}
},
"lower-case": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.1.tgz",
@ -4605,6 +4661,15 @@
"integrity": "sha512-UdS4swXs85fCGWWf6t6DMGgpN/vnlKeSGEQ7hJcrs7PBFoxoKLmibc3QRb7fwiYsjdL7PX8iI/TMSlZ90dgHhQ==",
"dev": true
},
"nopt": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/nopt/-/nopt-2.2.1.tgz",
"integrity": "sha1-KqCbfRdoSHs7ianFqlIzW/8Lrqc=",
"dev": true,
"requires": {
"abbrev": "1"
}
},
"normalize-package-data": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz",
@ -6401,6 +6466,12 @@
"integrity": "sha1-YaajIBBiKvoHljvzJSA88SI51gQ=",
"dev": true
},
"underscore.string": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.4.0.tgz",
"integrity": "sha1-jN2PusTi0uoefi6Al8QvRCKA+Fs=",
"dev": true
},
"union-value": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",

2
package.json
View File

@ -86,6 +86,7 @@
},
"devDependencies": {
"@fluentui/react": "^7.126.2",
"@types/lovefield": "^2.1.3",
"@types/nedb": "^1.8.9",
"@types/react": "^16.9.35",
"@types/react-dom": "^16.9.8",
@ -99,6 +100,7 @@
"hard-source-webpack-plugin": "^0.13.1",
"html-webpack-plugin": "^4.3.0",
"js-md5": "^0.7.3",
"lovefield": "^2.1.12",
"nedb": "^1.8.0",
"qrcode.react": "^1.0.0",
"react": "^16.13.1",

7
src/bridges/settings.ts
View File

@ -103,6 +103,13 @@ const settingsBridge = {
ipcRenderer.invoke("set-view-configs", view, configs)
},
getNeDBStatus: (): boolean => {
return ipcRenderer.sendSync("get-nedb-status")
},
setNeDBStatus: () => {
ipcRenderer.invoke("set-nedb-status")
},
getAll: () => {
return ipcRenderer.sendSync("get-all-settings") as Object
},

8
src/main/settings.ts
View File

@ -172,3 +172,11 @@ ipcMain.handle("set-view-configs", (_, view: ViewType, configs: ViewConfigs) =>
break
}
})
const NEDB_STATUS_STORE_KEY = "useNeDB"
ipcMain.on("get-nedb-status", (event) => {
event.returnValue = store.get(NEDB_STATUS_STORE_KEY, true)
})
ipcMain.handle("set-nedb-status", () => {
store.set(NEDB_STATUS_STORE_KEY, false)
})

9
src/main/update-scripts.ts
View File

@ -4,8 +4,17 @@ import { SchemaTypes } from "../schema-types"
export default function performUpdate(store: Store<SchemaTypes>) {
let version = store.get("version", null)
let useNeDB = store.get("useNeDB", undefined)
let currentVersion = app.getVersion()
if (useNeDB === undefined) {
const revs = version.split(".").map(s => parseInt(s))
if ((revs[0] === 0 && revs[1] < 8) || !app.isPackaged) {
store.set("useNeDB", true)
} else {
store.set("useNeDB", false)
}
}
if (version != currentVersion) {
store.set("version", currentVersion)
}

1
src/schema-types.ts
View File

@ -76,4 +76,5 @@ export type SchemaTypes = {
serviceConfigs: ServiceConfigs
filterType: number
listViewConfigs: ViewConfigs
useNeDB: boolean
}

96
src/scripts/db.ts
View File

@ -1,17 +1,41 @@
import Datastore from "nedb"
import lf from "lovefield"
import { RSSSource } from "./models/source"
import { RSSItem } from "./models/item"
export const sdb = new Datastore<RSSSource>({
filename: "sources",
autoload: true,
onload: (err) => {
if (err) window.console.log(err)
}
})
sdb.ensureIndex({ fieldName: "sid", unique: true })
sdb.ensureIndex({ fieldName: "url", unique: true })
//sdb.remove({}, { multi: true })
const sdbSchema = lf.schema.create("sourcesDB", 1)
sdbSchema.createTable("sources").
addColumn("sid", lf.Type.INTEGER).addPrimaryKey(["sid"], false).
addColumn("url", lf.Type.STRING).
addColumn("iconurl", lf.Type.STRING).
addColumn("name", lf.Type.STRING).
addColumn("openTarget", lf.Type.NUMBER).
addColumn("lastFetched", lf.Type.DATE_TIME).
addColumn("serviceRef", lf.Type.NUMBER).
addColumn("fetchFrequency", lf.Type.NUMBER).
addColumn("rules", lf.Type.OBJECT).
addNullable(["iconurl", "serviceRef", "rules"]).
addIndex("idxURL", ["url"], true)
const idbSchema = lf.schema.create("itemsDB", 1)
idbSchema.createTable("items").
addColumn("_id", lf.Type.INTEGER).addPrimaryKey(["_id"], true).
addColumn("source", lf.Type.INTEGER).
addColumn("title", lf.Type.STRING).
addColumn("link", lf.Type.STRING).
addColumn("date", lf.Type.DATE_TIME).
addColumn("fetchedDate", lf.Type.DATE_TIME).
addColumn("thumb", lf.Type.STRING).
addColumn("content", lf.Type.STRING).
addColumn("snippet", lf.Type.STRING).
addColumn("creator", lf.Type.STRING).
addColumn("hasRead", lf.Type.BOOLEAN).
addColumn("starred", lf.Type.BOOLEAN).
addColumn("hidden", lf.Type.BOOLEAN).
addColumn("notify", lf.Type.BOOLEAN).
addColumn("serviceRef", lf.Type.NUMBER).
addNullable(["thumb", "creator", "serviceRef"]).
addIndex("idxDate", ["date"], false, lf.Order.DESC)
export const idb = new Datastore<RSSItem>({
filename: "items",
@ -23,4 +47,54 @@ export const idb = new Datastore<RSSItem>({
idb.ensureIndex({ fieldName: "source" })
//idb.removeIndex("id")
//idb.update({}, {$unset: {id: true}}, {multi: true})
//idb.remove({}, { multi: true })
//idb.remove({}, { multi: true })
export let sourcesDB: lf.Database
export let sources: lf.schema.Table
export let itemsDB: lf.Database
export let items: lf.schema.Table
export async function init() {
sourcesDB = await sdbSchema.connect()
sources = sourcesDB.getSchema().table("sources")
itemsDB = await idbSchema.connect()
items = itemsDB.getSchema().table("items")
if (window.settings.getNeDBStatus()) {
const sdb = new Datastore<RSSSource>({
filename: "sources",
autoload: true,
onload: (err) => {
if (err) window.console.log(err)
}
})
const sourceDocs = await new Promise<RSSSource[]>(resolve => {
sdb.find({}, (_, docs) => {
resolve(docs)
})
})
const itemDocs = await new Promise<RSSItem[]>(resolve => {
idb.find({}, (_, docs) => {
resolve(docs)
})
})
const sRows = sourceDocs.map(doc => {
//doc.serviceRef = String(doc.serviceRef)
// @ts-ignore
delete doc._id
if (!doc.fetchFrequency) doc.fetchFrequency = 0
return sources.createRow(doc)
})
const iRows = itemDocs.map(doc => {
//doc.serviceRef = String(doc.serviceRef)
delete doc._id
doc.starred = Boolean(doc.starred)
doc.hidden = Boolean(doc.hidden)
doc.notify = Boolean(doc.notify)
return items.createRow(doc)
})
await Promise.all([
sourcesDB.insert().into(sources).values(sRows).exec(),
itemsDB.insert().into(items).values(iRows).exec()
])
window.settings.setNeDBStatus()
}
}

1
src/scripts/models/app.ts
View File

@ -301,7 +301,6 @@ export function initApp(): AppThunk {
dispatch(fixBrokenGroups())
await dispatch(fetchItems())
}).then(() => {
db.sdb.persistence.compactDatafile()
db.idb.persistence.compactDatafile()
dispatch(updateFavicon())
})

9
src/scripts/models/item.ts
View File

@ -1,7 +1,7 @@
import * as db from "../db"
import intl from "react-intl-universal"
import { domParser, htmlDecode, ActionStatus, AppThunk, platformCtrl } from "../utils"
import { RSSSource } from "./source"
import { RSSSource, updateSource } from "./source"
import { FeedActionTypes, INIT_FEED, LOAD_MORE, FilterType, initFeeds } from "./feed"
import Parser from "@yang991178/rss-parser"
import { pushNotification, setupAutoFetch } from "./app"
@ -19,9 +19,9 @@ export class RSSItem {
snippet: string
creator?: string
hasRead: boolean
starred?: true
hidden?: true
notify?: true
starred?: boolean
hidden?: boolean
notify?: boolean
serviceRef?: string | number
constructor (item: Parser.Item, source: RSSSource) {
@ -189,6 +189,7 @@ export function fetchItems(background = false, sids: number[] = null): AppThunk<
: sids.map(sid => sourcesState[sid]).filter(s => !s.serviceRef)
for (let source of sources) {
let promise = RSSSource.fetchItems(source)
promise.then(() => dispatch(updateSource({ ...source, lastFetched: new Date() })))
promise.finally(() => dispatch(fetchItemsIntermediate()))
promises.push(promise)
}

64
src/scripts/models/service.ts
View File

@ -79,46 +79,42 @@ function updateSources(hook: ServiceHooks["updateSources"]): AppThunk<Promise<vo
const forceSettings = () => {
if (!(getState().app.settings.saving)) dispatch(saveSettings())
}
let promises = sources.map(s => new Promise<RSSSource>((resolve, reject) => {
let promises = sources.map(async (s) => {
if (existing.has(s.serviceRef)) {
const doc = existing.get(s.serviceRef)
existing.delete(s.serviceRef)
resolve(doc)
return doc
} else {
db.sdb.findOne({ url: s.url }, (err, doc) => {
if (err) {
reject(err)
} else if (doc === null) {
// Create a new source
forceSettings()
dispatch(insertSource(s))
.then((inserted) => {
inserted.unreadCount = 0
resolve(inserted)
dispatch(addSourceSuccess(inserted, true))
window.settings.saveGroups(getState().groups)
dispatch(updateFavicon([inserted.sid]))
})
.catch((err) => {
reject(err)
})
} else if (doc.serviceRef !== s.serviceRef) {
// Mark an existing source as remote and remove all items
forceSettings()
doc.serviceRef = s.serviceRef
doc.unreadCount = 0
dispatch(updateSource(doc)).finally(() => {
db.idb.remove({ source: doc.sid }, { multi: true }, (err) => {
if (err) reject(err)
else resolve(doc)
})
const docs = (await db.sourcesDB.select().from(db.sources).where(
db.sources.url.eq(s.url)
).exec()) as RSSSource[]
if (docs.length === 0) {
// Create a new source
forceSettings()
const inserted = await dispatch(insertSource(s))
inserted.unreadCount = 0
dispatch(addSourceSuccess(inserted, true))
window.settings.saveGroups(getState().groups)
dispatch(updateFavicon([inserted.sid]))
return inserted
} else if (docs[0].serviceRef !== s.serviceRef) {
// Mark an existing source as remote and remove all items
const doc = docs[0]
forceSettings()
doc.serviceRef = s.serviceRef
doc.unreadCount = 0
await dispatch(updateSource(doc))
await new Promise((resolve, reject) => {
db.idb.remove({ source: doc.sid }, { multi: true }, (err) => {
if (err) reject(err)
else resolve(doc)
})
} else {
resolve(doc)
}
})
})
} else {
return docs[0]
}
}
}))
})
for (let [_, source] of existing) {
// Delete sources removed from the service side
forceSettings()

86
src/scripts/models/source.ts
View File

@ -19,7 +19,7 @@ export class RSSSource {
unreadCount: number
lastFetched: Date
serviceRef?: string | number
fetchFrequency?: number // in minutes
fetchFrequency: number // in minutes
rules?: SourceRule[]
constructor(url: string, name: string = null) {
@ -27,6 +27,7 @@ export class RSSSource {
this.name = name
this.openTarget = SourceOpenTarget.Local
this.lastFetched = new Date()
this.fetchFrequency = 0
}
static async fetchMetaData(source: RSSSource) {
@ -74,7 +75,6 @@ export class RSSSource {
static async fetchItems(source: RSSSource) {
let feed = await parseRSS(source.url)
db.sdb.update({ sid: source.sid }, { $set: { lastFetched: new Date() } })
return await this.checkItems(source, feed.items)
}
}
@ -160,24 +160,13 @@ export function updateUnreadCounts(): AppThunk<Promise<void>> {
}
export function initSources(): AppThunk<Promise<void>> {
return (dispatch) => {
return async (dispatch) => {
dispatch(initSourcesRequest())
return new Promise<void>((resolve, reject) => {
db.sdb.find({}).sort({ sid: 1 }).exec((err, sources) => {
if (err) {
dispatch(initSourcesFailure(err))
reject(err)
} else {
let p = sources.map(s => unreadCount(s))
Promise.all(p)
.then(values => {
dispatch(initSourcesSuccess(values))
resolve()
})
.catch(err => reject(err))
}
})
})
await db.init()
const sources = (await db.sourcesDB.select().from(db.sources).exec()) as RSSSource[]
const promises = sources.map(s => unreadCount(s))
const counted = await Promise.all(promises)
dispatch(initSourcesSuccess(counted))
}
}
@ -211,22 +200,17 @@ let insertPromises = Promise.resolve()
export function insertSource(source: RSSSource): AppThunk<Promise<RSSSource>> {
return (_, getState) => {
return new Promise((resolve, reject) => {
insertPromises = insertPromises.then(() => new Promise(innerResolve => {
insertPromises = insertPromises.then(async () => {
let sids = Object.values(getState().sources).map(s => s.sid)
source.sid = Math.max(...sids, -1) + 1
db.sdb.insert(source, (err, inserted) => {
if (err) {
if ((new RegExp(`^Can't insert key ${source.url},`)).test(err.message)) {
reject(intl.get("sources.exist"))
} else {
reject(err)
}
} else {
resolve(inserted)
}
innerResolve()
})
}))
const row = db.sources.createRow(source)
try {
const inserted = (await db.sourcesDB.insert().into(db.sources).values([row]).exec()) as RSSSource[]
resolve(inserted[0])
} catch {
reject(intl.get("sources.exist"))
}
})
})
}
}
@ -267,17 +251,13 @@ export function updateSourceDone(source: RSSSource): SourceActionTypes {
}
export function updateSource(source: RSSSource): AppThunk<Promise<void>> {
return (dispatch) => new Promise((resolve) => {
return async (dispatch) => {
let sourceCopy = { ...source }
delete sourceCopy.sid
delete sourceCopy.unreadCount
db.sdb.update({ sid: source.sid }, { $set: { ...sourceCopy }}, {}, err => {
if (!err) {
dispatch(updateSourceDone(source))
}
resolve()
})
})
const row = db.sources.createRow(sourceCopy)
await db.sourcesDB.insertOrReplace().into(db.sources).values([row]).exec()
dispatch(updateSourceDone(source))
}
}
export function deleteSourceDone(source: RSSSource): SourceActionTypes {
@ -297,17 +277,17 @@ export function deleteSource(source: RSSSource, batch = false): AppThunk<Promise
if (!batch) dispatch(saveSettings())
resolve()
} else {
db.sdb.remove({ sid: source.sid }, {}, (err) => {
if (err) {
console.log(err)
if (!batch) dispatch(saveSettings())
resolve()
} else {
dispatch(deleteSourceDone(source))
window.settings.saveGroups(getState().groups)
if (!batch) dispatch(saveSettings())
resolve()
}
db.sourcesDB.delete().from(db.sources).where(
db.sources.sid.eq(source.sid)
).exec().then(() => {
dispatch(deleteSourceDone(source))
window.settings.saveGroups(getState().groups)
if (!batch) dispatch(saveSettings())
resolve()
}).catch(err => {
console.log(err)
if (!batch) dispatch(saveSettings())
resolve()
})
}
})