FreshRSS
/
FluentReader
mirror of https://github.com/yang991178/fluent-reader.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use lovefield for items

This commit is contained in:
刘浩远 2020-08-31 15:59:39 +08:00
parent 8251bb25ac
commit 06757d0fcd
10 changed files with 160 additions and 252 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/containers/settings/app-container.tsx
View File

@ -15,16 +15,14 @@ const mapDispatchToProps = (dispatch: AppDispatch) => ({
window.settings.setFetchInterval(interval)
dispatch(setupAutoFetch())
},
deleteArticles: (days: number) => new Promise((resolve) => {
deleteArticles: async (days: number) => {
dispatch(saveSettings())
let date = new Date()
date.setTime(date.getTime() - days * 86400000)
db.idb.remove({ date: { $lt: date } }, { multi: true }, () => {
dispatch(updateUnreadCounts()).then(() => dispatch(saveSettings()))
db.idb.prependOnceListener("compaction.done", resolve)
db.idb.persistence.compactDatafile()
})
}),
await db.itemsDB.delete().from(db.items).where(db.items.date.lt(date)).exec()
await dispatch(updateUnreadCounts())
dispatch(saveSettings())
},
importAll: async () => {
dispatch(saveSettings())
let cancelled = await importAll()

18
src/scripts/db.ts
View File

@ -37,17 +37,6 @@ idbSchema.createTable("items").
addNullable(["thumb", "creator", "serviceRef"]).
addIndex("idxDate", ["date"], false, lf.Order.DESC)
export const idb = new Datastore<RSSItem>({
filename: "items",
autoload: true,
onload: (err) => {
if (err) window.console.log(err)
}
})
idb.ensureIndex({ fieldName: "source" })
//idb.removeIndex("id")
//idb.update({}, {$unset: {id: true}}, {multi: true})
//idb.remove({}, { multi: true })
export let sourcesDB: lf.Database
export let sources: lf.schema.Table
export let itemsDB: lf.Database
@ -66,6 +55,13 @@ export async function init() {
if (err) window.console.log(err)
}
})
const idb = new Datastore<RSSItem>({
filename: "items",
autoload: true,
onload: (err) => {
if (err) window.console.log(err)
}
})
const sourceDocs = await new Promise<RSSSource[]>(resolve => {
sdb.find({}, (_, docs) => {
resolve(docs)

1
src/scripts/models/app.ts
View File

@ -301,7 +301,6 @@ export function initApp(): AppThunk {
dispatch(fixBrokenGroups())
await dispatch(fetchItems())
}).then(() => {
db.idb.persistence.compactDatafile()
dispatch(updateFavicon())
})
}

54
src/scripts/models/feed.ts
View File

@ -1,4 +1,5 @@
import * as db from "../db"
import lf from "lovefield"
import { SourceActionTypes, INIT_SOURCES, ADD_SOURCE, DELETE_SOURCE } from "./source"
import { ItemActionTypes, FETCH_ITEMS, RSSItem, MARK_READ, MARK_UNREAD, TOGGLE_STARRED, TOGGLE_HIDDEN, applyItemReduction } from "./item"
import { ActionStatus, AppThunk, mergeSortedArrays } from "../utils"
@ -30,29 +31,25 @@ export class FeedFilter {
this.search = search
}
static toQueryObject(filter: FeedFilter) {
static toPredicates(filter: FeedFilter) {
let type = filter.type
let query = {
hasRead: false,
starred: true,
hidden: { $exists: false }
} as any
if (type & FilterType.ShowRead) delete query.hasRead
if (type & FilterType.ShowNotStarred) delete query.starred
if (type & FilterType.ShowHidden) delete query.hidden
const predicates = new Array<lf.Predicate>()
if (!(type & FilterType.ShowRead)) predicates.push(db.items.hasRead.eq(false))
if (!(type & FilterType.ShowNotStarred)) predicates.push(db.items.starred.eq(true))
if (!(type & FilterType.ShowHidden)) predicates.push(db.items.hidden.eq(false))
if (filter.search !== "") {
const flags = (type & FilterType.CaseInsensitive) ? "i" : ""
const regex = RegExp(filter.search, flags)
if (type & FilterType.FullSearch) {
query.$or = [
{ title: { $regex: regex } },
{ snippet: { $regex: regex } }
]
predicates.push(lf.op.or(
db.items.title.match(regex),
db.items.snippet.match(regex)
))
} else {
query.title = { $regex: regex }
predicates.push(db.items.title.match(regex))
}
}
return query
return predicates
}
static testItem(filter: FeedFilter, item: RSSItem) {
@ -99,24 +96,15 @@ export class RSSFeed {
this.filter = filter === null ? new FeedFilter() : filter
}
static loadFeed(feed: RSSFeed, init = false): Promise<RSSItem[]> {
return new Promise<RSSItem[]>((resolve, reject) => {
let query = {
source: { $in: feed.sids },
...FeedFilter.toQueryObject(feed.filter)
}
db.idb.find(query)
.sort({ date: -1 })
.skip(init ? 0 : feed.iids.length)
.limit(LOAD_QUANTITY)
.exec((err, docs) => {
if (err) {
reject(err)
} else {
resolve(docs)
}
})
})
static async loadFeed(feed: RSSFeed, init = false): Promise<RSSItem[]> {
const predicates = FeedFilter.toPredicates(feed.filter)
predicates.push(db.items.source.in(feed.sids))
return (await db.itemsDB.select().from(db.items).where(
lf.op.and.apply(null, predicates)
).orderBy(db.items.date, lf.Order.DESC)
.skip(init ? 0 : feed.iids.length)
.limit(LOAD_QUANTITY)
.exec()) as RSSItem[]
}
}

111
src/scripts/models/item.ts
View File

@ -1,7 +1,8 @@
import * as db from "../db"
import lf from "lovefield"
import intl from "react-intl-universal"
import { domParser, htmlDecode, ActionStatus, AppThunk, platformCtrl } from "../utils"
import { RSSSource, updateSource } from "./source"
import { RSSSource, updateSource, updateUnreadCounts } from "./source"
import { FeedActionTypes, INIT_FEED, LOAD_MORE, FilterType, initFeeds } from "./feed"
import Parser from "@yang991178/rss-parser"
import { pushNotification, setupAutoFetch } from "./app"
@ -19,9 +20,9 @@ export class RSSItem {
snippet: string
creator?: string
hasRead: boolean
starred?: boolean
hidden?: boolean
notify?: boolean
starred: boolean
hidden: boolean
notify: boolean
serviceRef?: string | number
constructor (item: Parser.Item, source: RSSSource) {
@ -36,6 +37,9 @@ export class RSSItem {
this.date = item.isoDate ? new Date(item.isoDate) : this.fetchedDate
this.creator = item.creator
this.hasRead = false
this.starred = false
this.hidden = false
this.notify = false
}
static parseContent(item: RSSItem, parsed: Parser.Item) {
@ -103,7 +107,6 @@ interface MarkReadAction {
interface MarkAllReadAction {
type: typeof MARK_ALL_READ,
sids: number[]
counts?: number[]
time?: number
before?: boolean
}
@ -159,17 +162,10 @@ export function fetchItemsIntermediate(): ItemActionTypes {
}
}
export function insertItems(items: RSSItem[]): Promise<RSSItem[]> {
return new Promise<RSSItem[]>((resolve, reject) => {
items.sort((a, b) => a.date.getTime() - b.date.getTime())
db.idb.insert(items, (err, inserted) => {
if (err) {
reject(err)
} else {
resolve(inserted)
}
})
})
export async function insertItems(items: RSSItem[]): Promise<RSSItem[]> {
items.sort((a, b) => a.date.getTime() - b.date.getTime())
const rows = items.map(item => db.items.createRow(item))
return (await db.itemsDB.insert().into(db.items).values(rows).exec()) as RSSItem[]
}
export function fetchItems(background = false, sids: number[] = null): AppThunk<Promise<void>> {
@ -244,7 +240,7 @@ const markUnreadDone = (item: RSSItem): ItemActionTypes => ({
export function markRead(item: RSSItem): AppThunk {
return (dispatch) => {
if (!item.hasRead) {
db.idb.update({ _id: item._id }, { $set: { hasRead: true } })
db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.hasRead, true).exec()
dispatch(markReadDone(item))
if (item.serviceRef) {
dispatch(dispatch(getServiceHooks()).markRead?.(item))
@ -262,52 +258,39 @@ export function markAllRead(sids: number[] = null, date: Date = null, before = t
}
const action = dispatch(getServiceHooks()).markAllRead?.(sids, date, before)
if (action) await dispatch(action)
let query = {
source: { $in: sids },
hasRead: false,
} as any
const predicates: lf.Predicate[] = [
db.items.source.in(sids),
db.items.hasRead.eq(false)
]
if (date) {
query.date = before ? { $lte: date } : { $gte: date }
predicates.push(before ? db.items.date.lte(date) : db.items.date.gte(date))
}
const callback = (items: RSSItem[] = null) => {
if (items) {
const counts = new Map<number, number>()
for (let sid of sids) {
counts.set(sid, 0)
}
for (let item of items) {
counts.set(item.source, counts.get(item.source) + 1)
}
dispatch({
type: MARK_ALL_READ,
sids: sids,
counts: sids.map(i => counts.get(i)),
time: date.getTime(),
before: before
})
} else {
dispatch({
type: MARK_ALL_READ,
sids: sids
})
}
if (!(state.page.filter.type & FilterType.ShowRead)) {
dispatch(initFeeds(true))
}
const query = lf.op.and.apply(null, predicates)
await db.itemsDB.update(db.items).set(db.items.hasRead, true).where(query).exec()
if (date) {
dispatch({
type: MARK_ALL_READ,
sids: sids,
time: date.getTime(),
before: before
})
dispatch(updateUnreadCounts())
} else {
dispatch({
type: MARK_ALL_READ,
sids: sids
})
}
if (!(state.page.filter.type & FilterType.ShowRead)) {
dispatch(initFeeds(true))
}
db.idb.update(query, { $set: { hasRead: true } }, { multi: true, returnUpdatedDocs: Boolean(date) },
(err, _, affectedDocuments) => {
if (err) console.log(err)
if (date) callback(affectedDocuments as unknown as RSSItem[])
})
if (!date) callback()
}
}
export function markUnread(item: RSSItem): AppThunk {
return (dispatch) => {
if (item.hasRead) {
db.idb.update({ _id: item._id }, { $set: { hasRead: false } })
db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.hasRead, false).exec()
dispatch(markUnreadDone(item))
if (item.serviceRef) {
dispatch(dispatch(getServiceHooks()).markUnread?.(item))
@ -324,9 +307,9 @@ const toggleStarredDone = (item: RSSItem): ItemActionTypes => ({
export function toggleStarred(item: RSSItem): AppThunk {
return (dispatch) => {
if (item.starred === true) {
db.idb.update({ _id: item._id }, { $unset: { starred: true } })
db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.starred, false).exec()
} else {
db.idb.update({ _id: item._id }, { $set: { starred: true } })
db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.starred, true).exec()
}
dispatch(toggleStarredDone(item))
if (item.serviceRef) {
@ -345,9 +328,9 @@ const toggleHiddenDone = (item: RSSItem): ItemActionTypes => ({
export function toggleHidden(item: RSSItem): AppThunk {
return (dispatch) => {
if (item.hidden === true) {
db.idb.update({ _id: item._id }, { $unset: { hidden: true } })
db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.hidden, true).exec()
} else {
db.idb.update({ _id: item._id }, { $set: { hidden: true } })
db.itemsDB.update(db.items).where(db.items._id.eq(item._id)).set(db.items.hidden, false).exec()
}
dispatch(toggleHiddenDone(item))
}
@ -384,13 +367,11 @@ export function applyItemReduction(item: RSSItem, type: string) {
break
}
case TOGGLE_STARRED: {
if (item.starred === true) delete nextItem.starred
else nextItem.starred = true
nextItem.starred = !item.starred
break
}
case TOGGLE_HIDDEN: {
if (item.hidden === true) delete nextItem.hidden
else nextItem.hidden = true
item.hidden = !item.hidden
break
}
}
@ -461,11 +442,7 @@ export function itemReducer(
if (item.hasOwnProperty("serviceRef")) {
const nextItem = { ...item }
nextItem.hasRead = !unreadSet.has(nextItem.serviceRef as number)
if (starredSet.has(item.serviceRef as number)) {
nextItem.starred = true
} else {
delete nextItem.starred
}
nextItem.starred = starredSet.has(item.serviceRef as number)
nextState[id] = nextItem
}
}

24
src/scripts/models/rule.ts
View File

@ -31,32 +31,16 @@ type ActionTransformType = {
}
const actionTransform: ActionTransformType = {
[ItemAction.Read]: (i, f) => {
if (f) {
i.hasRead = true
} else {
i.hasRead = false
}
i.hasRead = f
},
[ItemAction.Star]: (i, f) => {
if (f) {
i.starred = true
} else if (i.starred) {
delete i.starred
}
i.starred = f
},
[ItemAction.Hide]: (i, f) => {
if (f) {
i.hidden = true
} else if (i.hidden) {
delete i.hidden
}
i.hidden = f
},
[ItemAction.Notify]: (i, f) => {
if (f) {
i.notify = true
} else if (i.notify) {
delete i.notify
}
i.notify = f
},
}

64
src/scripts/models/service.ts
View File

@ -1,4 +1,5 @@
import * as db from "../db"
import lf from "lovefield"
import { SyncService, ServiceConfigs } from "../../schema-types"
import { AppThunk, ActionStatus } from "../utils"
import { RSSItem, insertItems, fetchItemsSuccess } from "./item"
@ -104,12 +105,8 @@ function updateSources(hook: ServiceHooks["updateSources"]): AppThunk<Promise<vo
doc.serviceRef = s.serviceRef
doc.unreadCount = 0
await dispatch(updateSource(doc))
await new Promise((resolve, reject) => {
db.idb.remove({ source: doc.sid }, { multi: true }, (err) => {
if (err) reject(err)
else resolve(doc)
})
})
await db.itemsDB.delete().from(db.items).where(db.items.source.eq(doc.sid)).exec()
return doc
} else {
return docs[0]
}
@ -141,38 +138,29 @@ function syncItems(hook: ServiceHooks["syncItems"]): AppThunk<Promise<void>> {
return async (dispatch, getState) => {
const state = getState()
const [unreadRefs, starredRefs] = await dispatch(hook())
const promises = new Array<Promise<number>>()
promises.push(new Promise((resolve) => {
db.idb.update({
serviceRef: { $exists: true, $in: unreadRefs },
hasRead: true
}, { $set: { hasRead: false } }, { multi: true }, (_, num) => resolve(num))
}))
promises.push(new Promise((resolve) => {
db.idb.update({
serviceRef: { $exists: true, $nin: unreadRefs },
hasRead: false
}, { $set: { hasRead: true } }, { multi: true }, (_, num) => resolve(num))
}))
promises.push(new Promise((resolve) => {
db.idb.update({
serviceRef: { $exists: true, $in: starredRefs },
starred: { $exists: false }
}, { $set: { starred: true } }, { multi: true }, (_, num) => resolve(num))
}))
promises.push(new Promise((resolve) => {
db.idb.update({
serviceRef: { $exists: true, $nin: starredRefs },
starred: true
}, { $unset: { starred: true } }, { multi: true }, (_, num) => resolve(num))
}))
const affected = (await Promise.all(promises)).reduce((a, b) => a + b, 0)
if (affected > 0) {
dispatch(syncLocalItems(unreadRefs, starredRefs))
if (!(state.page.filter.type & FilterType.ShowRead) || !(state.page.filter.type & FilterType.ShowNotStarred)) {
dispatch(initFeeds(true))
}
await dispatch(updateUnreadCounts())
const promises = [
db.itemsDB.update(db.items).set(db.items.hasRead, false).where(lf.op.and(
db.items.serviceRef.in(unreadRefs),
db.items.hasRead.eq(true)
)).exec(),
db.itemsDB.update(db.items).set(db.items.hasRead, true).where(lf.op.and(
lf.op.not(db.items.serviceRef.in(unreadRefs)),
db.items.hasRead.eq(false)
)).exec(),
db.itemsDB.update(db.items).set(db.items.starred, true).where(lf.op.and(
db.items.serviceRef.in(starredRefs),
db.items.starred.eq(false)
)).exec(),
db.itemsDB.update(db.items).set(db.items.starred, false).where(lf.op.and(
lf.op.not(db.items.serviceRef.in(starredRefs)),
db.items.hasRead.eq(true)
)).exec(),
]
await Promise.all(promises)
await dispatch(updateUnreadCounts())
dispatch(syncLocalItems(unreadRefs, starredRefs))
if (!(state.page.filter.type & FilterType.ShowRead) || !(state.page.filter.type & FilterType.ShowNotStarred)) {
dispatch(initFeeds(true))
}
}
}

32
src/scripts/models/services/feedbin.ts
View File

@ -1,5 +1,6 @@
import intl from "react-intl-universal"
import * as db from "../../db"
import lf from "lovefield"
import { ServiceHooks } from "../service"
import { ServiceConfigs, SyncService } from "../../../schema-types"
import { createSourceGroup } from "../group"
@ -145,9 +146,11 @@ export const feedbinServiceHooks: ServiceHooks = {
snippet: dom.documentElement.textContent.trim(),
creator: i.author,
hasRead: !unread.has(i.id),
starred: starred.has(i.id),
hidden: false,
notify: false,
serviceRef: i.id,
} as RSSItem
if (starred.has(i.id)) item.starred = true
if (i.images && i.images.original_url) {
item.thumb = i.images.original_url
} else {
@ -171,26 +174,21 @@ export const feedbinServiceHooks: ServiceHooks = {
}
},
markAllRead: (sids, date, before) => (_, getState) => new Promise(resolve => {
markAllRead: (sids, date, before) => async (_, getState) => {
const state = getState()
const configs = state.service as FeedbinConfigs
const query: any = {
source: { $in: sids },
hasRead: false,
serviceRef: { $exists: true }
}
const predicates: lf.Predicate[] = [
db.items.source.in(sids),
db.items.hasRead.eq(false),
db.items.serviceRef.isNotNull()
]
if (date) {
query.date = before ? { $lte: date } : { $gte: date }
predicates.push(before ? db.items.date.lte(date) : db.items.date.gte(date))
}
// @ts-ignore
db.idb.find(query, { serviceRef: 1 }, (err, docs) => {
resolve()
if (!err) {
const refs = docs.map(i => i.serviceRef as number)
markItems(configs, "unread", "DELETE", refs)
}
})
}),
const rows = await db.itemsDB.select(db.items.serviceRef).where(lf.op.and.apply(null, predicates)).exec()
const refs = rows.map(row => row["serviceRef"]) as number[]
markItems(configs, "unread", "DELETE", refs)
},
markRead: (item: RSSItem) => async (_, getState) => {
await markItems(getState().service as FeedbinConfigs, "unread", "DELETE", [item.serviceRef as number])

4
src/scripts/models/services/fever.ts
View File

@ -122,9 +122,11 @@ export const feverServiceHooks: ServiceHooks = {
snippet: htmlDecode(i.html).trim(),
creator: i.author,
hasRead: Boolean(i.is_read),
starred: Boolean(i.is_saved),
hidden: false,
notify: false,
serviceRef: typeof i.id === "string" ? parseInt(i.id) : i.id,
} as RSSItem
if (i.is_saved) item.starred = true
// Try to get the thumbnail of the item
let dom = domParser.parseFromString(item.content, "text/html")
let baseEl = dom.createElement('base')

92
src/scripts/models/source.ts
View File

@ -1,6 +1,7 @@
import Parser from "@yang991178/rss-parser"
import intl from "react-intl-universal"
import * as db from "../db"
import lf from "lovefield"
import { fetchFavicon, ActionStatus, AppThunk, parseRSS } from "../utils"
import { RSSItem, insertItems, ItemActionTypes, FETCH_ITEMS, MARK_READ, MARK_UNREAD, MARK_ALL_READ } from "./item"
import { saveSettings } from "./app"
@ -39,26 +40,20 @@ export class RSSSource {
return feed
}
private static checkItem(source: RSSSource, item: Parser.Item): Promise<RSSItem> {
return new Promise<RSSItem>((resolve, reject) => {
let i = new RSSItem(item, source)
db.idb.findOne({
source: i.source,
title: i.title,
date: i.date
},
(err, doc) => {
if (err) {
reject(err)
} else if (doc === null) {
RSSItem.parseContent(i, item)
if (source.rules) SourceRule.applyAll(source.rules, i)
resolve(i)
} else {
resolve(null)
}
})
})
private static async checkItem(source: RSSSource, item: Parser.Item): Promise<RSSItem> {
let i = new RSSItem(item, source)
const items = (await db.itemsDB.select().from(db.items).where(lf.op.and(
db.items.source.eq(i.source),
db.items.title.eq(i.title),
db.items.date.eq(i.date)
)).limit(1).exec()) as RSSItem[]
if (items.length === 0) {
RSSItem.parseContent(i, item)
if (source.rules) SourceRule.applyAll(source.rules, i)
return i
} else {
return null
}
}
static checkItems(source: RSSSource, items: Parser.Item[]): Promise<RSSItem[]> {
@ -138,17 +133,14 @@ export function initSourcesFailure(err): SourceActionTypes {
}
}
function unreadCount(source: RSSSource): Promise<RSSSource> {
return new Promise<RSSSource>((resolve, reject) => {
db.idb.count({ source: source.sid, hasRead: false }, (err, n) => {
if (err) {
reject(err)
} else {
source.unreadCount = n
resolve(source)
}
})
})
async function unreadCount(source: RSSSource): Promise<RSSSource> {
source.unreadCount = (await db.itemsDB.select(
lf.fn.count(db.items._id)
).from(db.items).where(lf.op.and(
db.items.source.eq(source.sid),
db.items.hasRead.eq(false)
)).exec())[0]["COUNT(_id)"]
return source
}
export function updateUnreadCounts(): AppThunk<Promise<void>> {
@ -268,30 +260,18 @@ export function deleteSourceDone(source: RSSSource): SourceActionTypes {
}
export function deleteSource(source: RSSSource, batch = false): AppThunk<Promise<void>> {
return (dispatch, getState) => {
return new Promise((resolve) => {
return async (dispatch, getState) => {
if (!batch) dispatch(saveSettings())
try {
await db.itemsDB.delete().from(db.items).where(db.items.source.eq(source.sid)).exec()
await db.sourcesDB.delete().from(db.sources).where(db.sources.sid.eq(source.sid)).exec()
dispatch(deleteSourceDone(source))
window.settings.saveGroups(getState().groups)
} catch (err) {
console.log(err)
} finally {
if (!batch) dispatch(saveSettings())
db.idb.remove({ source: source.sid }, { multi: true }, (err) => {
if (err) {
console.log(err)
if (!batch) dispatch(saveSettings())
resolve()
} else {
db.sourcesDB.delete().from(db.sources).where(
db.sources.sid.eq(source.sid)
).exec().then(() => {
dispatch(deleteSourceDone(source))
window.settings.saveGroups(getState().groups)
if (!batch) dispatch(saveSettings())
resolve()
}).catch(err => {
console.log(err)
if (!batch) dispatch(saveSettings())
resolve()
})
}
})
})
}
}
}
@ -398,9 +378,7 @@ export function sourceReducer(
action.sids.map((sid, i) => {
nextState[sid] = {
...state[sid],
unreadCount: action.counts
? state[sid].unreadCount - action.counts[i]
: 0
unreadCount: action.time ? state[sid].unreadCount : 0
}
})
return nextState