From 324c8c32e97611af37e763654946cd92d586cd71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=B5=A9=E8=BF=9C?= Date: Fri, 25 Dec 2020 20:47:21 +0800 Subject: [PATCH] add support for google reader api --- src/components/settings/service.tsx | 7 + src/components/settings/services/fever.tsx | 1 - src/components/settings/services/greader.tsx | 180 ++++++++++++ .../settings/services/inoreader.tsx | 187 ++++++++++++ src/containers/settings/service-container.tsx | 9 + src/schema-types.ts | 2 +- src/scripts/models/service.ts | 17 +- src/scripts/models/services/greader.ts | 278 ++++++++++++++++++ 8 files changed, 678 insertions(+), 3 deletions(-) create mode 100644 src/components/settings/services/greader.tsx create mode 100644 src/components/settings/services/inoreader.tsx create mode 100644 src/scripts/models/services/greader.ts diff --git a/src/components/settings/service.tsx b/src/components/settings/service.tsx index 2d220c3..473730f 100644 --- a/src/components/settings/service.tsx +++ b/src/components/settings/service.tsx @@ -4,6 +4,8 @@ import { ServiceConfigs, SyncService } from "../../schema-types" import { Stack, Icon, Link, Dropdown, IDropdownOption } from "@fluentui/react" import FeverConfigsTab from "./services/fever" import FeedbinConfigsTab from "./services/feedbin" +import GReaderConfigsTab from "./services/greader" +import InoreaderConfigsTab from "./services/inoreader" type ServiceTabProps = { configs: ServiceConfigs @@ -12,6 +14,7 @@ type ServiceTabProps = { remove: () => Promise blockActions: () => void authenticate: (configs: ServiceConfigs) => Promise + reauthenticate: (configs: ServiceConfigs) => Promise } export type ServiceConfigsTabProps = ServiceTabProps & { @@ -33,6 +36,8 @@ export class ServiceTab extends React.Component [ { key: SyncService.Fever, text: "Fever API" }, { key: SyncService.Feedbin, text: "Feedbin" }, + { key: SyncService.GReader, text: "Google Reader API (Beta)" }, + { key: SyncService.Inoreader, text: "Inoreader" }, { key: -1, text: intl.get("service.suggest") }, ] @@ -52,6 +57,8 @@ export class ServiceTab extends React.Component case SyncService.Feedbin: return + case SyncService.GReader: return + case SyncService.Inoreader: return default: return null } } diff --git a/src/components/settings/services/fever.tsx b/src/components/settings/services/fever.tsx index a9d00f7..9cc9144 100644 --- a/src/components/settings/services/fever.tsx +++ b/src/components/settings/services/fever.tsx @@ -7,7 +7,6 @@ import { SyncService } from "../../../schema-types" import { Stack, Icon, Label, TextField, PrimaryButton, DefaultButton, Checkbox, MessageBar, MessageBarType, Dropdown, IDropdownOption } from "@fluentui/react" import DangerButton from "../../utils/danger-button" import { urlTest } from "../../../scripts/utils" -import { exists } from "fs" type FeverConfigsTabState = { existing: boolean diff --git a/src/components/settings/services/greader.tsx b/src/components/settings/services/greader.tsx new file mode 100644 index 0000000..683cbdf --- /dev/null +++ b/src/components/settings/services/greader.tsx @@ -0,0 +1,180 @@ +import * as React from "react" +import intl from "react-intl-universal" +import { ServiceConfigsTabProps } from "../service" +import { GReaderConfigs } from "../../../scripts/models/services/greader" +import { SyncService } from "../../../schema-types" +import { Stack, Icon, Label, TextField, PrimaryButton, DefaultButton, Checkbox, MessageBar, MessageBarType, Dropdown, IDropdownOption } from "@fluentui/react" +import DangerButton from "../../utils/danger-button" +import { urlTest } from "../../../scripts/utils" + +type GReaderConfigsTabState = { + existing: boolean + endpoint: string + username: string + password: string + fetchLimit: number + importGroups: boolean +} + +class GReaderConfigsTab extends React.Component { + constructor(props: ServiceConfigsTabProps) { + super(props) + const configs = props.configs as GReaderConfigs + this.state = { + existing: configs.type === SyncService.GReader, + endpoint: configs.endpoint || "", + username: configs.username || "", + password: "", + fetchLimit: configs.fetchLimit || 250, + importGroups: true, + } + } + + fetchLimitOptions = (): IDropdownOption[] => [ + { key: 250, text: intl.get("service.fetchLimitNum", { count: 250 }) }, + { key: 500, text: intl.get("service.fetchLimitNum", { count: 500 }) }, + { key: 750, text: intl.get("service.fetchLimitNum", { count: 750 }) }, + { key: 1000, text: intl.get("service.fetchLimitNum", { count: 1000 }) }, + { key: 1500, text: intl.get("service.fetchLimitNum", { count: 1500 }) }, + { key: Number.MAX_SAFE_INTEGER, text: intl.get("service.fetchUnlimited") }, + ] + onFetchLimitOptionChange = (_, option: IDropdownOption) => { + this.setState({ fetchLimit: option.key as number }) + } + + handleInputChange = (event) => { + const name: string = event.target.name + // @ts-expect-error + this.setState({[name]: event.target.value}) + } + + checkNotEmpty = (v: string) => { + return (!this.state.existing && v.length == 0) ? intl.get("emptyField") : "" + } + + validateForm = () => { + return urlTest(this.state.endpoint.trim()) && (this.state.existing || (this.state.username && this.state.password)) + } + + save = async () => { + let configs: GReaderConfigs + if (this.state.existing) { + configs = { + ...this.props.configs, + endpoint: this.state.endpoint, + fetchLimit: this.state.fetchLimit + } as GReaderConfigs + } else { + configs = { + type: SyncService.GReader, + endpoint: this.state.endpoint, + username: this.state.username, + password: this.state.password, + fetchLimit: this.state.fetchLimit, + useInt64: !this.state.endpoint.endsWith("theoldreader.com") + } + if (this.state.importGroups) configs.importGroups = true + } + this.props.blockActions() + configs = await this.props.reauthenticate(configs) as GReaderConfigs + const valid = await this.props.authenticate(configs) + if (valid) { + this.props.save(configs) + this.setState({ existing: true }) + this.props.sync() + } else { + this.props.blockActions() + window.utils.showErrorBox(intl.get("service.failure"), intl.get("service.failureHint")) + } + } + + remove = async () => { + this.props.exit() + await this.props.remove() + } + + render() { + return <> + {!this.state.existing && ( + {intl.get("service.overwriteWarning")} + )} + + + + + + + + + urlTest(v.trim()) ? "" : intl.get("sources.badUrl")} + validateOnLoad={false} + name="endpoint" + value={this.state.endpoint} + onChange={this.handleInputChange} /> + + + + + + + + + + + + + + + + + + + + + + + + + + + {!this.state.existing && this.setState({importGroups: c})} />} + + + + + + {this.state.existing + ? + : + } + + + + + } +} + +export default GReaderConfigsTab \ No newline at end of file diff --git a/src/components/settings/services/inoreader.tsx b/src/components/settings/services/inoreader.tsx new file mode 100644 index 0000000..7858052 --- /dev/null +++ b/src/components/settings/services/inoreader.tsx @@ -0,0 +1,187 @@ +import * as React from "react" +import intl from "react-intl-universal" +import { ServiceConfigsTabProps } from "../service" +import { GReaderConfigs } from "../../../scripts/models/services/greader" +import { SyncService } from "../../../schema-types" +import { Stack, Icon, Label, TextField, PrimaryButton, DefaultButton, Checkbox, MessageBar, MessageBarType, Dropdown, IDropdownOption } from "@fluentui/react" +import DangerButton from "../../utils/danger-button" +import { urlTest } from "../../../scripts/utils" + +type GReaderConfigsTabState = { + existing: boolean + endpoint: string + username: string + password: string + fetchLimit: number + importGroups: boolean +} + +const endpointOptions: IDropdownOption[] = [ + "https://www.inoreader.com", + "https://www.innoreader.com", + "https://jp.inoreader.com" +].map(s => ({ key: s, text: s })) + +class InoreaderConfigsTab extends React.Component { + constructor(props: ServiceConfigsTabProps) { + super(props) + const configs = props.configs as GReaderConfigs + this.state = { + existing: configs.type === SyncService.Inoreader, + endpoint: configs.endpoint || "https://www.inoreader.com", + username: configs.username || "", + password: "", + fetchLimit: configs.fetchLimit || 250, + importGroups: true, + } + } + + fetchLimitOptions = (): IDropdownOption[] => [ + { key: 250, text: intl.get("service.fetchLimitNum", { count: 250 }) }, + { key: 500, text: intl.get("service.fetchLimitNum", { count: 500 }) }, + { key: 750, text: intl.get("service.fetchLimitNum", { count: 750 }) }, + { key: 1000, text: intl.get("service.fetchLimitNum", { count: 1000 }) }, + { key: 1500, text: intl.get("service.fetchLimitNum", { count: 1500 }) }, + { key: Number.MAX_SAFE_INTEGER, text: intl.get("service.fetchUnlimited") }, + ] + onFetchLimitOptionChange = (_, option: IDropdownOption) => { + this.setState({ fetchLimit: option.key as number }) + } + onEndpointChange = (_, option: IDropdownOption) => { + this.setState({ endpoint: option.key as string }) + } + + handleInputChange = (event) => { + const name: string = event.target.name + // @ts-expect-error + this.setState({[name]: event.target.value}) + } + + checkNotEmpty = (v: string) => { + return (!this.state.existing && v.length == 0) ? intl.get("emptyField") : "" + } + + validateForm = () => { + return this.state.existing || (this.state.username && this.state.password) + } + + save = async () => { + let configs: GReaderConfigs + if (this.state.existing) { + configs = { + ...this.props.configs, + endpoint: this.state.endpoint, + fetchLimit: this.state.fetchLimit + } as GReaderConfigs + } else { + configs = { + type: SyncService.Inoreader, + endpoint: this.state.endpoint, + username: this.state.username, + password: this.state.password, + fetchLimit: this.state.fetchLimit, + useInt64: true + } + if (this.state.importGroups) configs.importGroups = true + } + this.props.blockActions() + configs = await this.props.reauthenticate(configs) as GReaderConfigs + const valid = await this.props.authenticate(configs) + if (valid) { + this.props.save(configs) + this.setState({ existing: true }) + this.props.sync() + } else { + this.props.blockActions() + window.utils.showErrorBox(intl.get("service.failure"), intl.get("service.failureHint")) + } + } + + remove = async () => { + this.props.exit() + await this.props.remove() + } + + render() { + return <> + {!this.state.existing && ( + {intl.get("service.overwriteWarning")} + )} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {!this.state.existing && this.setState({importGroups: c})} />} + + + + + + {this.state.existing + ? + : + } + + + + + } +} + +export default InoreaderConfigsTab \ No newline at end of file diff --git a/src/containers/settings/service-container.tsx b/src/containers/settings/service-container.tsx index df5670b..6dd7475 100644 --- a/src/containers/settings/service-container.tsx +++ b/src/containers/settings/service-container.tsx @@ -25,6 +25,15 @@ const mapDispatchToProps = (dispatch: AppDispatch) => ({ const hooks = getServiceHooksFromType(configs.type) if (hooks.authenticate) return await hooks.authenticate(configs) else return true + }, + reauthenticate: async (configs: ServiceConfigs) => { + const hooks = getServiceHooksFromType(configs.type) + try { + if (hooks.reauthenticate) return await hooks.reauthenticate(configs) + } catch (err) { + console.log(err) + return configs + } } }) diff --git a/src/schema-types.ts b/src/schema-types.ts index 88ef709..2c81870 100644 --- a/src/schema-types.ts +++ b/src/schema-types.ts @@ -43,7 +43,7 @@ export const enum ImageCallbackTypes { } export const enum SyncService { - None, Fever, Feedbin + None, Fever, Feedbin, GReader, Inoreader } export interface ServiceConfigs { type: SyncService diff --git a/src/scripts/models/service.ts b/src/scripts/models/service.ts index b7ff848..22aa51c 100644 --- a/src/scripts/models/service.ts +++ b/src/scripts/models/service.ts @@ -6,14 +6,15 @@ import { RSSItem, insertItems, fetchItemsSuccess } from "./item" import { saveSettings, pushNotification } from "./app" import { deleteSource, updateUnreadCounts, RSSSource, insertSource, addSourceSuccess, updateSource, updateFavicon } from "./source" -import { FilterType, initFeeds } from "./feed" import { createSourceGroup, addSourceToGroup } from "./group" import { feverServiceHooks } from "./services/fever" import { feedbinServiceHooks } from "./services/feedbin" +import { gReaderServiceHooks } from "./services/greader" export interface ServiceHooks { authenticate?: (configs: ServiceConfigs) => Promise + reauthenticate?: (configs: ServiceConfigs) => Promise updateSources?: () => AppThunk]>> fetchItems?: () => AppThunk> syncItems?: () => AppThunk, Set]>> @@ -28,6 +29,9 @@ export function getServiceHooksFromType(type: SyncService): ServiceHooks { switch (type) { case SyncService.Fever: return feverServiceHooks case SyncService.Feedbin: return feedbinServiceHooks + case SyncService.GReader: + case SyncService.Inoreader: + return gReaderServiceHooks default: return {} } } @@ -47,6 +51,7 @@ export function syncWithService(background = false): AppThunk> { type: SYNC_SERVICE, status: ActionStatus.Request }) + if (hooks.reauthenticate) await dispatch(reauthenticate(hooks)) await dispatch(updateSources(hooks.updateSources)) await dispatch(syncItems(hooks.syncItems)) await dispatch(fetchItems(hooks.fetchItems, background)) @@ -68,6 +73,16 @@ export function syncWithService(background = false): AppThunk> { } } +function reauthenticate(hooks: ServiceHooks): AppThunk> { + return async (dispatch, getState) => { + let configs = getState().service + if (!(await hooks.authenticate(configs))) { + configs = await hooks.reauthenticate(configs) + dispatch(saveServiceConfigs(configs)) + } + } +} + function updateSources(hook: ServiceHooks["updateSources"]): AppThunk> { return async (dispatch, getState) => { const [sources, groupsMap] = await dispatch(hook()) diff --git a/src/scripts/models/services/greader.ts b/src/scripts/models/services/greader.ts new file mode 100644 index 0000000..d1c42cb --- /dev/null +++ b/src/scripts/models/services/greader.ts @@ -0,0 +1,278 @@ +import intl from "react-intl-universal" +import * as db from "../../db" +import lf from "lovefield" +import { ServiceHooks } from "../service" +import { ServiceConfigs, SyncService } from "../../../schema-types" +import { createSourceGroup } from "../group" +import { RSSSource } from "../source" +import { RSSItem } from "../item" +import { domParser } from "../../utils" +import { SourceRule } from "../rule" + +const ALL_TAG = "user/-/state/com.google/reading-list" +const READ_TAG = "user/-/state/com.google/read" +const STAR_TAG = "user/-/state/com.google/starred" + +export interface GReaderConfigs extends ServiceConfigs { + type: SyncService.GReader | SyncService.Inoreader + endpoint: string + username: string + password: string + fetchLimit: number + lastFetched?: number + lastId?: string + auth?: string + useInt64: boolean // The Old Reader uses ids longer than 64 bits +} + +async function fetchAPI(configs: GReaderConfigs, params: string, method="GET", body:BodyInit=null) { + const headers = new Headers() + if (configs.auth !== null) headers.set("Authorization", configs.auth) + if (configs.type == SyncService.Inoreader) { + headers.set("AppId", "999999298") + headers.set("AppKey", "KPbKYXTfgrKbwmroOeYC7mcW21ZRwF5Y") + } + return await fetch(configs.endpoint + params, { + method: method, + headers: headers, + body: body + }) +} + +async function fetchAll(configs: GReaderConfigs, params: string): Promise> { + let results = new Array() + let fetched: any[] + let continuation: string + do { + let p = params + if (continuation) p += `&c=${continuation}` + const response = await fetchAPI(configs, p) + const parsed = await response.json() + fetched = parsed.itemRefs + if (fetched) { + for (let i of fetched) { + results.push(i.id) + } + } + continuation = parsed.continuation + } while (continuation && fetched && fetched.length >= 1000) + return new Set(results) +} + +async function editTag(configs: GReaderConfigs, ref: string, tag: string, add=true) { + const body = new URLSearchParams(`i=${ref}&${add?"a":"r"}=${tag}`) + return await fetchAPI(configs, "/reader/api/0/edit-tag", "POST", body) +} + +function compactId(longId: string, useInt64: boolean) { + let parts = longId.split("/") + const last = parts[parts.length - 1] + if (!useInt64) return last + let i = BigInt("0x" + last) + return BigInt.asIntN(64, i).toString() +} + +const APIError = () => new Error(intl.get("service.failure")) + +export const gReaderServiceHooks: ServiceHooks = { + authenticate: async (configs: GReaderConfigs) => { + if (configs.auth !== null) { + try { + const result = await fetchAPI(configs, "/reader/api/0/user-info") + return result.status === 200 + } catch { + return false + } + } + }, + + reauthenticate: async (configs: GReaderConfigs): Promise => { + const body = new URLSearchParams() + body.append("Email", configs.username) + body.append("Passwd", configs.password) + const result = await fetchAPI(configs, "/accounts/ClientLogin", "POST", body) + if (result.status === 200) { + const text = await result.text() + const matches = text.match(/Auth=(\S+)/) + if (matches.length > 1) configs.auth = "GoogleLogin auth=" + matches[1] + return configs + } else { + throw APIError() + } + }, + + updateSources: () => async (dispatch, getState) => { + const configs = getState().service as GReaderConfigs + const response = await fetchAPI(configs, "/reader/api/0/subscription/list?output=json") + if (response.status !== 200) throw APIError() + const subscriptions: any[] = (await response.json()).subscriptions + let groupsMap: Map + if (configs.importGroups) { + groupsMap = new Map() + const groupSet = new Set() + for (let s of subscriptions) { + if (s.categories && s.categories.length > 0) { + const group: string = s.categories[0].label + if (!groupSet.has(group)) { + groupSet.add(group) + dispatch(createSourceGroup(group)) + } + groupsMap.set(s.id, group) + } + } + } + const sources = new Array() + subscriptions.forEach(s => { + const source = new RSSSource(s.url || s.htmlUrl, s.title) + source.serviceRef = s.id + // Omit duplicate sources in The Old Reader + if (configs.useInt64 || s.url != "http://blog.theoldreader.com/rss") { + sources.push(source) + } + }) + return [sources, groupsMap] + }, + + syncItems: () => async (_, getState) => { + const configs = getState().service as GReaderConfigs + if (configs.type == SyncService.Inoreader) { + return await Promise.all([ + fetchAll(configs, `/reader/api/0/stream/items/ids?output=json&xt=${READ_TAG}&n=1000`), + fetchAll(configs, `/reader/api/0/stream/items/ids?output=json&it=${STAR_TAG}&n=1000`) + ]) + } else { + return await Promise.all([ + fetchAll(configs, `/reader/api/0/stream/items/ids?output=json&s=${ALL_TAG}&xt=${READ_TAG}&n=1000`), + fetchAll(configs, `/reader/api/0/stream/items/ids?output=json&s=${STAR_TAG}&n=1000`) + ]) + } + }, + + fetchItems: () => async (_, getState) => { + const state = getState() + const configs = state.service as GReaderConfigs + const items = new Array() + let fetchedItems: any[] + let continuation: string + do { + try { + let params = "/reader/api/0/stream/contents?output=json&n=125" + if (configs.lastFetched) params += `&ot=${configs.lastFetched}` + if (continuation) params += `&c=${continuation}` + const response = await fetchAPI(configs, params) + let fetched = await response.json() + fetchedItems = fetched.items + for (let i of fetchedItems) { + i.id = compactId(i.id, configs.useInt64) + if (i.id === configs.lastId) { + break + } else { + items.push(i) + } + } + continuation = fetched.continuation + } catch { + break + } + } while ( + continuation && + fetchedItems && fetchedItems.length >= 125 && + items.length < configs.fetchLimit + ) + if (items.length > 0) { + configs.lastId = items[0].id + const fidMap = new Map() + for (let source of Object.values(state.sources)) { + if (source.serviceRef) { + fidMap.set(source.serviceRef, source) + } + } + const parsedItems = new Array() + items.map(i => { + const source = fidMap.get(i.origin.streamId) + if (source === undefined) return + const dom = domParser.parseFromString(i.summary.content, "text/html") + const item = { + source: source.sid, + title: i.title, + link: i.canonical[0].href, + date: new Date(i.published * 1000), + fetchedDate: new Date(parseInt(i.crawlTimeMsec)), + content: i.summary.content, + snippet: dom.documentElement.textContent.trim(), + creator: i.author, + hasRead: false, + starred: false, + hidden: false, + notify: false, + serviceRef: i.id + } as RSSItem + const baseEl = dom.createElement('base') + baseEl.setAttribute('href', item.link.split("/").slice(0, 3).join("/")) + dom.head.append(baseEl) + let img = dom.querySelector("img") + if (img && img.src) item.thumb = img.src + for (let c of i.categories) { + if (!item.hasRead && c.endsWith("/state/com.google/read")) item.hasRead = true + else if (!item.starred && c.endsWith("/state/com.google/starred")) item.starred = true + } + // Apply rules and sync back to the service + if (source.rules) SourceRule.applyAll(source.rules, item) + // TODO + parsedItems.push(item) + }) + if (parsedItems.length > 0) { + configs.lastFetched = Math.round(parsedItems[0].fetchedDate.getTime() / 1000) + } + return [parsedItems, configs] + } else { + return [[], configs] + } + }, + + markAllRead: (sids, date, before) => async (_, getState) => { + const state = getState() + const configs = state.service as GReaderConfigs + if (date) { + const predicates: lf.Predicate[] = [ + db.items.source.in(sids), + db.items.hasRead.eq(false), + db.items.serviceRef.isNotNull() + ] + if (date) { + predicates.push(before ? db.items.date.lte(date) : db.items.date.gte(date)) + } + const query = lf.op.and.apply(null, predicates) + const rows = await db.itemsDB.select(db.items.serviceRef).from(db.items).where(query).exec() + const refs = rows.map(row => row["serviceRef"]).join("&i=") + if (refs) { + editTag(getState().service as GReaderConfigs, refs, READ_TAG) + } + } else { + const sources = sids.map(sid => state.sources[sid]) + for (let source of sources) { + if (source.serviceRef) { + const body = new URLSearchParams() + body.set("s", source.serviceRef) + fetchAPI(configs, "/reader/api/0/mark-all-as-read", "POST", body) + } + } + } + }, + + markRead: (item: RSSItem) => async (_, getState) => { + await editTag(getState().service as GReaderConfigs, item.serviceRef, READ_TAG) + }, + + markUnread: (item: RSSItem) => async (_, getState) => { + await editTag(getState().service as GReaderConfigs, item.serviceRef, READ_TAG, false) + }, + + star: (item: RSSItem) => async (_, getState) => { + await editTag(getState().service as GReaderConfigs, item.serviceRef, STAR_TAG) + }, + + unstar: (item: RSSItem) => async (_, getState) => { + await editTag(getState().service as GReaderConfigs, item.serviceRef, STAR_TAG, false) + }, +}