From c1d2f4be136ee2d26f254c318ab90286341abb84 Mon Sep 17 00:00:00 2001 From: Mat <4396128+yawks@users.noreply.github.com> Date: Tue, 4 Oct 2022 13:05:21 +0200 Subject: [PATCH] add nextcloud service --- src/components/settings/service.tsx | 9 + .../settings/services/nextcloud.tsx | 253 +++++++++++++++ src/schema-types.ts | 1 + src/scripts/models/service.ts | 3 + src/scripts/models/services/nextcloud.ts | 305 ++++++++++++++++++ 5 files changed, 571 insertions(+) create mode 100644 src/components/settings/services/nextcloud.tsx create mode 100644 src/scripts/models/services/nextcloud.ts diff --git a/src/components/settings/service.tsx b/src/components/settings/service.tsx index 59097d0..7ff3d85 100644 --- a/src/components/settings/service.tsx +++ b/src/components/settings/service.tsx @@ -6,6 +6,7 @@ import FeverConfigsTab from "./services/fever" import FeedbinConfigsTab from "./services/feedbin" import GReaderConfigsTab from "./services/greader" import InoreaderConfigsTab from "./services/inoreader" +import NextcloudConfigsTab from "./services/nextcloud" type ServiceTabProps = { configs: ServiceConfigs @@ -41,6 +42,7 @@ export class ServiceTab extends React.Component< { key: SyncService.Feedbin, text: "Feedbin" }, { key: SyncService.GReader, text: "Google Reader API (Beta)" }, { key: SyncService.Inoreader, text: "Inoreader" }, + { key: SyncService.Nextcloud, text: "Nextcloud news API" }, { key: -1, text: intl.get("service.suggest") }, ] @@ -88,6 +90,13 @@ export class ServiceTab extends React.Component< exit={this.exitConfigsTab} /> ) + case SyncService.Nextcloud: + return ( + + ) default: return null } diff --git a/src/components/settings/services/nextcloud.tsx b/src/components/settings/services/nextcloud.tsx new file mode 100644 index 0000000..35386fa --- /dev/null +++ b/src/components/settings/services/nextcloud.tsx @@ -0,0 +1,253 @@ +import * as React from "react" +import intl from "react-intl-universal" +import { ServiceConfigsTabProps } from "../service" +import { NextcloudConfigs } from "../../../scripts/models/services/nextcloud" +import { SyncService } from "../../../schema-types" +import { + Stack, + Icon, + Label, + TextField, + PrimaryButton, + DefaultButton, + Checkbox, + MessageBar, + MessageBarType, + Dropdown, + IDropdownOption, +} from "@fluentui/react" +import DangerButton from "../../utils/danger-button" +import { urlTest } from "../../../scripts/utils" +import LiteExporter from "./lite-exporter" + +type NextcloudConfigsTabState = { + existing: boolean + endpoint: string + username: string + password: string + fetchLimit: number + importGroups: boolean +} + +class NextcloudConfigsTab extends React.Component< + ServiceConfigsTabProps, + NextcloudConfigsTabState +> { + constructor(props: ServiceConfigsTabProps) { + super(props) + const configs = props.configs as NextcloudConfigs + this.state = { + existing: configs.type === SyncService.Nextcloud, + endpoint: configs.endpoint || "https://yawks.net/nextcloud/", + username: configs.username || "", + password: "", + fetchLimit: configs.fetchLimit || 250, + importGroups: true, + } + } + + fetchLimitOptions = (): IDropdownOption[] => [ + { key: 250, text: intl.get("service.fetchLimitNum", { count: 250 }) }, + { key: 500, text: intl.get("service.fetchLimitNum", { count: 500 }) }, + { key: 750, text: intl.get("service.fetchLimitNum", { count: 750 }) }, + { key: 1000, text: intl.get("service.fetchLimitNum", { count: 1000 }) }, + { key: 1500, text: intl.get("service.fetchLimitNum", { count: 1500 }) }, + { + key: Number.MAX_SAFE_INTEGER, + text: intl.get("service.fetchUnlimited"), + }, + ] + onFetchLimitOptionChange = (_, option: IDropdownOption) => { + this.setState({ fetchLimit: option.key as number }) + } + + handleInputChange = event => { + const name: string = event.target.name + // @ts-expect-error + this.setState({ [name]: event.target.value }) + } + + checkNotEmpty = (v: string) => { + return !this.state.existing && v.length == 0 + ? intl.get("emptyField") + : "" + } + + validateForm = () => { + return ( + urlTest(this.state.endpoint.trim()) && + (this.state.existing || + (this.state.username && this.state.password)) + ) + } + + save = async () => { + let configs: NextcloudConfigs + if (this.state.existing) { + configs = { + ...this.props.configs, + endpoint: this.state.endpoint, + fetchLimit: this.state.fetchLimit, + } as NextcloudConfigs + if (this.state.password) configs.password = this.state.password + } else { + configs = { + type: SyncService.Nextcloud, + endpoint: this.state.endpoint + "index.php/apps/news/api/v1-3", + username: this.state.username, + password: this.state.password, + fetchLimit: this.state.fetchLimit, + } + if (this.state.importGroups) configs.importGroups = true + } + this.props.blockActions() + const valid = await this.props.authenticate(configs) + if (valid) { + this.props.save(configs) + this.setState({ existing: true }) + this.props.sync() + } else { + this.props.blockActions() + window.utils.showErrorBox( + intl.get("service.failure"), + intl.get("service.failureHint") + ) + } + } + + remove = async () => { + this.props.exit() + await this.props.remove() + } + + render() { + return ( + <> + {!this.state.existing && ( + + {intl.get("service.overwriteWarning")} + + )} + + + + + + + + + + + + urlTest(v.trim()) + ? "" + : intl.get("sources.badUrl") + } + validateOnLoad={false} + name="endpoint" + value={this.state.endpoint} + onChange={this.handleInputChange} + /> + + + + + + + + + + + + + + + + + + + + + + + + + + + {!this.state.existing && ( + + this.setState({ importGroups: c }) + } + /> + )} + + + + + + {this.state.existing ? ( + + ) : ( + + )} + + + {this.state.existing && ( + + )} + + + ) + } +} + +export default NextcloudConfigsTab diff --git a/src/schema-types.ts b/src/schema-types.ts index 45dcc77..6b05811 100644 --- a/src/schema-types.ts +++ b/src/schema-types.ts @@ -59,6 +59,7 @@ export const enum SyncService { Feedbin, GReader, Inoreader, + Nextcloud } export interface ServiceConfigs { type: SyncService diff --git a/src/scripts/models/service.ts b/src/scripts/models/service.ts index cc62d57..51c3987 100644 --- a/src/scripts/models/service.ts +++ b/src/scripts/models/service.ts @@ -18,6 +18,7 @@ import { createSourceGroup, addSourceToGroup } from "./group" import { feverServiceHooks } from "./services/fever" import { feedbinServiceHooks } from "./services/feedbin" import { gReaderServiceHooks } from "./services/greader" +import { nextcloudServiceHooks } from "./services/nextcloud" export interface ServiceHooks { authenticate?: (configs: ServiceConfigs) => Promise @@ -45,6 +46,8 @@ export function getServiceHooksFromType(type: SyncService): ServiceHooks { case SyncService.GReader: case SyncService.Inoreader: return gReaderServiceHooks + case SyncService.Nextcloud: + return nextcloudServiceHooks default: return {} } diff --git a/src/scripts/models/services/nextcloud.ts b/src/scripts/models/services/nextcloud.ts new file mode 100644 index 0000000..86ff545 --- /dev/null +++ b/src/scripts/models/services/nextcloud.ts @@ -0,0 +1,305 @@ +import intl from "react-intl-universal" +import * as db from "../../db" +import lf from "lovefield" +import { ServiceHooks } from "../service" +import { ServiceConfigs, SyncService } from "../../../schema-types" +import { createSourceGroup } from "../group" +import { RSSSource } from "../source" +import { domParser } from "../../utils" +import { RSSItem } from "../item" +import { SourceRule } from "../rule" + +export interface NextcloudConfigs extends ServiceConfigs { + type: SyncService.Nextcloud + endpoint: string + username: string + password: string + fetchLimit: number + lastModified?: number + lastId?: number +} + +async function fetchAPI(configs: NextcloudConfigs, params: string) { + const headers = new Headers() + headers.set( + "Authorization", + "Basic " + btoa(configs.username + ":" + configs.password) + ) + return await fetch(configs.endpoint + params, { headers: headers }) +} + +async function markItems( + configs: NextcloudConfigs, + type: string, + method: string, + refs: number[] +) { + const headers = new Headers() + headers.set( + "Authorization", + "Basic " + btoa(configs.username + ":" + configs.password) + ) + headers.set("Content-Type", "application/json; charset=utf-8") + const promises = new Array>() + while (refs.length > 0) { + const batch = new Array() + while (batch.length < 1000 && refs.length > 0) { + batch.push(refs.pop()) + } + const bodyObject: any = {} + bodyObject["itemIds"] = batch + promises.push( + fetch(configs.endpoint + "/items/" + type + "/multiple", { + method: method, + headers: headers, + body: JSON.stringify(bodyObject), + }) + ) + } + return await Promise.all(promises) +} + +const APIError = () => new Error(intl.get("service.failure")) + +export const nextcloudServiceHooks: ServiceHooks = { + authenticate: async (configs: NextcloudConfigs) => { + try { + const result = await fetchAPI(configs, "/version") + return result.status === 200 + } catch { + return false + } + }, + + updateSources: () => async (dispatch, getState) => { + const configs = getState().service as NextcloudConfigs + const response = await fetchAPI(configs, "/feeds") + if (response.status !== 200) throw APIError() + const feeds = await response.json() + let groupsMap: Map + let groupsByTagId: Map = new Map() + if (configs.importGroups) { + const foldersResponse = await fetchAPI(configs, "/folders") + if (foldersResponse.status !== 200) throw APIError() + const folders = await foldersResponse.json() + const foldersSet = new Set() + groupsMap = new Map() + for (let folder of folders.folders) { + const title = folder.name.trim() + if (!foldersSet.has(title)) { + foldersSet.add(title) + dispatch(createSourceGroup(title)) + } + groupsByTagId.set(String(folder.id), title) + } + } + const sources = feeds.feeds.map(s => { + const source = new RSSSource(s.url, s.title) + source.iconurl = s.faviconLink + source.serviceRef = String(s.id) + if (s.folderId && groupsByTagId.has(String(s.folderId))) { + groupsMap.set(String(s.id), groupsByTagId.get(String(s.folderId))) + } + return source + }) + return [sources, groupsMap] + }, + + syncItems: () => async (_, getState) => { + const configs = getState().service as NextcloudConfigs + const [unreadResponse, starredResponse]= await Promise.all([ + fetchAPI(configs, "/items?getRead=false&type=3&batchSize=-1"), + fetchAPI(configs, "/items?getRead=true&type=2&batchSize=-1"), + ]) + if (unreadResponse.status !== 200 || starredResponse.status !== 200) + throw APIError() + const unread = await unreadResponse.json() + const starred = await starredResponse.json() + return [ + new Set(unread.items.map(i => String(i.id))), + new Set(starred.items.map(i => String(i.id))), + ] + }, + + fetchItems: () => async (_, getState) => { + const state = getState() + const configs = state.service as NextcloudConfigs + let items = new Array() + configs.lastModified = configs.lastModified || 0 + configs.lastId = configs.lastId || 0 + let lastFetched: any + + if (!configs.lastModified || configs.lastModified == 0) { + //first sync + let min = Number.MAX_SAFE_INTEGER + do { + try { + const response = await fetchAPI( + configs, + "/items?getRead=true&type=3&batchSize=125&offset=" + min + ) + if (response.status !== 200) throw APIError() + lastFetched = await response.json() + items = [ ...items, ...lastFetched.items] + min = lastFetched.items.reduce((m, n) => Math.min(m, n.id), min) + } catch (error) { + console.log(error) + break + } + } while ( + lastFetched.items && + lastFetched.items.length >= 125 && + items.length < configs.fetchLimit + ) + } else { + //incremental sync + const response = await fetchAPI( + configs, + "/items/updated?lastModified="+configs.lastModified+"&type=3" + ) + if (response.status !== 200) throw APIError() + lastFetched = (await response.json()).items + items.push( + ...lastFetched.filter( + i => i.id > configs.lastId + ) + ) + + } + const previousLastModified = configs.lastModified + configs.lastModified = items.reduce( + (m, n) => Math.max(m, n.lastModified), + configs.lastModified + ) + configs.lastId = items.reduce( + (m, n) => Math.max(m, n.id), + configs.lastId + ) + console.log("last modified from "+ previousLastModified + " to " + configs.lastModified) + configs.lastModified++ //+1 to avoid fetching articles with same lastModified next time + if (items.length > 0) { + const fidMap = new Map() + for (let source of Object.values(state.sources)) { + if (source.serviceRef) { + fidMap.set(source.serviceRef, source) + } + } + + const parsedItems = new Array() + items.forEach(i => { + if (i.body === null || i.url === null) return + const unreadItem = i.unread + const starredItem = i.starred + const source = fidMap.get(String(i.feedId)) + const dom = domParser.parseFromString(i.body, "text/html") + const item = { + source: source.sid, + title: i.title, + link: i.url, + date: new Date(i.pubDate*1000), + fetchedDate: new Date(i.pubDate*1000), + content: i.body, + snippet: dom.documentElement.textContent.trim(), + creator: i.author, + hasRead: !i.unread, + starred: i.starred, + hidden: false, + notify: false, + serviceRef: String(i.id), + } as RSSItem + if (i.enclosureLink ) { + item.thumb = i.enclosureLink + } else { + let baseEl = dom.createElement("base") + baseEl.setAttribute( + "href", + item.link.split("/").slice(0, 3).join("/") + ) + dom.head.append(baseEl) + let img = dom.querySelector("img") + if (img && img.src) item.thumb = img.src + } + // Apply rules and sync back to the service + if (source.rules) SourceRule.applyAll(source.rules, item) + if (unreadItem && item.hasRead) + markItems( + configs, + item.hasRead ? "read" : "unread", + "POST", + [i.id] + ) + if (starredItem !== Boolean(item.starred)) + markItems( + configs, + item.starred ? "star" : "unstar", + "POST", + [i.id] + ) + + parsedItems.push(item) + }) + return [parsedItems, configs] + } else { + return [[], configs] + } + }, + + markAllRead: (sids, date, before) => async (_, getState) => { + const state = getState() + const configs = state.service as NextcloudConfigs + const predicates: lf.Predicate[] = [ + db.items.source.in(sids), + db.items.hasRead.eq(false), + db.items.serviceRef.isNotNull(), + ] + if (date) { + predicates.push( + before ? db.items.date.lte(date) : db.items.date.gte(date) + ) + } + const query = lf.op.and.apply(null, predicates) + const rows = await db.itemsDB + .select(db.items.serviceRef) + .from(db.items) + .where(query) + .exec() + const refs = rows.map(row => parseInt(row["serviceRef"])) + markItems(configs, "unread", "POST", refs) + }, + + markRead: (item: RSSItem) => async (_, getState) => { + await markItems( + getState().service as NextcloudConfigs, + "read", + "POST", + [parseInt(item.serviceRef)] + ) + }, + + markUnread: (item: RSSItem) => async (_, getState) => { + await markItems( + getState().service as NextcloudConfigs, + "unread", + "POST", + [parseInt(item.serviceRef)] + ) + }, + + star: (item: RSSItem) => async (_, getState) => { + await markItems( + getState().service as NextcloudConfigs, + "star", + "POST", + [parseInt(item.serviceRef)] + ) + }, + + unstar: (item: RSSItem) => async (_, getState) => { + await markItems( + getState().service as NextcloudConfigs, + "unstar", + "POST", + [parseInt(item.serviceRef)] + ) + }, +}