add mercury parser

2025-01-29 16:49:31 +01:00 · 2020-08-05 19:03:05 +08:00 · 2020-08-05 19:03:05 +08:00 · 38646b227c
commit 38646b227c
parent 52e0c1b90c
12 changed files with 111 additions and 48 deletions
--- a/README.md
+++ b/README.md
@ -72,6 +72,7 @@ npm run package-win
 - [Redux](https://github.com/reduxjs/redux)
 - [Fluent UI](https://github.com/microsoft/fluentui)
 - [NeDB](https://github.com/louischatriot/nedb)
+- [Mercury Parser](https://github.com/postlight/mercury-parser)

 ### License

--- a/dist/article/article.html
+++ b/dist/article/article.html
@ -3,13 +3,14 @@
 <head>
    <meta charset="UTF-8">
    <meta http-equiv="Content-Security-Policy"
-        content="default-src 'none'; script-src-elem 'sha256-HLvh6tC4kZKt81b6Yi9wjdyXvuwO6InxwRG96ZZjHrw='; img-src http://* https://*; style-src 'self' 'unsafe-inline'; frame-src http://* https://*; media-src http://* https://*">
+        content="default-src 'none'; script-src-elem 'sha256-sLDWrq1tUAO8IyyqmUckFqxbXYfZ2/3TEUmtxH8Unf0=' 'sha256-q3VKKMMe+ucICfT8N3WHLJdQovcvvHc0HbJ5N9uA3+w='; img-src http: https: data:; style-src 'self' 'unsafe-inline'; frame-src http: https:; media-src http: https:; connect-src https: http:">
    <title>Article</title>
    <link rel="stylesheet" href="article.css" />
+    <script integrity="sha256-sLDWrq1tUAO8IyyqmUckFqxbXYfZ2/3TEUmtxH8Unf0=" src="mercury.web.js"></script>
 </head>
 <body>
    <div id="main"></div>
-    <script integrity="sha256-HLvh6tC4kZKt81b6Yi9wjdyXvuwO6InxwRG96ZZjHrw=" src="article.js"></script>
+    <script integrity="sha256-q3VKKMMe+ucICfT8N3WHLJdQovcvvHc0HbJ5N9uA3+w=" src="article.js"></script>
    <!-- Run "cat article.js | openssl dgst -sha256 -binary | openssl enc -base64 -A" for hash -->
 </body>
 </html>
--- a/dist/article/article.js
+++ b/dist/article/article.js
@ -2,18 +2,33 @@ function get(name) {
    if (name = (new RegExp('[?&]' + encodeURIComponent(name) + '=([^&]*)')).exec(location.search))
        return decodeURIComponent(name[1]);
 }
+async function getArticle(url) {
+    let article = get("a")
+    if (get("m") === "1") {
+        return (await Mercury.parse(url, {html: article})).content || ""
+    } else {
+        return article
+    }
+}
 document.documentElement.style.fontSize = get("s") + "px"
-let html = get("h")
-let domParser = new DOMParser()
-let dom = domParser.parseFromString(html, "text/html")
-let baseEl = dom.createElement('base')
-baseEl.setAttribute('href', get("u").split("/").slice(0, 3).join("/"))
-dom.head.append(baseEl)
-for (let e of dom.querySelectorAll("*[src]")) {
-    e.src = e.src
-}
-for (let s of dom.querySelectorAll("script")) {
-    s.parentNode.removeChild(s)
-}
-let main = document.getElementById("main")
-main.innerHTML = dom.body.innerHTML
+let url = get("u")
+getArticle(url).then(article => {
+    let domParser = new DOMParser()
+    let dom = domParser.parseFromString(get("h"), "text/html")
+    dom.getElementsByTagName("article")[0].innerHTML = article
+    let baseEl = dom.createElement('base')
+    baseEl.setAttribute('href', url.split("/").slice(0, 3).join("/"))
+    dom.head.append(baseEl)
+    for (let s of dom.getElementsByTagName("script")) {
+        s.parentNode.removeChild(s)
+    }
+    for (let e of dom.querySelectorAll("*[src]")) {
+        e.src = e.src
+    }
+    for (let e of dom.querySelectorAll("*[href]")) {
+        e.href = e.href
+    }
+    let main = document.getElementById("main")
+    main.innerHTML = dom.body.innerHTML
+})
+
--- a/dist/article/mercury.web.js
+++ b/dist/article/mercury.web.js
--- a/dist/icons/fabric-icons-10-c4ded8e4.woff
+++ b/dist/icons/fabric-icons-10-c4ded8e4.woff
--- a/src/components/article.tsx
+++ b/src/components/article.tsx
@ -27,6 +27,8 @@ type ArticleProps = {
 type ArticleState = {
    fontSize: number
    loadWebpage: boolean
+    loadFull: boolean
+    fullContent: string
    loaded: boolean
    error: boolean
    errorDescription: string
@ -35,11 +37,13 @@ type ArticleState = {
 class Article extends React.Component<ArticleProps, ArticleState> {
    webview: Electron.WebviewTag
    
-    constructor(props) {
+    constructor(props: ArticleProps) {
        super(props)
        this.state = {
            fontSize: this.getFontSize(),
-            loadWebpage: this.props.source.openTarget === SourceOpenTarget.Webpage,
+            loadWebpage: props.source.openTarget === SourceOpenTarget.Webpage,
+            loadFull: props.source.openTarget === SourceOpenTarget.FullContent,
+            fullContent: "",
            loaded: false,
            error: false,
            errorDescription: "",
@ -47,6 +51,7 @@ class Article extends React.Component<ArticleProps, ArticleState> {
        window.utils.addWebviewContextListener(this.contextMenuHandler)
        window.utils.addWebviewKeydownListener(this.keyDownHandler)
        window.utils.addWebviewErrorListener(this.webviewError)
+        if (props.source.openTarget === SourceOpenTarget.FullContent) this.loadFull()
    }

    getFontSize = () => {
@ -87,6 +92,13 @@ class Article extends React.Component<ArticleProps, ArticleState> {
                iconProps: { iconName: this.props.item.hidden ? "View" : "Hide3" },
                onClick: () => { this.props.toggleHidden(this.props.item) }
            },
+            {
+                key: "fontMenu",
+                text: intl.get("article.fontSize"),
+                iconProps: { iconName: "FontSize" },
+                disabled: this.state.loadWebpage,
+                subMenuProps: this.fontMenuProps()
+            },
            {
                key: "divider_1",
                itemType: ContextualMenuItemType.Divider,
@ -117,6 +129,9 @@ class Article extends React.Component<ArticleProps, ArticleState> {
                case "l": case "L":
                    this.toggleWebpage()
                    break
+                case "w": case "W":
+                    this.toggleFull()
+                    break
                default:
                    const keyboardEvent = new KeyboardEvent("keydown", {
                        code: input.code,
@ -145,6 +160,8 @@ class Article extends React.Component<ArticleProps, ArticleState> {
        if (this.webview) {
            this.setState({loaded: false, error: false})
            this.webview.reload()
+        } else if (this.state.loadFull) {
+            this.loadFull()
        }
    }

@ -152,17 +169,23 @@ class Article extends React.Component<ArticleProps, ArticleState> {
        let webview = document.getElementById("article") as Electron.WebviewTag
        if (webview != this.webview) {
            this.webview = webview
-            webview.focus()
-            this.setState({loaded: false, error: false})
-            webview.addEventListener("did-stop-loading", this.webviewLoaded)
-            let card = document.querySelector(`#refocus div[data-iid="${this.props.item._id}"]`) as HTMLElement
-            // @ts-ignore
-            if (card) card.scrollIntoViewIfNeeded()
+            if (webview) {
+                webview.focus()
+                this.setState({loaded: false, error: false})
+                webview.addEventListener("did-stop-loading", this.webviewLoaded)
+                let card = document.querySelector(`#refocus div[data-iid="${this.props.item._id}"]`) as HTMLElement
+                // @ts-ignore
+                if (card) card.scrollIntoViewIfNeeded()
+            }
        }
    }
    componentDidUpdate = (prevProps: ArticleProps) => {
        if (prevProps.item._id != this.props.item._id) {
-            this.setState({loadWebpage: this.props.source.openTarget === SourceOpenTarget.Webpage})
+            this.setState({
+                loadWebpage: this.props.source.openTarget === SourceOpenTarget.Webpage,
+                loadFull: this.props.source.openTarget === SourceOpenTarget.FullContent,
+            })
+            if (this.props.source.openTarget === SourceOpenTarget.FullContent) this.loadFull()
        }
        this.componentDidMount()
    }
@ -174,17 +197,39 @@ class Article extends React.Component<ArticleProps, ArticleState> {

    toggleWebpage = () => {
        if (this.state.loadWebpage) {
-            this.setState({loadWebpage: false})
+            this.setState({ loadWebpage: false })
        } else if (this.props.item.link.startsWith("https://") || this.props.item.link.startsWith("http://")) {
-            this.setState({loadWebpage: true})
+            this.setState({ loadWebpage: true, loadFull: false })
        }
    }

-    articleView = () => "article/article.html?h=" + encodeURIComponent(renderToString(<>
-        <p className="title">{this.props.item.title}</p>
-        <p className="date">{this.props.item.date.toLocaleString(this.props.locale, {hour12: !this.props.locale.startsWith("zh")})}</p>
-        <article dangerouslySetInnerHTML={{__html: this.props.item.content}}></article>
-    </>)) + `&s=${this.state.fontSize}&u=${this.props.item.link}`
+    toggleFull = () => {
+        if (this.state.loadFull) {
+            this.setState({ loadFull: false })
+        } else if (this.props.item.link.startsWith("https://") || this.props.item.link.startsWith("http://")) {
+            this.setState({ loadFull: true, loadWebpage: false })
+            this.loadFull()
+        }
+    }
+    loadFull = async () => {
+        this.setState({ fullContent: "", loaded: false, error: false })
+        try {
+            const html = await (await fetch(this.props.item.link)).text()
+            this.setState({ fullContent: html })
+        } catch {
+            this.setState({ loaded: true, error: true, errorDescription: "MERCURY_PARSER_FAILURE" })
+        }
+    }
+
+    articleView = () => {
+        const a = encodeURIComponent(this.state.loadFull ? this.state.fullContent : this.props.item.content)
+        const h = encodeURIComponent(renderToString(<>
+            <p className="title">{this.props.item.title}</p>
+            <p className="date">{this.props.item.date.toLocaleString(this.props.locale, {hour12: !this.props.locale.startsWith("zh")})}</p>
+            <article></article>
+        </>))
+        return `article/article.html?a=${a}&h=${h}&s=${this.state.fontSize}&u=${this.props.item.link}&m=${this.state.loadFull?1:0}`
+    }
    
    render = () => (
        <FocusZone className="article">
@ -211,11 +256,10 @@ class Article extends React.Component<ArticleProps, ArticleState> {
                        iconProps={{iconName: this.props.item.starred ? "FavoriteStarFill" : "FavoriteStar"}}
                        onClick={() => this.props.toggleStarred(this.props.item)} />
                    <CommandBarButton
-                        title={intl.get("article.fontSize")}
-                        disabled={this.state.loadWebpage}
-                        iconProps={{iconName: "FontSize"}}
-                        menuIconProps={{style: {display: "none"}}}
-                        menuProps={this.fontMenuProps()} />
+                        title={intl.get("article.loadFull")}
+                        className={this.state.loadFull ? "active" : ""}
+                        iconProps={{iconName: "RawSource"}} 
+                        onClick={this.toggleFull} />
                    <CommandBarButton
                        title={intl.get("article.loadWebpage")}
                        className={this.state.loadWebpage ? "active" : ""}
@ -234,13 +278,13 @@ class Article extends React.Component<ArticleProps, ArticleState> {
                        onClick={this.props.dismiss} />
                </Stack>  
            </Stack>
-            <webview 
+            {(!this.state.loadFull || this.state.fullContent) && <webview 
                id="article"
                className={this.state.error ? "error" : ""}
                key={this.props.item._id + (this.state.loadWebpage ? "_" : "")}
                src={this.state.loadWebpage ? this.props.item.link : this.articleView()}
                webpreferences="contextIsolation,disableDialogs,autoplayPolicy=document-user-activation-required"
-                partition={this.state.loadWebpage ? "sandbox" : undefined} />
+                partition={this.state.loadWebpage ? "sandbox" : undefined} />}
            {this.state.error && (
                <Stack className="error-prompt" verticalAlign="center" horizontalAlign="center" tokens={{childrenGap: 12}}>
                    <Icon iconName="HeartBroken" style={{fontSize: 32}} />
--- a/src/components/cards/card.tsx
+++ b/src/components/cards/card.tsx
@ -30,16 +30,15 @@ export namespace Card {
        e.preventDefault()
        e.stopPropagation()
        switch (props.source.openTarget) {
-            case SourceOpenTarget.Local:
-            case SourceOpenTarget.Webpage: {
-                props.markRead(props.item)
-                props.showItem(props.feedId, props.item)
-                break
-            }
            case SourceOpenTarget.External: {
                openInBrowser(props, e)
                break
            }
+            default: {
+                props.markRead(props.item)
+                props.showItem(props.feedId, props.item)
+                break
+            }
        }
    }

--- a/src/components/settings/sources.tsx
+++ b/src/components/settings/sources.tsx
@ -121,6 +121,7 @@ class SourcesTab extends React.Component<SourcesTabProps, SourcesTabState> {

    sourceOpenTargetChoices = (): IChoiceGroupOption[] => [
        { key: String(SourceOpenTarget.Local), text: intl.get("sources.rssText") },
+        { key: String(SourceOpenTarget.FullContent), text: intl.get("article.loadFull") },
        { key: String(SourceOpenTarget.Webpage), text: intl.get("sources.loadWebpage") },
        { key: String(SourceOpenTarget.External), text: intl.get("openExternal") }
    ]
--- a/src/main/window.ts
+++ b/src/main/window.ts
@ -57,7 +57,6 @@ export class WindowManager {
                fullscreenable: false,
                show: false,
                webPreferences: {
-                    sandbox: true,
                    webviewTag: true,
                    enableRemoteModule: false,
                    contextIsolation: true,
--- a/src/scripts/i18n/en-US.json
+++ b/src/scripts/i18n/en-US.json
@ -64,6 +64,7 @@
        "unstar": "Remove star",
        "fontSize": "Font size",
        "loadWebpage": "Load webpage",
+        "loadFull": "Load full content",
        "notify": "Notify if fetched in background",
        "dontNotify": "Don't notify"
    },
--- a/src/scripts/i18n/zh-CN.json
+++ b/src/scripts/i18n/zh-CN.json
@ -64,6 +64,7 @@
        "unstar": "取消星标",
        "fontSize": "字体大小",
        "loadWebpage": "加载网页",
+        "loadFull": "抓取全文",
        "notify": "后台抓取时发送通知",
        "dontNotify": "不发送通知"
    },
--- a/src/scripts/models/source.ts
+++ b/src/scripts/models/source.ts
@ -6,8 +6,8 @@ import { RSSItem, insertItems, ItemActionTypes, FETCH_ITEMS, MARK_READ, MARK_UNR
 import { saveSettings } from "./app"
 import { SourceRule } from "./rule"

-export enum SourceOpenTarget {
-    Local, Webpage, External
+export const enum SourceOpenTarget {
+    Local, Webpage, External, FullContent
 }

 export class RSSSource {