Use Readability to extract text from HTML

public/index.html

@@ -47,6 +47,8 @@
     <script src="lib/pagination.js"></script>
     <script src="lib/toolcool-color-picker.js"></script>
     <script src="lib/svg-inject.js"></script>
+    <script src="lib/Readability.js"></script>
+    <script src="lib/Readability-readerable.js"></script>
     <script type="module" src="lib/structured-clone/monkey-patch.js"></script>
     <script type="module" src="lib/swiped-events.js"></script>
     <script type="module" src="lib/eventemitter.js"></script>

public/jsconfig.json

@@ -23,7 +23,8 @@
             "handlebars",
             "highlight.js",
             "localforage",
-            "pdfjs-dist"
+            "pdfjs-dist",
+            "@mozilla/readability"
         ]
     }
 }

public/lib/Readability-readerable.js

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2010 Arc90 Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This code is heavily based on Arc90's readability.js (1.7.1) script
+ * available at: http://code.google.com/p/arc90labs-readability
+ */
+
+var REGEXPS = {
+  // NOTE: These two regular expressions are duplicated in
+  // Readability.js. Please keep both copies in sync.
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
+  okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,
+};
+
+function isNodeVisible(node) {
+  // Have to null-check node.style and node.className.indexOf to deal with SVG and MathML nodes.
+  return (!node.style || node.style.display != "none")
+    && !node.hasAttribute("hidden")
+    //check for "fallback-image" so that wikimedia math images are displayed
+    && (!node.hasAttribute("aria-hidden") || node.getAttribute("aria-hidden") != "true" || (node.className && node.className.indexOf && node.className.indexOf("fallback-image") !== -1));
+}
+
+/**
+ * Decides whether or not the document is reader-able without parsing the whole thing.
+ * @param {Object} options Configuration object.
+ * @param {number} [options.minContentLength=140] The minimum node content length used to decide if the document is readerable.
+ * @param {number} [options.minScore=20] The minumum cumulated 'score' used to determine if the document is readerable.
+ * @param {Function} [options.visibilityChecker=isNodeVisible] The function used to determine if a node is visible.
+ * @return {boolean} Whether or not we suspect Readability.parse() will suceeed at returning an article object.
+ */
+function isProbablyReaderable(doc, options = {}) {
+  // For backward compatibility reasons 'options' can either be a configuration object or the function used
+  // to determine if a node is visible.
+  if (typeof options == "function") {
+    options = { visibilityChecker: options };
+  }
+
+  var defaultOptions = { minScore: 20, minContentLength: 140, visibilityChecker: isNodeVisible };
+  options = Object.assign(defaultOptions, options);
+
+  var nodes = doc.querySelectorAll("p, pre, article");
+
+  // Get <div> nodes which have <br> node(s) and append them into the `nodes` variable.
+  // Some articles' DOM structures might look like
+  // <div>
+  //   Sentences<br>
+  //   <br>
+  //   Sentences<br>
+  // </div>
+  var brNodes = doc.querySelectorAll("div > br");
+  if (brNodes.length) {
+    var set = new Set(nodes);
+    [].forEach.call(brNodes, function (node) {
+      set.add(node.parentNode);
+    });
+    nodes = Array.from(set);
+  }
+
+  var score = 0;
+  // This is a little cheeky, we use the accumulator 'score' to decide what to return from
+  // this callback:
+  return [].some.call(nodes, function (node) {
+    if (!options.visibilityChecker(node)) {
+      return false;
+    }
+
+    var matchString = node.className + " " + node.id;
+    if (REGEXPS.unlikelyCandidates.test(matchString) &&
+        !REGEXPS.okMaybeItsACandidate.test(matchString)) {
+      return false;
+    }
+
+    if (node.matches("li p")) {
+      return false;
+    }
+
+    var textContentLength = node.textContent.trim().length;
+    if (textContentLength < options.minContentLength) {
+      return false;
+    }
+
+    score += Math.sqrt(textContentLength - options.minContentLength);
+
+    if (score > options.minScore) {
+      return true;
+    }
+    return false;
+  });
+}
+
+if (typeof module === "object") {
+  /* global module */
+  module.exports = isProbablyReaderable;
+}

public/scripts/utils.js

@@ -1111,11 +1111,13 @@ export function uuidv4() {
     });
 }
 
-function postProcessText(text) {
+function postProcessText(text, collapse = true) {
     // Collapse multiple newlines into one
-    text = collapseNewlines(text);
+    if (collapse) {
-    // Trim leading and trailing whitespace, and remove empty lines
+        text = collapseNewlines(text);
-    text = text.split('\n').map(l => l.trim()).filter(Boolean).join('\n');
+        // Trim leading and trailing whitespace, and remove empty lines
+        text = text.split('\n').map(l => l.trim()).filter(Boolean).join('\n');
+    }
     // Remove carriage returns
     text = text.replace(/\r/g, '');
     // Normalize unicode spaces
@@ -1127,6 +1129,25 @@ function postProcessText(text) {
     return text;
 }
 
+/**
+ * Uses Readability.js to parse the text from a web page.
+ * @param {Document} document HTML document
+ * @param {string} [textSelector='body'] The fallback selector for the text to parse.
+ * @returns {Promise<string>} A promise that resolves to the parsed text.
+ */
+export async function getReadableText(document, textSelector = 'body') {
+    if (isProbablyReaderable(document)) {
+        const parser = new Readability(document);
+        const article = parser.parse();
+        return postProcessText(article.textContent, false);
+    }
+
+    const elements = document.querySelectorAll(textSelector);
+    const rawText = Array.from(elements).map(e => e.textContent).join('\n');
+    const text = postProcessText(rawText);
+    return text;
+}
+
 /**
  * Use pdf.js to load and parse text from PDF pages
  * @param {Blob} blob PDF file blob
@@ -1188,10 +1209,7 @@ export async function extractTextFromHTML(blob, textSelector = 'body') {
     const html = await blob.text();
     const domParser = new DOMParser();
     const document = domParser.parseFromString(DOMPurify.sanitize(html), 'text/html');
-    const elements = document.querySelectorAll(textSelector);
+    return await getReadableText(document, textSelector);
-    const rawText = Array.from(elements).map(e => e.textContent).join('\n');
-    const text = postProcessText(rawText);
-    return text;
 }
 
 /**
@@ -1205,6 +1223,6 @@ export async function extractTextFromMarkdown(blob) {
     const html = converter.makeHtml(markdown);
     const domParser = new DOMParser();
     const document = domParser.parseFromString(DOMPurify.sanitize(html), 'text/html');
-    const text = postProcessText(document.body.textContent);
+    const text = postProcessText(document.body.textContent, false);
     return text;
 }