unvendor: Replace Readability

.eslintrc.cjs

@@ -57,7 +57,6 @@ module.exports = {
                 showdownKatex: 'readonly',
                 SVGInject: 'readonly',
                 toastr: 'readonly',
-                Readability: 'readonly',
                 isProbablyReaderable: 'readonly',
                 SillyTavern: 'readonly',
             },

package-lock.json

@@ -13,6 +13,7 @@
                 "@adobe/css-tools": "^4.4.0",
                 "@agnai/sentencepiece-js": "^1.1.1",
                 "@agnai/web-tokenizers": "^0.1.3",
+                "@mozilla/readability": "^0.5.0",
                 "@zeldafan0225/ai_horde": "^5.1.0",
                 "archiver": "^7.0.1",
                 "bing-translate-api": "^2.9.1",
@@ -1038,6 +1039,15 @@
                 }
             }
         },
+        "node_modules/@mozilla/readability": {
+            "version": "0.5.0",
+            "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.5.0.tgz",
+            "integrity": "sha512-Z+CZ3QaosfFaTqvhQsIktyGrjFjSC0Fa4EMph4mqKnWhmyoGICsV/8QK+8HpXut6zV7zwfWwqDmEjtk1Qf6EgQ==",
+            "license": "Apache-2.0",
+            "engines": {
+                "node": ">=14.0.0"
+            }
+        },
         "node_modules/@nodelib/fs.scandir": {
             "version": "2.1.5",
             "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",

package.json

@@ -3,6 +3,7 @@
         "@adobe/css-tools": "^4.4.0",
         "@agnai/sentencepiece-js": "^1.1.1",
         "@agnai/web-tokenizers": "^0.1.3",
+        "@mozilla/readability": "^0.5.0",
         "@zeldafan0225/ai_horde": "^5.1.0",
         "archiver": "^7.0.1",
         "bing-translate-api": "^2.9.1",

public/global.d.ts

@@ -4,8 +4,6 @@ declare var Popper;
 declare var showdown;
 declare var showdownKatex;
 declare var SVGInject;
-declare var Readability;
-declare var isProbablyReaderable;
 declare var ai;
 
 declare var SillyTavern: {

public/index.html

@@ -6740,7 +6740,6 @@
 
     <!-- Script includes -->
     <script src="lib/polyfill.js"></script>
-    <script src="lib/diff_match_patch.js"></script>
     <script src="lib/jquery-3.5.1.min.js"></script>
     <script src="lib/jquery-ui.min.js"></script>
     <script src="lib/jquery.transit.min.js"></script>
@@ -6761,8 +6760,6 @@
     <script src="lib/pagination.js"></script>
     <script src="lib/toolcool-color-picker.js"></script>
     <script src="lib/svg-inject.js"></script>
-    <script src="lib/Readability.js"></script>
-    <script src="lib/Readability-readerable.js"></script>
     <script src="lib/jquery.izoomify.js"></script>
     <script type="module" src="lib/structured-clone/monkey-patch.js"></script>
     <script type="module" src="lib/swiped-events.js"></script>

public/jsconfig.json

@@ -21,8 +21,7 @@
             "showdown",
             "seedrandom",
             "showdown-katex",
-            "droll",
+            "droll"
-            "@mozilla/readability"
         ]
     }
 }

public/lib.js

@@ -12,6 +12,7 @@ import Bowser from 'bowser';
 import ePub from 'epubjs';
 import * as pdfjsLib from 'pdfjs-dist/webpack.mjs';
 import DiffMatchPatch from 'diff-match-patch';
+import { isProbablyReaderable, Readability } from '@mozilla/readability';
 
 /**
  * Expose the libraries to the 'window' object.
@@ -59,6 +60,8 @@ export default {
     ePub,
     pdfjsLib,
     DiffMatchPatch,
+    Readability,
+    isProbablyReaderable,
 };
 
 export {
@@ -72,4 +75,6 @@ export {
     ePub,
     pdfjsLib,
     DiffMatchPatch,
+    Readability,
+    isProbablyReaderable,
 };

public/lib/Readability-readerable.js

@@ -1,108 +0,0 @@
-/*
- * Copyright (c) 2010 Arc90 Inc
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * This code is heavily based on Arc90's readability.js (1.7.1) script
- * available at: http://code.google.com/p/arc90labs-readability
- */
-
-var REGEXPS = {
-  // NOTE: These two regular expressions are duplicated in
-  // Readability.js. Please keep both copies in sync.
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
-  okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,
-};
-
-function isNodeVisible(node) {
-  // Have to null-check node.style and node.className.indexOf to deal with SVG and MathML nodes.
-  return (!node.style || node.style.display != "none")
-    && !node.hasAttribute("hidden")
-    //check for "fallback-image" so that wikimedia math images are displayed
-    && (!node.hasAttribute("aria-hidden") || node.getAttribute("aria-hidden") != "true" || (node.className && node.className.indexOf && node.className.indexOf("fallback-image") !== -1));
-}
-
-/**
- * Decides whether or not the document is reader-able without parsing the whole thing.
- * @param {Object} options Configuration object.
- * @param {number} [options.minContentLength=140] The minimum node content length used to decide if the document is readerable.
- * @param {number} [options.minScore=20] The minumum cumulated 'score' used to determine if the document is readerable.
- * @param {Function} [options.visibilityChecker=isNodeVisible] The function used to determine if a node is visible.
- * @return {boolean} Whether or not we suspect Readability.parse() will suceeed at returning an article object.
- */
-function isProbablyReaderable(doc, options = {}) {
-  // For backward compatibility reasons 'options' can either be a configuration object or the function used
-  // to determine if a node is visible.
-  if (typeof options == "function") {
-    options = { visibilityChecker: options };
-  }
-
-  var defaultOptions = { minScore: 20, minContentLength: 140, visibilityChecker: isNodeVisible };
-  options = Object.assign(defaultOptions, options);
-
-  var nodes = doc.querySelectorAll("p, pre, article");
-
-  // Get <div> nodes which have <br> node(s) and append them into the `nodes` variable.
-  // Some articles' DOM structures might look like
-  // <div>
-  //   Sentences<br>
-  //   <br>
-  //   Sentences<br>
-  // </div>
-  var brNodes = doc.querySelectorAll("div > br");
-  if (brNodes.length) {
-    var set = new Set(nodes);
-    [].forEach.call(brNodes, function (node) {
-      set.add(node.parentNode);
-    });
-    nodes = Array.from(set);
-  }
-
-  var score = 0;
-  // This is a little cheeky, we use the accumulator 'score' to decide what to return from
-  // this callback:
-  return [].some.call(nodes, function (node) {
-    if (!options.visibilityChecker(node)) {
-      return false;
-    }
-
-    var matchString = node.className + " " + node.id;
-    if (REGEXPS.unlikelyCandidates.test(matchString) &&
-        !REGEXPS.okMaybeItsACandidate.test(matchString)) {
-      return false;
-    }
-
-    if (node.matches("li p")) {
-      return false;
-    }
-
-    var textContentLength = node.textContent.trim().length;
-    if (textContentLength < options.minContentLength) {
-      return false;
-    }
-
-    score += Math.sqrt(textContentLength - options.minContentLength);
-
-    if (score > options.minScore) {
-      return true;
-    }
-    return false;
-  });
-}
-
-if (typeof module === "object") {
-  /* global module */
-  module.exports = isProbablyReaderable;
-}

public/scripts/utils.js

@@ -1,4 +1,4 @@
-import { DOMPurify, ePub, pdfjsLib } from '../lib.js';
+import { DOMPurify, ePub, pdfjsLib, Readability, isProbablyReaderable } from '../lib.js';
 
 import { getContext } from './extensions.js';
 import { characters, getRequestHeaders, this_chid } from '../script.js';