save

resources/desktop/com.github.rssguard.appdata.xml

@@ -30,7 +30,7 @@
   <url type="donation">https://martinrotter.github.io/donate/</url>
   <content_rating type="oars-1.1" />
   <releases>
-    <release version="3.9.0" date="2021-05-04"/>
+    <release version="3.9.0" date="2021-05-07"/>
   </releases>
   <content_rating type="oars-1.0">
     <content_attribute id="violence-cartoon">none</content_attribute>

resources/rssguard.qrc

@@ -6,6 +6,7 @@
         <file>text/COPYING_GNU_GPL</file>
         <file>text/COPYING_GNU_GPL_HTML</file>
 
+        <file>scripts/adblock/adblock-server.js</file>
         <file>scripts/public_suffix_list.dat</file>
         
         <file>graphics/rssguard.ico</file>

resources/scripts/adblock/adblock-server.js

@@ -3,7 +3,7 @@
 // How to install:
 //     npm i -g @cliqz/adblocker
 //     npm i -g concat-stream
-//     npm i -g psl
+//     npm i -g tldts-experimental
 //     npm i -g node-fetch
 //
 // How to run:
@@ -19,84 +19,100 @@
 //       }' 'http://localhost:<port>'
 
 const fs = require('fs');
-const psl = require('psl');
+const tldts = require('tldts-experimental');
 const adblock = require('@cliqz/adblocker')
 const http = require('http');
 const concat = require('concat-stream');
 const constants = require('node:http2');
 const fetch = require("node-fetch");
+const cluster = require('cluster');
 
+const numCPUs = require('os').cpus().length;
 const port = process.argv[2];
 const filtersFile = process.argv[3];
 const engine = adblock.FiltersEngine.parse(fs.readFileSync(filtersFile, 'utf-8'));
 const hostname = '127.0.0.1';
 
-const server = http.createServer((req, res) => {
+if (cluster.isPrimary) {
-  try {
+  console.log(`Primary ${process.pid} is running`);
-    console.log(new Date());
 
-    const chunks = [];
+  // Fork workers.
-    req.on('data', chunk => chunks.push(chunk));
+  for (let i = 0; i < numCPUs; i++) {
-    req.on('end', () => {
+    cluster.fork();
+  }
+
+  cluster.on('exit', (worker, code, signal) => {
+    console.log(`worker ${worker.process.pid} died`);
+  });
+}
+else {
+  const server = http.createServer((req, res) => {
+    try {
       console.log(new Date());
 
-      try {
+      const chunks = [];
-        const jsonData = Buffer.concat(chunks);
+      req.on('data', chunk => chunks.push(chunk));
-        const jsonStruct = JSON.parse(jsonData.toString());
+      req.on('end', () => {
-
-        const askUrl = jsonStruct['url'];
-        const askFilter = jsonStruct['filter'];
-        const askCosmetic = jsonStruct['cosmetic'];
-        const askUrlType = jsonStruct['url_type'];
-        const fullUrl = new URL(askUrl);
-
-        resultJson = {};
-
-        if (askFilter) {
-          const adblockMatch = engine.match(adblock.Request.fromRawDetails({
-            type: askUrlType,
-            url: askUrl,
-          }));
-
-          resultJson["filter"] = adblockMatch;
-          console.log(`adblocker: Filter is:\n${JSON.stringify(adblockMatch)}.`)
-        }
-
-        if (askCosmetic) {
-          const adblockCosmetic = engine.getCosmeticsFilters({
-            url: askUrl,
-            hostname: fullUrl.hostname,
-            domain: psl.parse(fullUrl.hostname).domain
-          });
-
-          resultJson["cosmetic"] = adblockCosmetic;
-          console.log(`adblocker: Cosmetic is:\n${JSON.stringify(adblockCosmetic)}.`)
-        }
-
-        res.statusCode = 200;
-        res.setHeader('Content-Type', 'application/json');
-        res.end(JSON.stringify(resultJson));
-
         console.log(new Date());
-      }
-      catch (inner_error) {
-        console.error(`adblocker: ${inner_error}.`);
 
-        res.statusCode = 500;
+        try {
-        res.setHeader('Content-Type', 'text/plain');
+          const jsonData = Buffer.concat(chunks);
-        res.end(String(inner_error));
+          const jsonStruct = JSON.parse(jsonData.toString());
-      }
-    })
-  }
-  catch (error) {
-    console.error(`adblocker: ${inner_error}.`);
 
-    res.statusCode = 500;
+          const askUrl = jsonStruct['url'];
-    res.setHeader('Content-Type', 'text/plain');
+          const askFilter = jsonStruct['filter'];
-    res.end(String(error));
+          const askCosmetic = jsonStruct['cosmetic'];
-  }
+          const askUrlType = jsonStruct['url_type'];
-});
+          const fullUrl = new URL(askUrl);
 
-server.listen(port, hostname, () => {
+          resultJson = {};
-  console.log(`adblocker: Server started at local port ${port}.`);
+
-});
+          if (askFilter) {
+            const adblockMatch = engine.match(adblock.Request.fromRawDetails({
+              type: askUrlType,
+              url: askUrl,
+            }));
+
+            resultJson["filter"] = adblockMatch;
+            console.log(`adblocker: Filter is:\n${JSON.stringify(adblockMatch)}.`)
+          }
+
+          if (askCosmetic) {
+            const adblockCosmetic = engine.getCosmeticsFilters({
+              url: askUrl,
+              hostname: fullUrl.hostname,
+              domain: tldts.parse(fullUrl).domain
+            });
+
+            resultJson["cosmetic"] = adblockCosmetic;
+            console.log(`adblocker: Cosmetic is:\n${JSON.stringify(adblockCosmetic)}.`)
+          }
+
+          res.statusCode = 200;
+          res.setHeader('Content-Type', 'application/json');
+          res.end(JSON.stringify(resultJson));
+
+          console.log(new Date());
+        }
+        catch (inner_error) {
+          console.error(`adblocker: ${inner_error}.`);
+
+          res.statusCode = 500;
+          res.setHeader('Content-Type', 'text/plain');
+          res.end(String(inner_error));
+        }
+      })
+    }
+    catch (error) {
+      console.error(`adblocker: ${inner_error}.`);
+
+      res.statusCode = 500;
+      res.setHeader('Content-Type', 'text/plain');
+      res.end(String(error));
+    }
+  });
+
+  server.listen(port, hostname, () => {
+    console.log(`adblocker: Server started at local port ${port}.`);
+  });
+}

src/librssguard/definitions/definitions.h

@@ -14,13 +14,10 @@
 #define SERVICE_CODE_INOREADER  "inoreader"
 #define SERVICE_CODE_GMAIL      "gmail"
 
+#define ADBLOCK_SERVER_PORT                   "48484"
 #define ADBLOCK_HOWTO                         "https://github.com/martinrotter/rssguard/blob/master/resources/docs/Documentation.md#adblock"
-#define ADBLOCK_UPDATE_DAYS_INTERVAL          14
 #define ADBLOCK_ICON_ACTIVE                   "adblock"
 #define ADBLOCK_ICON_DISABLED                 "adblock-disabled"
-#define ADBLOCK_CUSTOMLIST_NAME               "customlist.txt"
-#define ADBLOCK_LISTS_SUBDIRECTORY            "adblock"
-#define ADBLOCK_EASYLIST_URL                  "https://easylist.to/easylist/easylist.txt"
 
 #define OAUTH_DECRYPTION_KEY                  11451167756100761335ul
 #define OAUTH_REDIRECT_URI                    "http://localhost"

src/librssguard/miscellaneous/application.cpp

@@ -78,7 +78,10 @@ Application::Application(const QString& id, int& argc, char** argv)
 
 #if defined(USE_WEBENGINE)
   m_webFactory->urlIinterceptor()->load();
-  m_webFactory->adBlock()->load(true);
+
+  QTimer::singleShot(3000, this, [=]() {
+    m_webFactory->adBlock()->load(true);
+  });
 #endif
 
   qDebugNN << LOGSEC_CORE

src/librssguard/miscellaneous/skinfactory.cpp

@@ -60,9 +60,10 @@ QString SkinFactory::selectedSkinName() const {
   return qApp->settings()->value(GROUP(GUI), SETTING(GUI::Skin)).toString();
 }
 
-QString SkinFactory::adBlockedPage(const QString& url) {
+QString SkinFactory::adBlockedPage(const QString& url, const QString& filter) {
   const QString& adblocked = currentSkin().m_adblocked.arg(tr("This page was blocked by AdBlock"),
-                                                           tr(R"(Blocked URL: "%1")").arg(url));
+                                                           tr(R"(Blocked URL: "%1"<br/>Used filter: "%2")").arg(url,
+                                                                                                                filter));
 
   return currentSkin().m_layoutMarkupWrapper.arg(tr("This page was blocked by AdBlock"), adblocked);
 }

src/librssguard/miscellaneous/skinfactory.h

@@ -50,7 +50,7 @@ class RSSGUARD_DLLSPEC SkinFactory : public QObject {
     // after application restart.
     QString selectedSkinName() const;
 
-    QString adBlockedPage(const QString& url);
+    QString adBlockedPage(const QString& url, const QString& filter);
 
     // Gets skin about a particular skin.
     Skin skinInfo(const QString& skin_name, bool* ok = nullptr) const;

src/librssguard/network-web/adblock/adblockmanager.cpp

@@ -3,6 +3,7 @@
 #include "network-web/adblock/adblockmanager.h"
 
 #include "exceptions/applicationexception.h"
+#include "exceptions/networkexception.h"
 #include "miscellaneous/application.h"
 #include "miscellaneous/settings.h"
 #include "network-web/adblock/adblockdialog.h"
@@ -15,7 +16,10 @@
 
 #include <QDateTime>
 #include <QDir>
+#include <QJsonDocument>
+#include <QJsonObject>
 #include <QMessageBox>
+#include <QProcess>
 #include <QTimer>
 #include <QUrlQuery>
 #include <QWebEngineProfile>
@@ -24,24 +28,43 @@ AdBlockManager::AdBlockManager(QObject* parent)
   : QObject(parent), m_loaded(false), m_enabled(false), m_interceptor(new AdBlockUrlInterceptor(this)) {
   m_adblockIcon = new AdBlockIcon(this);
   m_adblockIcon->setObjectName(QSL("m_adblockIconAction"));
-
   m_unifiedFiltersFile = qApp->userDataFolder() + QDir::separator() + QSL("adblock-unified-filters.txt");
+  m_serverProcess = new QProcess(this);
 }
 
-bool AdBlockManager::block(const AdblockRequestInfo& request) const {
+AdBlockManager::~AdBlockManager() {
+  if (m_serverProcess->state() == QProcess::ProcessState::Running) {
+    m_serverProcess->kill();
+  }
+}
+
+BlockingResult AdBlockManager::block(const AdblockRequestInfo& request) const {
   if (!isEnabled()) {
-    return false;
+    return { false };
   }
 
   const QString url_string = request.requestUrl().toEncoded().toLower();
   const QString url_scheme = request.requestUrl().scheme().toLower();
 
   if (!canRunOnScheme(url_scheme)) {
-    return false;
+    return { false };
   }
   else {
-    // TODO: start server if needed, call it.
+    if (m_serverProcess->state() == QProcess::ProcessState::Running) {
-    return false;
+      try {
+        auto result = askServerIfBlocked(url_string);
+
+        return result;
+      }
+      catch (const ApplicationException& ex) {
+        qCriticalNN << LOGSEC_ADBLOCK
+                    << "HTTP error when calling server:"
+                    << QUOTE_W_SPACE_DOT(ex.message());
+      }
+    }
+    else {
+      return { false };
+    }
   }
 }
 
@@ -123,8 +146,106 @@ void AdBlockManager::showDialog() {
   AdBlockDialog(qApp->mainFormWidget()).exec();
 }
 
+BlockingResult AdBlockManager::askServerIfBlocked(const QString& url) const {
+  QJsonObject req_obj;
+  QByteArray out;
+  QElapsedTimer tmr;
+
+  req_obj["url"] = url;
+  req_obj["filter"] = true;
+
+  tmr.start();
+
+  auto network_res = NetworkFactory::performNetworkOperation(QSL("http://%1:%2").arg(QHostAddress(QHostAddress::SpecialAddress::LocalHost).toString(),
+                                                                                     ADBLOCK_SERVER_PORT),
+                                                             500,
+                                                             QJsonDocument(req_obj).toJson(),
+                                                             out,
+                                                             QNetworkAccessManager::Operation::PostOperation,
+                                                             { {
+                                                               QSL(HTTP_HEADERS_CONTENT_TYPE).toLocal8Bit(),
+                                                               QSL("application/json").toLocal8Bit() } });
+
+  if (network_res.first == QNetworkReply::NetworkError::NoError) {
+    qDebugNN << LOGSEC_ADBLOCK
+             << "Query to server took "
+             << tmr.elapsed()
+             << " ms.";
+
+    QJsonObject out_obj = QJsonDocument::fromJson(out).object();
+    bool blocking = out_obj["filter"].toObject()["match"].toBool();
+
+    return {
+      blocking,
+      blocking
+          ? out_obj["filter"].toObject()["filter"].toObject()["filter"].toString()
+          : QString()
+    };
+  }
+  else {
+    throw NetworkException(network_res.first);
+  }
+}
+
 void AdBlockManager::restartServer() {
-  // TODO:
+  if (m_serverProcess->state() == QProcess::ProcessState::Running) {
+    m_serverProcess->kill();
+
+    if (!m_serverProcess->waitForFinished(1000)) {
+      m_serverProcess->deleteLater();
+      m_serverProcess = new QProcess(this);
+    }
+  }
+
+  QString temp_server = QDir::toNativeSeparators(IOFactory::getSystemFolder(QStandardPaths::StandardLocation::TempLocation)) +
+                        QDir::separator() +
+                        QSL("adblock-server.js");
+
+  if (!IOFactory::copyFile(QSL(":/scripts/adblock/adblock-server.js"), temp_server)) {
+    qCriticalNN << LOGSEC_ADBLOCK << "Failed to copy server file to TEMP.";
+  }
+
+#if defined(Q_OS_WIN)
+  m_serverProcess->setProgram(QSL("node.exe"));
+#else
+  m_serverProcess->setProgram(QSL("node"));
+#endif
+
+  m_serverProcess->setArguments({
+    QDir::toNativeSeparators(temp_server),
+    ADBLOCK_SERVER_PORT,
+    QDir::toNativeSeparators(m_unifiedFiltersFile)
+  });
+
+  m_serverProcess->setProcessEnvironment(QProcessEnvironment::systemEnvironment());
+
+  auto pe = m_serverProcess->processEnvironment();
+  QString node_path =
+#if defined(Q_OS_WIN)
+    pe.value(QSL("APPDATA")) +
+#elif defined(Q_OS_LINUX)
+    QSL("/usr/local/lib/node_modules") +
+#else
+    QDir::toNativeSeparators(IOFactory::getSystemFolder(QStandardPaths::StandardLocation::GenericDataLocation)) +
+#endif
+    QDir::separator() +
+    QSL("npm") +
+    QDir::separator() +
+    QSL("node_modules");
+
+  if (!pe.contains(QSL("NODE_PATH"))) {
+    pe.insert(QSL("NODE_PATH"), node_path);
+  }
+
+  m_serverProcess->setProcessEnvironment(pe);
+  m_serverProcess->setProcessChannelMode(QProcess::ProcessChannelMode::ForwardedErrorChannel);
+
+  if (!m_serverProcess->open()) {
+    qWarningNN << LOGSEC_ADBLOCK << "Failed to start server.";
+  }
+  else {
+    qDebugNN << LOGSEC_ADBLOCK << "Started server.";
+  }
 }
 
 void AdBlockManager::updateUnifiedFiltersFile() {
@@ -132,8 +253,7 @@ void AdBlockManager::updateUnifiedFiltersFile() {
     QFile::remove(m_unifiedFiltersFile);
   }
 
-  // TODO: generate file
+  QString unified_contents;
-  QByteArray unified_contents;
   auto filter_lists = filterLists();
 
   // Download filters one by one and append.
@@ -142,10 +262,16 @@ void AdBlockManager::updateUnifiedFiltersFile() {
     auto res = NetworkFactory::performNetworkOperation(filter_list_url,
                                                        2000,
                                                        {},
-                                                       out, QNetworkAccessManager::Operation::GetOperation);
+                                                       out,
+                                                       QNetworkAccessManager::Operation::GetOperation);
 
     if (res.first == QNetworkReply::NetworkError::NoError) {
-      unified_contents += out;
+      unified_contents = unified_contents.append(QString::fromUtf8(out));
+      unified_contents = unified_contents.append('\n');
+
+      qDebugNN << LOGSEC_ADBLOCK
+               << "Downloaded filter list from"
+               << QUOTE_W_SPACE_DOT(filter_list_url);
     }
     else {
       qWarningNN << LOGSEC_ADBLOCK
@@ -156,7 +282,7 @@ void AdBlockManager::updateUnifiedFiltersFile() {
     }
   }
 
-  unified_contents += customFilters().join(QSL("\n")).toUtf8();
+  unified_contents = unified_contents.append(customFilters().join(QSL("\n")));
 
   // Save.
   m_unifiedFiltersFile = IOFactory::getSystemFolder(QStandardPaths::StandardLocation::TempLocation) +
@@ -164,10 +290,9 @@ void AdBlockManager::updateUnifiedFiltersFile() {
                          QSL("adblock.filters");
 
   try {
-    IOFactory::writeFile(m_unifiedFiltersFile, unified_contents);
+    IOFactory::writeFile(m_unifiedFiltersFile, unified_contents.toUtf8());
 
     if (m_enabled) {
-      // TODO: re-start nodejs adblock server.
       restartServer();
     }
   }

src/librssguard/network-web/adblock/adblockmanager.h

@@ -6,15 +6,26 @@
 #include <QObject>
 
 class QUrl;
+class QProcess;
 class AdblockRequestInfo;
 class AdBlockUrlInterceptor;
 class AdBlockIcon;
 
+struct BlockingResult {
+  bool m_blocked;
+  QString m_blockedByFilter;
+
+  BlockingResult(bool blocked, QString blocked_by_filter = {})
+    : m_blocked(blocked), m_blockedByFilter(std::move(blocked_by_filter)) {}
+
+};
+
 class AdBlockManager : public QObject {
   Q_OBJECT
 
   public:
     explicit AdBlockManager(QObject* parent = nullptr);
+    virtual ~AdBlockManager();
 
     // If "initial_load" is false, then we want to explicitly turn off
     // Adblock if it is running or turn on when not running.
@@ -27,7 +38,7 @@ class AdBlockManager : public QObject {
     AdBlockIcon* adBlockIcon() const;
 
     // General methods for adblocking.
-    bool block(const AdblockRequestInfo& request) const;
+    BlockingResult block(const AdblockRequestInfo& request) const;
     QString elementHidingRulesForDomain(const QUrl& url) const;
 
     QStringList filterLists() const;
@@ -47,6 +58,8 @@ class AdBlockManager : public QObject {
     void enabledChanged(bool enabled);
 
   private:
+    BlockingResult askServerIfBlocked(const QString& url) const;
+
     void restartServer();
 
   private:
@@ -55,6 +68,7 @@ class AdBlockManager : public QObject {
     AdBlockIcon* m_adblockIcon;
     AdBlockUrlInterceptor* m_interceptor;
     QString m_unifiedFiltersFile;
+    QProcess* m_serverProcess;
 };
 
 inline AdBlockIcon* AdBlockManager::adBlockIcon() const {

src/librssguard/network-web/adblock/adblockurlinterceptor.cpp

@@ -27,7 +27,7 @@ AdBlockUrlInterceptor::AdBlockUrlInterceptor(AdBlockManager* manager)
   : UrlInterceptor(manager), m_manager(manager) {}
 
 void AdBlockUrlInterceptor::interceptRequest(QWebEngineUrlRequestInfo& info) {
-  if (m_manager->block(AdblockRequestInfo(info))) {
+  if (m_manager->block(AdblockRequestInfo(info)).m_blocked) {
     info.block(true);
 
     qWarningNN << LOGSEC_ADBLOCK << "Blocked request:" << QUOTE_W_SPACE_DOT(info.requestUrl().toString());

src/librssguard/network-web/webpage.cpp

@@ -48,9 +48,9 @@ bool WebPage::acceptNavigationRequest(const QUrl& url, NavigationType type, bool
   if (is_main_frame) {
     auto blocked = qApp->web()->adBlock()->block(AdblockRequestInfo(url));
 
-    if (blocked) {
+    if (blocked.m_blocked) {
       // This website is entirely blocked.
-      setHtml(qApp->skins()->adBlockedPage(url.toString()),
+      setHtml(qApp->skins()->adBlockedPage(url.toString(), blocked.m_blockedByFilter),
               QUrl::fromUserInput(INTERNAL_URL_ADBLOCKED));
       return false;
     }