integrate nodejs in a better way, fix APP_REVISION missing

This commit is contained in:
Martin Rotter 2022-02-10 12:19:13 +01:00
parent da7ffc34a1
commit b2343a0189
15 changed files with 158 additions and 108 deletions

View File

@ -164,6 +164,24 @@ else()
endif()
endif()
# Load git commit hash.
if(REVISION_FROM_GIT AND EXISTS "${CMAKE_SOURCE_DIR}/.git")
execute_process(COMMAND "git" "rev-parse" "--short" "HEAD"
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE APP_REVISION
ERROR_QUIET
OUTPUT_STRIP_TRAILING_WHITESPACE
)
message(STATUS "Detected git revision: '${APP_REVISION}'.")
else()
set(APP_REVISION "")
endif()
if(NOT USE_WEBENGINE)
set(APP_REVISION "${APP_REVISION}-nowebengine")
endif()
# Pass common defines.
add_compile_definitions(
APP_NAME="${APP_NAME}"

View File

@ -26,7 +26,7 @@
<url type="donation">https://github.com/sponsors/martinrotter</url>
<content_rating type="oars-1.1" />
<releases>
<release version="4.1.2" date="2022-02-09"/>
<release version="4.1.2" date="2022-02-10"/>
</releases>
<content_rating type="oars-1.0">
<content_attribute id="violence-cartoon">none</content_attribute>

View File

@ -17,6 +17,7 @@ There is a [Discord server](https://discord.gg/7xbVMPPNqH) for user communicatio
- [Built-in Web Browser with AdBlock](#webb)
- [Minor Features](#mife)
- [Files Downloader](#downl)
- [Node.js](#node)
- [Labels](#lbls)
- [Skins](#skin)
- [GUI Tweaking](#guit)
@ -426,18 +427,13 @@ If you're not sure which version to use, **use the WebEngine-based RSS Guard**.
#### AdBlock <a id="adbl"></a>
[Web-based variant](#webb) of RSS Guard offers ad-blocking functionality via [Adblocker](https://github.com/cliqz-oss/adblocker). Adblocker offers similar performance to [uBlock Origin](https://github.com/gorhill/uBlock).
If you want to enable AdBlock in RSS Guard you need to do this:
1. Have [Node.js](https://nodejs.org) with [NPM](https://www.npmjs.com) (which is usually included in Node.js installer) installed. Also you need to have paths `node.exe` and `npm` added to your system `PATH` environment available.
2. The implementation requires additional [npm](https://www.npmjs.com) modules to be installed. You see the list of needed modules near the top of [this](https://github.com/martinrotter/rssguard/blob/master/resources/scripts/adblock/adblock-server.js) file.
I understand that the above installation is not trivial, but it is necessary evil to have up-to-date and modern implementation of AdBlock in RSS Guard. Previous, `C++`-based, implementation was buggy, slow, and hard to maintain.
If you want to use AdBlock, you need to have [Node.js](#node) installed.
You can find elaborate lists of AdBlock rules [here](https://easylist.to). You can just copy direct hyperlinks to those lists and paste them into the "Filter lists" text-box as shown below. Remember to always separate individual links with newlines. Same applies to "Custom filters", where you can insert individual filters, for example [filter](https://adblockplus.org/filter-cheatsheet) "idnes" to block all URLs with "idnes" in them.
<img alt="alt-img" src="images/adblock.png" width="350px">
The way ad-blocking internally works is that RSS Guard starts local HTTP browser which provides ad-blocking API, which is subsequently called by RSS Guard. There is some caching done in between, which speeds up some ad-blocking decisions.
The way ad-blocking internally works is that RSS Guard starts local `HTTP` browser which provides ad-blocking API, which is subsequently called by RSS Guard. There is some caching done in between, which speeds up some ad-blocking decisions.
## Minor Features <a id="mife"></a>
@ -452,6 +448,12 @@ You can right click on any item in embedded web browser and hit `Save as` button
You can download up to 6 files simultaneously.
### Node.js <a id="node"></a>
RSS Guard integrates [`Node.js`](https://nodejs.org). Go to `Node.js` section of `Settings` dialog to see more.
`Node.js` is used for some advanced functionality like [AdBlock](#adbl).
### Labels <a id="lbls"></a>
RSS Guard supports labels (tags). Any number of tags can be assigned to any article.

View File

@ -1,11 +1,10 @@
// Simple local HTTP server providing ad-blocking functionality via https://github.com/cliqz-oss/adblocker
//
// How to install:
// npm i -g tldts-experimental
// npm i -g @cliqz/adblocker
// npm i @cliqz/adblocker
//
// How to run:
// NODE_PATH="C:\Users\<user>\AppData\Roaming\npm\node_modules" node ./adblock-server.js "<port>" "<filters-file-path>"
// NODE_PATH="..." node ./adblock-server.js "<port>" "<filters-file-path>"
//
// How to use:
// curl -i -X POST --data '

View File

@ -514,24 +514,6 @@ set(GMAIL_CLIENT_SECRET "" CACHE STRING "GMail client secret")
set(INOREADER_CLIENT_ID "" CACHE STRING "Inoreader client ID")
set(INOREADER_CLIENT_SECRET "" CACHE STRING "Inoreader client secret")
# Load git commit hash.
if(REVISION_FROM_GIT AND EXISTS "${CMAKE_SOURCE_DIR}/.git")
execute_process(COMMAND "git" "rev-parse" "--short" "HEAD"
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE APP_REVISION
ERROR_QUIET
OUTPUT_STRIP_TRAILING_WHITESPACE
)
message(STATUS "Detected git revision: '${APP_REVISION}'.")
else()
set(APP_REVISION "")
endif()
if(NOT USE_WEBENGINE)
set(APP_REVISION "${APP_REVISION}-nowebengine")
endif()
# Bundle icons on some platforms which do not provide system-wide icon themes.
if(APPLE OR WIN32 OR OS2)
qt_add_resources(SOURCES ${CMAKE_SOURCE_DIR}/resources/icons.qrc)

View File

@ -368,7 +368,7 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
<< feed->customId() << " stored in DB.";
if (updated_messages.first > 0) {
m_results.appendUpdatedFeed(QPair<QString, int>(feed->title(), updated_messages.first));
m_results.appendUpdatedFeed({ feed->title(), updated_messages.first });
}
}
catch (const FeedFetchException& feed_ex) {
@ -412,8 +412,7 @@ void FeedDownloader::finalizeUpdate() {
emit updateFinished(m_results);
}
bool FeedDownloader::isCacheSynchronizationRunning() const
{
bool FeedDownloader::isCacheSynchronizationRunning() const {
return m_isCacheSynchronizationRunning;
}

View File

@ -16,7 +16,7 @@
#define SERVICE_CODE_REDDIT "reddit"
#define ADBLOCK_SERVER_PORT 48484
#define ADBLOCK_HOWTO "https://github.com/martinrotter/rssguard/blob/master/resources/docs/Documentation.md#adblock"
#define ADBLOCK_HOWTO "https://github.com/martinrotter/rssguard/blob/master/resources/docs/Documentation.md#adbl"
#define ADBLOCK_ICON_ACTIVE "adblock"
#define ADBLOCK_ICON_DISABLED "adblock-disabled"

View File

@ -9,6 +9,7 @@ SettingsGeneral::SettingsGeneral(Settings* settings, QWidget* parent)
: SettingsPanel(settings, parent), m_ui(new Ui::SettingsGeneral) {
m_ui->setupUi(this);
m_ui->m_checkAutostart->setText(m_ui->m_checkAutostart->text().arg(QSL(APP_NAME)));
m_ui->m_checkForUpdatesOnStart->setText(m_ui->m_checkForUpdatesOnStart->text().arg(QSL(APP_NAME)));
connect(m_ui->m_checkAutostart, &QCheckBox::stateChanged, this, &SettingsGeneral::dirtifySettings);
connect(m_ui->m_checkForUpdatesOnStart, &QCheckBox::stateChanged, this, &SettingsGeneral::dirtifySettings);

View File

@ -21,7 +21,7 @@
<item row="1" column="0">
<widget class="QCheckBox" name="m_checkForUpdatesOnStart">
<property name="text">
<string>Check for updates on application startup</string>
<string>Check for %1 updates on application startup</string>
</property>
</widget>
</item>

View File

@ -60,6 +60,7 @@ Application::Application(const QString& id, int& argc, char** argv)
m_mainForm = nullptr;
m_trayIcon = nullptr;
m_settings = Settings::setupSettings(this);
m_nodejs = new NodeJs(m_settings, this);
m_webFactory = new WebFactory(this);
m_system = new SystemFactory(this);
m_skins = new SkinFactory(this);
@ -68,7 +69,6 @@ Application::Application(const QString& id, int& argc, char** argv)
m_database = new DatabaseFactory(this);
m_downloadManager = nullptr;
m_notifications = new NotificationFactory(this);
m_nodejs = new NodeJs(m_settings, this);
m_shouldRestart = false;
determineFirstRuns();
@ -663,13 +663,9 @@ void Application::downloadRequested(QWebEngineDownloadItem* download_item) {
void Application::onAdBlockFailure() {
qApp->showGuiMessage(Notification::Event::GeneralEvent, {
tr("AdBlock needs to be configured"),
tr("AdBlock component is not configured properly."),
QSystemTrayIcon::MessageIcon::Critical },
{}, {
tr("Configure now"),
[=]() {
m_webFactory->adBlock()->showDialog();
} });
tr("AdBlock component is not configured properly. Go to \"Settings\" -> \"Node.js\" and check "
"if your Node.js is properly configured."),
QSystemTrayIcon::MessageIcon::Critical }, {});
qApp->settings()->setValue(GROUP(AdBlock), AdBlock::AdBlockEnabled, false);
}

View File

@ -15,6 +15,16 @@
NodeJs::NodeJs(Settings* settings, QObject* parent) : QObject(parent), m_settings(settings) {}
void NodeJs::runScript(QProcess* proc, const QString& script, const QStringList& arguments) const {
QStringList arg = { script }; arg.append(arguments);
QProcessEnvironment env;
QString node_modules = processedPackageFolder() + QDir::separator() + QSL("node_modules");
env.insert(QSL("NODE_PATH"), node_modules);
IOFactory::startProcess(proc, nodeJsExecutable(), arg, env);
}
QString NodeJs::nodeJsExecutable() const {
return QDir::toNativeSeparators(m_settings->value(GROUP(Node), SETTING(Node::NodeJsExecutable)).toString());
}
@ -94,6 +104,8 @@ void NodeJs::installUpdatePackage(const PackageMetadata& pkg) {
break;
case PackageStatus::UpToDate:
qDebugNN << LOGSEC_NODEJS << "Package" << QUOTE_W_SPACE(pkg.m_name) << "is up-to-date.";
emit packageInstalledUpdated(pkg);
break;
@ -101,8 +113,6 @@ void NodeJs::installUpdatePackage(const PackageMetadata& pkg) {
}
void NodeJs::installPackage(const PackageMetadata& pkg) {
// npm install --prefix "." @cliqz/adblocker@">=1.0.0 <2.0.0" --production --save-exact
//https://docs.npmjs.com/cli/v8/commands/npm-install
try {
QProcess* proc = new QProcess();
@ -132,6 +142,8 @@ void NodeJs::installPackage(const PackageMetadata& pkg) {
emit packageError(pkg, sndr->errorString());
});
qDebugNN << LOGSEC_NODEJS << "Installing package" << QUOTE_W_SPACE_DOT(pkg.m_name);
IOFactory::startProcess(proc,
npmExecutable(),
{ QSL("install"), QSL("--production"),
@ -139,11 +151,9 @@ void NodeJs::installPackage(const PackageMetadata& pkg) {
QSL("--prefix"), processedPackageFolder() });
}
catch (const ProcessException& ex) {
qCriticalNN << LOGSEC_NODEJS << "Package" << QUOTE_W_SPACE(pkg.m_name)
"was not installed, error:" << QUOTE_W_SPACE_DOT(ex.message());
emit packageError(pkg, ex.message());
}
}
void NodeJs::updatePackage(const PackageMetadata& pkg) {
// npm update --prefix "." @cliqz/adblocker@">=1.0.0 <2.0.0" --production --save-exact
//https://docs.npmjs.com/cli/v8/commands/npm-update
}

View File

@ -6,35 +6,38 @@
#include <QObject>
class Settings;
class QProcess;
class NodeJs : public QObject {
Q_OBJECT
struct PackageMetadata {
public:
// Name of package.
QString m_name;
// Version description. This could be fixed version or empty
// string (latest version) or perhaps version range.
QString m_version;
};
enum class PackageStatus {
// Package not installed.
NotInstalled,
// Package installed but out-of-date.
OutOfDate,
// Package installed and up-to-date.
UpToDate
};
public:
struct PackageMetadata {
public:
// Name of package.
QString m_name;
// Version description. This could be fixed version or empty
// string (latest version) or perhaps version range.
QString m_version;
};
enum class PackageStatus {
// Package not installed.
NotInstalled,
// Package installed but out-of-date.
OutOfDate,
// Package installed and up-to-date.
UpToDate
};
explicit NodeJs(Settings* settings, QObject* parent = nullptr);
void runScript(QProcess* proc, const QString& script, const QStringList& arguments) const;
QString nodeJsExecutable() const;
void setNodeJsExecutable(const QString& exe) const;
@ -67,7 +70,6 @@ class NodeJs : public QObject {
private:
void installPackage(const PackageMetadata& pkg);
void updatePackage(const PackageMetadata& pkg);
Settings* m_settings;
};

View File

@ -112,7 +112,7 @@ void AdBlockDialog::onAdBlockProcessTerminated() {
m_ui.m_cbEnable->setChecked(false);
m_ui.m_lblTestResult->setStatus(WidgetWithStatus::StatusType::Error,
tr("There is error, check application log for more details and "
"head to online documentation. Also make sure that Node.js is installed."),
"head to online documentation."),
tr("ERROR!"));
}

View File

@ -25,11 +25,14 @@
#include <QWebEngineProfile>
AdBlockManager::AdBlockManager(QObject* parent)
: QObject(parent), m_loaded(false), m_enabled(false), m_interceptor(new AdBlockUrlInterceptor(this)),
: QObject(parent), m_loaded(false), m_enabled(false), m_installing(false), m_interceptor(new AdBlockUrlInterceptor(this)),
m_serverProcess(nullptr), m_cacheBlocks({}) {
m_adblockIcon = new AdBlockIcon(this);
m_adblockIcon->setObjectName(QSL("m_adblockIconAction"));
m_unifiedFiltersFile = qApp->userDataFolder() + QDir::separator() + QSL("adblock-unified-filters.txt");
connect(qApp->nodejs(), &NodeJs::packageInstalledUpdated, this, &AdBlockManager::onPackageReady);
connect(qApp->nodejs(), &NodeJs::packageError, this, &AdBlockManager::onPackageError);
}
AdBlockManager::~AdBlockManager() {
@ -98,16 +101,9 @@ void AdBlockManager::setEnabled(bool enabled) {
emit enabledChanged(m_enabled);
if (m_enabled) {
try {
updateUnifiedFiltersFileAndStartServer();
}
catch (const ApplicationException& ex) {
qCriticalNN << LOGSEC_ADBLOCK
<< "Failed to write unified filters to file or re-start server, error:"
<< QUOTE_W_SPACE_DOT(ex.message());
m_enabled = false;
emit enabledChanged(m_enabled);
if (!m_installing) {
m_installing = true;
qApp->nodejs()->installUpdatePackage({ QSL(CLIQZ_ADBLOCKED_PACKAGE), QSL(CLIQZ_ADBLOCKED_VERSION) });
}
}
else {
@ -179,6 +175,37 @@ void AdBlockManager::showDialog() {
AdBlockDialog(qApp->mainFormWidget()).exec();
}
void AdBlockManager::onPackageReady(const NodeJs::PackageMetadata& pkg) {
if (pkg.m_name == QSL(CLIQZ_ADBLOCKED_PACKAGE)) {
m_installing = false;
if (m_enabled) {
try {
updateUnifiedFiltersFileAndStartServer();
}
catch (const ApplicationException& ex) {
qCriticalNN << LOGSEC_ADBLOCK
<< "Failed to setup filters and start server:"
<< QUOTE_W_SPACE_DOT(ex.message());
m_enabled = false;
emit enabledChanged(m_enabled);
}
}
}
}
void AdBlockManager::onPackageError(const NodeJs::PackageMetadata& pkg, const QString& error) {
if (pkg.m_name == QSL(CLIQZ_ADBLOCKED_PACKAGE)) {
m_installing = false;
m_enabled = false;
qCriticalNN << LOGSEC_ADBLOCK << "Needed Node.js packages were not installed:" << QUOTE_W_SPACE_DOT(error);
emit processTerminated();
}
}
void AdBlockManager::onServerProcessFinished(int exit_code, QProcess::ExitStatus exit_status) {
Q_UNUSED(exit_status)
killServer();
@ -281,48 +308,54 @@ QProcess* AdBlockManager::startServer(int port) {
QProcess* proc = new QProcess(this);
#if defined(Q_OS_WIN)
proc->setProgram(QSL("node.exe"));
#else
proc->setProgram(QSL("node"));
#endif
proc->setProcessChannelMode(QProcess::ProcessChannelMode::ForwardedErrorChannel);
proc->setArguments({
QDir::toNativeSeparators(temp_server),
connect(proc, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished), this, &AdBlockManager::onServerProcessFinished);
qApp->nodejs()->runScript(proc, QDir::toNativeSeparators(temp_server), {
QString::number(port),
QDir::toNativeSeparators(m_unifiedFiltersFile)
});
proc->setProcessEnvironment(QProcessEnvironment::systemEnvironment());
/*
#if defined(Q_OS_WIN)
proc->setProgram(QSL("node.exe"));
#else
proc->setProgram(QSL("node"));
#endif
auto pe = proc->processEnvironment();
proc->setArguments({
QDir::toNativeSeparators(temp_server),
QString::number(port),
QDir::toNativeSeparators(m_unifiedFiltersFile)
});
if (!pe.contains(QSL("NODE_PATH"))) {
try {
proc->setProcessEnvironment(QProcessEnvironment::systemEnvironment());
auto pe = proc->processEnvironment();
if (!pe.contains(QSL("NODE_PATH"))) {
try {
const QString system_node_prefix = IOFactory::startProcessGetOutput(
#if defined(Q_OS_WIN)
#if defined(Q_OS_WIN)
QSL("npm.cmd")
#else
#else
QSL("npm")
#endif
#endif
, { QSL("root"), QSL("--quiet"), QSL("-g") }
);
if (!system_node_prefix.isEmpty()) {
pe.insert(QSL("NODE_PATH"), system_node_prefix.simplified());
}
}
catch (const ApplicationException& ex) {
}
catch (const ApplicationException& ex) {
qWarningNN << LOGSEC_ADBLOCK << "Failed to get NPM root path:" << QUOTE_W_SPACE_DOT(ex.message());
}
}
}
}
proc->setProcessEnvironment(pe);
proc->setProcessChannelMode(QProcess::ProcessChannelMode::ForwardedErrorChannel);
connect(proc, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished), this, &AdBlockManager::onServerProcessFinished);
proc->open();
proc->setProcessEnvironment(pe);
*/
qDebugNN << LOGSEC_ADBLOCK << "Attempting to start AdBlock server.";
return proc;

View File

@ -5,9 +5,14 @@
#include <QObject>
#include "miscellaneous/nodejs.h"
#include <QHash>
#include <QProcess>
#define CLIQZ_ADBLOCKED_PACKAGE "@cliqz/adblocker"
#define CLIQZ_ADBLOCKED_VERSION "1.23.5"
class QUrl;
class AdblockRequestInfo;
class AdBlockUrlInterceptor;
@ -66,6 +71,8 @@ class AdBlockManager : public QObject {
void processTerminated();
private slots:
void onPackageReady(const NodeJs::PackageMetadata& pkg);
void onPackageError(const NodeJs::PackageMetadata& pkg, const QString& error);
void onServerProcessFinished(int exit_code, QProcess::ExitStatus exit_status);
private:
@ -79,6 +86,7 @@ class AdBlockManager : public QObject {
private:
bool m_loaded;
bool m_enabled;
bool m_installing;
AdBlockIcon* m_adblockIcon;
AdBlockUrlInterceptor* m_interceptor;
QString m_unifiedFiltersFile;