FreshRSS/rssguard
1
0
Fork 0
mirror of https://github.com/martinrotter/rssguard.git synced 2025-02-05 20:03:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

save

Browse Source
This commit is contained in:
Martin Rotter 2021-05-07 06:56:21 +02:00
parent f344ace62e
commit 3df1bc75ce
11 changed files with 248 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
resources/desktop/com.github.rssguard.appdata.xml
View File

@ -30,7 +30,7 @@
<url type="donation">https://martinrotter.github.io/donate/</url>
<content_rating type="oars-1.1" />
<releases>
<release version="3.9.0" date="2021-05-04"/>
<release version="3.9.0" date="2021-05-07"/>
</releases>
<content_rating type="oars-1.0">
<content_attribute id="violence-cartoon">none</content_attribute>

1
resources/rssguard.qrc
View File

@ -6,6 +6,7 @@
<file>text/COPYING_GNU_GPL</file>
<file>text/COPYING_GNU_GPL_HTML</file>
<file>scripts/adblock/adblock-server.js</file>
<file>scripts/public_suffix_list.dat</file>
<file>graphics/rssguard.ico</file>

144
resources/scripts/adblock/adblock-server.js
View File

@ -3,7 +3,7 @@
// How to install:
// npm i -g @cliqz/adblocker
// npm i -g concat-stream
// npm i -g psl
// npm i -g tldts-experimental
// npm i -g node-fetch
//
// How to run:
@ -19,84 +19,100 @@
// }' 'http://localhost:<port>'
const fs = require('fs');
const psl = require('psl');
const tldts = require('tldts-experimental');
const adblock = require('@cliqz/adblocker')
const http = require('http');
const concat = require('concat-stream');
const constants = require('node:http2');
const fetch = require("node-fetch");
const cluster = require('cluster');
const numCPUs = require('os').cpus().length;
const port = process.argv[2];
const filtersFile = process.argv[3];
const engine = adblock.FiltersEngine.parse(fs.readFileSync(filtersFile, 'utf-8'));
const hostname = '127.0.0.1';
const server = http.createServer((req, res) => {
try {
console.log(new Date());
if (cluster.isPrimary) {
console.log(`Primary ${process.pid} is running`);
const chunks = [];
req.on('data', chunk => chunks.push(chunk));
req.on('end', () => {
// Fork workers.
for (let i = 0; i < numCPUs; i++) {
cluster.fork();
}
cluster.on('exit', (worker, code, signal) => {
console.log(`worker ${worker.process.pid} died`);
});
}
else {
const server = http.createServer((req, res) => {
try {
console.log(new Date());
try {
const jsonData = Buffer.concat(chunks);
const jsonStruct = JSON.parse(jsonData.toString());
const askUrl = jsonStruct['url'];
const askFilter = jsonStruct['filter'];
const askCosmetic = jsonStruct['cosmetic'];
const askUrlType = jsonStruct['url_type'];
const fullUrl = new URL(askUrl);
resultJson = {};
if (askFilter) {
const adblockMatch = engine.match(adblock.Request.fromRawDetails({
type: askUrlType,
url: askUrl,
}));
resultJson["filter"] = adblockMatch;
console.log(`adblocker: Filter is:\n${JSON.stringify(adblockMatch)}.`)
}
if (askCosmetic) {
const adblockCosmetic = engine.getCosmeticsFilters({
url: askUrl,
hostname: fullUrl.hostname,
domain: psl.parse(fullUrl.hostname).domain
});
resultJson["cosmetic"] = adblockCosmetic;
console.log(`adblocker: Cosmetic is:\n${JSON.stringify(adblockCosmetic)}.`)
}
res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(resultJson));
const chunks = [];
req.on('data', chunk => chunks.push(chunk));
req.on('end', () => {
console.log(new Date());
}
catch (inner_error) {
console.error(`adblocker: ${inner_error}.`);
res.statusCode = 500;
res.setHeader('Content-Type', 'text/plain');
res.end(String(inner_error));
}
})
}
catch (error) {
console.error(`adblocker: ${inner_error}.`);
try {
const jsonData = Buffer.concat(chunks);
const jsonStruct = JSON.parse(jsonData.toString());
res.statusCode = 500;
res.setHeader('Content-Type', 'text/plain');
res.end(String(error));
}
});
const askUrl = jsonStruct['url'];
const askFilter = jsonStruct['filter'];
const askCosmetic = jsonStruct['cosmetic'];
const askUrlType = jsonStruct['url_type'];
const fullUrl = new URL(askUrl);
server.listen(port, hostname, () => {
console.log(`adblocker: Server started at local port ${port}.`);
});
resultJson = {};
if (askFilter) {
const adblockMatch = engine.match(adblock.Request.fromRawDetails({
type: askUrlType,
url: askUrl,
}));
resultJson["filter"] = adblockMatch;
console.log(`adblocker: Filter is:\n${JSON.stringify(adblockMatch)}.`)
}
if (askCosmetic) {
const adblockCosmetic = engine.getCosmeticsFilters({
url: askUrl,
hostname: fullUrl.hostname,
domain: tldts.parse(fullUrl).domain
});
resultJson["cosmetic"] = adblockCosmetic;
console.log(`adblocker: Cosmetic is:\n${JSON.stringify(adblockCosmetic)}.`)
}
res.statusCode = 200;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(resultJson));
console.log(new Date());
}
catch (inner_error) {
console.error(`adblocker: ${inner_error}.`);
res.statusCode = 500;
res.setHeader('Content-Type', 'text/plain');
res.end(String(inner_error));
}
})
}
catch (error) {
console.error(`adblocker: ${inner_error}.`);
res.statusCode = 500;
res.setHeader('Content-Type', 'text/plain');
res.end(String(error));
}
});
server.listen(port, hostname, () => {
console.log(`adblocker: Server started at local port ${port}.`);
});
}

5
src/librssguard/definitions/definitions.h
View File

@ -14,13 +14,10 @@
#define SERVICE_CODE_INOREADER "inoreader"
#define SERVICE_CODE_GMAIL "gmail"
#define ADBLOCK_SERVER_PORT "48484"
#define ADBLOCK_HOWTO "https://github.com/martinrotter/rssguard/blob/master/resources/docs/Documentation.md#adblock"
#define ADBLOCK_UPDATE_DAYS_INTERVAL 14
#define ADBLOCK_ICON_ACTIVE "adblock"
#define ADBLOCK_ICON_DISABLED "adblock-disabled"
#define ADBLOCK_CUSTOMLIST_NAME "customlist.txt"
#define ADBLOCK_LISTS_SUBDIRECTORY "adblock"
#define ADBLOCK_EASYLIST_URL "https://easylist.to/easylist/easylist.txt"
#define OAUTH_DECRYPTION_KEY 11451167756100761335ul
#define OAUTH_REDIRECT_URI "http://localhost"

5
src/librssguard/miscellaneous/application.cpp
View File

@ -78,7 +78,10 @@ Application::Application(const QString& id, int& argc, char** argv)
#if defined(USE_WEBENGINE)
m_webFactory->urlIinterceptor()->load();
m_webFactory->adBlock()->load(true);
QTimer::singleShot(3000, this, [=]() {
m_webFactory->adBlock()->load(true);
});
#endif
qDebugNN << LOGSEC_CORE

5
src/librssguard/miscellaneous/skinfactory.cpp
View File

@ -60,9 +60,10 @@ QString SkinFactory::selectedSkinName() const {
return qApp->settings()->value(GROUP(GUI), SETTING(GUI::Skin)).toString();
}
QString SkinFactory::adBlockedPage(const QString& url) {
QString SkinFactory::adBlockedPage(const QString& url, const QString& filter) {
const QString& adblocked = currentSkin().m_adblocked.arg(tr("This page was blocked by AdBlock"),
tr(R"(Blocked URL: "%1")").arg(url));
tr(R"(Blocked URL: "%1"<br/>Used filter: "%2")").arg(url,
filter));
return currentSkin().m_layoutMarkupWrapper.arg(tr("This page was blocked by AdBlock"), adblocked);
}

2
src/librssguard/miscellaneous/skinfactory.h
View File

@ -50,7 +50,7 @@ class RSSGUARD_DLLSPEC SkinFactory : public QObject {
// after application restart.
QString selectedSkinName() const;
QString adBlockedPage(const QString& url);
QString adBlockedPage(const QString& url, const QString& filter);
// Gets skin about a particular skin.
Skin skinInfo(const QString& skin_name, bool* ok = nullptr) const;

153
src/librssguard/network-web/adblock/adblockmanager.cpp
View File

@ -3,6 +3,7 @@
#include "network-web/adblock/adblockmanager.h"
#include "exceptions/applicationexception.h"
#include "exceptions/networkexception.h"
#include "miscellaneous/application.h"
#include "miscellaneous/settings.h"
#include "network-web/adblock/adblockdialog.h"
@ -15,7 +16,10 @@
#include <QDateTime>
#include <QDir>
#include <QJsonDocument>
#include <QJsonObject>
#include <QMessageBox>
#include <QProcess>
#include <QTimer>
#include <QUrlQuery>
#include <QWebEngineProfile>
@ -24,24 +28,43 @@ AdBlockManager::AdBlockManager(QObject* parent)
: QObject(parent), m_loaded(false), m_enabled(false), m_interceptor(new AdBlockUrlInterceptor(this)) {
m_adblockIcon = new AdBlockIcon(this);
m_adblockIcon->setObjectName(QSL("m_adblockIconAction"));
m_unifiedFiltersFile = qApp->userDataFolder() + QDir::separator() + QSL("adblock-unified-filters.txt");
m_serverProcess = new QProcess(this);
}
bool AdBlockManager::block(const AdblockRequestInfo& request) const {
AdBlockManager::~AdBlockManager() {
if (m_serverProcess->state() == QProcess::ProcessState::Running) {
m_serverProcess->kill();
}
}
BlockingResult AdBlockManager::block(const AdblockRequestInfo& request) const {
if (!isEnabled()) {
return false;
return { false };
}
const QString url_string = request.requestUrl().toEncoded().toLower();
const QString url_scheme = request.requestUrl().scheme().toLower();
if (!canRunOnScheme(url_scheme)) {
return false;
return { false };
}
else {
// TODO: start server if needed, call it.
return false;
if (m_serverProcess->state() == QProcess::ProcessState::Running) {
try {
auto result = askServerIfBlocked(url_string);
return result;
}
catch (const ApplicationException& ex) {
qCriticalNN << LOGSEC_ADBLOCK
<< "HTTP error when calling server:"
<< QUOTE_W_SPACE_DOT(ex.message());
}
}
else {
return { false };
}
}
}
@ -123,8 +146,106 @@ void AdBlockManager::showDialog() {
AdBlockDialog(qApp->mainFormWidget()).exec();
}
BlockingResult AdBlockManager::askServerIfBlocked(const QString& url) const {
QJsonObject req_obj;
QByteArray out;
QElapsedTimer tmr;
req_obj["url"] = url;
req_obj["filter"] = true;
tmr.start();
auto network_res = NetworkFactory::performNetworkOperation(QSL("http://%1:%2").arg(QHostAddress(QHostAddress::SpecialAddress::LocalHost).toString(),
ADBLOCK_SERVER_PORT),
500,
QJsonDocument(req_obj).toJson(),
out,
QNetworkAccessManager::Operation::PostOperation,
{ {
QSL(HTTP_HEADERS_CONTENT_TYPE).toLocal8Bit(),
QSL("application/json").toLocal8Bit() } });
if (network_res.first == QNetworkReply::NetworkError::NoError) {
qDebugNN << LOGSEC_ADBLOCK
<< "Query to server took "
<< tmr.elapsed()
<< " ms.";
QJsonObject out_obj = QJsonDocument::fromJson(out).object();
bool blocking = out_obj["filter"].toObject()["match"].toBool();
return {
blocking,
blocking
? out_obj["filter"].toObject()["filter"].toObject()["filter"].toString()
: QString()
};
}
else {
throw NetworkException(network_res.first);
}
}
void AdBlockManager::restartServer() {
// TODO:
if (m_serverProcess->state() == QProcess::ProcessState::Running) {
m_serverProcess->kill();
if (!m_serverProcess->waitForFinished(1000)) {
m_serverProcess->deleteLater();
m_serverProcess = new QProcess(this);
}
}
QString temp_server = QDir::toNativeSeparators(IOFactory::getSystemFolder(QStandardPaths::StandardLocation::TempLocation)) +
QDir::separator() +
QSL("adblock-server.js");
if (!IOFactory::copyFile(QSL(":/scripts/adblock/adblock-server.js"), temp_server)) {
qCriticalNN << LOGSEC_ADBLOCK << "Failed to copy server file to TEMP.";
}
#if defined(Q_OS_WIN)
m_serverProcess->setProgram(QSL("node.exe"));
#else
m_serverProcess->setProgram(QSL("node"));
#endif
m_serverProcess->setArguments({
QDir::toNativeSeparators(temp_server),
ADBLOCK_SERVER_PORT,
QDir::toNativeSeparators(m_unifiedFiltersFile)
});
m_serverProcess->setProcessEnvironment(QProcessEnvironment::systemEnvironment());
auto pe = m_serverProcess->processEnvironment();
QString node_path =
#if defined(Q_OS_WIN)
pe.value(QSL("APPDATA")) +
#elif defined(Q_OS_LINUX)
QSL("/usr/local/lib/node_modules") +
#else
QDir::toNativeSeparators(IOFactory::getSystemFolder(QStandardPaths::StandardLocation::GenericDataLocation)) +
#endif
QDir::separator() +
QSL("npm") +
QDir::separator() +
QSL("node_modules");
if (!pe.contains(QSL("NODE_PATH"))) {
pe.insert(QSL("NODE_PATH"), node_path);
}
m_serverProcess->setProcessEnvironment(pe);
m_serverProcess->setProcessChannelMode(QProcess::ProcessChannelMode::ForwardedErrorChannel);
if (!m_serverProcess->open()) {
qWarningNN << LOGSEC_ADBLOCK << "Failed to start server.";
}
else {
qDebugNN << LOGSEC_ADBLOCK << "Started server.";
}
}
void AdBlockManager::updateUnifiedFiltersFile() {
@ -132,8 +253,7 @@ void AdBlockManager::updateUnifiedFiltersFile() {
QFile::remove(m_unifiedFiltersFile);
}
// TODO: generate file
QByteArray unified_contents;
QString unified_contents;
auto filter_lists = filterLists();
// Download filters one by one and append.
@ -142,10 +262,16 @@ void AdBlockManager::updateUnifiedFiltersFile() {
auto res = NetworkFactory::performNetworkOperation(filter_list_url,
2000,
{},
out, QNetworkAccessManager::Operation::GetOperation);
out,
QNetworkAccessManager::Operation::GetOperation);
if (res.first == QNetworkReply::NetworkError::NoError) {
unified_contents += out;
unified_contents = unified_contents.append(QString::fromUtf8(out));
unified_contents = unified_contents.append('\n');
qDebugNN << LOGSEC_ADBLOCK
<< "Downloaded filter list from"
<< QUOTE_W_SPACE_DOT(filter_list_url);
}
else {
qWarningNN << LOGSEC_ADBLOCK
@ -156,7 +282,7 @@ void AdBlockManager::updateUnifiedFiltersFile() {
}
}
unified_contents += customFilters().join(QSL("\n")).toUtf8();
unified_contents = unified_contents.append(customFilters().join(QSL("\n")));
// Save.
m_unifiedFiltersFile = IOFactory::getSystemFolder(QStandardPaths::StandardLocation::TempLocation) +
@ -164,10 +290,9 @@ void AdBlockManager::updateUnifiedFiltersFile() {
QSL("adblock.filters");
try {
IOFactory::writeFile(m_unifiedFiltersFile, unified_contents);
IOFactory::writeFile(m_unifiedFiltersFile, unified_contents.toUtf8());
if (m_enabled) {
// TODO: re-start nodejs adblock server.
restartServer();
}
}

16
src/librssguard/network-web/adblock/adblockmanager.h
View File

@ -6,15 +6,26 @@
#include <QObject>
class QUrl;
class QProcess;
class AdblockRequestInfo;
class AdBlockUrlInterceptor;
class AdBlockIcon;
struct BlockingResult {
bool m_blocked;
QString m_blockedByFilter;
BlockingResult(bool blocked, QString blocked_by_filter = {})
: m_blocked(blocked), m_blockedByFilter(std::move(blocked_by_filter)) {}
};
class AdBlockManager : public QObject {
Q_OBJECT
public:
explicit AdBlockManager(QObject* parent = nullptr);
virtual ~AdBlockManager();
// If "initial_load" is false, then we want to explicitly turn off
// Adblock if it is running or turn on when not running.
@ -27,7 +38,7 @@ class AdBlockManager : public QObject {
AdBlockIcon* adBlockIcon() const;
// General methods for adblocking.
bool block(const AdblockRequestInfo& request) const;
BlockingResult block(const AdblockRequestInfo& request) const;
QString elementHidingRulesForDomain(const QUrl& url) const;
QStringList filterLists() const;
@ -47,6 +58,8 @@ class AdBlockManager : public QObject {
void enabledChanged(bool enabled);
private:
BlockingResult askServerIfBlocked(const QString& url) const;
void restartServer();
private:
@ -55,6 +68,7 @@ class AdBlockManager : public QObject {
AdBlockIcon* m_adblockIcon;
AdBlockUrlInterceptor* m_interceptor;
QString m_unifiedFiltersFile;
QProcess* m_serverProcess;
};
inline AdBlockIcon* AdBlockManager::adBlockIcon() const {

2
src/librssguard/network-web/adblock/adblockurlinterceptor.cpp
View File

@ -27,7 +27,7 @@ AdBlockUrlInterceptor::AdBlockUrlInterceptor(AdBlockManager* manager)
: UrlInterceptor(manager), m_manager(manager) {}
void AdBlockUrlInterceptor::interceptRequest(QWebEngineUrlRequestInfo& info) {
if (m_manager->block(AdblockRequestInfo(info))) {
if (m_manager->block(AdblockRequestInfo(info)).m_blocked) {
info.block(true);
qWarningNN << LOGSEC_ADBLOCK << "Blocked request:" << QUOTE_W_SPACE_DOT(info.requestUrl().toString());

4
src/librssguard/network-web/webpage.cpp
View File

@ -48,9 +48,9 @@ bool WebPage::acceptNavigationRequest(const QUrl& url, NavigationType type, bool
if (is_main_frame) {
auto blocked = qApp->web()->adBlock()->block(AdblockRequestInfo(url));
if (blocked) {
if (blocked.m_blocked) {
// This website is entirely blocked.
setHtml(qApp->skins()->adBlockedPage(url.toString()),
setHtml(qApp->skins()->adBlockedPage(url.toString(), blocked.m_blockedByFilter),
QUrl::fromUserInput(INTERNAL_URL_ADBLOCKED));
return false;
}