Great refactorings for Adblock - fully working site blocking with correct error pages, removed some unused stuff.

This commit is contained in:
Martin Rotter 2021-01-08 08:56:18 +01:00
parent a4562a7800
commit c789f14b1d
16 changed files with 219 additions and 79 deletions

View File

@ -30,7 +30,7 @@
<url type="donation">https://martinrotter.github.io/donate/</url>
<content_rating type="oars-1.1" />
<releases>
<release version="3.8.4" date="2021-01-07"/>
<release version="3.8.4" date="2021-01-08"/>
</releases>
<content_rating type="oars-1.0">
<content_attribute id="violence-cartoon">none</content_attribute>

View File

@ -408,6 +408,7 @@ equals(USE_WEBENGINE, true) {
network-web/adblock/adblocksubscription.h \
network-web/adblock/adblocktreewidget.h \
network-web/adblock/adblockurlinterceptor.h \
network-web/adblock/adblockrequestinfo.h \
network-web/urlinterceptor.h \
network-web/networkurlinterceptor.h \
gui/treewidget.h
@ -422,6 +423,7 @@ equals(USE_WEBENGINE, true) {
network-web/adblock/adblocksubscription.cpp \
network-web/adblock/adblocktreewidget.cpp \
network-web/adblock/adblockurlinterceptor.cpp \
network-web/adblock/adblockrequestinfo.cpp \
network-web/networkurlinterceptor.cpp \
gui/treewidget.cpp

View File

@ -24,6 +24,7 @@
#include "network-web/adblock/adblockdialog.h"
#include "network-web/adblock/adblockicon.h"
#include "network-web/adblock/adblockmatcher.h"
#include "network-web/adblock/adblockrequestinfo.h"
#include "network-web/adblock/adblocksubscription.h"
#include "network-web/adblock/adblockurlinterceptor.h"
#include "network-web/networkurlinterceptor.h"
@ -38,7 +39,6 @@
#include <QTimer>
#include <QUrlQuery>
#include <QWebEngineProfile>
#include <QWebEngineUrlRequestInfo>
AdBlockManager::AdBlockManager(QObject* parent)
: QObject(parent), m_loaded(false), m_enabled(false), m_matcher(new AdBlockMatcher(this)),
@ -55,43 +55,25 @@ QList<AdBlockSubscription*> AdBlockManager::subscriptions() const {
return m_subscriptions;
}
bool AdBlockManager::block(QWebEngineUrlRequestInfo& request) {
const AdBlockRule* AdBlockManager::block(const AdblockRequestInfo& request) {
QMutexLocker locker(&m_mutex);
if (!isEnabled()) {
return false;
}
const QString urlString = request.requestUrl().toEncoded().toLower();
const QString urlDomain = request.requestUrl().host().toLower();
const QString urlScheme = request.requestUrl().scheme().toLower();
const QString url_string = request.requestUrl().toEncoded().toLower();
const QString url_domain = request.requestUrl().host().toLower();
const QString url_scheme = request.requestUrl().scheme().toLower();
if (!canRunOnScheme(urlScheme) || !canBeBlocked(request.firstPartyUrl())) {
return false;
if (!canRunOnScheme(url_scheme) || !canBeBlocked(request.firstPartyUrl())) {
return nullptr;
}
else {
const AdBlockRule* blocked_rule = m_matcher->match(request, url_domain, url_string);
bool res = false;
const AdBlockRule* blockedRule = m_matcher->match(request, urlDomain, urlString);
if (blockedRule != nullptr) {
if (request.resourceType() == QWebEngineUrlRequestInfo::ResourceTypeMainFrame) {
QUrlQuery query;
QUrl url(QSL("%1://%2").arg(APP_LOW_NAME, ADBLOCK_ADBLOCKED_PAGE));
query.addQueryItem(QSL("rule"), blockedRule->filter());
query.addQueryItem(QSL("subscription"), blockedRule->subscription()->title());
url.setQuery(query);
res = true;
request.redirect(url);
}
else {
res = true;
request.block(true);
}
return blocked_rule;
}
return res;
}
QStringList AdBlockManager::disabledRules() const {

View File

@ -26,7 +26,7 @@
#include <QStringList>
class QUrl;
class QWebEngineUrlRequestInfo;
class AdblockRequestInfo;
class AdBlockMatcher;
class AdBlockCustomList;
class AdBlockSubscription;
@ -48,7 +48,9 @@ class AdBlockManager : public QObject {
void load(bool initial_load);
void save();
bool block(QWebEngineUrlRequestInfo& request);
// General method for adblocking. Returns pointer to rule if request should
// be blocked.
const AdBlockRule* block(const AdblockRequestInfo& request);
bool isEnabled() const;
bool canRunOnScheme(const QString& scheme) const;

View File

@ -17,12 +17,12 @@
// You should have received a copy of the GNU General Public License
// along with RSS Guard. If not, see <http://www.gnu.org/licenses/>.
#include "network-web/adblock/adblockmanager.h"
#include "network-web/adblock/adblockmatcher.h"
#include "network-web/adblock/adblockrule.h"
#include "network-web/adblock/adblocksubscription.h"
#include "definitions/definitions.h"
#include "network-web/adblock/adblockmanager.h"
#include "network-web/adblock/adblockrule.h"
#include "network-web/adblock/adblocksubscription.h"
AdBlockMatcher::AdBlockMatcher(AdBlockManager* manager)
: QObject(manager), m_manager(manager) {}
@ -31,7 +31,7 @@ AdBlockMatcher::~AdBlockMatcher() {
clear();
}
const AdBlockRule* AdBlockMatcher::match(const QWebEngineUrlRequestInfo& request, const QString& urlDomain,
const AdBlockRule* AdBlockMatcher::match(const AdblockRequestInfo& request, const QString& urlDomain,
const QString& urlString) const {
// Exception rules.
if (m_networkExceptionTree.find(request, urlDomain, urlString) != nullptr) {
@ -194,7 +194,6 @@ void AdBlockMatcher::update() {
// (In my testings, 4931 is the number that makes it crash).
// So let's split it by 1000 selectors.
int hidingRulesCount = 0;
QHashIterator<QString, const AdBlockRule*> it(cssRulesHash);
while (it.hasNext()) {

View File

@ -27,7 +27,7 @@
#include <QObject>
#include <QVector>
class QWebEngineUrlRequestInfo;
class AdblockRequestInfo;
class AdBlockManager;
class AdBlockMatcher : public QObject {
@ -37,7 +37,7 @@ class AdBlockMatcher : public QObject {
explicit AdBlockMatcher(AdBlockManager* manager);
virtual ~AdBlockMatcher();
const AdBlockRule* match(const QWebEngineUrlRequestInfo& request, const QString& urlDomain, const QString& urlString) const;
const AdBlockRule* match(const AdblockRequestInfo& request, const QString& urlDomain, const QString& urlString) const;
bool adBlockDisabledForUrl(const QUrl& url) const;
bool elemHideDisabledForUrl(const QUrl& url) const;
@ -51,14 +51,12 @@ class AdBlockMatcher : public QObject {
private:
AdBlockManager* m_manager;
QVector<AdBlockRule*> m_createdRules;
QVector<const AdBlockRule*> m_networkExceptionRules;
QVector<const AdBlockRule*> m_networkBlockRules;
QVector<const AdBlockRule*> m_domainRestrictedCssRules;
QVector<const AdBlockRule*> m_documentRules;
QVector<const AdBlockRule*> m_elemhideRules;
QString m_elementHidingRules;
AdBlockSearchTree m_networkBlockTree;
AdBlockSearchTree m_networkExceptionTree;

View File

@ -0,0 +1,79 @@
// For license of this file, see <project-root-folder>/LICENSE.md.
#include "network-web/adblock/adblockrequestinfo.h"
#include "definitions/definitions.h"
AdblockRequestInfo::AdblockRequestInfo(const QWebEngineUrlRequestInfo& webengine_info) {
initialize(webengine_info);
}
AdblockRequestInfo::AdblockRequestInfo(const QUrl& url) {
initialize(url);
}
QWebEngineUrlRequestInfo::ResourceType AdblockRequestInfo::resourceType() const {
return m_resourceType;
}
void AdblockRequestInfo::setResourceType(const QWebEngineUrlRequestInfo::ResourceType& resourceType) {
m_resourceType = resourceType;
}
QWebEngineUrlRequestInfo::NavigationType AdblockRequestInfo::navigationType() const {
return m_navigationType;
}
void AdblockRequestInfo::setNavigationType(const QWebEngineUrlRequestInfo::NavigationType& navigationType) {
m_navigationType = navigationType;
}
QUrl AdblockRequestInfo::requestUrl() const {
return m_requestUrl;
}
void AdblockRequestInfo::setRequestUrl(const QUrl& requestUrl) {
m_requestUrl = requestUrl;
}
QUrl AdblockRequestInfo::firstPartyUrl() const {
return m_firstPartyUrl;
}
void AdblockRequestInfo::setFirstPartyUrl(const QUrl& firstPartyUrl) {
m_firstPartyUrl = firstPartyUrl;
}
QUrl AdblockRequestInfo::initiator() const {
return m_initiator;
}
void AdblockRequestInfo::setInitiator(const QUrl& initiator) {
m_initiator = initiator;
}
QByteArray AdblockRequestInfo::requestMethod() const {
return m_requestMethod;
}
void AdblockRequestInfo::setRequestMethod(const QByteArray& requestMethod) {
m_requestMethod = requestMethod;
}
void AdblockRequestInfo::initialize(const QWebEngineUrlRequestInfo& webengine_info) {
setFirstPartyUrl(webengine_info.firstPartyUrl());
setInitiator(webengine_info.initiator());
setNavigationType(webengine_info.navigationType());
setRequestMethod(webengine_info.requestMethod());
setRequestUrl(webengine_info.requestUrl());
setResourceType(webengine_info.resourceType());
}
void AdblockRequestInfo::initialize(const QUrl& url) {
setFirstPartyUrl(url);
setInitiator(url);
setNavigationType(QWebEngineUrlRequestInfo::NavigationType::NavigationTypeTyped);
setRequestMethod(QSL("GET").toLocal8Bit());
setRequestUrl(url);
setResourceType(QWebEngineUrlRequestInfo::ResourceType::ResourceTypeMainFrame);
}

View File

@ -0,0 +1,44 @@
// For license of this file, see <project-root-folder>/LICENSE.md.
#ifndef ADBLOCKREQUESTINFO_H
#define ADBLOCKREQUESTINFO_H
#include <QWebEngineUrlRequestInfo>
class AdblockRequestInfo {
public:
explicit AdblockRequestInfo(const QWebEngineUrlRequestInfo& webengine_info);
explicit AdblockRequestInfo(const QUrl& url);
QWebEngineUrlRequestInfo::ResourceType resourceType() const;
void setResourceType(const QWebEngineUrlRequestInfo::ResourceType& resourceType);
QWebEngineUrlRequestInfo::NavigationType navigationType() const;
void setNavigationType(const QWebEngineUrlRequestInfo::NavigationType& navigationType);
QUrl requestUrl() const;
void setRequestUrl(const QUrl& requestUrl);
QUrl firstPartyUrl() const;
void setFirstPartyUrl(const QUrl& firstPartyUrl);
QUrl initiator() const;
void setInitiator(const QUrl& initiator);
QByteArray requestMethod() const;
void setRequestMethod(const QByteArray& requestMethod);
private:
void initialize(const QWebEngineUrlRequestInfo& webengine_info);
void initialize(const QUrl& url);
private:
QWebEngineUrlRequestInfo::ResourceType m_resourceType;
QWebEngineUrlRequestInfo::NavigationType m_navigationType;
QUrl m_requestUrl;
QUrl m_firstPartyUrl;
QUrl m_initiator;
QByteArray m_requestMethod;
};
#endif // ADBLOCKREQUESTINFO_H

View File

@ -49,6 +49,7 @@
#include "network-web/adblock/adblockrule.h"
#include "definitions/definitions.h"
#include "network-web/adblock/adblockrequestinfo.h"
#include "network-web/adblock/adblocksubscription.h"
#include <QRegularExpression>
@ -56,7 +57,6 @@
#include <QStringList>
#include <QUrl>
#include <QWebEnginePage>
#include <QWebEngineUrlRequestInfo>
static QString toSecondLevelDomain(const QUrl& url) {
const QString topLevelDomain = url.topLevelDomain();
@ -178,12 +178,14 @@ bool AdBlockRule::urlMatch(const QUrl& url) const {
}
}
bool AdBlockRule::networkMatch(const QWebEngineUrlRequestInfo& request, const QString& domain, const QString& encodedUrl) const {
bool AdBlockRule::networkMatch(const AdblockRequestInfo& request,
const QString& domain,
const QString& encoded_url) const {
if (m_type == CssRule || !m_isEnabled || m_isInternalDisabled) {
return false;
}
bool matched = stringMatch(domain, encodedUrl);
bool matched = stringMatch(domain, encoded_url);
if (matched) {
// Check domain restrictions.
@ -277,7 +279,7 @@ bool AdBlockRule::matchDomain(const QString& domain) const {
return false;
}
bool AdBlockRule::matchThirdParty(const QWebEngineUrlRequestInfo& request) const {
bool AdBlockRule::matchThirdParty(const AdblockRequestInfo& request) const {
// Third-party matching should be performed on second-level domains.
const QString firstPartyHost = toSecondLevelDomain(request.firstPartyUrl());
const QString host = toSecondLevelDomain(request.requestUrl());
@ -286,43 +288,43 @@ bool AdBlockRule::matchThirdParty(const QWebEngineUrlRequestInfo& request) const
return hasException(ThirdPartyOption) ? !match : match;
}
bool AdBlockRule::matchObject(const QWebEngineUrlRequestInfo& request) const {
bool AdBlockRule::matchObject(const AdblockRequestInfo& request) const {
bool match = request.resourceType() == QWebEngineUrlRequestInfo::ResourceTypeObject;
return hasException(ObjectOption) ? !match : match;
}
bool AdBlockRule::matchSubdocument(const QWebEngineUrlRequestInfo& request) const {
bool AdBlockRule::matchSubdocument(const AdblockRequestInfo& request) const {
bool match = request.resourceType() == QWebEngineUrlRequestInfo::ResourceTypeSubFrame;
return hasException(SubdocumentOption) ? !match : match;
}
bool AdBlockRule::matchXmlHttpRequest(const QWebEngineUrlRequestInfo& request) const {
bool AdBlockRule::matchXmlHttpRequest(const AdblockRequestInfo& request) const {
bool match = request.resourceType() == QWebEngineUrlRequestInfo::ResourceTypeXhr;
return hasException(XMLHttpRequestOption) ? !match : match;
}
bool AdBlockRule::matchImage(const QWebEngineUrlRequestInfo& request) const {
bool AdBlockRule::matchImage(const AdblockRequestInfo& request) const {
bool match = request.resourceType() == QWebEngineUrlRequestInfo::ResourceTypeImage;
return hasException(ImageOption) ? !match : match;
}
bool AdBlockRule::matchScript(const QWebEngineUrlRequestInfo& request) const {
bool AdBlockRule::matchScript(const AdblockRequestInfo& request) const {
bool match = request.resourceType() == QWebEngineUrlRequestInfo::ResourceTypeScript;
return hasException(ScriptOption) ? !match : match;
}
bool AdBlockRule::matchStyleSheet(const QWebEngineUrlRequestInfo& request) const {
bool AdBlockRule::matchStyleSheet(const AdblockRequestInfo& request) const {
bool match = request.resourceType() == QWebEngineUrlRequestInfo::ResourceTypeStylesheet;
return hasException(StyleSheetOption) ? !match : match;
}
bool AdBlockRule::matchObjectSubrequest(const QWebEngineUrlRequestInfo& request) const {
bool AdBlockRule::matchObjectSubrequest(const AdblockRequestInfo& request) const {
bool match = request.resourceType() == QWebEngineUrlRequestInfo::ResourceTypeSubResource;
return hasException(ObjectSubrequestOption) ? !match : match;

View File

@ -53,7 +53,7 @@
#include <QStringMatcher>
class QUrl;
class QWebEngineUrlRequestInfo;
class AdblockRequestInfo;
class AdBlockSubscription;
class AdBlockRule {
@ -88,17 +88,17 @@ class AdBlockRule {
bool isInternalDisabled() const;
bool urlMatch(const QUrl& url) const;
bool networkMatch(const QWebEngineUrlRequestInfo& request, const QString& domain, const QString& encodedUrl) const;
bool networkMatch(const AdblockRequestInfo& request, const QString& domain, const QString& encoded_url) const;
bool matchDomain(const QString& domain) const;
bool matchThirdParty(const QWebEngineUrlRequestInfo& request) const;
bool matchObject(const QWebEngineUrlRequestInfo& request) const;
bool matchSubdocument(const QWebEngineUrlRequestInfo& request) const;
bool matchXmlHttpRequest(const QWebEngineUrlRequestInfo& request) const;
bool matchImage(const QWebEngineUrlRequestInfo& request) const;
bool matchScript(const QWebEngineUrlRequestInfo& request) const;
bool matchStyleSheet(const QWebEngineUrlRequestInfo& request) const;
bool matchObjectSubrequest(const QWebEngineUrlRequestInfo& request) const;
bool matchThirdParty(const AdblockRequestInfo& request) const;
bool matchObject(const AdblockRequestInfo& request) const;
bool matchSubdocument(const AdblockRequestInfo& request) const;
bool matchXmlHttpRequest(const AdblockRequestInfo& request) const;
bool matchImage(const AdblockRequestInfo& request) const;
bool matchScript(const AdblockRequestInfo& request) const;
bool matchStyleSheet(const AdblockRequestInfo& request) const;
bool matchObjectSubrequest(const AdblockRequestInfo& request) const;
protected:
bool matchDomain(const QString& pattern, const QString& domain) const;
@ -116,6 +116,7 @@ class AdBlockRule {
StringContainsMatchRule = 4,
Invalid = 5
};
enum RuleOption {
DomainRestrictedOption = 1,
ThirdPartyOption = 2,
@ -160,14 +161,12 @@ class AdBlockRule {
// Case sensitivity for string matching
Qt::CaseSensitivity m_caseSensitivity;
bool m_isEnabled;
bool m_isException;
bool m_isInternalDisabled;
QStringList m_allowedDomains;
QStringList m_blockedDomains;
QString m_regexPattern;
QList<QStringMatcher> matchers;
friend class AdBlockMatcher;

View File

@ -17,12 +17,11 @@
// You should have received a copy of the GNU General Public License
// along with RSS Guard. If not, see <http://www.gnu.org/licenses/>.
#include "network-web/adblock/adblockrule.h"
#include "network-web/adblock/adblocksearchtree.h"
#include "definitions/definitions.h"
#include <QWebEngineUrlRequestInfo>
#include "network-web/adblock/adblockrequestinfo.h"
#include "network-web/adblock/adblockrule.h"
AdBlockSearchTree::AdBlockSearchTree() : m_root(new Node) {}
@ -67,7 +66,9 @@ bool AdBlockSearchTree::add(const AdBlockRule* rule) {
return true;
}
const AdBlockRule* AdBlockSearchTree::find(const QWebEngineUrlRequestInfo& request, const QString& domain, const QString& urlString) const {
const AdBlockRule* AdBlockSearchTree::find(const AdblockRequestInfo& request,
const QString& domain,
const QString& urlString) const {
int len = urlString.size();
if (len <= 0) {
@ -85,8 +86,9 @@ const AdBlockRule* AdBlockSearchTree::find(const QWebEngineUrlRequestInfo& reque
return nullptr;
}
const AdBlockRule* AdBlockSearchTree::prefixSearch(const QWebEngineUrlRequestInfo& request, const QString& domain,
const QString& urlString, const QString& choppedUrlString, int len) const {
const AdBlockRule* AdBlockSearchTree::prefixSearch(const AdblockRequestInfo& request, const QString& domain,
const QString& urlString, const QString& choppedUrlString,
int len) const {
if (len <= 0) {
return nullptr;
}

View File

@ -23,7 +23,7 @@
#include <QChar>
#include <QHash>
class QWebEngineUrlRequestInfo;
class AdblockRequestInfo;
class AdBlockRule;
class AdBlockSearchTree {
@ -34,20 +34,21 @@ class AdBlockSearchTree {
void clear();
bool add(const AdBlockRule* rule);
const AdBlockRule* find(const QWebEngineUrlRequestInfo& request, const QString& domain, const QString& urlString) const;
const AdBlockRule* find(const AdblockRequestInfo& request, const QString& domain, const QString& urlString) const;
private:
struct Node {
QChar c;
const AdBlockRule* rule;
QHash<QChar, Node*> children;
Node() : c(0), rule(0) { }
};
const AdBlockRule* prefixSearch(const QWebEngineUrlRequestInfo& request, const QString& domain,
const QString& urlString, const QString& choppedUrlString, int len) const;
const AdBlockRule* prefixSearch(const AdblockRequestInfo& request, const QString& domain,
const QString& urlString, const QString& choppedUrlString,
int len) const;
void deleteNode(Node* node);

View File

@ -19,11 +19,17 @@
#include "network-web/adblock/adblockurlinterceptor.h"
#include "definitions/definitions.h"
#include "network-web/adblock/adblockmanager.h"
#include "network-web/adblock/adblockrequestinfo.h"
AdBlockUrlInterceptor::AdBlockUrlInterceptor(AdBlockManager* manager)
: UrlInterceptor(manager), m_manager(manager) {}
void AdBlockUrlInterceptor::interceptRequest(QWebEngineUrlRequestInfo& info) {
m_manager->block(info);
if (m_manager->block(AdblockRequestInfo(info)) != nullptr) {
info.block(true);
qWarning() << LOGSEC_ADBLOCK << "Blocked request:" << QUOTE_W_SPACE_DOT(info.requestUrl().toString());
}
}

View File

@ -24,10 +24,10 @@
#include "network-web/urlinterceptor.h"
NetworkUrlInterceptor::NetworkUrlInterceptor(QObject* parent)
: QWebEngineUrlRequestInterceptor(parent), m_sendDNT(false) {}
: QWebEngineUrlRequestInterceptor(parent), m_sendDnt(false) {}
void NetworkUrlInterceptor::interceptRequest(QWebEngineUrlRequestInfo& info) {
if (m_sendDNT) {
if (m_sendDnt) {
info.setHttpHeader(QByteArrayLiteral("DNT"), QByteArrayLiteral("1"));
}
@ -49,5 +49,5 @@ void NetworkUrlInterceptor::removeUrlInterceptor(UrlInterceptor* interceptor) {
}
void NetworkUrlInterceptor::load() {
m_sendDNT = qApp->settings()->value(GROUP(Browser), SETTING(Browser::SendDNT)).toBool();
m_sendDnt = qApp->settings()->value(GROUP(Browser), SETTING(Browser::SendDNT)).toBool();
}

View File

@ -39,7 +39,7 @@ class NetworkUrlInterceptor : public QWebEngineUrlRequestInterceptor {
private:
QList<UrlInterceptor*> m_interceptors;
bool m_sendDNT;
bool m_sendDnt;
};
#endif // NETWORKURLINTERCEPTOR_H

View File

@ -4,12 +4,19 @@
#include "definitions/definitions.h"
#include "gui/webviewer.h"
#include "miscellaneous/application.h"
#include "network-web/adblock/adblockmanager.h"
#include "network-web/adblock/adblockrequestinfo.h"
#include "network-web/adblock/adblockrule.h"
#include "network-web/adblock/adblocksubscription.h"
#include "network-web/webfactory.h"
#include "services/abstract/rootitem.h"
#include "services/abstract/serviceroot.h"
#include <QString>
#include <QStringList>
#include <QUrl>
#include <QUrlQuery>
WebPage::WebPage(QObject* parent) : QWebEnginePage(parent) {
setBackgroundColor(Qt::transparent);
@ -22,6 +29,23 @@ WebViewer* WebPage::view() const {
bool WebPage::acceptNavigationRequest(const QUrl& url, NavigationType type, bool isMainFrame) {
const RootItem* root = view()->root();
if (isMainFrame) {
auto* adblock_rule = qApp->web()->adBlock()->block(AdblockRequestInfo(url));
if (adblock_rule != nullptr) {
// This website is entirely blocked.
QUrlQuery query;
QUrl new_url(QSL("%1:///%2/").arg(APP_LOW_NAME, ADBLOCK_ADBLOCKED_PAGE));
query.addQueryItem(QSL("rule"), adblock_rule->filter());
query.addQueryItem(QSL("subscription"), adblock_rule->subscription()->title());
new_url.setQuery(query);
setUrl(new_url);
return false;
}
}
if (url.toString().startsWith(INTERNAL_URL_PASSATTACHMENT) &&
root != nullptr &&
root->getParentServiceRoot()->downloadAttachmentOnMyOwn(url)) {