FreshRSS/rssguard
1
0
Fork 0
mirror of https://github.com/martinrotter/rssguard.git synced 2025-02-02 10:27:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

make new scraping feature more robust

Browse Source
This commit is contained in:
Martin Rotter 2024-03-08 08:36:32 +01:00
parent 47714ac590
commit 35b378d8b1
5 changed files with 34 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
resources/scripts/builtin_js/observer.js
View File

@ -3,7 +3,7 @@ const targetNode = document;
const waitTimeMs = 5000;
const idleIdString = "iiddllee";
var scrollings = 0; // Change to higher number to force some scrollings.
var scrollings = 1; // Change to higher number to force some scrollings.
var lastResourceTime = new Date();
// Setup DOM observer and observe for changes in elements only.

47
src/librssguard/gui/webviewers/webengine/webengineviewer.cpp
View File

@ -202,37 +202,50 @@ QUrl WebEngineViewer::url() const {
return QWebEngineView::url();
}
QByteArray WebEngineViewer::getJsEnabledHtml(const QString& url) {
QByteArray WebEngineViewer::getJsEnabledHtml(const QString& url, bool worker_thread) {
WebEnginePage* page = new WebEnginePage();
WebEngineViewer* viewer = nullptr;
QMetaObject::invokeMethod(
qApp,
[&] {
// NOTE: Must be created on main thread.
viewer = new WebEngineViewer();
},
Qt::ConnectionType::BlockingQueuedConnection);
if (worker_thread) {
QMetaObject::invokeMethod(
qApp,
[&] {
// NOTE: Must be created on main thread.
viewer = new WebEngineViewer();
},
Qt::ConnectionType::BlockingQueuedConnection);
viewer->moveToThread(qApp->thread());
page->moveToThread(qApp->thread());
viewer->moveToThread(qApp->thread());
page->moveToThread(qApp->thread());
}
else {
viewer = new WebEngineViewer();
}
viewer->setPage(page);
viewer->setAttribute(Qt::WidgetAttribute::WA_DontShowOnScreen, true);
viewer->setAttribute(Qt::WidgetAttribute::WA_DeleteOnClose, true);
QMetaObject::invokeMethod(viewer, "show", Qt::ConnectionType::BlockingQueuedConnection);
QString html;
QMetaObject::invokeMethod(page,
"pageHtml",
Qt::ConnectionType::BlockingQueuedConnection,
Q_RETURN_ARG(QString, html),
Q_ARG(QString, url));
if (worker_thread) {
QMetaObject::invokeMethod(viewer, "show", Qt::ConnectionType::BlockingQueuedConnection);
QMetaObject::invokeMethod(page,
"pageHtml",
Qt::ConnectionType::BlockingQueuedConnection,
Q_RETURN_ARG(QString, html),
Q_ARG(QString, url));
}
else {
viewer->show();
html = page->pageHtml(url);
}
page->deleteLater();
viewer->close();
IOFactory::writeFile("a.html", html.toUtf8());
return html.toUtf8();
}

2
src/librssguard/gui/webviewers/webengine/webengineviewer.h
View File

@ -37,7 +37,7 @@ class WebEngineViewer : public QWebEngineView, public WebViewer {
virtual QString html() const;
virtual QUrl url() const;
static QByteArray getJsEnabledHtml(const QString& url);
static QByteArray getJsEnabledHtml(const QString& url, bool worker_thread);
signals:
void pageTitleChanged(const QString& new_title);

2
src/librssguard/services/standard/standardfeed.cpp
View File

@ -297,7 +297,7 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type,
}
else if (source_type == StandardFeed::SourceType::EmbeddedBrowser) {
#if defined(NO_LITE)
feed_contents = WebEngineViewer::getJsEnabledHtml(source);
feed_contents = WebEngineViewer::getJsEnabledHtml(source, false);
#else
throw ApplicationException(tr("this source type cannot be used on 'lite' %1 build").arg(QSL(APP_NAME)));
#endif

2
src/librssguard/services/standard/standardserviceroot.cpp
View File

@ -242,7 +242,7 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
}
else if (f->sourceType() == StandardFeed::SourceType::EmbeddedBrowser) {
#if defined(NO_LITE)
feed_contents = WebEngineViewer::getJsEnabledHtml(f->source());
feed_contents = WebEngineViewer::getJsEnabledHtml(f->source(), true);
#else
throw ApplicationException(tr("this source type cannot be used on 'lite' %1 build").arg(QSL(APP_NAME)));
#endif