FreshRSS/rssguard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Concurrent import (#864)

Browse Source 
* save

* save

* save
This commit is contained in:
martinrotter 2023-01-03 13:07:34 +01:00 committed by GitHub
parent e4e98b861a
commit 8347a8cb2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 192 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -128,6 +128,7 @@ set(QT_COMPONENTS
Sql
Widgets
Xml
Concurrent
)
if(NOT OS2)

2
resources/desktop/rssguard.metainfo.xml.in
View File

@ -60,7 +60,7 @@
<content_rating type="oars-1.0" />
<content_rating type="oars-1.1" />
<releases>
<release version="4.2.7" date="2022-12-21" />
<release version="4.2.7" date="2023-01-03" />
</releases>
<provides>
<binary>@APP_LOW_NAME@</binary>

1
src/librssguard/CMakeLists.txt
View File

@ -625,6 +625,7 @@ target_link_libraries(rssguard PUBLIC
Qt${QT_VERSION_MAJOR}::Sql
Qt${QT_VERSION_MAJOR}::Widgets
Qt${QT_VERSION_MAJOR}::Xml
Qt${QT_VERSION_MAJOR}::Concurrent
)
if(QT_VERSION_MAJOR EQUAL 6)

2
src/librssguard/miscellaneous/application.cpp
View File

@ -216,6 +216,8 @@ Application::Application(const QString& id, int& argc, char** argv, const QStrin
qDebugNN << LOGSEC_CORE << "OpenSSL version:" << QUOTE_W_SPACE_DOT(QSslSocket::sslLibraryVersionString());
qDebugNN << LOGSEC_CORE << "OpenSSL supported:" << QUOTE_W_SPACE_DOT(QSslSocket::supportsSsl());
qDebugNN << LOGSEC_CORE << "Global thread pool has"
<< NONQUOTE_W_SPACE(QThreadPool::globalInstance()->maxThreadCount()) << "threads.";
}
Application::~Application() {

3
src/librssguard/network-web/basenetworkaccessmanager.cpp
View File

@ -17,7 +17,6 @@ BaseNetworkAccessManager::BaseNetworkAccessManager(QObject* parent)
}
void BaseNetworkAccessManager::loadSettings() {
QNetworkProxy new_proxy;
const QNetworkProxy::ProxyType selected_proxy_type =
static_cast<QNetworkProxy::ProxyType>(qApp->settings()->value(GROUP(Proxy), SETTING(Proxy::Type)).toInt());
@ -55,10 +54,10 @@ QNetworkReply* BaseNetworkAccessManager::createRequest(QNetworkAccessManager::Op
#if defined(Q_OS_WIN)
new_request.setAttribute(QNetworkRequest::Attribute::HttpPipeliningAllowedAttribute, true);
#endif
#if QT_VERSION >= 0x050F00 // Qt >= 5.15.0
new_request.setAttribute(QNetworkRequest::Attribute::Http2AllowedAttribute, m_enableHttp2);
#endif
#endif
new_request.setRawHeader(HTTP_HEADERS_COOKIE, QSL("JSESSIONID= ").toLocal8Bit());

21
src/librssguard/services/standard/gui/formstandardimportexport.cpp
View File

@ -125,15 +125,19 @@ void FormStandardImportExport::onParsingStarted() {
m_ui->m_buttonBox->button(QDialogButtonBox::StandardButton::Ok)->setEnabled(false);
}
void FormStandardImportExport::onParsingFinished(int count_failed, int count_succeeded, bool parsing_error) {
Q_UNUSED(count_failed)
Q_UNUSED(count_succeeded)
void FormStandardImportExport::onParsingFinished(int count_failed, int count_succeeded) {
m_ui->m_progressBar->setVisible(false);
m_ui->m_progressBar->setValue(0);
m_model->checkAllItems();
if (!parsing_error) {
if (count_failed > 0 && count_succeeded == 0) {
m_ui->m_groupFeeds->setEnabled(false);
m_ui->m_groupFetchMetadata->setEnabled(false);
m_ui->m_lblResult->setStatus(WidgetWithStatus::StatusType::Error,
tr("Some feeds were not loaded properly or import file is corrupted."),
tr("Some feeds were not loaded properly or import file is corrupted."));
}
else {
m_ui->m_lblResult->setStatus(WidgetWithStatus::StatusType::Ok, tr("Feeds were loaded."), tr("Feeds were loaded."));
m_ui->m_groupFeeds->setEnabled(true);
m_ui->m_groupFetchMetadata->setEnabled(true);
@ -141,13 +145,6 @@ void FormStandardImportExport::onParsingFinished(int count_failed, int count_suc
m_ui->m_treeFeeds->setModel(m_model);
m_ui->m_treeFeeds->expandAll();
}
else {
m_ui->m_groupFeeds->setEnabled(false);
m_ui->m_groupFetchMetadata->setEnabled(false);
m_ui->m_lblResult->setStatus(WidgetWithStatus::StatusType::Error,
tr("Error, file is not well-formed. Select another file."),
tr("Error occurred. File is not well-formed. Select another file."));
}
m_ui->m_buttonBox->button(QDialogButtonBox::StandardButton::Ok)->setEnabled(true);
}

2
src/librssguard/services/standard/gui/formstandardimportexport.h
View File

@ -31,7 +31,7 @@ class FormStandardImportExport : public QDialog {
void selectFile();
void onParsingStarted();
void onParsingFinished(int count_failed, int count_succeeded, bool parsing_error);
void onParsingFinished(int count_failed, int count_succeeded);
void onParsingProgress(int completed, int total);
void onPostProcessScriptChanged(const QString& new_pp);

267
src/librssguard/services/standard/standardfeedsimportexportmodel.cpp
View File

@ -2,6 +2,7 @@
#include "services/standard/standardfeedsimportexportmodel.h"
#include "3rd-party/boolinq/boolinq.h"
#include "definitions/definitions.h"
#include "exceptions/applicationexception.h"
#include "miscellaneous/application.h"
@ -16,11 +17,35 @@
#include <QDomElement>
#include <QLocale>
#include <QStack>
#include <QtConcurrent/QtConcurrentMap>
FeedsImportExportModel::FeedsImportExportModel(QObject* parent)
: AccountCheckSortedModel(parent), m_mode(Mode::Import) {}
: AccountCheckSortedModel(parent), m_mode(Mode::Import) {
connect(&m_watcherLookup, &QFutureWatcher<bool>::progressValueChanged, this, [=](int prog) {
emit parsingProgress(prog, m_lookup.size());
});
connect(&m_watcherLookup, &QFutureWatcher<bool>::finished, this, [=]() {
auto res = m_watcherLookup.future().results();
int number_error = boolinq::from(res).count([](bool rs) {
return !rs;
});
emit layoutChanged();
emit parsingFinished(number_error, m_lookup.size() - number_error);
// Done, remove lookups.
m_lookup.clear();
});
}
FeedsImportExportModel::~FeedsImportExportModel() {
if (m_watcherLookup.isRunning()) {
m_watcherLookup.cancel();
m_watcherLookup.waitForFinished();
}
if (sourceModel() != nullptr && sourceModel()->rootItem() != nullptr && m_mode == Mode::Import) {
// Delete all model items, but only if we are in import mode. Export mode shares
// root item with main feed model, thus cannot be deleted from memory now.
@ -145,27 +170,107 @@ bool FeedsImportExportModel::exportToOMPL20(QByteArray& result, bool export_icon
return true;
}
bool FeedsImportExportModel::produceFeed(const FeedLookup& feed_lookup) {
StandardFeed* new_feed = nullptr;
try {
if (feed_lookup.fetch_metadata_online) {
new_feed = StandardFeed::guessFeed(StandardFeed::SourceType::Url,
feed_lookup.url,
feed_lookup.post_process_script,
{},
{},
feed_lookup.custom_proxy);
new_feed->setSource(feed_lookup.url);
new_feed->setPostProcessScript(feed_lookup.post_process_script);
}
else {
new_feed = new StandardFeed(feed_lookup.parent);
if (feed_lookup.opml_element.isNull()) {
new_feed->setSource(feed_lookup.url);
new_feed->setTitle(feed_lookup.url);
new_feed->setIcon(qApp->icons()->fromTheme(QSL("application-rss+xml")));
new_feed->setEncoding(QSL(DEFAULT_FEED_ENCODING));
new_feed->setPostProcessScript(feed_lookup.post_process_script);
}
else {
QString feed_title = feed_lookup.opml_element.attribute(QSL("text"));
QString feed_encoding = feed_lookup.opml_element.attribute(QSL("encoding"), QSL(DEFAULT_FEED_ENCODING));
QString feed_type = feed_lookup.opml_element.attribute(QSL("version"), QSL(DEFAULT_FEED_TYPE)).toUpper();
QString feed_description = feed_lookup.opml_element.attribute(QSL("description"));
QIcon feed_icon =
qApp->icons()->fromByteArray(feed_lookup.opml_element.attribute(QSL("rssguard:icon")).toLocal8Bit());
StandardFeed::SourceType source_type =
StandardFeed::SourceType(feed_lookup.opml_element.attribute(QSL("rssguard:xmlUrlType")).toInt());
QString post_process = feed_lookup.opml_element.attribute(QSL("rssguard:postProcess"));
new_feed->setTitle(feed_title);
new_feed->setDescription(feed_description);
new_feed->setEncoding(feed_encoding);
new_feed->setSource(feed_lookup.url);
new_feed->setSourceType(source_type);
new_feed->setPostProcessScript(feed_lookup.post_process_script.isEmpty() ? post_process
: feed_lookup.post_process_script);
if (!feed_icon.isNull()) {
new_feed->setIcon(feed_icon);
}
if (feed_type == QL1S("RSS1")) {
new_feed->setType(StandardFeed::Type::Rdf);
}
else if (feed_type == QL1S("JSON")) {
new_feed->setType(StandardFeed::Type::Json);
}
else if (feed_type == QL1S("ATOM")) {
new_feed->setType(StandardFeed::Type::Atom10);
}
else {
new_feed->setType(StandardFeed::Type::Rss2X);
}
}
}
QMutexLocker mtx(&m_mtxLookup);
feed_lookup.parent->appendChild(new_feed);
return true;
}
catch (const ApplicationException& ex) {
qCriticalNN << LOGSEC_CORE << "Cannot fetch medatada for feed:" << QUOTE_W_SPACE(feed_lookup.url)
<< "with error:" << QUOTE_W_SPACE_DOT(ex.message());
if (new_feed != nullptr) {
new_feed->deleteLater();
}
return false;
}
}
void FeedsImportExportModel::importAsOPML20(const QByteArray& data,
bool fetch_metadata_online,
const QString& post_process_script) {
emit parsingStarted();
emit layoutAboutToBeChanged();
setRootItem(nullptr);
emit layoutChanged();
QDomDocument opml_document;
if (!opml_document.setContent(data)) {
emit parsingFinished(0, 0, true);
emit parsingFinished(0, 0);
}
if (opml_document.documentElement().isNull() || opml_document.documentElement().tagName() != QSL("opml") ||
opml_document.documentElement().elementsByTagName(QSL("body")).size() != 1) {
// This really is not an OPML file.
emit parsingFinished(0, 0, true);
emit parsingFinished(0, 0);
}
int completed = 0, total = 0, succeded = 0, failed = 0;
int completed = 0, total = 0;
auto* root_item = new StandardServiceRoot();
QStack<RootItem*> model_items;
QNetworkProxy custom_proxy;
@ -180,6 +285,8 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data,
elements_to_process.push(opml_document.documentElement().elementsByTagName(QSL("body")).at(0).toElement());
total = opml_document.elementsByTagName(QSL("outline")).size();
QList<FeedLookup> lookup;
while (!elements_to_process.isEmpty()) {
RootItem* active_model_item = model_items.pop();
QDomElement active_element = elements_to_process.pop();
@ -197,76 +304,18 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data,
// This is FEED.
// Add feed and end this iteration.
QString feed_url = child_element.attribute(QSL("xmlUrl"));
bool add_offline_anyway = true;
if (!feed_url.isEmpty()) {
try {
if (fetch_metadata_online) {
StandardFeed* guessed = StandardFeed::guessFeed(StandardFeed::SourceType::Url,
feed_url,
post_process_script,
{},
{},
custom_proxy);
FeedLookup f;
guessed->setSource(feed_url);
guessed->setPostProcessScript(post_process_script);
f.custom_proxy = custom_proxy;
f.fetch_metadata_online = fetch_metadata_online;
f.opml_element = child_element;
f.parent = active_model_item;
f.post_process_script = post_process_script;
f.url = feed_url;
active_model_item->appendChild(guessed);
succeded++;
add_offline_anyway = false;
}
}
catch (const ApplicationException& ex) {
qCriticalNN << LOGSEC_CORE << "Cannot fetch medatada for feed:" << QUOTE_W_SPACE(feed_url)
<< "with error:" << QUOTE_W_SPACE_DOT(ex.message());
}
if (add_offline_anyway) {
QString feed_title = child_element.attribute(QSL("text"));
QString feed_encoding = child_element.attribute(QSL("encoding"), QSL(DEFAULT_FEED_ENCODING));
QString feed_type = child_element.attribute(QSL("version"), QSL(DEFAULT_FEED_TYPE)).toUpper();
QString feed_description = child_element.attribute(QSL("description"));
QIcon feed_icon =
qApp->icons()->fromByteArray(child_element.attribute(QSL("rssguard:icon")).toLocal8Bit());
StandardFeed::SourceType source_type =
StandardFeed::SourceType(child_element.attribute(QSL("rssguard:xmlUrlType")).toInt());
QString post_process = child_element.attribute(QSL("rssguard:postProcess"));
auto* new_feed = new StandardFeed(active_model_item);
new_feed->setTitle(feed_title);
new_feed->setDescription(feed_description);
new_feed->setEncoding(feed_encoding);
new_feed->setSource(feed_url);
new_feed->setSourceType(source_type);
new_feed->setPostProcessScript(post_process);
if (!feed_icon.isNull()) {
new_feed->setIcon(feed_icon);
}
if (feed_type == QL1S("RSS1")) {
new_feed->setType(StandardFeed::Type::Rdf);
}
else if (feed_type == QL1S("JSON")) {
new_feed->setType(StandardFeed::Type::Json);
}
else if (feed_type == QL1S("ATOM")) {
new_feed->setType(StandardFeed::Type::Atom10);
}
else {
new_feed->setType(StandardFeed::Type::Rss2X);
}
active_model_item->appendChild(new_feed);
if (fetch_metadata_online) {
failed++;
}
else {
succeded++;
}
}
lookup.append(f);
}
}
else {
@ -313,8 +362,18 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data,
setRootItem(root_item);
emit layoutChanged();
emit parsingFinished(failed, succeded, false);
m_lookup.clear();
m_lookup.append(lookup);
QFuture<bool> fut = QtConcurrent::mapped(m_lookup, [=](const FeedLookup& lookup) {
return produceFeed(lookup);
});
m_watcherLookup.setFuture(fut);
if (!fetch_metadata_online) {
m_watcherLookup.waitForFinished();
}
}
bool FeedsImportExportModel::exportToTxtURLPerLine(QByteArray& result) {
@ -335,7 +394,7 @@ void FeedsImportExportModel::importAsTxtURLPerLine(const QByteArray& data,
setRootItem(nullptr);
emit layoutChanged();
int completed = 0, succeded = 0, failed = 0;
int completed = 0;
auto* root_item = new StandardServiceRoot();
QNetworkProxy custom_proxy;
@ -344,51 +403,22 @@ void FeedsImportExportModel::importAsTxtURLPerLine(const QByteArray& data,
}
QList<QByteArray> urls = data.split('\n');
QList<FeedLookup> lookup;
for (const QByteArray& url : urls) {
if (!url.isEmpty()) {
bool add_offline_anyway = true;
FeedLookup f;
try {
if (fetch_metadata_online) {
StandardFeed* guessed =
StandardFeed::guessFeed(StandardFeed::SourceType::Url, url, post_process_script, {}, {}, custom_proxy);
f.custom_proxy = custom_proxy;
f.fetch_metadata_online = fetch_metadata_online;
f.parent = root_item;
f.post_process_script = post_process_script;
f.url = url;
guessed->setSource(url);
guessed->setPostProcessScript(post_process_script);
root_item->appendChild(guessed);
succeded++;
add_offline_anyway = false;
}
}
catch (const ApplicationException& ex) {
qCriticalNN << LOGSEC_CORE << "Cannot fetch medatada for feed:" << QUOTE_W_SPACE(url)
<< "with error:" << QUOTE_W_SPACE_DOT(ex.message());
}
if (add_offline_anyway) {
auto* feed = new StandardFeed();
feed->setSource(url);
feed->setTitle(url);
feed->setIcon(qApp->icons()->fromTheme(QSL("application-rss+xml")));
feed->setEncoding(QSL(DEFAULT_FEED_ENCODING));
root_item->appendChild(feed);
if (fetch_metadata_online) {
failed++;
}
else {
succeded++;
}
}
qApp->processEvents();
lookup.append(f);
}
else {
qWarningNN << LOGSEC_CORE << "Detected empty URL when parsing input TXT [one URL per line] data.";
failed++;
}
emit parsingProgress(++completed, urls.size());
@ -398,8 +428,19 @@ void FeedsImportExportModel::importAsTxtURLPerLine(const QByteArray& data,
emit layoutAboutToBeChanged();
setRootItem(root_item);
emit layoutChanged();
emit parsingFinished(failed, succeded, false);
m_lookup.clear();
m_lookup.append(lookup);
QFuture<bool> fut = QtConcurrent::mapped(m_lookup, [=](const FeedLookup& lookup) {
return produceFeed(lookup);
});
m_watcherLookup.setFuture(fut);
if (!fetch_metadata_online) {
m_watcherLookup.waitForFinished();
}
}
FeedsImportExportModel::Mode FeedsImportExportModel::mode() const {

23
src/librssguard/services/standard/standardfeedsimportexportmodel.h
View File

@ -5,6 +5,21 @@
#include "services/abstract/accountcheckmodel.h"
#include <QDomElement>
#include <QFutureWatcher>
#include <QNetworkProxy>
class StandardFeed;
struct FeedLookup {
RootItem* parent;
QDomElement opml_element;
QString url;
bool fetch_metadata_online;
QNetworkProxy custom_proxy;
QString post_process_script;
};
class FeedsImportExportModel : public AccountCheckSortedModel {
Q_OBJECT
@ -32,9 +47,15 @@ class FeedsImportExportModel : public AccountCheckSortedModel {
signals:
void parsingStarted();
void parsingProgress(int completed, int total);
void parsingFinished(int count_failed, int count_succeeded, bool parsing_error);
void parsingFinished(int count_failed, int count_succeeded);
private:
bool produceFeed(const FeedLookup& feed_lookup);
private:
QMutex m_mtxLookup;
QList<FeedLookup> m_lookup;
QFutureWatcher<bool> m_watcherLookup;
Mode m_mode;
};