diff --git a/CMakeLists.txt b/CMakeLists.txt index d70d0f9ec..824071a34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,6 +128,7 @@ set(QT_COMPONENTS Sql Widgets Xml + Concurrent ) if(NOT OS2) diff --git a/resources/desktop/rssguard.metainfo.xml.in b/resources/desktop/rssguard.metainfo.xml.in index b0df0c45c..d135fa012 100644 --- a/resources/desktop/rssguard.metainfo.xml.in +++ b/resources/desktop/rssguard.metainfo.xml.in @@ -60,7 +60,7 @@ - + @APP_LOW_NAME@ diff --git a/src/librssguard/CMakeLists.txt b/src/librssguard/CMakeLists.txt index d1a4d250f..fa5cce10e 100644 --- a/src/librssguard/CMakeLists.txt +++ b/src/librssguard/CMakeLists.txt @@ -625,6 +625,7 @@ target_link_libraries(rssguard PUBLIC Qt${QT_VERSION_MAJOR}::Sql Qt${QT_VERSION_MAJOR}::Widgets Qt${QT_VERSION_MAJOR}::Xml + Qt${QT_VERSION_MAJOR}::Concurrent ) if(QT_VERSION_MAJOR EQUAL 6) diff --git a/src/librssguard/miscellaneous/application.cpp b/src/librssguard/miscellaneous/application.cpp index 7cf1a27e3..136794885 100644 --- a/src/librssguard/miscellaneous/application.cpp +++ b/src/librssguard/miscellaneous/application.cpp @@ -216,6 +216,8 @@ Application::Application(const QString& id, int& argc, char** argv, const QStrin qDebugNN << LOGSEC_CORE << "OpenSSL version:" << QUOTE_W_SPACE_DOT(QSslSocket::sslLibraryVersionString()); qDebugNN << LOGSEC_CORE << "OpenSSL supported:" << QUOTE_W_SPACE_DOT(QSslSocket::supportsSsl()); + qDebugNN << LOGSEC_CORE << "Global thread pool has" + << NONQUOTE_W_SPACE(QThreadPool::globalInstance()->maxThreadCount()) << "threads."; } Application::~Application() { diff --git a/src/librssguard/network-web/basenetworkaccessmanager.cpp b/src/librssguard/network-web/basenetworkaccessmanager.cpp index ef51ab3a5..847a202bb 100644 --- a/src/librssguard/network-web/basenetworkaccessmanager.cpp +++ b/src/librssguard/network-web/basenetworkaccessmanager.cpp @@ -17,7 +17,6 @@ BaseNetworkAccessManager::BaseNetworkAccessManager(QObject* parent) } void BaseNetworkAccessManager::loadSettings() { - QNetworkProxy new_proxy; const QNetworkProxy::ProxyType selected_proxy_type = static_cast(qApp->settings()->value(GROUP(Proxy), SETTING(Proxy::Type)).toInt()); @@ -55,10 +54,10 @@ QNetworkReply* BaseNetworkAccessManager::createRequest(QNetworkAccessManager::Op #if defined(Q_OS_WIN) new_request.setAttribute(QNetworkRequest::Attribute::HttpPipeliningAllowedAttribute, true); +#endif #if QT_VERSION >= 0x050F00 // Qt >= 5.15.0 new_request.setAttribute(QNetworkRequest::Attribute::Http2AllowedAttribute, m_enableHttp2); -#endif #endif new_request.setRawHeader(HTTP_HEADERS_COOKIE, QSL("JSESSIONID= ").toLocal8Bit()); diff --git a/src/librssguard/services/standard/gui/formstandardimportexport.cpp b/src/librssguard/services/standard/gui/formstandardimportexport.cpp index 808c2be10..cbc2018c4 100644 --- a/src/librssguard/services/standard/gui/formstandardimportexport.cpp +++ b/src/librssguard/services/standard/gui/formstandardimportexport.cpp @@ -125,15 +125,19 @@ void FormStandardImportExport::onParsingStarted() { m_ui->m_buttonBox->button(QDialogButtonBox::StandardButton::Ok)->setEnabled(false); } -void FormStandardImportExport::onParsingFinished(int count_failed, int count_succeeded, bool parsing_error) { - Q_UNUSED(count_failed) - Q_UNUSED(count_succeeded) - +void FormStandardImportExport::onParsingFinished(int count_failed, int count_succeeded) { m_ui->m_progressBar->setVisible(false); m_ui->m_progressBar->setValue(0); m_model->checkAllItems(); - if (!parsing_error) { + if (count_failed > 0 && count_succeeded == 0) { + m_ui->m_groupFeeds->setEnabled(false); + m_ui->m_groupFetchMetadata->setEnabled(false); + m_ui->m_lblResult->setStatus(WidgetWithStatus::StatusType::Error, + tr("Some feeds were not loaded properly or import file is corrupted."), + tr("Some feeds were not loaded properly or import file is corrupted.")); + } + else { m_ui->m_lblResult->setStatus(WidgetWithStatus::StatusType::Ok, tr("Feeds were loaded."), tr("Feeds were loaded.")); m_ui->m_groupFeeds->setEnabled(true); m_ui->m_groupFetchMetadata->setEnabled(true); @@ -141,13 +145,6 @@ void FormStandardImportExport::onParsingFinished(int count_failed, int count_suc m_ui->m_treeFeeds->setModel(m_model); m_ui->m_treeFeeds->expandAll(); } - else { - m_ui->m_groupFeeds->setEnabled(false); - m_ui->m_groupFetchMetadata->setEnabled(false); - m_ui->m_lblResult->setStatus(WidgetWithStatus::StatusType::Error, - tr("Error, file is not well-formed. Select another file."), - tr("Error occurred. File is not well-formed. Select another file.")); - } m_ui->m_buttonBox->button(QDialogButtonBox::StandardButton::Ok)->setEnabled(true); } diff --git a/src/librssguard/services/standard/gui/formstandardimportexport.h b/src/librssguard/services/standard/gui/formstandardimportexport.h index 1a3bf3af8..7b1da9d76 100644 --- a/src/librssguard/services/standard/gui/formstandardimportexport.h +++ b/src/librssguard/services/standard/gui/formstandardimportexport.h @@ -31,7 +31,7 @@ class FormStandardImportExport : public QDialog { void selectFile(); void onParsingStarted(); - void onParsingFinished(int count_failed, int count_succeeded, bool parsing_error); + void onParsingFinished(int count_failed, int count_succeeded); void onParsingProgress(int completed, int total); void onPostProcessScriptChanged(const QString& new_pp); diff --git a/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp b/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp index 9eb47ccd0..9e8446670 100644 --- a/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp +++ b/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp @@ -2,6 +2,7 @@ #include "services/standard/standardfeedsimportexportmodel.h" +#include "3rd-party/boolinq/boolinq.h" #include "definitions/definitions.h" #include "exceptions/applicationexception.h" #include "miscellaneous/application.h" @@ -16,11 +17,35 @@ #include #include #include +#include FeedsImportExportModel::FeedsImportExportModel(QObject* parent) - : AccountCheckSortedModel(parent), m_mode(Mode::Import) {} + : AccountCheckSortedModel(parent), m_mode(Mode::Import) { + + connect(&m_watcherLookup, &QFutureWatcher::progressValueChanged, this, [=](int prog) { + emit parsingProgress(prog, m_lookup.size()); + }); + + connect(&m_watcherLookup, &QFutureWatcher::finished, this, [=]() { + auto res = m_watcherLookup.future().results(); + int number_error = boolinq::from(res).count([](bool rs) { + return !rs; + }); + + emit layoutChanged(); + emit parsingFinished(number_error, m_lookup.size() - number_error); + + // Done, remove lookups. + m_lookup.clear(); + }); +} FeedsImportExportModel::~FeedsImportExportModel() { + if (m_watcherLookup.isRunning()) { + m_watcherLookup.cancel(); + m_watcherLookup.waitForFinished(); + } + if (sourceModel() != nullptr && sourceModel()->rootItem() != nullptr && m_mode == Mode::Import) { // Delete all model items, but only if we are in import mode. Export mode shares // root item with main feed model, thus cannot be deleted from memory now. @@ -145,27 +170,107 @@ bool FeedsImportExportModel::exportToOMPL20(QByteArray& result, bool export_icon return true; } +bool FeedsImportExportModel::produceFeed(const FeedLookup& feed_lookup) { + StandardFeed* new_feed = nullptr; + + try { + if (feed_lookup.fetch_metadata_online) { + new_feed = StandardFeed::guessFeed(StandardFeed::SourceType::Url, + feed_lookup.url, + feed_lookup.post_process_script, + {}, + {}, + feed_lookup.custom_proxy); + + new_feed->setSource(feed_lookup.url); + new_feed->setPostProcessScript(feed_lookup.post_process_script); + } + else { + new_feed = new StandardFeed(feed_lookup.parent); + + if (feed_lookup.opml_element.isNull()) { + new_feed->setSource(feed_lookup.url); + new_feed->setTitle(feed_lookup.url); + new_feed->setIcon(qApp->icons()->fromTheme(QSL("application-rss+xml"))); + new_feed->setEncoding(QSL(DEFAULT_FEED_ENCODING)); + new_feed->setPostProcessScript(feed_lookup.post_process_script); + } + else { + QString feed_title = feed_lookup.opml_element.attribute(QSL("text")); + QString feed_encoding = feed_lookup.opml_element.attribute(QSL("encoding"), QSL(DEFAULT_FEED_ENCODING)); + QString feed_type = feed_lookup.opml_element.attribute(QSL("version"), QSL(DEFAULT_FEED_TYPE)).toUpper(); + QString feed_description = feed_lookup.opml_element.attribute(QSL("description")); + QIcon feed_icon = + qApp->icons()->fromByteArray(feed_lookup.opml_element.attribute(QSL("rssguard:icon")).toLocal8Bit()); + StandardFeed::SourceType source_type = + StandardFeed::SourceType(feed_lookup.opml_element.attribute(QSL("rssguard:xmlUrlType")).toInt()); + QString post_process = feed_lookup.opml_element.attribute(QSL("rssguard:postProcess")); + + new_feed->setTitle(feed_title); + new_feed->setDescription(feed_description); + new_feed->setEncoding(feed_encoding); + new_feed->setSource(feed_lookup.url); + new_feed->setSourceType(source_type); + new_feed->setPostProcessScript(feed_lookup.post_process_script.isEmpty() ? post_process + : feed_lookup.post_process_script); + + if (!feed_icon.isNull()) { + new_feed->setIcon(feed_icon); + } + + if (feed_type == QL1S("RSS1")) { + new_feed->setType(StandardFeed::Type::Rdf); + } + else if (feed_type == QL1S("JSON")) { + new_feed->setType(StandardFeed::Type::Json); + } + else if (feed_type == QL1S("ATOM")) { + new_feed->setType(StandardFeed::Type::Atom10); + } + else { + new_feed->setType(StandardFeed::Type::Rss2X); + } + } + } + + QMutexLocker mtx(&m_mtxLookup); + feed_lookup.parent->appendChild(new_feed); + + return true; + } + catch (const ApplicationException& ex) { + qCriticalNN << LOGSEC_CORE << "Cannot fetch medatada for feed:" << QUOTE_W_SPACE(feed_lookup.url) + << "with error:" << QUOTE_W_SPACE_DOT(ex.message()); + + if (new_feed != nullptr) { + new_feed->deleteLater(); + } + + return false; + } +} + void FeedsImportExportModel::importAsOPML20(const QByteArray& data, bool fetch_metadata_online, const QString& post_process_script) { emit parsingStarted(); emit layoutAboutToBeChanged(); - setRootItem(nullptr); emit layoutChanged(); + QDomDocument opml_document; if (!opml_document.setContent(data)) { - emit parsingFinished(0, 0, true); + emit parsingFinished(0, 0); } if (opml_document.documentElement().isNull() || opml_document.documentElement().tagName() != QSL("opml") || opml_document.documentElement().elementsByTagName(QSL("body")).size() != 1) { // This really is not an OPML file. - emit parsingFinished(0, 0, true); + emit parsingFinished(0, 0); } - int completed = 0, total = 0, succeded = 0, failed = 0; + int completed = 0, total = 0; auto* root_item = new StandardServiceRoot(); QStack model_items; QNetworkProxy custom_proxy; @@ -180,6 +285,8 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data, elements_to_process.push(opml_document.documentElement().elementsByTagName(QSL("body")).at(0).toElement()); total = opml_document.elementsByTagName(QSL("outline")).size(); + QList lookup; + while (!elements_to_process.isEmpty()) { RootItem* active_model_item = model_items.pop(); QDomElement active_element = elements_to_process.pop(); @@ -197,76 +304,18 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data, // This is FEED. // Add feed and end this iteration. QString feed_url = child_element.attribute(QSL("xmlUrl")); - bool add_offline_anyway = true; if (!feed_url.isEmpty()) { - try { - if (fetch_metadata_online) { - StandardFeed* guessed = StandardFeed::guessFeed(StandardFeed::SourceType::Url, - feed_url, - post_process_script, - {}, - {}, - custom_proxy); + FeedLookup f; - guessed->setSource(feed_url); - guessed->setPostProcessScript(post_process_script); + f.custom_proxy = custom_proxy; + f.fetch_metadata_online = fetch_metadata_online; + f.opml_element = child_element; + f.parent = active_model_item; + f.post_process_script = post_process_script; + f.url = feed_url; - active_model_item->appendChild(guessed); - succeded++; - add_offline_anyway = false; - } - } - catch (const ApplicationException& ex) { - qCriticalNN << LOGSEC_CORE << "Cannot fetch medatada for feed:" << QUOTE_W_SPACE(feed_url) - << "with error:" << QUOTE_W_SPACE_DOT(ex.message()); - } - - if (add_offline_anyway) { - QString feed_title = child_element.attribute(QSL("text")); - QString feed_encoding = child_element.attribute(QSL("encoding"), QSL(DEFAULT_FEED_ENCODING)); - QString feed_type = child_element.attribute(QSL("version"), QSL(DEFAULT_FEED_TYPE)).toUpper(); - QString feed_description = child_element.attribute(QSL("description")); - QIcon feed_icon = - qApp->icons()->fromByteArray(child_element.attribute(QSL("rssguard:icon")).toLocal8Bit()); - StandardFeed::SourceType source_type = - StandardFeed::SourceType(child_element.attribute(QSL("rssguard:xmlUrlType")).toInt()); - QString post_process = child_element.attribute(QSL("rssguard:postProcess")); - auto* new_feed = new StandardFeed(active_model_item); - - new_feed->setTitle(feed_title); - new_feed->setDescription(feed_description); - new_feed->setEncoding(feed_encoding); - new_feed->setSource(feed_url); - new_feed->setSourceType(source_type); - new_feed->setPostProcessScript(post_process); - - if (!feed_icon.isNull()) { - new_feed->setIcon(feed_icon); - } - - if (feed_type == QL1S("RSS1")) { - new_feed->setType(StandardFeed::Type::Rdf); - } - else if (feed_type == QL1S("JSON")) { - new_feed->setType(StandardFeed::Type::Json); - } - else if (feed_type == QL1S("ATOM")) { - new_feed->setType(StandardFeed::Type::Atom10); - } - else { - new_feed->setType(StandardFeed::Type::Rss2X); - } - - active_model_item->appendChild(new_feed); - - if (fetch_metadata_online) { - failed++; - } - else { - succeded++; - } - } + lookup.append(f); } } else { @@ -313,8 +362,18 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data, setRootItem(root_item); - emit layoutChanged(); - emit parsingFinished(failed, succeded, false); + m_lookup.clear(); + m_lookup.append(lookup); + + QFuture fut = QtConcurrent::mapped(m_lookup, [=](const FeedLookup& lookup) { + return produceFeed(lookup); + }); + + m_watcherLookup.setFuture(fut); + + if (!fetch_metadata_online) { + m_watcherLookup.waitForFinished(); + } } bool FeedsImportExportModel::exportToTxtURLPerLine(QByteArray& result) { @@ -335,7 +394,7 @@ void FeedsImportExportModel::importAsTxtURLPerLine(const QByteArray& data, setRootItem(nullptr); emit layoutChanged(); - int completed = 0, succeded = 0, failed = 0; + int completed = 0; auto* root_item = new StandardServiceRoot(); QNetworkProxy custom_proxy; @@ -344,51 +403,22 @@ void FeedsImportExportModel::importAsTxtURLPerLine(const QByteArray& data, } QList urls = data.split('\n'); + QList lookup; for (const QByteArray& url : urls) { if (!url.isEmpty()) { - bool add_offline_anyway = true; + FeedLookup f; - try { - if (fetch_metadata_online) { - StandardFeed* guessed = - StandardFeed::guessFeed(StandardFeed::SourceType::Url, url, post_process_script, {}, {}, custom_proxy); + f.custom_proxy = custom_proxy; + f.fetch_metadata_online = fetch_metadata_online; + f.parent = root_item; + f.post_process_script = post_process_script; + f.url = url; - guessed->setSource(url); - guessed->setPostProcessScript(post_process_script); - - root_item->appendChild(guessed); - succeded++; - add_offline_anyway = false; - } - } - catch (const ApplicationException& ex) { - qCriticalNN << LOGSEC_CORE << "Cannot fetch medatada for feed:" << QUOTE_W_SPACE(url) - << "with error:" << QUOTE_W_SPACE_DOT(ex.message()); - } - - if (add_offline_anyway) { - auto* feed = new StandardFeed(); - - feed->setSource(url); - feed->setTitle(url); - feed->setIcon(qApp->icons()->fromTheme(QSL("application-rss+xml"))); - feed->setEncoding(QSL(DEFAULT_FEED_ENCODING)); - root_item->appendChild(feed); - - if (fetch_metadata_online) { - failed++; - } - else { - succeded++; - } - } - - qApp->processEvents(); + lookup.append(f); } else { qWarningNN << LOGSEC_CORE << "Detected empty URL when parsing input TXT [one URL per line] data."; - failed++; } emit parsingProgress(++completed, urls.size()); @@ -398,8 +428,19 @@ void FeedsImportExportModel::importAsTxtURLPerLine(const QByteArray& data, emit layoutAboutToBeChanged(); setRootItem(root_item); - emit layoutChanged(); - emit parsingFinished(failed, succeded, false); + + m_lookup.clear(); + m_lookup.append(lookup); + + QFuture fut = QtConcurrent::mapped(m_lookup, [=](const FeedLookup& lookup) { + return produceFeed(lookup); + }); + + m_watcherLookup.setFuture(fut); + + if (!fetch_metadata_online) { + m_watcherLookup.waitForFinished(); + } } FeedsImportExportModel::Mode FeedsImportExportModel::mode() const { diff --git a/src/librssguard/services/standard/standardfeedsimportexportmodel.h b/src/librssguard/services/standard/standardfeedsimportexportmodel.h index ff9a04a44..08813262d 100644 --- a/src/librssguard/services/standard/standardfeedsimportexportmodel.h +++ b/src/librssguard/services/standard/standardfeedsimportexportmodel.h @@ -5,6 +5,21 @@ #include "services/abstract/accountcheckmodel.h" +#include +#include +#include + +class StandardFeed; + +struct FeedLookup { + RootItem* parent; + QDomElement opml_element; + QString url; + bool fetch_metadata_online; + QNetworkProxy custom_proxy; + QString post_process_script; +}; + class FeedsImportExportModel : public AccountCheckSortedModel { Q_OBJECT @@ -32,9 +47,15 @@ class FeedsImportExportModel : public AccountCheckSortedModel { signals: void parsingStarted(); void parsingProgress(int completed, int total); - void parsingFinished(int count_failed, int count_succeeded, bool parsing_error); + void parsingFinished(int count_failed, int count_succeeded); private: + bool produceFeed(const FeedLookup& feed_lookup); + + private: + QMutex m_mtxLookup; + QList m_lookup; + QFutureWatcher m_watcherLookup; Mode m_mode; };