From 6100e56985735f2d0b7f10484c7a0dcbd43cc394 Mon Sep 17 00:00:00 2001 From: Martin Rotter Date: Mon, 16 Oct 2023 15:18:02 +0200 Subject: [PATCH] some refactoring in flow of url/script/post-process-script --- .../services/standard/standardfeed.cpp | 22 ++++++------- .../services/standard/standardfeed.h | 18 +++++------ .../services/standard/standardserviceroot.cpp | 32 +++++++++---------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/librssguard/services/standard/standardfeed.cpp b/src/librssguard/services/standard/standardfeed.cpp index 24035a1ad..8af13fbce 100644 --- a/src/librssguard/services/standard/standardfeed.cpp +++ b/src/librssguard/services/standard/standardfeed.cpp @@ -268,13 +268,13 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type, qDebugNN << LOGSEC_CORE << "Running custom script for guessing" << QUOTE_W_SPACE(source) << "to obtain feed data."; // Use script to generate feed file. - feed_contents = generateFeedFileWithScript(source, timeout).toUtf8(); + feed_contents = generateFeedFileWithScript(source, timeout); } if (!post_process_script.simplified().isEmpty()) { qDebugNN << LOGSEC_CORE << "Post-processing obtained feed data with custom script for guessing" << QUOTE_W_SPACE_DOT(post_process_script); - feed_contents = postProcessFeedFileWithScript(post_process_script, feed_contents, timeout).toUtf8(); + feed_contents = postProcessFeedFileWithScript(post_process_script, feed_contents, timeout); } StandardFeed* feed = nullptr; @@ -382,11 +382,11 @@ QStringList StandardFeed::prepareExecutionLine(const QString& execution_line) { return qApp->replaceDataUserDataFolderPlaceholder(args); } -QString StandardFeed::runScriptProcess(const QStringList& cmd_args, - const QString& working_directory, - int run_timeout, - bool provide_input, - const QString& input) { +QByteArray StandardFeed::runScriptProcess(const QStringList& cmd_args, + const QString& working_directory, + int run_timeout, + bool provide_input, + const QString& input) { QProcess process; if (provide_input) { @@ -450,7 +450,7 @@ QString StandardFeed::runScriptProcess(const QStringList& cmd_args, } } -QString StandardFeed::generateFeedFileWithScript(const QString& execution_line, int run_timeout) { +QByteArray StandardFeed::generateFeedFileWithScript(const QString& execution_line, int run_timeout) { auto prepared_query = prepareExecutionLine(execution_line); if (prepared_query.isEmpty()) { @@ -460,9 +460,9 @@ QString StandardFeed::generateFeedFileWithScript(const QString& execution_line, return runScriptProcess(prepared_query, qApp->userDataFolder(), run_timeout, false); } -QString StandardFeed::postProcessFeedFileWithScript(const QString& execution_line, - const QString& raw_feed_data, - int run_timeout) { +QByteArray StandardFeed::postProcessFeedFileWithScript(const QString& execution_line, + const QString& raw_feed_data, + int run_timeout) { auto prepared_query = prepareExecutionLine(execution_line); if (prepared_query.isEmpty()) { diff --git a/src/librssguard/services/standard/standardfeed.h b/src/librssguard/services/standard/standardfeed.h index de51ceed2..f29ff830c 100644 --- a/src/librssguard/services/standard/standardfeed.h +++ b/src/librssguard/services/standard/standardfeed.h @@ -94,15 +94,15 @@ class StandardFeed : public Feed { // Scraping + post+processing. static QStringList prepareExecutionLine(const QString& execution_line); - static QString generateFeedFileWithScript(const QString& execution_line, int run_timeout); - static QString postProcessFeedFileWithScript(const QString& execution_line, - const QString& raw_feed_data, - int run_timeout); - static QString runScriptProcess(const QStringList& cmd_args, - const QString& working_directory, - int run_timeout, - bool provide_input, - const QString& input = {}); + static QByteArray generateFeedFileWithScript(const QString& execution_line, int run_timeout); + static QByteArray postProcessFeedFileWithScript(const QString& execution_line, + const QString& raw_feed_data, + int run_timeout); + static QByteArray runScriptProcess(const QStringList& cmd_args, + const QString& working_directory, + int run_timeout, + bool provide_input, + const QString& input = {}); public slots: void fetchMetadataForItself(); diff --git a/src/librssguard/services/standard/standardserviceroot.cpp b/src/librssguard/services/standard/standardserviceroot.cpp index aa714a4ba..18222fa0a 100644 --- a/src/librssguard/services/standard/standardserviceroot.cpp +++ b/src/librssguard/services/standard/standardserviceroot.cpp @@ -156,13 +156,13 @@ QList StandardServiceRoot::obtainNewMessages(Feed* feed, Q_UNUSED(tagged_messages) StandardFeed* f = static_cast(feed); + QByteArray feed_contents; QString formatted_feed_contents; int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt(); if (f->sourceType() == StandardFeed::SourceType::Url) { qDebugNN << LOGSEC_CORE << "Downloading URL" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data."; - QByteArray feed_contents; QList> headers; headers << NetworkFactory::generateBasicAuthHeader(f->protection(), f->username(), f->password()); @@ -198,25 +198,13 @@ QList StandardServiceRoot::obtainNewMessages(Feed* feed, qWarningNN << LOGSEC_CORE << "This feed is gzipped."; #endif } - - // Encode downloaded data for further parsing. - QTextCodec* codec = QTextCodec::codecForName(f->encoding().toLocal8Bit()); - - if (codec == nullptr) { - // No suitable codec for this encoding was found. - // Use non-converted data. - formatted_feed_contents = feed_contents; - } - else { - formatted_feed_contents = codec->toUnicode(feed_contents); - } } else { qDebugNN << LOGSEC_CORE << "Running custom script" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data."; // Use script to generate feed file. try { - formatted_feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout); + feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout); } catch (const ScriptException& ex) { qCriticalNN << LOGSEC_CORE << "Custom script for generating feed file failed:" << QUOTE_W_SPACE_DOT(ex.message()); @@ -230,8 +218,8 @@ QList StandardServiceRoot::obtainNewMessages(Feed* feed, << QUOTE_W_SPACE_DOT(f->postProcessScript()); try { - formatted_feed_contents = - StandardFeed::postProcessFeedFileWithScript(f->postProcessScript(), formatted_feed_contents, download_timeout); + feed_contents = + StandardFeed::postProcessFeedFileWithScript(f->postProcessScript(), feed_contents, download_timeout); } catch (const ScriptException& ex) { qCriticalNN << LOGSEC_CORE << "Post-processing script for feed file failed:" << QUOTE_W_SPACE_DOT(ex.message()); @@ -240,6 +228,18 @@ QList StandardServiceRoot::obtainNewMessages(Feed* feed, } } + // Encode obtained data for further parsing. + QTextCodec* codec = QTextCodec::codecForName(f->encoding().toLocal8Bit()); + + if (codec == nullptr) { + // No suitable codec for this encoding was found. + // Use UTF-8. + formatted_feed_contents = QString::fromUtf8(feed_contents); + } + else { + formatted_feed_contents = codec->toUnicode(feed_contents); + } + // Feed data are downloaded and encoded. // Parse data and obtain messages. QList messages;