FreshRSS/rssguard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

some refactoring in flow of url/script/post-process-script

Browse Source
This commit is contained in:
Martin Rotter 2023-10-16 15:18:02 +02:00
parent 8720fe663d
commit 6100e56985
3 changed files with 36 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/librssguard/services/standard/standardfeed.cpp
View File

@ -268,13 +268,13 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type,
qDebugNN << LOGSEC_CORE << "Running custom script for guessing" << QUOTE_W_SPACE(source) << "to obtain feed data.";
// Use script to generate feed file.
feed_contents = generateFeedFileWithScript(source, timeout).toUtf8();
feed_contents = generateFeedFileWithScript(source, timeout);
}
if (!post_process_script.simplified().isEmpty()) {
qDebugNN << LOGSEC_CORE << "Post-processing obtained feed data with custom script for guessing"
<< QUOTE_W_SPACE_DOT(post_process_script);
feed_contents = postProcessFeedFileWithScript(post_process_script, feed_contents, timeout).toUtf8();
feed_contents = postProcessFeedFileWithScript(post_process_script, feed_contents, timeout);
}
StandardFeed* feed = nullptr;
@ -382,7 +382,7 @@ QStringList StandardFeed::prepareExecutionLine(const QString& execution_line) {
return qApp->replaceDataUserDataFolderPlaceholder(args);
}
QString StandardFeed::runScriptProcess(const QStringList& cmd_args,
QByteArray StandardFeed::runScriptProcess(const QStringList& cmd_args,
const QString& working_directory,
int run_timeout,
bool provide_input,
@ -450,7 +450,7 @@ QString StandardFeed::runScriptProcess(const QStringList& cmd_args,
}
}
QString StandardFeed::generateFeedFileWithScript(const QString& execution_line, int run_timeout) {
QByteArray StandardFeed::generateFeedFileWithScript(const QString& execution_line, int run_timeout) {
auto prepared_query = prepareExecutionLine(execution_line);
if (prepared_query.isEmpty()) {
@ -460,7 +460,7 @@ QString StandardFeed::generateFeedFileWithScript(const QString& execution_line,
return runScriptProcess(prepared_query, qApp->userDataFolder(), run_timeout, false);
}
QString StandardFeed::postProcessFeedFileWithScript(const QString& execution_line,
QByteArray StandardFeed::postProcessFeedFileWithScript(const QString& execution_line,
const QString& raw_feed_data,
int run_timeout) {
auto prepared_query = prepareExecutionLine(execution_line);

6
src/librssguard/services/standard/standardfeed.h
View File

@ -94,11 +94,11 @@ class StandardFeed : public Feed {
// Scraping + post+processing.
static QStringList prepareExecutionLine(const QString& execution_line);
static QString generateFeedFileWithScript(const QString& execution_line, int run_timeout);
static QString postProcessFeedFileWithScript(const QString& execution_line,
static QByteArray generateFeedFileWithScript(const QString& execution_line, int run_timeout);
static QByteArray postProcessFeedFileWithScript(const QString& execution_line,
const QString& raw_feed_data,
int run_timeout);
static QString runScriptProcess(const QStringList& cmd_args,
static QByteArray runScriptProcess(const QStringList& cmd_args,
const QString& working_directory,
int run_timeout,
bool provide_input,

32
src/librssguard/services/standard/standardserviceroot.cpp
View File

@ -156,13 +156,13 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
Q_UNUSED(tagged_messages)
StandardFeed* f = static_cast<StandardFeed*>(feed);
QByteArray feed_contents;
QString formatted_feed_contents;
int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
if (f->sourceType() == StandardFeed::SourceType::Url) {
qDebugNN << LOGSEC_CORE << "Downloading URL" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data.";
QByteArray feed_contents;
QList<QPair<QByteArray, QByteArray>> headers;
headers << NetworkFactory::generateBasicAuthHeader(f->protection(), f->username(), f->password());
@ -198,25 +198,13 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
qWarningNN << LOGSEC_CORE << "This feed is gzipped.";
#endif
}
// Encode downloaded data for further parsing.
QTextCodec* codec = QTextCodec::codecForName(f->encoding().toLocal8Bit());
if (codec == nullptr) {
// No suitable codec for this encoding was found.
// Use non-converted data.
formatted_feed_contents = feed_contents;
}
else {
formatted_feed_contents = codec->toUnicode(feed_contents);
}
}
else {
qDebugNN << LOGSEC_CORE << "Running custom script" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data.";
// Use script to generate feed file.
try {
formatted_feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout);
feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout);
}
catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE << "Custom script for generating feed file failed:" << QUOTE_W_SPACE_DOT(ex.message());
@ -230,8 +218,8 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
<< QUOTE_W_SPACE_DOT(f->postProcessScript());
try {
formatted_feed_contents =
StandardFeed::postProcessFeedFileWithScript(f->postProcessScript(), formatted_feed_contents, download_timeout);
feed_contents =
StandardFeed::postProcessFeedFileWithScript(f->postProcessScript(), feed_contents, download_timeout);
}
catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE << "Post-processing script for feed file failed:" << QUOTE_W_SPACE_DOT(ex.message());
@ -240,6 +228,18 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
}
}
// Encode obtained data for further parsing.
QTextCodec* codec = QTextCodec::codecForName(f->encoding().toLocal8Bit());
if (codec == nullptr) {
// No suitable codec for this encoding was found.
// Use UTF-8.
formatted_feed_contents = QString::fromUtf8(feed_contents);
}
else {
formatted_feed_contents = codec->toUnicode(feed_contents);
}
// Feed data are downloaded and encoded.
// Parse data and obtain messages.
QList<Message> messages;