FreshRSS/rssguard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

some refactoring in flow of url/script/post-process-script

Browse Source
This commit is contained in:
Martin Rotter 2023-10-16 15:18:02 +02:00
parent 8720fe663d
commit 6100e56985
3 changed files with 36 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/librssguard/services/standard/standardfeed.cpp
View File

@ -268,13 +268,13 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type,
qDebugNN << LOGSEC_CORE << "Running custom script for guessing" << QUOTE_W_SPACE(source) << "to obtain feed data."; qDebugNN << LOGSEC_CORE << "Running custom script for guessing" << QUOTE_W_SPACE(source) << "to obtain feed data.";
// Use script to generate feed file. // Use script to generate feed file.
feed_contents = generateFeedFileWithScript(source, timeout).toUtf8(); feed_contents = generateFeedFileWithScript(source, timeout);
} }
if (!post_process_script.simplified().isEmpty()) { if (!post_process_script.simplified().isEmpty()) {
qDebugNN << LOGSEC_CORE << "Post-processing obtained feed data with custom script for guessing" qDebugNN << LOGSEC_CORE << "Post-processing obtained feed data with custom script for guessing"
<< QUOTE_W_SPACE_DOT(post_process_script); << QUOTE_W_SPACE_DOT(post_process_script);
feed_contents = postProcessFeedFileWithScript(post_process_script, feed_contents, timeout).toUtf8(); feed_contents = postProcessFeedFileWithScript(post_process_script, feed_contents, timeout);
} }
StandardFeed* feed = nullptr; StandardFeed* feed = nullptr;
@ -382,11 +382,11 @@ QStringList StandardFeed::prepareExecutionLine(const QString& execution_line) {
return qApp->replaceDataUserDataFolderPlaceholder(args); return qApp->replaceDataUserDataFolderPlaceholder(args);
} }
QString StandardFeed::runScriptProcess(const QStringList& cmd_args, QByteArray StandardFeed::runScriptProcess(const QStringList& cmd_args,
const QString& working_directory, const QString& working_directory,
int run_timeout, int run_timeout,
bool provide_input, bool provide_input,
const QString& input) { const QString& input) {
QProcess process; QProcess process;
if (provide_input) { if (provide_input) {
@ -450,7 +450,7 @@ QString StandardFeed::runScriptProcess(const QStringList& cmd_args,
} }
} }
QString StandardFeed::generateFeedFileWithScript(const QString& execution_line, int run_timeout) { QByteArray StandardFeed::generateFeedFileWithScript(const QString& execution_line, int run_timeout) {
auto prepared_query = prepareExecutionLine(execution_line); auto prepared_query = prepareExecutionLine(execution_line);
if (prepared_query.isEmpty()) { if (prepared_query.isEmpty()) {
@ -460,9 +460,9 @@ QString StandardFeed::generateFeedFileWithScript(const QString& execution_line,
return runScriptProcess(prepared_query, qApp->userDataFolder(), run_timeout, false); return runScriptProcess(prepared_query, qApp->userDataFolder(), run_timeout, false);
} }
QString StandardFeed::postProcessFeedFileWithScript(const QString& execution_line, QByteArray StandardFeed::postProcessFeedFileWithScript(const QString& execution_line,
const QString& raw_feed_data, const QString& raw_feed_data,
int run_timeout) { int run_timeout) {
auto prepared_query = prepareExecutionLine(execution_line); auto prepared_query = prepareExecutionLine(execution_line);
if (prepared_query.isEmpty()) { if (prepared_query.isEmpty()) {

18
src/librssguard/services/standard/standardfeed.h
View File

@ -94,15 +94,15 @@ class StandardFeed : public Feed {
// Scraping + post+processing. // Scraping + post+processing.
static QStringList prepareExecutionLine(const QString& execution_line); static QStringList prepareExecutionLine(const QString& execution_line);
static QString generateFeedFileWithScript(const QString& execution_line, int run_timeout); static QByteArray generateFeedFileWithScript(const QString& execution_line, int run_timeout);
static QString postProcessFeedFileWithScript(const QString& execution_line, static QByteArray postProcessFeedFileWithScript(const QString& execution_line,
const QString& raw_feed_data, const QString& raw_feed_data,
int run_timeout); int run_timeout);
static QString runScriptProcess(const QStringList& cmd_args, static QByteArray runScriptProcess(const QStringList& cmd_args,
const QString& working_directory, const QString& working_directory,
int run_timeout, int run_timeout,
bool provide_input, bool provide_input,
const QString& input = {}); const QString& input = {});
public slots: public slots:
void fetchMetadataForItself(); void fetchMetadataForItself();

32
src/librssguard/services/standard/standardserviceroot.cpp
View File

@ -156,13 +156,13 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
Q_UNUSED(tagged_messages) Q_UNUSED(tagged_messages)
StandardFeed* f = static_cast<StandardFeed*>(feed); StandardFeed* f = static_cast<StandardFeed*>(feed);
QByteArray feed_contents;
QString formatted_feed_contents; QString formatted_feed_contents;
int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt(); int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
if (f->sourceType() == StandardFeed::SourceType::Url) { if (f->sourceType() == StandardFeed::SourceType::Url) {
qDebugNN << LOGSEC_CORE << "Downloading URL" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data."; qDebugNN << LOGSEC_CORE << "Downloading URL" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data.";
QByteArray feed_contents;
QList<QPair<QByteArray, QByteArray>> headers; QList<QPair<QByteArray, QByteArray>> headers;
headers << NetworkFactory::generateBasicAuthHeader(f->protection(), f->username(), f->password()); headers << NetworkFactory::generateBasicAuthHeader(f->protection(), f->username(), f->password());
@ -198,25 +198,13 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
qWarningNN << LOGSEC_CORE << "This feed is gzipped."; qWarningNN << LOGSEC_CORE << "This feed is gzipped.";
#endif #endif
} }
// Encode downloaded data for further parsing.
QTextCodec* codec = QTextCodec::codecForName(f->encoding().toLocal8Bit());
if (codec == nullptr) {
// No suitable codec for this encoding was found.
// Use non-converted data.
formatted_feed_contents = feed_contents;
}
else {
formatted_feed_contents = codec->toUnicode(feed_contents);
}
} }
else { else {
qDebugNN << LOGSEC_CORE << "Running custom script" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data."; qDebugNN << LOGSEC_CORE << "Running custom script" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data.";
// Use script to generate feed file. // Use script to generate feed file.
try { try {
formatted_feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout); feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout);
} }
catch (const ScriptException& ex) { catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE << "Custom script for generating feed file failed:" << QUOTE_W_SPACE_DOT(ex.message()); qCriticalNN << LOGSEC_CORE << "Custom script for generating feed file failed:" << QUOTE_W_SPACE_DOT(ex.message());
@ -230,8 +218,8 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
<< QUOTE_W_SPACE_DOT(f->postProcessScript()); << QUOTE_W_SPACE_DOT(f->postProcessScript());
try { try {
formatted_feed_contents = feed_contents =
StandardFeed::postProcessFeedFileWithScript(f->postProcessScript(), formatted_feed_contents, download_timeout); StandardFeed::postProcessFeedFileWithScript(f->postProcessScript(), feed_contents, download_timeout);
} }
catch (const ScriptException& ex) { catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE << "Post-processing script for feed file failed:" << QUOTE_W_SPACE_DOT(ex.message()); qCriticalNN << LOGSEC_CORE << "Post-processing script for feed file failed:" << QUOTE_W_SPACE_DOT(ex.message());
@ -240,6 +228,18 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
} }
} }
// Encode obtained data for further parsing.
QTextCodec* codec = QTextCodec::codecForName(f->encoding().toLocal8Bit());
if (codec == nullptr) {
// No suitable codec for this encoding was found.
// Use UTF-8.
formatted_feed_contents = QString::fromUtf8(feed_contents);
}
else {
formatted_feed_contents = codec->toUnicode(feed_contents);
}
// Feed data are downloaded and encoded. // Feed data are downloaded and encoded.
// Parse data and obtain messages. // Parse data and obtain messages.
QList<Message> messages; QList<Message> messages;