FreshRSS/rssguard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

apply gzip automatically after data is downloaded for all scenarios

Browse Source
This commit is contained in:
Martin Rotter 2023-10-16 15:34:45 +02:00
parent a4bf57f971
commit 075b249f23
2 changed files with 42 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/librssguard/services/standard/standardfeed.cpp
View File

@ -19,6 +19,10 @@
#include "services/standard/parsers/rssparser.h"
#include "services/standard/parsers/sitemapparser.h"
#if defined(ENABLE_COMPRESSED_SITEMAP)
#include "3rd-party/qcompressor/qcompressor.h"
#endif
#include <QCommandLineParser>
#include <QDomDocument>
#include <QDomElement>
@ -274,6 +278,25 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type,
feed_contents = generateFeedFileWithScript(source, timeout);
}
// Sitemap parser supports gzip-encoded data too.
// We need to decode it here before encoding
// stuff kicks in.
if (SitemapParser::isGzip(feed_contents)) {
#if defined(ENABLE_COMPRESSED_SITEMAP)
qWarningNN << LOGSEC_CORE << "Decompressing gzipped feed data.";
QByteArray uncompressed_feed_contents;
if (!QCompressor::gzipDecompress(feed_contents, uncompressed_feed_contents)) {
throw ApplicationException("gzip decompression failed");
}
feed_contents = uncompressed_feed_contents;
#else
qWarningNN << LOGSEC_CORE << "This feed is gzipped.";
#endif
}
if (!post_process_script.simplified().isEmpty()) {
qDebugNN << LOGSEC_CORE << "Post-processing obtained feed data with custom script for guessing"
<< QUOTE_W_SPACE_DOT(post_process_script);

33
src/librssguard/services/standard/standardserviceroot.cpp
View File

@ -184,20 +184,6 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
<< "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
throw FeedFetchException(Feed::Status::NetworkError, NetworkFactory::networkErrorText(network_result));
}
// Sitemap parser supports gzip-encoded data too.
if (SitemapParser::isGzip(feed_contents)) {
#if defined(ENABLE_COMPRESSED_SITEMAP)
qWarningNN << LOGSEC_CORE << "Decompressing gzipped feed data.";
QByteArray uncompressed_feed_contents;
QCompressor::gzipDecompress(feed_contents, uncompressed_feed_contents);
feed_contents = uncompressed_feed_contents;
#else
qWarningNN << LOGSEC_CORE << "This feed is gzipped.";
#endif
}
}
else if (f->sourceType() == StandardFeed::SourceType::LocalFile) {
feed_contents = IOFactory::readFile(feed->source());
@ -216,6 +202,25 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
}
}
// Sitemap parser supports gzip-encoded data too.
// We need to decode it here before encoding
// stuff kicks in.
if (SitemapParser::isGzip(feed_contents)) {
#if defined(ENABLE_COMPRESSED_SITEMAP)
qWarningNN << LOGSEC_CORE << "Decompressing gzipped feed data.";
QByteArray uncompressed_feed_contents;
if (!QCompressor::gzipDecompress(feed_contents, uncompressed_feed_contents)) {
throw ApplicationException("gzip decompression failed");
}
feed_contents = uncompressed_feed_contents;
#else
qWarningNN << LOGSEC_CORE << "This feed is gzipped.";
#endif
}
if (!f->postProcessScript().simplified().isEmpty()) {
qDebugNN << LOGSEC_CORE << "We will process feed data with post-process script"
<< QUOTE_W_SPACE_DOT(f->postProcessScript());