apply gzip automatically after data is downloaded for all scenarios
This commit is contained in:
parent
a4bf57f971
commit
075b249f23
|
@ -19,6 +19,10 @@
|
||||||
#include "services/standard/parsers/rssparser.h"
|
#include "services/standard/parsers/rssparser.h"
|
||||||
#include "services/standard/parsers/sitemapparser.h"
|
#include "services/standard/parsers/sitemapparser.h"
|
||||||
|
|
||||||
|
#if defined(ENABLE_COMPRESSED_SITEMAP)
|
||||||
|
#include "3rd-party/qcompressor/qcompressor.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
#include <QCommandLineParser>
|
#include <QCommandLineParser>
|
||||||
#include <QDomDocument>
|
#include <QDomDocument>
|
||||||
#include <QDomElement>
|
#include <QDomElement>
|
||||||
|
@ -274,6 +278,25 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type,
|
||||||
feed_contents = generateFeedFileWithScript(source, timeout);
|
feed_contents = generateFeedFileWithScript(source, timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sitemap parser supports gzip-encoded data too.
|
||||||
|
// We need to decode it here before encoding
|
||||||
|
// stuff kicks in.
|
||||||
|
if (SitemapParser::isGzip(feed_contents)) {
|
||||||
|
#if defined(ENABLE_COMPRESSED_SITEMAP)
|
||||||
|
qWarningNN << LOGSEC_CORE << "Decompressing gzipped feed data.";
|
||||||
|
|
||||||
|
QByteArray uncompressed_feed_contents;
|
||||||
|
|
||||||
|
if (!QCompressor::gzipDecompress(feed_contents, uncompressed_feed_contents)) {
|
||||||
|
throw ApplicationException("gzip decompression failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
feed_contents = uncompressed_feed_contents;
|
||||||
|
#else
|
||||||
|
qWarningNN << LOGSEC_CORE << "This feed is gzipped.";
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
if (!post_process_script.simplified().isEmpty()) {
|
if (!post_process_script.simplified().isEmpty()) {
|
||||||
qDebugNN << LOGSEC_CORE << "Post-processing obtained feed data with custom script for guessing"
|
qDebugNN << LOGSEC_CORE << "Post-processing obtained feed data with custom script for guessing"
|
||||||
<< QUOTE_W_SPACE_DOT(post_process_script);
|
<< QUOTE_W_SPACE_DOT(post_process_script);
|
||||||
|
|
|
@ -184,20 +184,6 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
|
||||||
<< "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
|
<< "during fetching of new messages for feed" << QUOTE_W_SPACE_DOT(feed->source());
|
||||||
throw FeedFetchException(Feed::Status::NetworkError, NetworkFactory::networkErrorText(network_result));
|
throw FeedFetchException(Feed::Status::NetworkError, NetworkFactory::networkErrorText(network_result));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sitemap parser supports gzip-encoded data too.
|
|
||||||
if (SitemapParser::isGzip(feed_contents)) {
|
|
||||||
#if defined(ENABLE_COMPRESSED_SITEMAP)
|
|
||||||
qWarningNN << LOGSEC_CORE << "Decompressing gzipped feed data.";
|
|
||||||
|
|
||||||
QByteArray uncompressed_feed_contents;
|
|
||||||
QCompressor::gzipDecompress(feed_contents, uncompressed_feed_contents);
|
|
||||||
|
|
||||||
feed_contents = uncompressed_feed_contents;
|
|
||||||
#else
|
|
||||||
qWarningNN << LOGSEC_CORE << "This feed is gzipped.";
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else if (f->sourceType() == StandardFeed::SourceType::LocalFile) {
|
else if (f->sourceType() == StandardFeed::SourceType::LocalFile) {
|
||||||
feed_contents = IOFactory::readFile(feed->source());
|
feed_contents = IOFactory::readFile(feed->source());
|
||||||
|
@ -216,6 +202,25 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sitemap parser supports gzip-encoded data too.
|
||||||
|
// We need to decode it here before encoding
|
||||||
|
// stuff kicks in.
|
||||||
|
if (SitemapParser::isGzip(feed_contents)) {
|
||||||
|
#if defined(ENABLE_COMPRESSED_SITEMAP)
|
||||||
|
qWarningNN << LOGSEC_CORE << "Decompressing gzipped feed data.";
|
||||||
|
|
||||||
|
QByteArray uncompressed_feed_contents;
|
||||||
|
|
||||||
|
if (!QCompressor::gzipDecompress(feed_contents, uncompressed_feed_contents)) {
|
||||||
|
throw ApplicationException("gzip decompression failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
feed_contents = uncompressed_feed_contents;
|
||||||
|
#else
|
||||||
|
qWarningNN << LOGSEC_CORE << "This feed is gzipped.";
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
if (!f->postProcessScript().simplified().isEmpty()) {
|
if (!f->postProcessScript().simplified().isEmpty()) {
|
||||||
qDebugNN << LOGSEC_CORE << "We will process feed data with post-process script"
|
qDebugNN << LOGSEC_CORE << "We will process feed data with post-process script"
|
||||||
<< QUOTE_W_SPACE_DOT(f->postProcessScript());
|
<< QUOTE_W_SPACE_DOT(f->postProcessScript());
|
||||||
|
|
Loading…
Reference in New Issue