Improve data downloading.

This commit is contained in:
Martin Rotter 2014-08-20 17:52:44 +02:00
parent 6ef2a8dcbb
commit d59d0cee3e
6 changed files with 47 additions and 117 deletions

View File

@ -145,12 +145,12 @@ QPair<FeedsModelFeed*, QNetworkReply::NetworkError> FeedsModelFeed::guessFeed(co
} }
QByteArray feed_contents; QByteArray feed_contents;
if ((result.second = NetworkFactory::downloadFeedFile(url, if ((result.second = NetworkFactory::downloadFile(url,
qApp->settings()->value(APP_CFG_FEEDS, "feed_update_timeout", DOWNLOAD_TIMEOUT).toInt(), qApp->settings()->value(APP_CFG_FEEDS, "feed_update_timeout", DOWNLOAD_TIMEOUT).toInt(),
feed_contents, feed_contents,
!username.isEmpty(), !username.isEmpty(),
username, username,
password)) == QNetworkReply::NoError) { password)) == QNetworkReply::NoError) {
// Feed XML was obtained, now we need to try to guess // Feed XML was obtained, now we need to try to guess
// its encoding before we can read further data. // its encoding before we can read further data.
QString xml_schema_encoding; QString xml_schema_encoding;
@ -349,12 +349,9 @@ QVariant FeedsModelFeed::data(int column, int role) const {
void FeedsModelFeed::update() { void FeedsModelFeed::update() {
QByteArray feed_contents; QByteArray feed_contents;
int download_timeout = qApp->settings()->value(APP_CFG_FEEDS, "feed_update_timeout", DOWNLOAD_TIMEOUT).toInt(); int download_timeout = qApp->settings()->value(APP_CFG_FEEDS, "feed_update_timeout", DOWNLOAD_TIMEOUT).toInt();
QNetworkReply::NetworkError download_result = NetworkFactory::downloadFeedFile(url(), QNetworkReply::NetworkError download_result = NetworkFactory::downloadFile(url(), download_timeout,
download_timeout, feed_contents, passwordProtected(),
feed_contents, username(), password());
passwordProtected(),
username(),
password());
if (download_result != QNetworkReply::NoError) { if (download_result != QNetworkReply::NoError) {
qWarning("Error during fetching of new messages for feed '%s' (id %d).", qWarning("Error during fetching of new messages for feed '%s' (id %d).",

View File

@ -152,7 +152,7 @@ QPair<UpdateInfo, QNetworkReply::NetworkError> SystemFactory::checkForUpdates()
QPair<UpdateInfo, QNetworkReply::NetworkError> result; QPair<UpdateInfo, QNetworkReply::NetworkError> result;
QByteArray releases_xml; QByteArray releases_xml;
result.second = NetworkFactory::downloadFeedFile(RELEASES_LIST, DOWNLOAD_TIMEOUT, releases_xml); result.second = NetworkFactory::downloadFile(RELEASES_LIST, DOWNLOAD_TIMEOUT, releases_xml);
if (result.second == QNetworkReply::NoError) { if (result.second == QNetworkReply::NoError) {
result.first = parseUpdatesFile(releases_xml); result.first = parseUpdatesFile(releases_xml);

View File

@ -23,10 +23,8 @@
Downloader::Downloader(QObject *parent) Downloader::Downloader(QObject *parent)
: QObject(parent), : QObject(parent), m_activeReply(NULL), m_downloadManager(new SilentNetworkAccessManager(this)),
m_activeReply(NULL), m_timer(new QTimer(this)), m_lastOutputData(QByteArray()), m_lastOutputError(QNetworkReply::NoError) {
m_downloadManager(new SilentNetworkAccessManager(this)),
m_timer(new QTimer(this)) {
m_timer->setInterval(DOWNLOAD_TIMEOUT); m_timer->setInterval(DOWNLOAD_TIMEOUT);
m_timer->setSingleShot(true); m_timer->setSingleShot(true);
@ -39,7 +37,7 @@ Downloader::~Downloader() {
m_downloadManager->deleteLater(); m_downloadManager->deleteLater();
} }
void Downloader::downloadFile(const QString &url, bool protected_contents, void Downloader::downloadFile(const QString &url, int timeout, bool protected_contents,
const QString &username, const QString &password) { const QString &username, const QString &password) {
QNetworkRequest request; QNetworkRequest request;
QObject originatingObject; QObject originatingObject;
@ -75,13 +73,13 @@ void Downloader::finished(QNetworkReply *reply) {
else { else {
// No redirection is indicated. Final file is obtained in our "reply" object. // No redirection is indicated. Final file is obtained in our "reply" object.
// Read the data into output buffer. // Read the data into output buffer.
QByteArray output = reply->readAll(); m_lastOutputData = reply->readAll();
QNetworkReply::NetworkError reply_error = reply->error(); m_lastOutputError = reply->error();
m_activeReply->deleteLater(); m_activeReply->deleteLater();
m_activeReply = NULL; m_activeReply = NULL;
emit completed(reply_error, output); emit completed(m_lastOutputError, m_lastOutputData);
} }
} }
@ -107,3 +105,11 @@ void Downloader::runGetRequest(const QNetworkRequest &request) {
connect(m_activeReply, SIGNAL(downloadProgress(qint64,qint64)), connect(m_activeReply, SIGNAL(downloadProgress(qint64,qint64)),
this, SLOT(progressInternal(qint64,qint64))); this, SLOT(progressInternal(qint64,qint64)));
} }
QNetworkReply::NetworkError Downloader::lastOutputError() const {
return m_lastOutputError;
}
QByteArray Downloader::lastOutputData() const {
return m_lastOutputData;
}

View File

@ -37,9 +37,13 @@ class Downloader : public QObject {
explicit Downloader(QObject *parent = 0); explicit Downloader(QObject *parent = 0);
virtual ~Downloader(); virtual ~Downloader();
// Access to last received full output data/error.
QByteArray lastOutputData() const;
QNetworkReply::NetworkError lastOutputError() const;
public slots: public slots:
// Performs asynchronous download of given file. Redirections are handled. // Performs asynchronous download of given file. Redirections are handled.
void downloadFile(const QString &url, bool protected_contents = false, void downloadFile(const QString &url, int timeout = DOWNLOAD_TIMEOUT, bool protected_contents = false,
const QString &username = QString(), const QString &password = QString()); const QString &username = QString(), const QString &password = QString());
signals: signals:
@ -64,6 +68,9 @@ class Downloader : public QObject {
QNetworkReply *m_activeReply; QNetworkReply *m_activeReply;
SilentNetworkAccessManager *m_downloadManager; SilentNetworkAccessManager *m_downloadManager;
QTimer *m_timer; QTimer *m_timer;
QByteArray m_lastOutputData;
QNetworkReply::NetworkError m_lastOutputError;
}; };
#endif // DOWNLOADER_H #endif // DOWNLOADER_H

View File

@ -20,6 +20,7 @@
#include "definitions/definitions.h" #include "definitions/definitions.h"
#include "miscellaneous/settings.h" #include "miscellaneous/settings.h"
#include "network-web/silentnetworkaccessmanager.h" #include "network-web/silentnetworkaccessmanager.h"
#include "network-web/downloader.h"
#include <QEventLoop> #include <QEventLoop>
#include <QTimer> #include <QTimer>
@ -104,7 +105,7 @@ QNetworkReply::NetworkError NetworkFactory::downloadIcon(const QString &url,
QString google_s2_with_url = QString("http://www.google.com/s2/favicons?domain=%1").arg(Qt::escape(url)); QString google_s2_with_url = QString("http://www.google.com/s2/favicons?domain=%1").arg(Qt::escape(url));
#endif #endif
QByteArray icon_data; QByteArray icon_data;
QNetworkReply::NetworkError network_result = downloadFeedFile(google_s2_with_url, timeout, icon_data); QNetworkReply::NetworkError network_result = downloadFile(google_s2_with_url, timeout, icon_data);
if (network_result == QNetworkReply::NoError) { if (network_result == QNetworkReply::NoError) {
QPixmap icon_pixmap; QPixmap icon_pixmap;
@ -115,101 +116,20 @@ QNetworkReply::NetworkError NetworkFactory::downloadIcon(const QString &url,
return network_result; return network_result;
} }
QNetworkReply::NetworkError NetworkFactory::downloadFeedFile(const QString &url, QNetworkReply::NetworkError NetworkFactory::downloadFile(const QString &url, int timeout,
int timeout, QByteArray &output, bool protected_contents,
QByteArray &output, const QString &username, const QString &password) {
bool protected_contents, // Here, we want to achieve "synchronous" approach because we want synchronout download API for
const QString &username, // some use-cases too.
const QString &password) { Downloader downloader;
// Original asynchronous behavior of QNetworkAccessManager
// is replaced by synchronous behavior in order to make
// process of downloading of a file easier to understand.
// Make necessary variables.
SilentNetworkAccessManager manager;
QEventLoop loop; QEventLoop loop;
QTimer timer;
QNetworkRequest request;
QNetworkReply *reply;
QObject originatingObject;
// Set credential information as originating object. // We need to quit event loop when the download finishes.
originatingObject.setProperty("protected", protected_contents); QObject::connect(&downloader, SIGNAL(completed(QNetworkReply::NetworkError)), &loop, SLOT(quit()));
originatingObject.setProperty("username", username);
originatingObject.setProperty("password", password);
request.setOriginatingObject(&originatingObject);
// Set url for this reques. downloader.downloadFile(url, timeout, protected_contents, username, password);
request.setUrl(url); loop.exec();
output = downloader.lastOutputData();
// Create necessary connections. return downloader.lastOutputError();
QObject::connect(&timer, SIGNAL(timeout()), &loop, SLOT(quit()));
QObject::connect(&manager, SIGNAL(finished(QNetworkReply*)), &loop, SLOT(quit()));
forever {
// This timer fires just ONCE.
timer.setSingleShot(true);
// Try to open communication channel.
reply = manager.get(request);
// Start the timeout timer.
timer.start(timeout);
// Enter the event loop.
loop.exec();
// At this point one of two things happened:
// a) file download was completed,
// b) communication timed-out.
if (timer.isActive()) {
// Download is complete because timer is still running.
timer.stop();
}
else {
if (reply != NULL) {
delete reply;
reply = NULL;
}
// Timer already fired. Download is NOT successful.
return QNetworkReply::TimeoutError;
}
// In this phase, some part of downloading process is completed.
QUrl redirection_url = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
if (redirection_url.isValid()) {
// Communication indicates that HTTP redirection is needed.
// Setup redirection URL and download again.
request.setUrl(redirection_url);
delete reply;
reply = NULL;
}
else {
// No redirection is indicated. Final file is obtained
// in our "reply" object.
break;
}
}
// Read the data into output buffer.
output = reply->readAll();
QNetworkReply::NetworkError reply_error = reply->error();
qDebug("File '%s' fetched with status '%s' (code %d).",
qPrintable(url),
qPrintable(networkErrorText(reply_error)),
reply_error);
// Delete needed stuff and exit.
if (reply != NULL) {
delete reply;
reply = NULL;
}
return reply_error;
} }

View File

@ -41,7 +41,7 @@ class NetworkFactory {
// Performs SYNCHRONOUS download of file with given URL // Performs SYNCHRONOUS download of file with given URL
// and given timeout. // and given timeout.
static QNetworkReply::NetworkError downloadFeedFile(const QString &url, static QNetworkReply::NetworkError downloadFile(const QString &url,
int timeout, int timeout,
QByteArray &output, QByteArray &output,
bool protected_contents = false, bool protected_contents = false,