FreshRSS/rssguard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

minor modernization of feed updating code

Browse Source
This commit is contained in:
Martin Rotter 2022-08-27 13:33:21 +02:00
parent 0edb3a2839
commit 69a8fc7ea6
11 changed files with 216 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

199
src/librssguard/core/feeddownloader.cpp
View File

@ -21,7 +21,8 @@
#include <QThread>
FeedDownloader::FeedDownloader()
: QObject(), m_isCacheSynchronizationRunning(false), m_stopCacheSynchronization(false), m_mutex(new QMutex()), m_feedsUpdated(0), m_feedsOriginalCount(0) {
: QObject(), m_isCacheSynchronizationRunning(false), m_stopCacheSynchronization(false), m_mutex(new QMutex()),
m_feedsUpdated(0), m_feedsOriginalCount(0) {
qRegisterMetaType<FeedDownloadResults>("FeedDownloadResults");
}
@ -40,8 +41,8 @@ void FeedDownloader::synchronizeAccountCaches(const QList<CacheForServiceRoot*>&
m_isCacheSynchronizationRunning = true;
for (CacheForServiceRoot* cache : caches) {
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "Synchronizing cache back to server on thread" << QUOTE_W_SPACE_DOT(QThread::currentThreadId());
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Synchronizing cache back to server on thread"
<< QUOTE_W_SPACE_DOT(QThread::currentThreadId());
cache->saveAllCachedData(false);
if (m_stopCacheSynchronization) {
@ -72,9 +73,8 @@ void FeedDownloader::updateFeeds(const QList<Feed*>& feeds) {
qDebugNN << LOGSEC_FEEDDOWNLOADER << "No feeds to update in worker thread, aborting update.";
}
else {
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "Starting feed updates from worker in thread: '"
<< QThread::currentThreadId() << "'.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Starting feed updates from worker in thread: '" << QThread::currentThreadId()
<< "'.";
// Job starts now.
emit updateStarted();
@ -102,11 +102,11 @@ void FeedDownloader::updateFeeds(const QList<Feed*>& feeds) {
synchronizeAccountCaches(caches.values(), false);
QHash<ServiceRoot*, ApplicationException> errored_roots;
auto roots = feeds_per_root.uniqueKeys();
bool is_main_thread = QThread::currentThread() == qApp->thread();
QSqlDatabase database = is_main_thread ?
qApp->database()->driver()->connection(metaObject()->className()) :
qApp->database()->driver()->connection(QSL("feed_upd"));
QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className())
: qApp->database()->driver()->connection(QSL("feed_upd"));
for (auto* rt : roots) {
// Obtain lists of local IDs.
@ -124,41 +124,60 @@ void FeedDownloader::updateFeeds(const QList<Feed*>& feeds) {
QHash<ServiceRoot::BagOfMessages, QStringList> per_feed_states;
per_feed_states.insert(ServiceRoot::BagOfMessages::Read,
DatabaseQueries::bagOfMessages(database,
ServiceRoot::BagOfMessages::Read,
fd));
DatabaseQueries::bagOfMessages(database, ServiceRoot::BagOfMessages::Read, fd));
per_feed_states.insert(ServiceRoot::BagOfMessages::Unread,
DatabaseQueries::bagOfMessages(database,
ServiceRoot::BagOfMessages::Unread,
fd));
DatabaseQueries::bagOfMessages(database, ServiceRoot::BagOfMessages::Unread, fd));
per_feed_states.insert(ServiceRoot::BagOfMessages::Starred,
DatabaseQueries::bagOfMessages(database,
ServiceRoot::BagOfMessages::Starred,
fd));
DatabaseQueries::bagOfMessages(database, ServiceRoot::BagOfMessages::Starred, fd));
per_acc_states.insert(fd->customId(), per_feed_states);
}
stated_messages.insert(rt, per_acc_states);
}
rt->aboutToBeginFeedFetching(feeds_per_root.values(rt),
stated_messages.value(rt),
tagged_messages.value(rt));
try {
rt->aboutToBeginFeedFetching(feeds_per_root.values(rt), stated_messages.value(rt), tagged_messages.value(rt));
}
catch (const ApplicationException& ex) {
// Common error showed, all feeds from the root are errored now!
errored_roots.insert(rt, ex);
}
}
while (!m_feeds.isEmpty()) {
auto n_f = m_feeds.takeFirst();
auto n_r = n_f->getParentServiceRoot();
updateOneFeed(n_f->getParentServiceRoot(),
n_f,
stated_messages.value(n_f->getParentServiceRoot()).value(n_f->customId()),
tagged_messages.value(n_f->getParentServiceRoot()));
if (errored_roots.contains(n_r)) {
// This feed is errored because its account errored when preparing feed update.
ApplicationException root_ex = errored_roots.value(n_r);
skipFeedUpdateWithError(n_r, n_f, root_ex);
}
else {
updateOneFeed(n_r, n_f, stated_messages.value(n_r).value(n_f->customId()), tagged_messages.value(n_r));
}
}
}
finalizeUpdate();
}
void FeedDownloader::skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const ApplicationException& ex) {
const FeedFetchException* fetch_ex = dynamic_cast<const FeedFetchException*>(&ex);
if (fetch_ex != nullptr) {
feed->setStatus(fetch_ex->feedStatus(), fetch_ex->message());
}
else {
feed->setStatus(Feed::Status::OtherError, ex.message());
}
acc->itemChanged({feed});
emit updateProgress(feed, ++m_feedsUpdated, m_feedsOriginalCount);
}
void FeedDownloader::stopRunningUpdate() {
m_stopCacheSynchronization = true;
m_feeds.clear();
@ -169,29 +188,25 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "Downloading new messages for feed ID '"
<< feed->customId() << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '"
<< QThread::currentThreadId() << "'.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloading new messages for feed ID '" << feed->customId() << "' URL: '"
<< feed->source() << "' title: '" << feed->title() << "' in thread: '" << QThread::currentThreadId() << "'.";
int acc_id = feed->getParentServiceRoot()->accountId();
QElapsedTimer tmr; tmr.start();
QElapsedTimer tmr;
tmr.start();
try {
bool is_main_thread = QThread::currentThread() == qApp->thread();
QSqlDatabase database = is_main_thread ?
qApp->database()->driver()->connection(metaObject()->className()) :
qApp->database()->driver()->connection(QSL("feed_upd"));
QList<Message> msgs = feed->getParentServiceRoot()->obtainNewMessages(feed,
stated_messages,
tagged_messages);
QSqlDatabase database = is_main_thread ? qApp->database()->driver()->connection(metaObject()->className())
: qApp->database()->driver()->connection(QSL("feed_upd"));
QList<Message> msgs = feed->getParentServiceRoot()->obtainNewMessages(feed, stated_messages, tagged_messages);
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded " << msgs.size() << " messages for feed ID '"
<< feed->customId() << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '"
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Downloaded " << msgs.size() << " messages for feed ID '" << feed->customId()
<< "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '"
<< QThread::currentThreadId() << "'. Operation took " << tmr.nsecsElapsed() / 1000 << " microseconds.";
bool fix_future_datetimes = qApp->settings()->value(GROUP(Messages),
SETTING(Messages::FixupFutureArticleDateTimes)).toBool();
bool fix_future_datetimes =
qApp->settings()->value(GROUP(Messages), SETTING(Messages::FixupFutureArticleDateTimes)).toBool();
// Now, sanitize messages (tweak encoding etc.).
for (auto& msg : msgs) {
@ -214,7 +229,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
MessageFilter::initializeFilteringEngine(filter_engine, &msg_obj);
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Setting up JS evaluation took " << tmr.nsecsElapsed() / 1000 << " microseconds.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Setting up JS evaluation took " << tmr.nsecsElapsed() / 1000
<< " microseconds.";
QList<Message> read_msgs, important_msgs;
@ -247,8 +263,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
try {
MessageObject::FilteringAction decision = msg_filter->filterMessage(&filter_engine);
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "Running filter script, it took " << tmr.nsecsElapsed() / 1000 << " microseconds.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Running filter script, it took " << tmr.nsecsElapsed() / 1000
<< " microseconds.";
switch (decision) {
case MessageObject::FilteringAction::Accept:
@ -265,8 +281,7 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
}
catch (const FilteringException& ex) {
qCriticalNN << LOGSEC_FEEDDOWNLOADER
<< "Error when evaluating filtering JS function: "
<< QUOTE_W_SPACE_DOT(ex.message())
<< "Error when evaluating filtering JS function: " << QUOTE_W_SPACE_DOT(ex.message())
<< " Accepting message.";
continue;
}
@ -277,13 +292,15 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
}
if (!msg_backup.m_isRead && msg_orig->m_isRead) {
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_backup.m_customId << "' was marked as read by message scripts.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_backup.m_customId
<< "' was marked as read by message scripts.";
read_msgs << *msg_orig;
}
if (!msg_backup.m_isImportant && msg_orig->m_isImportant) {
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_backup.m_customId << "' was marked as important by message scripts.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Message with custom ID: '" << msg_backup.m_customId
<< "' was marked as important by message scripts.";
important_msgs << *msg_orig;
}
@ -294,10 +311,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
// Label is not there anymore, it was deassigned.
lbl->deassignFromMessage(*msg_orig);
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "It was detected that label" << QUOTE_W_SPACE(lbl->customId())
<< "was DEASSIGNED from message" << QUOTE_W_SPACE(msg_orig->m_customId)
<< "by message filter(s).";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "It was detected that label" << QUOTE_W_SPACE(lbl->customId())
<< "was DEASSIGNED from message" << QUOTE_W_SPACE(msg_orig->m_customId) << "by message filter(s).";
}
}
@ -307,10 +322,8 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
// was newly assigned.
lbl->assignToMessage(*msg_orig);
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "It was detected that label" << QUOTE_W_SPACE(lbl->customId())
<< "was ASSIGNED to message" << QUOTE_W_SPACE(msg_orig->m_customId)
<< "by message filter(s).";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "It was detected that label" << QUOTE_W_SPACE(lbl->customId())
<< "was ASSIGNED to message" << QUOTE_W_SPACE(msg_orig->m_customId) << "by message filter(s).";
}
}
@ -322,8 +335,7 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
if (!read_msgs.isEmpty()) {
// Now we push new read states to the service.
if (feed->getParentServiceRoot()->onBeforeSetMessagesRead(feed, read_msgs, RootItem::ReadStatus::Read)) {
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "Notified services about messages marked as read by message filters.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Notified services about messages marked as read by message filters.";
}
else {
qCriticalNN << LOGSEC_FEEDDOWNLOADER
@ -333,9 +345,11 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
if (!important_msgs.isEmpty()) {
// Now we push new read states to the service.
auto list = boolinq::from(important_msgs).select([](const Message& msg) {
return ImportanceChange(msg, RootItem::Importance::Important);
}).toStdList();
auto list = boolinq::from(important_msgs)
.select([](const Message& msg) {
return ImportanceChange(msg, RootItem::Importance::Important);
})
.toStdList();
QList<ImportanceChange> chngs = FROM_STD_LIST(QList<ImportanceChange>, list);
if (feed->getParentServiceRoot()->onBeforeSwitchMessageImportance(feed, chngs)) {
@ -352,56 +366,47 @@ void FeedDownloader::updateOneFeed(ServiceRoot* acc,
removeDuplicateMessages(msgs);
// Now make sure, that messages are actually stored to SQL in a locked state.
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Saving messages of feed ID '"
<< feed->customId() << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '"
<< QThread::currentThreadId() << "'.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Saving messages of feed ID '" << feed->customId() << "' URL: '"
<< feed->source() << "' title: '" << feed->title() << "' in thread: '" << QThread::currentThreadId()
<< "'.";
tmr.restart();
auto updated_messages = acc->updateMessages(msgs, feed, false);
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "Updating messages in DB took " << tmr.nsecsElapsed() / 1000 << " microseconds.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Updating messages in DB took " << tmr.nsecsElapsed() / 1000
<< " microseconds.";
if (feed->status() != Feed::Status::NewMessages) {
feed->setStatus(updated_messages.first > 0 || updated_messages.second > 0
? Feed::Status::NewMessages
: Feed::Status::Normal);
feed->setStatus(updated_messages.first > 0 || updated_messages.second > 0 ? Feed::Status::NewMessages
: Feed::Status::Normal);
}
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< updated_messages << " messages for feed "
<< feed->customId() << " stored in DB.";
qDebugNN << LOGSEC_FEEDDOWNLOADER << updated_messages << " messages for feed " << feed->customId()
<< " stored in DB.";
if (updated_messages.first > 0) {
m_results.appendUpdatedFeed({ feed->title(), updated_messages.first });
m_results.appendUpdatedFeed({feed->title(), updated_messages.first});
}
}
catch (const FeedFetchException& feed_ex) {
qCriticalNN << LOGSEC_NETWORK
<< "Error when fetching feed:"
<< QUOTE_W_SPACE(feed_ex.feedStatus())
<< "message:"
<< QUOTE_W_SPACE_DOT(feed_ex.message());
qCriticalNN << LOGSEC_NETWORK << "Error when fetching feed:" << QUOTE_W_SPACE(feed_ex.feedStatus())
<< "message:" << QUOTE_W_SPACE_DOT(feed_ex.message());
feed->setStatus(feed_ex.feedStatus(), feed_ex.message());
}
catch (const ApplicationException& app_ex) {
qCriticalNN << LOGSEC_NETWORK
<< "Unknown error when fetching feed:"
<< "message:"
<< QUOTE_W_SPACE_DOT(app_ex.message());
qCriticalNN << LOGSEC_NETWORK << "Unknown error when fetching feed:"
<< "message:" << QUOTE_W_SPACE_DOT(app_ex.message());
feed->setStatus(Feed::Status::OtherError, app_ex.message());
}
feed->getParentServiceRoot()->itemChanged({ feed });
feed->getParentServiceRoot()->itemChanged({feed});
m_feedsUpdated++;
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "Made progress in feed updates, total feeds count "
<< m_feedsUpdated << "/" << m_feedsOriginalCount << " (id of feed is "
<< feed->id() << ").";
qDebugNN << LOGSEC_FEEDDOWNLOADER << "Made progress in feed updates, total feeds count " << m_feedsUpdated << "/"
<< m_feedsOriginalCount << " (id of feed is " << feed->id() << ").";
emit updateProgress(feed, m_feedsUpdated, m_feedsOriginalCount);
}
@ -429,22 +434,22 @@ void FeedDownloader::removeDuplicateMessages(QList<Message>& messages) {
if (message.m_id > 0) {
is_duplicate = [](const Message& a, const Message& b) {
return a.m_id == b.m_id;
};
return a.m_id == b.m_id;
};
}
else if (message.m_customId.isEmpty()) {
is_duplicate = [](const Message& a, const Message& b) {
return std::tie(a.m_title, a.m_url, a.m_author) == std::tie(b.m_title, b.m_url, b.m_author);
};
return std::tie(a.m_title, a.m_url, a.m_author) == std::tie(b.m_title, b.m_url, b.m_author);
};
}
else {
is_duplicate = [](const Message& a, const Message& b) {
return a.m_customId == b.m_customId;
};
return a.m_customId == b.m_customId;
};
}
auto next_idx = idx + 1; // Index of next message to check after removing all duplicates.
auto last_idx = idx; // Index of the last kept duplicate.
auto last_idx = idx; // Index of the last kept duplicate.
idx = next_idx;
@ -506,9 +511,11 @@ void FeedDownloadResults::appendUpdatedFeed(const QPair<QString, int>& feed) {
}
void FeedDownloadResults::sort() {
std::sort(m_updatedFeeds.begin(), m_updatedFeeds.end(), [](const QPair<QString, int>& lhs, const QPair<QString, int>& rhs) {
return lhs.second > rhs.second;
});
std::sort(m_updatedFeeds.begin(),
m_updatedFeeds.end(),
[](const QPair<QString, int>& lhs, const QPair<QString, int>& rhs) {
return lhs.second > rhs.second;
});
}
void FeedDownloadResults::clear() {

5
src/librssguard/core/feeddownloader.h
View File

@ -8,6 +8,7 @@
#include <QPair>
#include "core/message.h"
#include "exceptions/applicationexception.h"
#include "services/abstract/cacheforserviceroot.h"
#include "services/abstract/feed.h"
@ -25,7 +26,6 @@ class FeedDownloadResults {
void clear();
private:
// QString represents title if the feed, int represents count of newly downloaded messages.
QList<QPair<QString, int>> m_updatedFeeds;
};
@ -33,7 +33,7 @@ class FeedDownloadResults {
// This class offers means to "update" feeds and "special" categories.
// NOTE: This class is used within separate thread.
class FeedDownloader : public QObject {
Q_OBJECT
Q_OBJECT
public:
explicit FeedDownloader();
@ -54,6 +54,7 @@ class FeedDownloader : public QObject {
void updateProgress(const Feed* feed, int current, int total);
private:
void skipFeedUpdateWithError(ServiceRoot* acc, Feed* feed, const ApplicationException& ex);
void updateOneFeed(ServiceRoot* acc,
Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,

2
src/librssguard/exceptions/applicationexception.cpp
View File

@ -4,6 +4,8 @@
ApplicationException::ApplicationException(QString message) : m_message(std::move(message)) {}
ApplicationException::~ApplicationException() {}
QString ApplicationException::message() const {
return m_message;
}

1
src/librssguard/exceptions/applicationexception.h
View File

@ -8,6 +8,7 @@
class ApplicationException {
public:
explicit ApplicationException(QString message = {});
virtual ~ApplicationException();
QString message() const;

7
src/librssguard/network-web/downloader.cpp
View File

@ -18,7 +18,7 @@ Downloader::Downloader(QObject* parent)
: QObject(parent), m_activeReply(nullptr), m_downloadManager(new SilentNetworkAccessManager(this)),
m_timer(new QTimer(this)), m_inputData(QByteArray()), m_inputMultipartData(nullptr), m_targetProtected(false),
m_targetUsername(QString()), m_targetPassword(QString()), m_lastOutputData({}),
m_lastOutputError(QNetworkReply::NetworkError::NoError), m_lastHttpStatusCode(0) {
m_lastOutputError(QNetworkReply::NetworkError::NoError), m_lastHttpStatusCode(0), m_lastHeaders({}) {
m_timer->setInterval(DOWNLOAD_TIMEOUT);
m_timer->setSingleShot(true);
@ -215,6 +215,7 @@ void Downloader::finished() {
m_lastContentType = reply->header(QNetworkRequest::KnownHeaders::ContentTypeHeader);
m_lastOutputError = reply->error();
m_lastHttpStatusCode = reply->attribute(QNetworkRequest::Attribute::HttpStatusCodeAttribute).toInt();
m_lastHeaders = reply->rawHeaderPairs();
// original_url = m_activeReply->property("original_url").toUrl();
@ -336,6 +337,10 @@ void Downloader::runGetRequest(const QNetworkRequest& request) {
connect(m_activeReply, &QNetworkReply::finished, this, &Downloader::finished);
}
QList<QNetworkReply::RawHeaderPair> Downloader::lastHeaders() const {
return m_lastHeaders;
}
int Downloader::lastHttpStatusCode() const {
return m_lastHttpStatusCode;
}

2
src/librssguard/network-web/downloader.h
View File

@ -30,6 +30,7 @@ class Downloader : public QObject {
QVariant lastContentType() const;
QList<QNetworkCookie> lastCookies() const;
int lastHttpStatusCode() const;
QList<QNetworkReply::RawHeaderPair> lastHeaders() const;
void setProxy(const QNetworkProxy& proxy);
@ -115,6 +116,7 @@ class Downloader : public QObject {
int m_lastHttpStatusCode;
QVariant m_lastContentType;
QList<QNetworkCookie> m_lastCookies;
QList<QNetworkReply::RawHeaderPair> m_lastHeaders;
};
#endif // DOWNLOADER_H

5
src/librssguard/network-web/networkfactory.cpp
View File

@ -271,6 +271,7 @@ NetworkResult NetworkFactory::performNetworkOperation(const QString& url,
result.m_contentType = downloader.lastContentType().toString();
result.m_cookies = downloader.lastCookies();
result.m_httpCode = downloader.lastHttpStatusCode();
result.m_headers = downloader.lastHeaders();
return result;
}
@ -311,12 +312,14 @@ NetworkResult NetworkFactory::performNetworkOperation(const QString& url,
result.m_contentType = downloader.lastContentType().toString();
result.m_cookies = downloader.lastCookies();
result.m_httpCode = downloader.lastHttpStatusCode();
result.m_headers = downloader.lastHeaders();
return result;
}
NetworkResult::NetworkResult()
: m_networkError(QNetworkReply::NetworkError::NoError), m_httpCode(0), m_contentType(QString()), m_cookies({}) {}
: m_networkError(QNetworkReply::NetworkError::NoError), m_httpCode(0), m_contentType(QString()), m_cookies({}),
m_headers({}) {}
NetworkResult::NetworkResult(QNetworkReply::NetworkError err,
int http_code,

1
src/librssguard/network-web/networkfactory.h
View File

@ -18,6 +18,7 @@ struct NetworkResult {
int m_httpCode;
QString m_contentType;
QList<QNetworkCookie> m_cookies;
QList<QNetworkReply::RawHeaderPair> m_headers;
explicit NetworkResult();
explicit NetworkResult(QNetworkReply::NetworkError err,

242
src/librssguard/services/greader/greadernetwork.cpp
View File

@ -23,9 +23,8 @@
GreaderNetwork::GreaderNetwork(QObject* parent)
: QObject(parent), m_root(nullptr), m_service(GreaderServiceRoot::Service::FreshRss), m_username(QString()),
m_password(QString()), m_baseUrl(QString()), m_batchSize(GREADER_DEFAULT_BATCH_SIZE),
m_downloadOnlyUnreadMessages(false), m_prefetchedMessages({}), m_prefetchedStatus(Feed::Status::Normal),
m_performGlobalFetching(false), m_intelligentSynchronization(true),
m_newerThanFilter(QDate::currentDate().addYears(-1)),
m_downloadOnlyUnreadMessages(false), m_prefetchedMessages({}), m_performGlobalFetching(false),
m_intelligentSynchronization(true), m_newerThanFilter(QDate::currentDate().addYears(-1)),
m_oauth(new OAuth2Service(QSL(INO_OAUTH_AUTH_URL), QSL(INO_OAUTH_TOKEN_URL), {}, {}, QSL(INO_OAUTH_SCOPE), this)) {
initializeOauth();
clearCredentials();
@ -125,17 +124,12 @@ QVariantHash GreaderNetwork::userInfo(const QNetworkProxy& proxy) {
proxy);
if (res.m_networkError != QNetworkReply::NetworkError::NoError) {
throw NetworkException(res.m_networkError);
throw NetworkException(res.m_networkError, output);
}
return QJsonDocument::fromJson(output).object().toVariantHash();
}
void GreaderNetwork::clearPrefetchedMessages() {
m_prefetchedMessages.clear();
m_prefetchedStatus = Feed::Status::Normal;
}
void GreaderNetwork::prepareFeedFetching(GreaderServiceRoot* root,
const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>&
@ -145,114 +139,95 @@ void GreaderNetwork::prepareFeedFetching(GreaderServiceRoot* root,
Q_UNUSED(tagged_messages)
m_prefetchedMessages.clear();
m_prefetchedStatus = Feed::Status::Normal;
try {
double perc_of_fetching = (feeds.size() * 1.0) / root->getSubTreeFeeds().size();
double perc_of_fetching = (feeds.size() * 1.0) / root->getSubTreeFeeds().size();
m_performGlobalFetching = perc_of_fetching > GREADER_GLOBAL_UPDATE_THRES;
m_performGlobalFetching = perc_of_fetching > GREADER_GLOBAL_UPDATE_THRES;
qDebugNN << LOGSEC_GREADER << "Percentage of feeds for fetching:" << QUOTE_W_SPACE_DOT(perc_of_fetching * 100.0);
qDebugNN << LOGSEC_GREADER << "Percentage of feeds for fetching:" << QUOTE_W_SPACE_DOT(perc_of_fetching * 100.0);
auto remote_starred_ids_list = itemIds(QSL(GREADER_API_FULL_STATE_IMPORTANT), false, proxy, -1, m_newerThanFilter);
auto remote_starred_ids_list = itemIds(QSL(GREADER_API_FULL_STATE_IMPORTANT), false, proxy, -1, m_newerThanFilter);
for (int i = 0; i < remote_starred_ids_list.size(); i++) {
remote_starred_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_starred_ids_list.at(i)));
for (int i = 0; i < remote_starred_ids_list.size(); i++) {
remote_starred_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_starred_ids_list.at(i)));
}
QSet<QString> remote_starred_ids = FROM_LIST_TO_SET(QSet<QString>, remote_starred_ids_list);
QSet<QString> local_starred_ids;
QList<QHash<ServiceRoot::BagOfMessages, QStringList>> all_states = stated_messages.values();
for (auto& lst : all_states) {
auto s = lst.value(ServiceRoot::BagOfMessages::Starred);
local_starred_ids.unite(FROM_LIST_TO_SET(QSet<QString>, s));
}
auto starred_to_download((remote_starred_ids - local_starred_ids).unite(local_starred_ids - remote_starred_ids));
auto to_download = starred_to_download;
if (m_performGlobalFetching) {
qWarningNN << LOGSEC_GREADER << "Performing global contents fetching.";
QStringList remote_all_ids_list =
m_downloadOnlyUnreadMessages
? QStringList()
: itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), false, proxy, -1, m_newerThanFilter);
QStringList remote_unread_ids_list =
itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), true, proxy, -1, m_newerThanFilter);
for (int i = 0; i < remote_all_ids_list.size(); i++) {
remote_all_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_all_ids_list.at(i)));
}
QSet<QString> remote_starred_ids = FROM_LIST_TO_SET(QSet<QString>, remote_starred_ids_list);
QSet<QString> local_starred_ids;
QList<QHash<ServiceRoot::BagOfMessages, QStringList>> all_states = stated_messages.values();
for (int i = 0; i < remote_unread_ids_list.size(); i++) {
remote_unread_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_unread_ids_list.at(i)));
}
QSet<QString> remote_all_ids = FROM_LIST_TO_SET(QSet<QString>, remote_all_ids_list);
QSet<QString> remote_unread_ids = FROM_LIST_TO_SET(QSet<QString>, remote_unread_ids_list);
QSet<QString> remote_read_ids = remote_all_ids - remote_unread_ids;
QSet<QString> local_unread_ids;
QSet<QString> local_read_ids;
for (auto& lst : all_states) {
auto s = lst.value(ServiceRoot::BagOfMessages::Starred);
auto u = lst.value(ServiceRoot::BagOfMessages::Unread);
auto r = lst.value(ServiceRoot::BagOfMessages::Read);
local_starred_ids.unite(FROM_LIST_TO_SET(QSet<QString>, s));
local_unread_ids.unite(FROM_LIST_TO_SET(QSet<QString>, u));
local_read_ids.unite(FROM_LIST_TO_SET(QSet<QString>, r));
}
auto starred_to_download((remote_starred_ids - local_starred_ids).unite(local_starred_ids - remote_starred_ids));
auto to_download = starred_to_download;
if (m_performGlobalFetching) {
qWarningNN << LOGSEC_GREADER << "Performing global contents fetching.";
QStringList remote_all_ids_list =
m_downloadOnlyUnreadMessages
? QStringList()
: itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), false, proxy, -1, m_newerThanFilter);
QStringList remote_unread_ids_list =
itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), true, proxy, -1, m_newerThanFilter);
for (int i = 0; i < remote_all_ids_list.size(); i++) {
remote_all_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_all_ids_list.at(i)));
}
for (int i = 0; i < remote_unread_ids_list.size(); i++) {
remote_unread_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_unread_ids_list.at(i)));
}
QSet<QString> remote_all_ids = FROM_LIST_TO_SET(QSet<QString>, remote_all_ids_list);
QSet<QString> remote_unread_ids = FROM_LIST_TO_SET(QSet<QString>, remote_unread_ids_list);
QSet<QString> remote_read_ids = remote_all_ids - remote_unread_ids;
QSet<QString> local_unread_ids;
QSet<QString> local_read_ids;
for (auto& lst : all_states) {
auto u = lst.value(ServiceRoot::BagOfMessages::Unread);
auto r = lst.value(ServiceRoot::BagOfMessages::Read);
local_unread_ids.unite(FROM_LIST_TO_SET(QSet<QString>, u));
local_read_ids.unite(FROM_LIST_TO_SET(QSet<QString>, r));
}
if (!m_downloadOnlyUnreadMessages) {
to_download += remote_all_ids - local_read_ids - local_unread_ids;
}
else {
to_download += remote_unread_ids - local_read_ids - local_unread_ids;
}
auto moved_read = local_read_ids.intersect(remote_unread_ids);
to_download += moved_read;
if (!m_downloadOnlyUnreadMessages) {
auto moved_unread = local_unread_ids.intersect(remote_read_ids);
to_download += moved_unread;
}
if (!m_downloadOnlyUnreadMessages) {
to_download += remote_all_ids - local_read_ids - local_unread_ids;
}
else {
qWarningNN << LOGSEC_GREADER << "Performing feed-based contents fetching.";
to_download += remote_unread_ids - local_read_ids - local_unread_ids;
}
Feed::Status error;
QList<QString> to_download_list(to_download.values());
auto moved_read = local_read_ids.intersect(remote_unread_ids);
if (!to_download_list.isEmpty()) {
if (m_service == GreaderServiceRoot::Service::Reedah) {
for (int i = 0; i < to_download_list.size(); i++) {
to_download_list.replace(i, convertLongStreamIdToShortStreamId(to_download_list.at(i)));
}
to_download += moved_read;
if (!m_downloadOnlyUnreadMessages) {
auto moved_unread = local_unread_ids.intersect(remote_read_ids);
to_download += moved_unread;
}
}
else {
qWarningNN << LOGSEC_GREADER << "Performing feed-based contents fetching.";
}
QList<QString> to_download_list(to_download.values());
if (!to_download_list.isEmpty()) {
if (m_service == GreaderServiceRoot::Service::Reedah) {
for (int i = 0; i < to_download_list.size(); i++) {
to_download_list.replace(i, convertLongStreamIdToShortStreamId(to_download_list.at(i)));
}
m_prefetchedMessages = itemContents(root, to_download_list, error, proxy);
}
}
catch (const FeedFetchException& fex) {
m_prefetchedStatus = fex.feedStatus();
qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for common stream:" << QUOTE_W_SPACE_DOT(fex.message());
}
catch (const NetworkException& nex) {
m_prefetchedStatus = Feed::Status::NetworkError;
qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for common stream:" << QUOTE_W_SPACE_DOT(nex.message());
}
catch (const ApplicationException& aex) {
m_prefetchedStatus = Feed::Status::OtherError;
qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for common stream:" << QUOTE_W_SPACE_DOT(aex.message());
m_prefetchedMessages = itemContents(root, to_download_list, proxy);
}
}
@ -261,50 +236,19 @@ QList<Message> GreaderNetwork::getMessagesIntelligently(ServiceRoot* root,
const QHash<ServiceRoot::BagOfMessages, QStringList>&
stated_messages,
const QHash<QString, QStringList>& tagged_messages,
Feed::Status& error,
const QNetworkProxy& proxy) {
Q_UNUSED(tagged_messages)
QList<Message> msgs;
if (m_prefetchedStatus != Feed::Status::Normal) {
error = m_prefetchedStatus;
return msgs;
}
if (!m_performGlobalFetching) {
// 1. Get unread IDs for a feed.
// 2. Get read IDs for a feed.
// 3. Download messages/contents for missing or changed IDs.
// 4. Add prefetched starred msgs.
QStringList remote_all_ids_list, remote_unread_ids_list;
try {
remote_all_ids_list =
m_downloadOnlyUnreadMessages ? QStringList() : itemIds(stream_id, false, proxy, -1, m_newerThanFilter);
remote_unread_ids_list = itemIds(stream_id, true, proxy, -1, m_newerThanFilter);
}
catch (const FeedFetchException& fex) {
error = fex.feedStatus();
qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for specific stream:" << QUOTE_W_SPACE_DOT(fex.message());
return msgs;
}
catch (const NetworkException& nex) {
error = Feed::Status::NetworkError;
qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for specific stream:" << QUOTE_W_SPACE_DOT(nex.message());
return msgs;
}
catch (const ApplicationException& aex) {
error = Feed::Status::OtherError;
qCriticalNN << LOGSEC_CORE << "Failed to fetch item IDs for specific stream:" << QUOTE_W_SPACE_DOT(aex.message());
return msgs;
}
QStringList remote_all_ids_list =
m_downloadOnlyUnreadMessages ? QStringList() : itemIds(stream_id, false, proxy, -1, m_newerThanFilter);
QStringList remote_unread_ids_list = itemIds(stream_id, true, proxy, -1, m_newerThanFilter);
// Convert item IDs to long form.
for (int i = 0; i < remote_all_ids_list.size(); i++) {
@ -356,7 +300,7 @@ QList<Message> GreaderNetwork::getMessagesIntelligently(ServiceRoot* root,
}
}
msgs = itemContents(root, to_download_list, error, proxy);
msgs = itemContents(root, to_download_list, proxy);
}
}
@ -443,7 +387,7 @@ QStringList GreaderNetwork::itemIds(const QString& stream_id,
if (result_stream.m_networkError != QNetworkReply::NetworkError::NoError) {
qCriticalNN << LOGSEC_GREADER << "Cannot download item IDs for " << QUOTE_NO_SPACE(stream_id)
<< ", network error:" << QUOTE_W_SPACE_DOT(result_stream.m_networkError);
throw NetworkException(result_stream.m_networkError);
throw NetworkException(result_stream.m_networkError, output_stream);
}
else {
ids.append(decodeItemIds(output_stream, continuation));
@ -456,13 +400,11 @@ QStringList GreaderNetwork::itemIds(const QString& stream_id,
QList<Message> GreaderNetwork::itemContents(ServiceRoot* root,
const QList<QString>& stream_ids,
Feed::Status& error,
const QNetworkProxy& proxy) {
QString continuation;
if (!ensureLogin(proxy)) {
error = Feed::Status::AuthError;
return {};
throw FeedFetchException(Feed::Status::AuthError, tr("login failed"));
}
QList<Message> msgs;
@ -518,8 +460,7 @@ QList<Message> GreaderNetwork::itemContents(ServiceRoot* root,
if (result_stream.m_networkError != QNetworkReply::NetworkError::NoError) {
qCriticalNN << LOGSEC_GREADER << "Cannot download messages for " << batch_ids
<< ", network error:" << QUOTE_W_SPACE_DOT(result_stream.m_networkError);
error = Feed::Status::NetworkError;
return {};
throw NetworkException(result_stream.m_networkError, output_stream);
}
else {
msgs.append(decodeStreamContents(root, output_stream, QString(), continuation));
@ -528,19 +469,14 @@ QList<Message> GreaderNetwork::itemContents(ServiceRoot* root,
while (!continuation.isEmpty());
}
error = Feed::Status::Normal;
return msgs;
}
QList<Message> GreaderNetwork::streamContents(ServiceRoot* root,
const QString& stream_id,
Feed::Status& error,
const QNetworkProxy& proxy) {
QList<Message> GreaderNetwork::streamContents(ServiceRoot* root, const QString& stream_id, const QNetworkProxy& proxy) {
QString continuation;
if (!ensureLogin(proxy)) {
error = Feed::Status::AuthError;
return {};
throw FeedFetchException(Feed::Status::AuthError, tr("login failed"));
}
QList<Message> msgs;
@ -589,8 +525,7 @@ QList<Message> GreaderNetwork::streamContents(ServiceRoot* root,
if (result_stream.m_networkError != QNetworkReply::NetworkError::NoError) {
qCriticalNN << LOGSEC_GREADER << "Cannot download messages for " << QUOTE_NO_SPACE(stream_id)
<< ", network error:" << QUOTE_W_SPACE_DOT(result_stream.m_networkError);
error = Feed::Status::NetworkError;
return {};
throw NetworkException(result_stream.m_networkError, output_stream);
}
else {
msgs.append(decodeStreamContents(root, output_stream, stream_id, continuation));
@ -598,7 +533,6 @@ QList<Message> GreaderNetwork::streamContents(ServiceRoot* root,
}
while (!continuation.isEmpty() && msgs.size() < target_msgs_size);
error = Feed::Status::Normal;
return msgs;
}
@ -608,7 +542,7 @@ RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNe
if (!ensureLogin(proxy)) {
qCriticalNN << LOGSEC_GREADER << "Cannot get feed tree, not logged-in.";
throw ApplicationException(tr("you are not logged-in, maybe wrong credentials"));
throw ApplicationException(tr("login failed"));
}
QByteArray output_labels;
@ -627,8 +561,7 @@ RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNe
qCriticalNN << LOGSEC_GREADER
<< "Cannot get labels tree, network error:" << QUOTE_W_SPACE_DOT(result_labels.m_networkError);
throw NetworkException(result_labels.m_networkError,
tr("cannot get list of labels, HTTP code '%1'").arg(result_labels.m_httpCode));
throw NetworkException(result_labels.m_networkError, output_labels);
}
full_url = generateFullUrl(Operations::SubscriptionList);
@ -648,8 +581,7 @@ RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNe
qCriticalNN << LOGSEC_GREADER
<< "Cannot get feed tree, network error:" << QUOTE_W_SPACE_DOT(result_feeds.m_networkError);
throw NetworkException(result_labels.m_networkError,
tr("cannot get list of feeds, HTTP code '%1'").arg(result_feeds.m_httpCode));
throw NetworkException(result_labels.m_networkError, output_feeds);
}
return decodeTagsSubscriptions(output_labels, output_feeds, obtain_icons, proxy);
@ -1097,6 +1029,10 @@ void GreaderNetwork::clearCredentials() {
m_authAuth = m_authSid = m_authToken = QString();
}
void GreaderNetwork::clearPrefetchedMessages() {
m_prefetchedMessages.clear();
}
QString GreaderNetwork::sanitizedBaseUrl() const {
QString base_url = m_service == GreaderServiceRoot::Service::Inoreader ? QSL(GREADER_URL_INOREADER) : m_baseUrl;

8
src/librssguard/services/greader/greadernetwork.h
View File

@ -47,7 +47,6 @@ class GreaderNetwork : public QObject {
const QString& stream_id,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages,
Feed::Status& error,
const QNetworkProxy& proxy);
RootItem* categoriesFeedsLabelsTree(bool obtain_icons, const QNetworkProxy& proxy);
@ -95,12 +94,8 @@ class GreaderNetwork : public QObject {
QDate newer_than = {});
QList<Message> itemContents(ServiceRoot* root,
const QList<QString>& stream_ids,
Feed::Status& error,
const QNetworkProxy& proxy);
QList<Message> streamContents(ServiceRoot* root,
const QString& stream_id,
Feed::Status& error,
const QNetworkProxy& proxy);
QList<Message> streamContents(ServiceRoot* root, const QString& stream_id, const QNetworkProxy& proxy);
QNetworkReply::NetworkError clientLogin(const QNetworkProxy& proxy);
QDate newerThanFilter() const;
@ -148,7 +143,6 @@ class GreaderNetwork : public QObject {
QString m_authAuth;
QString m_authToken;
QList<Message> m_prefetchedMessages;
Feed::Status m_prefetchedStatus;
bool m_performGlobalFetching;
bool m_intelligentSynchronization;
QDate m_newerThanFilter;

18
src/librssguard/services/greader/greaderserviceroot.cpp
View File

@ -130,27 +130,17 @@ QList<Message> GreaderServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>&
stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
Feed::Status error = Feed::Status::Normal;
QList<Message> msgs;
if (m_network->intelligentSynchronization()) {
msgs = m_network->getMessagesIntelligently(this,
feed->customId(),
stated_messages,
tagged_messages,
error,
networkProxy());
msgs =
m_network->getMessagesIntelligently(this, feed->customId(), stated_messages, tagged_messages, networkProxy());
}
else {
msgs = m_network->streamContents(this, feed->customId(), error, networkProxy());
msgs = m_network->streamContents(this, feed->customId(), networkProxy());
}
if (error != Feed::Status::NewMessages && error != Feed::Status::Normal) {
throw FeedFetchException(error);
}
else {
return msgs;
}
return msgs;
}
bool GreaderServiceRoot::wantsBaggedIdsOfExistingMessages() const {