FreshRSS/rssguard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

preload all local IDs, fix feedslist item reloading upon feed fetching, add handler to allow plugins to perform plugin-wide feed fetching befor individual feeds are fetched

Browse Source
This commit is contained in:
Martin Rotter 2021-07-30 08:34:07 +02:00
parent dfc28bcd4e
commit 79827ad6af
20 changed files with 284 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
resources/desktop/com.github.rssguard.appdata.xml
View File

@ -30,7 +30,7 @@
<url type="donation">https://martinrotter.github.io/donate/</url>
<content_rating type="oars-1.1" />
<releases>
<release version="3.9.2" date="2021-07-28"/>
<release version="3.9.2" date="2021-07-30"/>
</releases>
<content_rating type="oars-1.0">
<content_attribute id="violence-cartoon">none</content_attribute>

142
src/librssguard/core/feeddownloader.cpp
View File

@ -38,31 +38,7 @@ bool FeedDownloader::isUpdateRunning() const {
return !m_feeds.isEmpty();
}
void FeedDownloader::updateAvailableFeeds() {
for (const Feed* feed : qAsConst(m_feeds)) {
auto* cache = dynamic_cast<CacheForServiceRoot*>(feed->getParentServiceRoot());
if (cache != nullptr) {
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "Saving cache for feed with DB ID '" << feed->id()
<< "' and title '" << feed->title() << "'.";
cache->saveAllCachedData(false);
}
if (m_stopCacheSynchronization) {
qWarningNN << LOGSEC_FEEDDOWNLOADER << "Aborting cache synchronization.";
m_stopCacheSynchronization = false;
break;
}
}
while (!m_feeds.isEmpty()) {
updateOneFeed(m_feeds.takeFirst());
}
}
void FeedDownloader::synchronizeAccountCaches(const QList<CacheForServiceRoot*>& caches) {
void FeedDownloader::synchronizeAccountCaches(const QList<CacheForServiceRoot*>& caches, bool emit_signals) {
m_isCacheSynchronizationRunning = true;
for (CacheForServiceRoot* cache : caches) {
@ -80,7 +56,10 @@ void FeedDownloader::synchronizeAccountCaches(const QList<CacheForServiceRoot*>&
m_isCacheSynchronizationRunning = false;
qDebugNN << LOGSEC_FEEDDOWNLOADER << "All caches synchronized.";
emit cachesSynchronized();
if (emit_signals) {
emit cachesSynchronized();
}
}
void FeedDownloader::updateFeeds(const QList<Feed*>& feeds) {
@ -100,8 +79,81 @@ void FeedDownloader::updateFeeds(const QList<Feed*>& feeds) {
// Job starts now.
emit updateStarted();
QSet<CacheForServiceRoot*> caches;
QMultiHash<ServiceRoot*, Feed*> feeds_per_root;
updateAvailableFeeds();
// 1. key - account.
// 2. key - feed custom ID.
// 3. key - msg state.
QHash<ServiceRoot*, QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>> stated_messages;
// 1. key - account.
// 2. key - label custom ID.
QHash<ServiceRoot*, QHash<QString, QStringList>> tagged_messages;
for (auto* fd : feeds) {
CacheForServiceRoot* fd_cache = fd->getParentServiceRoot()->toCache();
if (fd_cache != nullptr) {
caches.insert(fd_cache);
}
feeds_per_root.insert(fd->getParentServiceRoot(), fd);
}
synchronizeAccountCaches(QList<CacheForServiceRoot*>(caches.begin(), caches.end()), false);
auto roots = feeds_per_root.uniqueKeys();
bool is_main_thread = QThread::currentThread() == qApp->thread();
QSqlDatabase database = is_main_thread ?
qApp->database()->driver()->connection(metaObject()->className()) :
qApp->database()->driver()->connection(QSL("feed_upd"));
for (auto* rt : roots) {
// Obtain lists of local IDs.
if (rt->wantsBaggedIdsOfExistingMessages()) {
// Tagged messages for the account.
tagged_messages.insert(rt, DatabaseQueries::bagsOfMessages(database, rt->labelsNode()->labels()));
QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>> per_acc_states;
// This account has activated intelligent downloading of messages.
// Prepare bags.
auto fds = feeds_per_root.values(rt);
for (Feed* fd : fds) {
QHash<ServiceRoot::BagOfMessages, QStringList> per_feed_states;
per_feed_states.insert(ServiceRoot::BagOfMessages::Read,
DatabaseQueries::bagOfMessages(database,
ServiceRoot::BagOfMessages::Read,
fd));
per_feed_states.insert(ServiceRoot::BagOfMessages::Unread,
DatabaseQueries::bagOfMessages(database,
ServiceRoot::BagOfMessages::Unread,
fd));
per_feed_states.insert(ServiceRoot::BagOfMessages::Starred,
DatabaseQueries::bagOfMessages(database,
ServiceRoot::BagOfMessages::Starred,
fd));
per_acc_states.insert(fd->customId(), per_feed_states);
}
stated_messages.insert(rt, per_acc_states);
}
rt->aboutToBeginFeedFetching(feeds_per_root.values(rt),
stated_messages.value(rt),
tagged_messages.value(rt));
}
while (!m_feeds.isEmpty()) {
auto n_f = m_feeds.takeFirst();
updateOneFeed(n_f,
stated_messages.value(n_f->getParentServiceRoot()).value(n_f->customId()),
tagged_messages.value(n_f->getParentServiceRoot()));
}
}
finalizeUpdate();
@ -113,7 +165,9 @@ void FeedDownloader::stopRunningUpdate() {
m_feedsOriginalCount = m_feedsUpdated = 0;
}
void FeedDownloader::updateOneFeed(Feed* feed) {
void FeedDownloader::updateOneFeed(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
qDebugNN << LOGSEC_FEEDDOWNLOADER
<< "Downloading new messages for feed ID '"
<< feed->customId() << "' URL: '" << feed->source() << "' title: '" << feed->title() << "' in thread: '"
@ -127,33 +181,7 @@ void FeedDownloader::updateOneFeed(Feed* feed) {
QSqlDatabase database = is_main_thread ?
qApp->database()->driver()->connection(metaObject()->className()) :
qApp->database()->driver()->connection(QSL("feed_upd"));
QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>> stated_messages;
QHash<QString, QStringList> tagged_messages;
if (feed->getParentServiceRoot()->wantsBaggedIdsOfExistingMessages()) {
// This account has activated intelligent downloading of messages.
// Prepare bags.
QHash<ServiceRoot::BagOfMessages, QStringList> per_feed_states;
per_feed_states.insert(ServiceRoot::BagOfMessages::Read,
DatabaseQueries::bagOfMessages(database,
ServiceRoot::BagOfMessages::Read,
feed));
per_feed_states.insert(ServiceRoot::BagOfMessages::Unread,
DatabaseQueries::bagOfMessages(database,
ServiceRoot::BagOfMessages::Unread,
feed));
per_feed_states.insert(ServiceRoot::BagOfMessages::Starred,
DatabaseQueries::bagOfMessages(database,
ServiceRoot::BagOfMessages::Starred,
feed));
stated_messages.insert(feed->customId(), per_feed_states);
tagged_messages = DatabaseQueries::bagsOfMessages(database,
feed->getParentServiceRoot()->labelsNode()->labels());
}
QList<Message> msgs = feed->getParentServiceRoot()->obtainNewMessages({ feed },
QList<Message> msgs = feed->getParentServiceRoot()->obtainNewMessages(feed,
stated_messages,
tagged_messages);
@ -343,7 +371,6 @@ void FeedDownloader::updateOneFeed(Feed* feed) {
<< QUOTE_W_SPACE_DOT(feed_ex.message());
feed->setStatus(feed_ex.feedStatus(), feed_ex.message());
feed->getParentServiceRoot()->itemChanged({ feed });
}
catch (const ApplicationException& app_ex) {
@ -353,9 +380,10 @@ void FeedDownloader::updateOneFeed(Feed* feed) {
<< QUOTE_W_SPACE_DOT(app_ex.message());
feed->setStatus(Feed::Status::OtherError, app_ex.message());
feed->getParentServiceRoot()->itemChanged({ feed });
}
feed->getParentServiceRoot()->itemChanged({ feed });
m_feedsUpdated++;
qDebugNN << LOGSEC_FEEDDOWNLOADER

7
src/librssguard/core/feeddownloader.h
View File

@ -43,7 +43,7 @@ class FeedDownloader : public QObject {
bool isCacheSynchronizationRunning() const;
public slots:
void synchronizeAccountCaches(const QList<CacheForServiceRoot*>& caches);
void synchronizeAccountCaches(const QList<CacheForServiceRoot*>& caches, bool emit_signals);
void updateFeeds(const QList<Feed*>& feeds);
void stopRunningUpdate();
@ -54,8 +54,9 @@ class FeedDownloader : public QObject {
void updateProgress(const Feed* feed, int current, int total);
private:
void updateOneFeed(Feed* feed);
void updateAvailableFeeds();
void updateOneFeed(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages);
void finalizeUpdate();
bool m_isCacheSynchronizationRunning;

3
src/librssguard/miscellaneous/feedreader.cpp
View File

@ -88,7 +88,8 @@ void FeedReader::updateFeeds(const QList<Feed*>& feeds) {
void FeedReader::synchronizeMessageData(const QList<CacheForServiceRoot*>& caches) {
QMetaObject::invokeMethod(m_feedDownloader, "synchronizeAccountCaches",
Qt::ConnectionType::QueuedConnection,
Q_ARG(QList<CacheForServiceRoot*>, caches));
Q_ARG(QList<CacheForServiceRoot*>, caches),
Q_ARG(bool, true));
}
void FeedReader::initializeFeedDownloader() {

8
src/librssguard/services/abstract/serviceroot.cpp
View File

@ -315,6 +315,14 @@ bool ServiceRoot::wantsBaggedIdsOfExistingMessages() const {
return false;
}
void ServiceRoot::aboutToBeginFeedFetching(const QList<Feed*>& feeds,
const QHash<QString, QHash<BagOfMessages, QStringList>>& stated_msgs,
const QHash<QString, QStringList>& tagged_msgs) {
Q_UNUSED(feeds)
Q_UNUSED(stated_msgs)
Q_UNUSED(tagged_msgs)
}
void ServiceRoot::itemChanged(const QList<RootItem*>& items) {
emit dataChanged(items);
}

7
src/librssguard/services/abstract/serviceroot.h
View File

@ -65,6 +65,9 @@ class ServiceRoot : public RootItem {
virtual QVariantHash customDatabaseData() const;
virtual void setCustomDatabaseData(const QVariantHash& data);
virtual bool wantsBaggedIdsOfExistingMessages() const;
virtual void aboutToBeginFeedFetching(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_msgs,
const QHash<QString, QStringList>& tagged_msgs);
// Returns list of specific actions for "Add new item" main window menu.
// So typical list of returned actions could look like:
@ -101,8 +104,8 @@ class ServiceRoot : public RootItem {
// Obtains list of messages.
// Throws exception subclassed from ApplicationException, preferably FeedFetchException
// if any problems arise.
virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
virtual QList<Message> obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) = 0;
// This method should prepare messages for given "item" (download them maybe?)

20
src/librssguard/services/feedly/feedlyserviceroot.cpp
View File

@ -72,24 +72,18 @@ void FeedlyServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool());
}
QList<Message> FeedlyServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
QList<Message> FeedlyServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
Q_UNUSED(stated_messages)
Q_UNUSED(tagged_messages)
QList<Message> messages;
for (Feed* feed : feeds) {
try {
messages << m_network->streamContents(feed->customId());
}
catch (const ApplicationException& ex) {
throw FeedFetchException(Feed::Status::NetworkError, ex.message());
}
try {
return m_network->streamContents(feed->customId());
}
catch (const ApplicationException& ex) {
throw FeedFetchException(Feed::Status::NetworkError, ex.message());
}
return messages;
}
void FeedlyServiceRoot::start(bool freshly_activated) {

4
src/librssguard/services/feedly/feedlyserviceroot.h
View File

@ -23,8 +23,8 @@ class FeedlyServiceRoot : public ServiceRoot, public CacheForServiceRoot {
virtual LabelOperation supportedLabelOperations() const;
virtual QVariantHash customDatabaseData() const;
virtual void setCustomDatabaseData(const QVariantHash& data);
virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
virtual QList<Message> obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages);
FeedlyNetwork* network() const;

17
src/librssguard/services/gmail/gmailserviceroot.cpp
View File

@ -74,22 +74,17 @@ void GmailServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
m_network->oauth()->setRedirectUrl(data["redirect_uri"].toString());
}
QList<Message> GmailServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
QList<Message> GmailServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
Q_UNUSED(stated_messages)
Q_UNUSED(tagged_messages)
QList<Message> messages;
Feed::Status error = Feed::Status::Normal;
QList<Message> messages = network()->messages(feed->customId(), error, networkProxy());
for (Feed* feed : feeds) {
Feed::Status error = Feed::Status::Normal;
messages << network()->messages(feed->customId(), error, networkProxy());
if (error == Feed::Status::NetworkError || error == Feed::Status::AuthError || error == Feed::Status::ParsingError) {
throw FeedFetchException(error);
}
if (error != Feed::Status::NewMessages && error != Feed::Status::Normal) {
throw FeedFetchException(error);
}
return messages;

4
src/librssguard/services/gmail/gmailserviceroot.h
View File

@ -31,8 +31,8 @@ class GmailServiceRoot : public ServiceRoot, public CacheForServiceRoot {
virtual void saveAllCachedData(bool ignore_errors);
virtual QVariantHash customDatabaseData() const;
virtual void setCustomDatabaseData(const QVariantHash& data);
virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
virtual QList<Message> obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages);
protected:

38
src/librssguard/services/greader/greaderserviceroot.cpp
View File

@ -57,35 +57,29 @@ void GreaderServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool());
}
QList<Message> GreaderServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
QList<Message> GreaderServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
Q_UNUSED(stated_messages)
Q_UNUSED(tagged_messages)
QList<Message> messages;
Feed::Status error = Feed::Status::Normal;
for (Feed* feed : feeds) {
Feed::Status error = Feed::Status::Normal;
if (true /* intelligent downloading */ ) {
messages << network()->getMessagesIntelligently(this,
feed->customId(),
stated_messages.value(feed->customId()),
tagged_messages,
error,
networkProxy());
}
else {
messages << network()->streamContents(this, feed->customId(), error, networkProxy());
}
if (error == Feed::Status::NetworkError || error == Feed::Status::AuthError) {
throw FeedFetchException(error);
}
if (true /* intelligent downloading */ ) {
return network()->getMessagesIntelligently(this,
feed->customId(),
stated_messages,
tagged_messages,
error,
networkProxy());
}
else {
return network()->streamContents(this, feed->customId(), error, networkProxy());
}
return messages;
if (error != Feed::Status::NewMessages && error != Feed::Status::Normal) {
throw FeedFetchException(error);
}
}
bool GreaderServiceRoot::wantsBaggedIdsOfExistingMessages() const {

4
src/librssguard/services/greader/greaderserviceroot.h
View File

@ -31,8 +31,8 @@ class GreaderServiceRoot : public ServiceRoot, public CacheForServiceRoot {
virtual LabelOperation supportedLabelOperations() const;
virtual QVariantHash customDatabaseData() const;
virtual void setCustomDatabaseData(const QVariantHash& data);
virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
virtual QList<Message> obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages);
virtual bool wantsBaggedIdsOfExistingMessages() const;

17
src/librssguard/services/inoreader/inoreaderserviceroot.cpp
View File

@ -56,22 +56,17 @@ void InoreaderServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
m_network->oauth()->setRedirectUrl(data["redirect_uri"].toString());
}
QList<Message> InoreaderServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
QList<Message> InoreaderServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
Q_UNUSED(stated_messages)
Q_UNUSED(tagged_messages)
QList<Message> messages;
Feed::Status error = Feed::Status::Normal;
QList<Message> messages = network()->messages(this, feed->customId(), error);
for (Feed* feed : feeds) {
Feed::Status error = Feed::Status::Normal;
messages << network()->messages(this, feed->customId(), error);
if (error == Feed::Status::NetworkError || error == Feed::Status::AuthError) {
throw FeedFetchException(error);
}
if (error != Feed::Status::NewMessages && error != Feed::Status::Normal) {
throw FeedFetchException(error);
}
return messages;

4
src/librssguard/services/inoreader/inoreaderserviceroot.h
View File

@ -29,8 +29,8 @@ class InoreaderServiceRoot : public ServiceRoot, public CacheForServiceRoot {
virtual void saveAllCachedData(bool ignore_errors);
virtual QVariantHash customDatabaseData() const;
virtual void setCustomDatabaseData(const QVariantHash& data);
virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
virtual QList<Message> obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages);
protected:

22
src/librssguard/services/owncloud/owncloudserviceroot.cpp
View File

@ -150,24 +150,18 @@ void OwnCloudServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool());
}
QList<Message> OwnCloudServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
QList<Message> OwnCloudServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
Q_UNUSED(stated_messages)
Q_UNUSED(tagged_messages)
QList<Message> msgs;
OwnCloudGetMessagesResponse messages = network()->getMessages(feed->customNumericId(), networkProxy());
for (Feed* feed : feeds) {
OwnCloudGetMessagesResponse messages = network()->getMessages(feed->customNumericId(), networkProxy());
if (messages.networkError() != QNetworkReply::NetworkError::NoError) {
throw FeedFetchException(Feed::Status::NetworkError);
}
else {
msgs << messages.messages();
}
if (messages.networkError() != QNetworkReply::NetworkError::NoError) {
throw FeedFetchException(Feed::Status::NetworkError);
}
else {
return messages.messages();
}
return msgs;
}

4
src/librssguard/services/owncloud/owncloudserviceroot.h
View File

@ -28,8 +28,8 @@ class OwnCloudServiceRoot : public ServiceRoot, public CacheForServiceRoot {
virtual void saveAllCachedData(bool ignore_errors);
virtual QVariantHash customDatabaseData() const;
virtual void setCustomDatabaseData(const QVariantHash& data);
virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
virtual QList<Message> obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages);
OwnCloudNetworkFactory* network() const;

210
src/librssguard/services/standard/standardserviceroot.cpp
View File

@ -144,134 +144,128 @@ Qt::ItemFlags StandardServiceRoot::additionalFlags() const {
return Qt::ItemFlag::ItemIsDropEnabled;
}
QList<Message> StandardServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
Q_UNUSED(stated_messages)
Q_UNUSED(tagged_messages)
QList<Message> msgs;
StandardFeed* f = static_cast<StandardFeed*>(feed);
QString formatted_feed_contents;
int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
for (Feed* f : feeds) {
StandardFeed* feed = static_cast<StandardFeed*>(f);
QString formatted_feed_contents;
int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
if (f->sourceType() == StandardFeed::SourceType::Url) {
qDebugNN << LOGSEC_CORE
<< "Downloading URL"
<< QUOTE_W_SPACE(feed->source())
<< "to obtain feed data.";
if (feed->sourceType() == StandardFeed::SourceType::Url) {
qDebugNN << LOGSEC_CORE
<< "Downloading URL"
<< QUOTE_W_SPACE(feed->source())
<< "to obtain feed data.";
QByteArray feed_contents;
QList<QPair<QByteArray, QByteArray>> headers;
QByteArray feed_contents;
QList<QPair<QByteArray, QByteArray>> headers;
headers << NetworkFactory::generateBasicAuthHeader(f->username(), f->password());
headers << NetworkFactory::generateBasicAuthHeader(feed->username(), feed->password());
auto network_result = NetworkFactory::performNetworkOperation(feed->source(),
download_timeout,
{},
feed_contents,
QNetworkAccessManager::Operation::GetOperation,
headers,
false,
{},
{},
networkProxy()).first;
auto network_result = NetworkFactory::performNetworkOperation(feed->source(),
download_timeout,
{},
feed_contents,
QNetworkAccessManager::Operation::GetOperation,
headers,
false,
{},
{},
networkProxy()).first;
if (network_result != QNetworkReply::NetworkError::NoError) {
qWarningNN << LOGSEC_CORE
<< "Error"
<< QUOTE_W_SPACE(network_result)
<< "during fetching of new messages for feed"
<< QUOTE_W_SPACE_DOT(feed->source());
throw FeedFetchException(Feed::Status::NetworkError, NetworkFactory::networkErrorText(network_result));
}
if (network_result != QNetworkReply::NetworkError::NoError) {
qWarningNN << LOGSEC_CORE
<< "Error"
<< QUOTE_W_SPACE(network_result)
<< "during fetching of new messages for feed"
<< QUOTE_W_SPACE_DOT(feed->source());
throw FeedFetchException(Feed::Status::NetworkError, NetworkFactory::networkErrorText(network_result));
}
// Encode downloaded data for further parsing.
QTextCodec* codec = QTextCodec::codecForName(f->encoding().toLocal8Bit());
// Encode downloaded data for further parsing.
QTextCodec* codec = QTextCodec::codecForName(feed->encoding().toLocal8Bit());
if (codec == nullptr) {
// No suitable codec for this encoding was found.
// Use non-converted data.
formatted_feed_contents = feed_contents;
}
else {
formatted_feed_contents = codec->toUnicode(feed_contents);
}
if (codec == nullptr) {
// No suitable codec for this encoding was found.
// Use non-converted data.
formatted_feed_contents = feed_contents;
}
else {
qDebugNN << LOGSEC_CORE
<< "Running custom script"
<< QUOTE_W_SPACE(feed->source())
<< "to obtain feed data.";
// Use script to generate feed file.
try {
formatted_feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout);
}
catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE
<< "Custom script for generating feed file failed:"
<< QUOTE_W_SPACE_DOT(ex.message());
throw FeedFetchException(Feed::Status::OtherError, ex.message());
}
formatted_feed_contents = codec->toUnicode(feed_contents);
}
}
else {
qDebugNN << LOGSEC_CORE
<< "Running custom script"
<< QUOTE_W_SPACE(feed->source())
<< "to obtain feed data.";
if (!feed->postProcessScript().simplified().isEmpty()) {
qDebugNN << LOGSEC_CORE
<< "We will process feed data with post-process script"
<< QUOTE_W_SPACE_DOT(feed->postProcessScript());
try {
formatted_feed_contents = StandardFeed::postProcessFeedFileWithScript(feed->postProcessScript(),
formatted_feed_contents,
download_timeout);
}
catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE
<< "Post-processing script for feed file failed:"
<< QUOTE_W_SPACE_DOT(ex.message());
throw FeedFetchException(Feed::Status::OtherError, ex.message());
}
// Use script to generate feed file.
try {
formatted_feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout);
}
catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE
<< "Custom script for generating feed file failed:"
<< QUOTE_W_SPACE_DOT(ex.message());
// Feed data are downloaded and encoded.
// Parse data and obtain messages.
QList<Message> messages;
switch (feed->type()) {
case StandardFeed::Type::Rss0X:
case StandardFeed::Type::Rss2X:
messages = RssParser(formatted_feed_contents).messages();
break;
case StandardFeed::Type::Rdf:
messages = RdfParser(formatted_feed_contents).messages();
break;
case StandardFeed::Type::Atom10:
messages = AtomParser(formatted_feed_contents).messages();
break;
case StandardFeed::Type::Json:
messages = JsonParser(formatted_feed_contents).messages();
break;
default:
break;
throw FeedFetchException(Feed::Status::OtherError, ex.message());
}
for (Message& mess : messages) {
mess.m_feedId = f->customId();
}
msgs << messages;
}
return msgs;
if (!f->postProcessScript().simplified().isEmpty()) {
qDebugNN << LOGSEC_CORE
<< "We will process feed data with post-process script"
<< QUOTE_W_SPACE_DOT(f->postProcessScript());
try {
formatted_feed_contents = StandardFeed::postProcessFeedFileWithScript(f->postProcessScript(),
formatted_feed_contents,
download_timeout);
}
catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE
<< "Post-processing script for feed file failed:"
<< QUOTE_W_SPACE_DOT(ex.message());
throw FeedFetchException(Feed::Status::OtherError, ex.message());
}
}
// Feed data are downloaded and encoded.
// Parse data and obtain messages.
QList<Message> messages;
switch (f->type()) {
case StandardFeed::Type::Rss0X:
case StandardFeed::Type::Rss2X:
messages = RssParser(formatted_feed_contents).messages();
break;
case StandardFeed::Type::Rdf:
messages = RdfParser(formatted_feed_contents).messages();
break;
case StandardFeed::Type::Atom10:
messages = AtomParser(formatted_feed_contents).messages();
break;
case StandardFeed::Type::Json:
messages = JsonParser(formatted_feed_contents).messages();
break;
default:
break;
}
for (Message& mess : messages) {
mess.m_feedId = feed->customId();
}
return messages;
}
QList<QAction*> StandardServiceRoot::getContextMenuForFeed(StandardFeed* feed) {

4
src/librssguard/services/standard/standardserviceroot.h
View File

@ -32,8 +32,8 @@ class StandardServiceRoot : public ServiceRoot {
virtual bool supportsFeedAdding() const;
virtual bool supportsCategoryAdding() const;
virtual Qt::ItemFlags additionalFlags() const;
virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
virtual QList<Message> obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages);
QList<QAction*> serviceMenu();

43
src/librssguard/services/tt-rss/ttrssserviceroot.cpp
View File

@ -214,38 +214,35 @@ void TtRssServiceRoot::setCustomDatabaseData(const QVariantHash& data) {
m_network->setDownloadOnlyUnreadMessages(data["download_only_unread"].toBool());
}
QList<Message> TtRssServiceRoot::obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
QList<Message> TtRssServiceRoot::obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages) {
Q_UNUSED(stated_messages)
Q_UNUSED(tagged_messages)
QList<Message> messages;
int newly_added_messages = 0;
int limit = network()->batchSize() <= 0 ? TTRSS_MAX_MESSAGES : network()->batchSize();
int skip = 0;
for (Feed* feed : feeds) {
int newly_added_messages = 0;
int limit = network()->batchSize() <= 0 ? TTRSS_MAX_MESSAGES : network()->batchSize();
int skip = 0;
do {
TtRssGetHeadlinesResponse headlines = network()->getHeadlines(feed->customNumericId(), limit, skip,
true, true, false,
network()->downloadOnlyUnreadMessages(),
networkProxy());
do {
TtRssGetHeadlinesResponse headlines = network()->getHeadlines(feed->customNumericId(), limit, skip,
true, true, false,
network()->downloadOnlyUnreadMessages(),
networkProxy());
if (network()->lastError() != QNetworkReply::NetworkError::NoError) {
throw FeedFetchException(Feed::Status::NetworkError, headlines.error());
}
else {
QList<Message> new_messages = headlines.messages(this);
messages << new_messages;
newly_added_messages = new_messages.size();
skip += newly_added_messages;
}
if (network()->lastError() != QNetworkReply::NetworkError::NoError) {
throw FeedFetchException(Feed::Status::NetworkError, headlines.error());
}
else {
QList<Message> new_messages = headlines.messages(this);
messages << new_messages;
newly_added_messages = new_messages.size();
skip += newly_added_messages;
}
while (newly_added_messages > 0 && (network()->batchSize() <= 0 || messages.size() < network()->batchSize()));
}
while (newly_added_messages > 0 && (network()->batchSize() <= 0 || messages.size() < network()->batchSize()));
return messages;
}

4
src/librssguard/services/tt-rss/ttrssserviceroot.h
View File

@ -33,8 +33,8 @@ class TtRssServiceRoot : public ServiceRoot, public CacheForServiceRoot {
virtual void saveAllCachedData(bool ignore_errors);
virtual QVariantHash customDatabaseData() const;
virtual void setCustomDatabaseData(const QVariantHash& data);
virtual QList<Message> obtainNewMessages(const QList<Feed*>& feeds,
const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
virtual QList<Message> obtainNewMessages(Feed* feed,
const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
const QHash<QString, QStringList>& tagged_messages);
// Access to network.