FreshRSS/rssguard
1
0
Fork 0
mirror of https://github.com/martinrotter/rssguard.git synced 2025-02-06 04:14:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

very experimental solution for #451

Browse Source
This commit is contained in:
Martin Rotter 2021-07-25 09:08:43 +02:00
parent 1be05409a9
commit af30552364
5 changed files with 50 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
src/librssguard/database/databasequeries.cpp
View File

@ -79,7 +79,7 @@ bool DatabaseQueries::deassignLabelFromMessage(const QSqlDatabase& db, Label* la
q.setForwardOnly(true); q.setForwardOnly(true);
q.prepare("DELETE FROM LabelsInMessages WHERE label = :label AND message = :message AND account_id = :account_id;"); q.prepare("DELETE FROM LabelsInMessages WHERE label = :label AND message = :message AND account_id = :account_id;");
q.bindValue(QSL(":label"), label->customId()); q.bindValue(QSL(":label"), label->customId());
q.bindValue(QSL(":message"), msg.m_customId); q.bindValue(QSL(":message"), msg.m_customId.isEmpty() ? QString::number(msg.m_id) : msg.m_customId);
q.bindValue(QSL(":account_id"), label->getParentServiceRoot()->accountId()); q.bindValue(QSL(":account_id"), label->getParentServiceRoot()->accountId());
return q.exec(); return q.exec();
@ -91,7 +91,7 @@ bool DatabaseQueries::assignLabelToMessage(const QSqlDatabase& db, Label* label,
q.setForwardOnly(true); q.setForwardOnly(true);
q.prepare("DELETE FROM LabelsInMessages WHERE label = :label AND message = :message AND account_id = :account_id;"); q.prepare("DELETE FROM LabelsInMessages WHERE label = :label AND message = :message AND account_id = :account_id;");
q.bindValue(QSL(":label"), label->customId()); q.bindValue(QSL(":label"), label->customId());
q.bindValue(QSL(":message"), msg.m_customId); q.bindValue(QSL(":message"), msg.m_customId.isEmpty() ? QString::number(msg.m_id) : msg.m_customId);
q.bindValue(QSL(":account_id"), label->getParentServiceRoot()->accountId()); q.bindValue(QSL(":account_id"), label->getParentServiceRoot()->accountId());
auto succ = q.exec(); auto succ = q.exec();
@ -99,7 +99,7 @@ bool DatabaseQueries::assignLabelToMessage(const QSqlDatabase& db, Label* label,
if (succ) { if (succ) {
q.prepare("INSERT INTO LabelsInMessages (label, message, account_id) VALUES (:label, :message, :account_id);"); q.prepare("INSERT INTO LabelsInMessages (label, message, account_id) VALUES (:label, :message, :account_id);");
q.bindValue(QSL(":label"), label->customId()); q.bindValue(QSL(":label"), label->customId());
q.bindValue(QSL(":message"), msg.m_customId); q.bindValue(QSL(":message"), msg.m_customId.isEmpty() ? QString::number(msg.m_id) : msg.m_customId);
q.bindValue(QSL(":account_id"), label->getParentServiceRoot()->accountId()); q.bindValue(QSL(":account_id"), label->getParentServiceRoot()->accountId());
succ = q.exec(); succ = q.exec();
@ -170,7 +170,7 @@ QList<Label*> DatabaseQueries::getLabelsForMessage(const QSqlDatabase& db,
q.prepare("SELECT DISTINCT label FROM LabelsInMessages WHERE message = :message AND account_id = :account_id;"); q.prepare("SELECT DISTINCT label FROM LabelsInMessages WHERE message = :message AND account_id = :account_id;");
q.bindValue(QSL(":account_id"), msg.m_accountId); q.bindValue(QSL(":account_id"), msg.m_accountId);
q.bindValue(QSL(":message"), msg.m_customId); q.bindValue(QSL(":message"), msg.m_customId.isEmpty() ? QString::number(msg.m_id) : msg.m_customId);
if (q.exec()) { if (q.exec()) {
auto iter = boolinq::from(installed_labels); auto iter = boolinq::from(installed_labels);
@ -1024,8 +1024,7 @@ QHash<QString, QStringList> DatabaseQueries::bagsOfMessages(const QSqlDatabase&
QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db, QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
const QList<Message>& messages, const QList<Message>& messages,
const QString& feed_custom_id, Feed* feed,
int account_id,
const QString& url, const QString& url,
bool force_update, bool force_update,
bool* ok) { bool* ok) {
@ -1036,6 +1035,8 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
bool use_transactions = qApp->settings()->value(GROUP(Database), SETTING(Database::UseTransactions)).toBool(); bool use_transactions = qApp->settings()->value(GROUP(Database), SETTING(Database::UseTransactions)).toBool();
QPair<int, int> updated_messages = { 0, 0 }; QPair<int, int> updated_messages = { 0, 0 };
int account_id = feed->getParentServiceRoot()->accountId();
auto feed_custom_id = feed->customId();
// Prepare queries. // Prepare queries.
QSqlQuery query_select_with_url(db); QSqlQuery query_select_with_url(db);
@ -1058,14 +1059,14 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
// When we have custom ID of the message, we can check directly for existence // When we have custom ID of the message, we can check directly for existence
// of that particular message. // of that particular message.
query_select_with_custom_id.setForwardOnly(true); query_select_with_custom_id.setForwardOnly(true);
query_select_with_custom_id.prepare("SELECT id, date_created, is_read, is_important, contents, feed FROM Messages " query_select_with_custom_id.prepare("SELECT id, date_created, is_read, is_important, contents, feed, title FROM Messages "
"WHERE custom_id = :custom_id AND account_id = :account_id;"); "WHERE custom_id = :custom_id AND account_id = :account_id;");
// In some case, messages are already stored in the DB and they all have primary DB ID. // In some case, messages are already stored in the DB and they all have primary DB ID.
// This is particularly the case when user runs some message filter manually on existing messages // This is particularly the case when user runs some message filter manually on existing messages
// of some feed. // of some feed.
query_select_with_id.setForwardOnly(true); query_select_with_id.setForwardOnly(true);
query_select_with_id.prepare("SELECT date_created, is_read, is_important, contents, feed FROM Messages " query_select_with_id.prepare("SELECT date_created, is_read, is_important, contents, feed, title FROM Messages "
"WHERE id = :id AND account_id = :account_id;"); "WHERE id = :id AND account_id = :account_id;");
// Used to insert new messages. // Used to insert new messages.
@ -1090,6 +1091,7 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
for (Message message : messages) { for (Message message : messages) {
// Check if messages contain relative URLs and if they do, then replace them. // Check if messages contain relative URLs and if they do, then replace them.
if (message.m_url.startsWith(QL1S("//"))) { if (message.m_url.startsWith(QL1S("//"))) {
// TODO: This probably should be replace with HTTPS or taken scheme from feed's URL?
message.m_url = QString(URI_SCHEME_HTTP) + message.m_url.mid(2); message.m_url = QString(URI_SCHEME_HTTP) + message.m_url.mid(2);
} }
else if (message.m_url.startsWith(QL1S("/"))) { else if (message.m_url.startsWith(QL1S("/"))) {
@ -1109,6 +1111,7 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
bool is_important_existing_message = false; bool is_important_existing_message = false;
QString contents_existing_message; QString contents_existing_message;
QString feed_id_existing_message; QString feed_id_existing_message;
QString title_existing_message;
if (message.m_id > 0) { if (message.m_id > 0) {
// We recognize directly existing message. // We recognize directly existing message.
@ -1128,6 +1131,7 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
is_important_existing_message = query_select_with_id.value(2).toBool(); is_important_existing_message = query_select_with_id.value(2).toBool();
contents_existing_message = query_select_with_id.value(3).toString(); contents_existing_message = query_select_with_id.value(3).toString();
feed_id_existing_message = query_select_with_id.value(4).toString(); feed_id_existing_message = query_select_with_id.value(4).toString();
title_existing_message = query_select_with_id.value(5).toString();
qDebugNN << LOGSEC_DB qDebugNN << LOGSEC_DB
<< "Message with these attributes is already present in DB and has DB ID '" << "Message with these attributes is already present in DB and has DB ID '"
@ -1144,8 +1148,9 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
query_select_with_id.finish(); query_select_with_id.finish();
} }
else if (message.m_customId.isEmpty()) { else if (message.m_customId.isEmpty()) {
// We need to recognize existing messages according URL & AUTHOR & TITLE. // We need to recognize existing messages according to URL & AUTHOR & TITLE.
// NOTE: This particularly concerns messages from standard account. // NOTE: This concerns articles from RSS/ATOM/JSON which do not
// provide unique ID/GUID.
query_select_with_url.bindValue(QSL(":feed"), unnulifyString(feed_custom_id)); query_select_with_url.bindValue(QSL(":feed"), unnulifyString(feed_custom_id));
query_select_with_url.bindValue(QSL(":title"), unnulifyString(message.m_title)); query_select_with_url.bindValue(QSL(":title"), unnulifyString(message.m_title));
query_select_with_url.bindValue(QSL(":url"), unnulifyString(message.m_url)); query_select_with_url.bindValue(QSL(":url"), unnulifyString(message.m_url));
@ -1153,13 +1158,13 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
query_select_with_url.bindValue(QSL(":account_id"), account_id); query_select_with_url.bindValue(QSL(":account_id"), account_id);
qDebugNN << LOGSEC_DB qDebugNN << LOGSEC_DB
<< "Checking if message with title '" << "Checking if message with title "
<< message.m_title << QUOTE_NO_SPACE(message.m_title)
<< "', url '" << ", url "
<< message.m_url << QUOTE_NO_SPACE(message.m_url)
<< "' and author '" << "' and author "
<< message.m_author << QUOTE_NO_SPACE(message.m_author)
<< "' is present in DB."; << " is present in DB.";
if (query_select_with_url.exec() && query_select_with_url.next()) { if (query_select_with_url.exec() && query_select_with_url.next()) {
id_existing_message = query_select_with_url.value(0).toInt(); id_existing_message = query_select_with_url.value(0).toInt();
@ -1168,6 +1173,7 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
is_important_existing_message = query_select_with_url.value(3).toBool(); is_important_existing_message = query_select_with_url.value(3).toBool();
contents_existing_message = query_select_with_url.value(4).toString(); contents_existing_message = query_select_with_url.value(4).toString();
feed_id_existing_message = query_select_with_url.value(5).toString(); feed_id_existing_message = query_select_with_url.value(5).toString();
title_existing_message = unnulifyString(message.m_title);
qDebugNN << LOGSEC_DB qDebugNN << LOGSEC_DB
<< "Message with these attributes is already present in DB and has DB ID '" << "Message with these attributes is already present in DB and has DB ID '"
@ -1201,6 +1207,7 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
is_important_existing_message = query_select_with_custom_id.value(3).toBool(); is_important_existing_message = query_select_with_custom_id.value(3).toBool();
contents_existing_message = query_select_with_custom_id.value(4).toString(); contents_existing_message = query_select_with_custom_id.value(4).toString();
feed_id_existing_message = query_select_with_custom_id.value(5).toString(); feed_id_existing_message = query_select_with_custom_id.value(5).toString();
title_existing_message = query_select_with_custom_id.value(6).toString();
qDebugNN << LOGSEC_DB qDebugNN << LOGSEC_DB
<< "Message with custom ID" << "Message with custom ID"
@ -1224,23 +1231,32 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
// Message is already in the DB. // Message is already in the DB.
// //
// Now, we update it if at least one of next conditions is true: // Now, we update it if at least one of next conditions is true:
// 1) Message has custom ID AND (its date OR read status OR starred status are changed or message // 1) FOR SYNCHRONIZED SERVICES: Message has custom ID AND (its date OR read status OR starred status are changed
// was moved from one feed to another - this can particularly happen in Gmail feeds). // or message was moved from one feed to another - this can particularly happen in Gmail feeds).
// //
// 2) Message has its date fetched from feed AND its date is different from date in DB and contents is changed. // 2) FOR NON-SYNCHRONIZED SERVICES (RSS/ATOM/JSON): Message has custom ID/GUID and its title or contents are changed.
// //
// 3) Message update is force, we want to overwrite message as some arbitrary atribute was changed, // 3) FOR ALL SERVICES: Message has its date fetched from feed AND its date is different
// from date in DB and contents is changed.
//
// 4) FOR ALL SERVICES: Message update is forced, we want to overwrite message as some arbitrary atribute was changed,
// this particularly happens when manual message filter execution happens. // this particularly happens when manual message filter execution happens.
if (/* 1 */ (!message.m_customId.isEmpty() && (message.m_created.toMSecsSinceEpoch() != date_existing_message || if (/* 1 */ (!message.m_customId.isEmpty() && feed->getParentServiceRoot()->isSyncable() &&
message.m_isRead != is_read_existing_message || (message.m_created.toMSecsSinceEpoch() != date_existing_message ||
message.m_isImportant != is_important_existing_message || message.m_isRead != is_read_existing_message ||
message.m_feedId != feed_id_existing_message || message.m_isImportant != is_important_existing_message ||
message.m_contents != contents_existing_message)) || message.m_feedId != feed_id_existing_message ||
message.m_title != title_existing_message ||
message.m_contents != contents_existing_message)) ||
/* 2 */ (message.m_createdFromFeed && message.m_created.toMSecsSinceEpoch() != date_existing_message && /* 2 */ (!message.m_customId.isEmpty() && !feed->getParentServiceRoot()->isSyncable() &&
(message.m_title != title_existing_message ||
message.m_contents != contents_existing_message)) ||
/* 3 */ (message.m_createdFromFeed && message.m_created.toMSecsSinceEpoch() != date_existing_message &&
message.m_contents != contents_existing_message) || message.m_contents != contents_existing_message) ||
/* 3 */ force_update) { /* 4 */ force_update) {
// Message exists, it is changed, update it. // Message exists, it is changed, update it.
query_update.bindValue(QSL(":title"), unnulifyString(message.m_title)); query_update.bindValue(QSL(":title"), unnulifyString(message.m_title));
query_update.bindValue(QSL(":is_read"), int(message.m_isRead)); query_update.bindValue(QSL(":is_read"), int(message.m_isRead));
@ -1251,7 +1267,7 @@ QPair<int, int> DatabaseQueries::updateMessages(QSqlDatabase db,
query_update.bindValue(QSL(":date_created"), message.m_created.toMSecsSinceEpoch()); query_update.bindValue(QSL(":date_created"), message.m_created.toMSecsSinceEpoch());
query_update.bindValue(QSL(":contents"), unnulifyString(message.m_contents)); query_update.bindValue(QSL(":contents"), unnulifyString(message.m_contents));
query_update.bindValue(QSL(":enclosures"), Enclosures::encodeEnclosuresToString(message.m_enclosures)); query_update.bindValue(QSL(":enclosures"), Enclosures::encodeEnclosuresToString(message.m_enclosures));
query_update.bindValue(QSL(":feed"), unnulifyString(feed_id_existing_message)); query_update.bindValue(QSL(":feed"), unnulifyString(message.m_feedId));
query_update.bindValue(QSL(":score"), message.m_score); query_update.bindValue(QSL(":score"), message.m_score);
query_update.bindValue(QSL(":id"), id_existing_message); query_update.bindValue(QSL(":id"), id_existing_message);

4
src/librssguard/database/databasequeries.h
View File

@ -113,8 +113,8 @@ class DatabaseQueries {
static void createOverwriteAccount(const QSqlDatabase& db, ServiceRoot* account); static void createOverwriteAccount(const QSqlDatabase& db, ServiceRoot* account);
// Returns counts of updated messages <unread, all>. // Returns counts of updated messages <unread, all>.
static QPair<int, int> updateMessages(QSqlDatabase db, const QList<Message>& messages, const QString& feed_custom_id, static QPair<int, int> updateMessages(QSqlDatabase db, const QList<Message>& messages, Feed* feed,
int account_id, const QString& url, bool force_update, bool* ok = nullptr); const QString& url, bool force_update, bool* ok = nullptr);
static bool deleteAccount(const QSqlDatabase& db, int account_id); static bool deleteAccount(const QSqlDatabase& db, int account_id);
static bool deleteAccountData(const QSqlDatabase& db, int account_id, bool delete_messages_too); static bool deleteAccountData(const QSqlDatabase& db, int account_id, bool delete_messages_too);
static bool cleanLabelledMessages(const QSqlDatabase& db, bool clean_read_only, Label* label); static bool cleanLabelledMessages(const QSqlDatabase& db, bool clean_read_only, Label* label);

4
src/librssguard/services/abstract/feed.cpp
View File

@ -205,14 +205,12 @@ QPair<int, int> Feed::updateMessages(const QList<Message>& messages, bool force_
<< QUOTE_W_SPACE_DOT(is_main_thread); << QUOTE_W_SPACE_DOT(is_main_thread);
bool ok = false; bool ok = false;
QString custom_id = customId();
int account_id = getParentServiceRoot()->accountId();
QSqlDatabase database = is_main_thread ? QSqlDatabase database = is_main_thread ?
qApp->database()->driver()->connection(metaObject()->className()) : qApp->database()->driver()->connection(metaObject()->className()) :
qApp->database()->driver()->connection(QSL("feed_upd")); qApp->database()->driver()->connection(QSL("feed_upd"));
updated_messages = DatabaseQueries::updateMessages(database, messages, updated_messages = DatabaseQueries::updateMessages(database, messages,
custom_id, account_id, this,
source(), force_update, source(), force_update,
&ok); &ok);

1
src/librssguard/services/standard/atomparser.cpp
View File

@ -68,6 +68,7 @@ Message AtomParser::extractMessage(const QDomElement& msg_element, QDateTime cur
new_message.m_title = qApp->web()->unescapeHtml(qApp->web()->stripTags(title)); new_message.m_title = qApp->web()->unescapeHtml(qApp->web()->stripTags(title));
new_message.m_contents = summary; new_message.m_contents = summary;
new_message.m_author = qApp->web()->unescapeHtml(messageAuthor(msg_element)); new_message.m_author = qApp->web()->unescapeHtml(messageAuthor(msg_element));
new_message.m_customId = msg_element.elementsByTagNameNS(m_atomNamespace, QSL("id")).at(0).toElement().text();
QString raw_contents; QString raw_contents;
QTextStream str(&raw_contents); QTextStream str(&raw_contents);

1
src/librssguard/services/standard/rssparser.cpp
View File

@ -34,6 +34,7 @@ Message RssParser::extractMessage(const QDomElement& msg_element, QDateTime curr
QString elem_enclosure = msg_element.namedItem(QSL("enclosure")).toElement().attribute(QSL("url")); QString elem_enclosure = msg_element.namedItem(QSL("enclosure")).toElement().attribute(QSL("url"));
QString elem_enclosure_type = msg_element.namedItem(QSL("enclosure")).toElement().attribute(QSL("type")); QString elem_enclosure_type = msg_element.namedItem(QSL("enclosure")).toElement().attribute(QSL("type"));
new_message.m_customId = msg_element.namedItem(QSL("guid")).toElement().text();
new_message.m_url = msg_element.namedItem(QSL("link")).toElement().text(); new_message.m_url = msg_element.namedItem(QSL("link")).toElement().text();
if (new_message.m_url.isEmpty() && !new_message.m_enclosures.isEmpty()) { if (new_message.m_url.isEmpty() && !new_message.m_enclosures.isEmpty()) {