diff --git a/src/librssguard/database/databasefactory.cpp b/src/librssguard/database/databasefactory.cpp index 8e7b505e8..9b7af1286 100644 --- a/src/librssguard/database/databasefactory.cpp +++ b/src/librssguard/database/databasefactory.cpp @@ -78,6 +78,10 @@ DatabaseDriver* DatabaseFactory::driverForType(DatabaseDriver::DriverType d) con }); } +QString DatabaseFactory::escapeQuery(const QString& query) { + return QString(query).replace(QSL("'"), QSL("''")).replace(QSL("\""), QSL("\\\"")); +} + DatabaseDriver::DriverType DatabaseFactory::activeDatabaseDriver() const { return m_dbDriver->driverType(); } diff --git a/src/librssguard/database/databasefactory.h b/src/librssguard/database/databasefactory.h index 3c172e384..2c24c6bbe 100644 --- a/src/librssguard/database/databasefactory.h +++ b/src/librssguard/database/databasefactory.h @@ -24,6 +24,8 @@ class DatabaseFactory : public QObject { DatabaseDriver* driver() const; DatabaseDriver* driverForType(DatabaseDriver::DriverType d) const; + static QString escapeQuery(const QString& query); + private: void determineDriver(); diff --git a/src/librssguard/database/databasequeries.cpp b/src/librssguard/database/databasequeries.cpp index 5bde0afb5..377b863d0 100755 --- a/src/librssguard/database/databasequeries.cpp +++ b/src/librssguard/database/databasequeries.cpp @@ -1093,6 +1093,8 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, return updated_messages; } + QVector msgs_to_insert; + for (Message message : messages) { int id_existing_message = -1; qint64 date_existing_message = 0; @@ -1315,49 +1317,59 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, } } else { - // Message with this URL is not fetched in this feed yet. - query_insert.bindValue(QSL(":feed"), unnulifyString(feed_custom_id)); - query_insert.bindValue(QSL(":title"), unnulifyString(message.m_title)); - query_insert.bindValue(QSL(":is_read"), int(message.m_isRead)); - query_insert.bindValue(QSL(":is_important"), int(message.m_isImportant)); - query_insert.bindValue(QSL(":is_deleted"), int(message.m_isDeleted)); - query_insert.bindValue(QSL(":url"), unnulifyString( message.m_url)); - query_insert.bindValue(QSL(":author"), unnulifyString(message.m_author)); - query_insert.bindValue(QSL(":date_created"), message.m_created.toMSecsSinceEpoch()); - query_insert.bindValue(QSL(":contents"), unnulifyString(message.m_contents)); - query_insert.bindValue(QSL(":enclosures"), Enclosures::encodeEnclosuresToString(message.m_enclosures)); - query_insert.bindValue(QSL(":custom_id"), unnulifyString(message.m_customId)); - query_insert.bindValue(QSL(":custom_hash"), unnulifyString(message.m_customHash)); - query_insert.bindValue(QSL(":score"), message.m_score); - query_insert.bindValue(QSL(":account_id"), account_id); + msgs_to_insert.append(message); - if (query_insert.exec() && query_insert.numRowsAffected() == 1) { - if (!message.m_isRead) { + if (!message.m_isRead) { + updated_messages.first++; + } + + updated_messages.second++; + + /* + // Message with this URL is not fetched in this feed yet. + query_insert.bindValue(QSL(":feed"), unnulifyString(feed_custom_id)); + query_insert.bindValue(QSL(":title"), unnulifyString(message.m_title)); + query_insert.bindValue(QSL(":is_read"), int(message.m_isRead)); + query_insert.bindValue(QSL(":is_important"), int(message.m_isImportant)); + query_insert.bindValue(QSL(":is_deleted"), int(message.m_isDeleted)); + query_insert.bindValue(QSL(":url"), unnulifyString( message.m_url)); + query_insert.bindValue(QSL(":author"), unnulifyString(message.m_author)); + query_insert.bindValue(QSL(":date_created"), message.m_created.toMSecsSinceEpoch()); + query_insert.bindValue(QSL(":contents"), unnulifyString(message.m_contents)); + query_insert.bindValue(QSL(":enclosures"), Enclosures::encodeEnclosuresToString(message.m_enclosures)); + query_insert.bindValue(QSL(":custom_id"), unnulifyString(message.m_customId)); + query_insert.bindValue(QSL(":custom_hash"), unnulifyString(message.m_customHash)); + query_insert.bindValue(QSL(":score"), message.m_score); + query_insert.bindValue(QSL(":account_id"), account_id); + + if (query_insert.exec() && query_insert.numRowsAffected() == 1) { + if (!message.m_isRead) { updated_messages.first++; - } + } - updated_messages.second++; + updated_messages.second++; - if (query_insert.lastInsertId().isValid()) { + if (query_insert.lastInsertId().isValid()) { id_existing_message = query_insert.lastInsertId().toInt(); - } + } - qDebugNN << LOGSEC_DB + qDebugNN << LOGSEC_DB << "Adding new message with title" << QUOTE_W_SPACE(message.m_title) << ", URL" << QUOTE_W_SPACE(message.m_url) << "to DB."; - } - else if (query_insert.lastError().isValid()) { - qWarningNN << LOGSEC_DB + } + else if (query_insert.lastError().isValid()) { + qWarningNN << LOGSEC_DB << "Failed to insert new message to DB:" << QUOTE_W_SPACE(query_insert.lastError().text()) << "- message title is" << QUOTE_W_SPACE_DOT(message.m_title); - } + } - query_insert.finish(); + query_insert.finish(); + */ } // Update labels assigned to message. @@ -1380,6 +1392,58 @@ QPair DatabaseQueries::updateMessages(QSqlDatabase db, } } + if (!msgs_to_insert.isEmpty()) { + QString bulk_insert = QSL("INSERT INTO Messages " + "(feed, title, is_read, is_important, is_deleted, url, author, score, date_created, contents, enclosures, custom_id, custom_hash, account_id) " + "VALUES %1;"); + + for (int i = 0; i < msgs_to_insert.size(); i += 1000) { + QStringList vals; + auto next_batch = msgs_to_insert.mid(i, 1000); + + for (const Message& msg: next_batch) { + if (msg.m_title.isEmpty()) { + qCriticalNN << LOGSEC_DB + << "Message" + << QUOTE_W_SPACE(msg.m_customId) + << "will not be inserted to DB because it does not meet DB constraints."; + continue; + } + + vals.append(QSL("\n(':feed', ':title', :is_read, :is_important, :is_deleted, " + "':url', ':author', :score, :date_created, ':contents', ':enclosures', " + "':custom_id', ':custom_hash', :account_id)") + .replace(QSL(":feed"), unnulifyString(feed_custom_id)) + .replace(QSL(":title"), DatabaseFactory::escapeQuery(unnulifyString(msg.m_title))) + .replace(QSL(":is_read"), QString::number(int(msg.m_isRead))) + .replace(QSL(":is_important"), QString::number(int(msg.m_isImportant))) + .replace(QSL(":is_deleted"), QString::number(int(msg.m_isDeleted))) + .replace(QSL(":url"), DatabaseFactory::escapeQuery(unnulifyString(msg.m_url))) + .replace(QSL(":author"), DatabaseFactory::escapeQuery(unnulifyString(msg.m_author))) + .replace(QSL(":date_created"), QString::number(msg.m_created.toMSecsSinceEpoch())) + .replace(QSL(":contents"), DatabaseFactory::escapeQuery(unnulifyString(msg.m_contents))) + .replace(QSL(":enclosures"), Enclosures::encodeEnclosuresToString(msg.m_enclosures)) + .replace(QSL(":custom_id"), unnulifyString(msg.m_customId)) + .replace(QSL(":custom_hash"), unnulifyString(msg.m_customHash)) + .replace(QSL(":score"), QString::number(msg.m_score)) + .replace(QSL(":account_id"), QString::number(account_id))); + } + + QString final_bulk = bulk_insert.arg(vals.join(QSL(", "))); + auto bulk_error = db.exec(final_bulk).lastError(); + + if (bulk_error.isValid()) { + QString txt = bulk_error.text() + bulk_error.databaseText(); + + IOFactory::writeFile("aa.sql", final_bulk.toUtf8()); + + qCriticalNN << LOGSEC_DB + << "Failed bulk insert of articles:" + << QUOTE_W_SPACE_DOT(txt); + } + } + } + // Now, fixup custom IDS for messages which initially did not have them, // just to keep the data consistent. if (db.exec("UPDATE Messages " diff --git a/src/librssguard/database/sqlitedriver.cpp b/src/librssguard/database/sqlitedriver.cpp index 0fa70b18e..067077991 100755 --- a/src/librssguard/database/sqlitedriver.cpp +++ b/src/librssguard/database/sqlitedriver.cpp @@ -146,6 +146,11 @@ QSqlDatabase SqliteDriver::connection(const QString& connection_name, DesiredSto << "seems to be established."; } + QSqlQuery query_db(database); + + query_db.setForwardOnly(true); + setPragmas(query_db); + return database; } } @@ -382,6 +387,9 @@ void SqliteDriver::setPragmas(QSqlQuery& query) { query.exec(QSL("PRAGMA synchronous = OFF")); query.exec(QSL("PRAGMA journal_mode = MEMORY")); query.exec(QSL("PRAGMA page_size = 4096")); + + //query.exec(QSL("PRAGMA locking_mode = EXCLUSIVE")); + query.exec(QSL("PRAGMA cache_size = 16384")); query.exec(QSL("PRAGMA count_changes = OFF")); query.exec(QSL("PRAGMA temp_store = MEMORY"));