FreshRSS/rssguard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

github releases scraper

Browse Source
This commit is contained in:
Martin Rotter 2021-03-04 09:19:22 +01:00
parent db156ce4a7
commit c67e0e3157
9 changed files with 108 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
resources/desktop/com.github.rssguard.appdata.xml
View File

@ -30,7 +30,7 @@
<url type="donation">https://martinrotter.github.io/donate/</url>
<content_rating type="oars-1.1" />
<releases>
<release version="3.9.0" date="2021-03-02"/>
<release version="3.9.0" date="2021-03-04"/>
</releases>
<content_rating type="oars-1.0">
<content_attribute id="violence-cartoon">none</content_attribute>

1
resources/docs/Message-filters.md
View File

@ -49,6 +49,7 @@ Here is the reference of methods and properties of some types available in your
| `String url` | URL of the message. |
| `String author` | Author of the message. |
| `String contents` | Contents of the message. |
| `Number score` | Arbitrary number in range <0.0, 100.0>. You can use this number to order messages in a custom fashion as this attribute also has its own column in messages list. |
| `Date created` | Date/time of the message. |
| `Boolean isRead` | Is message read? |
| `Boolean isImportant` | Is message important? |

32
resources/scripts/scrapers/github-releases.py Executable file
View File

@ -0,0 +1,32 @@
# Sample file: https://api.github.com/repos/<user>/<repo>/releases
#
# File is downloaded and we expect its contents via stdin.
# This script produces JSON feed.
import re
import json
import sys
import distutils.util
from datetime import datetime
leave_out_prereleases = distutils.util.strtobool(sys.argv[1])
input_data = sys.stdin.read()
json_data = json.loads(input_data)
json_feed = "{{\"title\": \"{title}\", \"items\": [{items}]}}"
items = list()
for rel in json_data:
if rel['prerelease'] and leave_out_prereleases:
continue
article_url = json.dumps(rel['url'])
article_title = json.dumps(rel['tag_name'])
article_time = json.dumps(datetime.strptime(rel["published_at"], "%Y-%m-%dT%H:%M:%SZ").isoformat())
items.append("{{\"title\": {title}, \"content_html\": {html}, \"url\": {url}, \"date_published\": {date}}}".format(title=article_title,
html=article_title,
url=article_url,
date=article_time))
json_feed = json_feed.format(title = "Releases", items = ", ".join(items))
print(json_feed)

5
src/librssguard/core/messagesforfiltersmodel.cpp
View File

@ -10,7 +10,7 @@
MessagesForFiltersModel::MessagesForFiltersModel(QObject* parent) : QAbstractTableModel(parent) {
m_headerData << tr("Read") << tr("Important") << tr("In recycle bin") << tr("Title")
<< tr("URL") << tr("Author") << tr("Created on");
<< tr("URL") << tr("Author") << tr("Created on") << tr("Score");
}
void MessagesForFiltersModel::setMessages(const QList<Message>& messages) {
@ -75,6 +75,9 @@ QVariant MessagesForFiltersModel::data(const QModelIndex& index, int role) const
case MFM_MODEL_CREATED:
return msg.m_created;
case MFM_MODEL_SCORE:
return msg.m_score;
}
break;

22
src/librssguard/core/messagesmodelsqllayer.cpp
View File

@ -4,6 +4,7 @@
#include "definitions/definitions.h"
#include "miscellaneous/application.h"
#include "miscellaneous/databasequeries.h"
MessagesModelSqlLayer::MessagesModelSqlLayer()
: m_filter(QSL(DEFAULT_SQL_MESSAGES_FILTER)), m_fieldNames({}), m_orderByNames({}),
@ -11,31 +12,14 @@ MessagesModelSqlLayer::MessagesModelSqlLayer()
m_db = qApp->database()->connection(QSL("MessagesModel"));
// Used in <x>: SELECT <x1>, <x2> FROM ....;
m_fieldNames[MSG_DB_ID_INDEX] = "Messages.id";
m_fieldNames[MSG_DB_READ_INDEX] = "Messages.is_read";
m_fieldNames[MSG_DB_IMPORTANT_INDEX] = "Messages.is_important";
m_fieldNames[MSG_DB_DELETED_INDEX] = "Messages.is_deleted";
m_fieldNames[MSG_DB_PDELETED_INDEX] = "Messages.is_pdeleted";
m_fieldNames[MSG_DB_FEED_CUSTOM_ID_INDEX] = "Messages.feed";
m_fieldNames[MSG_DB_TITLE_INDEX] = "Messages.title";
m_fieldNames[MSG_DB_URL_INDEX] = "Messages.url";
m_fieldNames[MSG_DB_AUTHOR_INDEX] = "Messages.author";
m_fieldNames[MSG_DB_DCREATED_INDEX] = "Messages.date_created";
m_fieldNames[MSG_DB_CONTENTS_INDEX] = "Messages.contents";
m_fieldNames[MSG_DB_ENCLOSURES_INDEX] = "Messages.enclosures";
m_fieldNames[MSG_DB_SCORE_INDEX] = "Messages.score";
m_fieldNames[MSG_DB_ACCOUNT_ID_INDEX] = "Messages.account_id";
m_fieldNames[MSG_DB_CUSTOM_ID_INDEX] = "Messages.custom_id";
m_fieldNames[MSG_DB_CUSTOM_HASH_INDEX] = "Messages.custom_hash";
m_fieldNames[MSG_DB_FEED_TITLE_INDEX] = "Feeds.title";
m_fieldNames[MSG_DB_HAS_ENCLOSURES] = "CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures";
m_fieldNames = DatabaseQueries::messageTableAttributes(false);
// Used in <x>: SELECT ... FROM ... ORDER BY <x1> DESC, <x2> ASC;
m_orderByNames[MSG_DB_ID_INDEX] = "Messages.id";
m_orderByNames[MSG_DB_READ_INDEX] = "Messages.is_read";
m_orderByNames[MSG_DB_IMPORTANT_INDEX] = "Messages.is_important";
m_orderByNames[MSG_DB_PDELETED_INDEX] = "Messages.is_pdeleted";
m_orderByNames[MSG_DB_DELETED_INDEX] = "Messages.is_deleted";
m_orderByNames[MSG_DB_PDELETED_INDEX] = "Messages.is_pdeleted";
m_orderByNames[MSG_DB_FEED_CUSTOM_ID_INDEX] = "Messages.feed";
m_orderByNames[MSG_DB_TITLE_INDEX] = "Messages.title";
m_orderByNames[MSG_DB_URL_INDEX] = "Messages.url";

1
src/librssguard/definitions/definitions.h
View File

@ -229,6 +229,7 @@
#define MFM_MODEL_URL 4
#define MFM_MODEL_AUTHOR 5
#define MFM_MODEL_CREATED 6
#define MFM_MODEL_SCORE 7
#if defined(Q_OS_LINUX)
#define OS_ID "Linux"

1
src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp
View File

@ -52,6 +52,7 @@ FormMessageFiltersManager::FormMessageFiltersManager(FeedReader* reader, const Q
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_ISDELETED, QHeaderView::ResizeMode::ResizeToContents);
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_AUTHOR, QHeaderView::ResizeMode::ResizeToContents);
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_CREATED, QHeaderView::ResizeMode::ResizeToContents);
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_SCORE, QHeaderView::ResizeMode::ResizeToContents);
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_TITLE, QHeaderView::ResizeMode::Interactive);
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_URL, QHeaderView::ResizeMode::Interactive);

92
src/librssguard/miscellaneous/databasequeries.cpp
View File

@ -8,14 +8,36 @@
#include "miscellaneous/iconfactory.h"
#include "network-web/oauth2service.h"
#include "services/abstract/category.h"
#include "services/standard/standardcategory.h"
#include "services/standard/standardfeed.h"
#include "services/standard/standardserviceroot.h"
#include <QSqlDriver>
#include <QUrl>
#include <QVariant>
QMap<int, QString> DatabaseQueries::messageTableAttributes(bool only_msg_table) {
QMap<int, QString> field_names;
field_names[MSG_DB_ID_INDEX] = "Messages.id";
field_names[MSG_DB_READ_INDEX] = "Messages.is_read";
field_names[MSG_DB_IMPORTANT_INDEX] = "Messages.is_important";
field_names[MSG_DB_DELETED_INDEX] = "Messages.is_deleted";
field_names[MSG_DB_PDELETED_INDEX] = "Messages.is_pdeleted";
field_names[MSG_DB_FEED_CUSTOM_ID_INDEX] = "Messages.feed";
field_names[MSG_DB_TITLE_INDEX] = "Messages.title";
field_names[MSG_DB_URL_INDEX] = "Messages.url";
field_names[MSG_DB_AUTHOR_INDEX] = "Messages.author";
field_names[MSG_DB_DCREATED_INDEX] = "Messages.date_created";
field_names[MSG_DB_CONTENTS_INDEX] = "Messages.contents";
field_names[MSG_DB_ENCLOSURES_INDEX] = "Messages.enclosures";
field_names[MSG_DB_SCORE_INDEX] = "Messages.score";
field_names[MSG_DB_ACCOUNT_ID_INDEX] = "Messages.account_id";
field_names[MSG_DB_CUSTOM_ID_INDEX] = "Messages.custom_id";
field_names[MSG_DB_CUSTOM_HASH_INDEX] = "Messages.custom_hash";
field_names[MSG_DB_FEED_TITLE_INDEX] = only_msg_table ? "Messages.feed" : "Feeds.title";
field_names[MSG_DB_HAS_ENCLOSURES] = "CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures";
return field_names;
}
QString DatabaseQueries::serializeCustomData(const QVariantHash& data) {
if (!data.isEmpty()) {
return QString::fromUtf8(QJsonDocument::fromVariant(data).toJson(QJsonDocument::JsonFormat::Indented));
@ -115,7 +137,7 @@ bool DatabaseQueries::setLabelsForMessage(const QSqlDatabase& db, const QList<La
return true;
}
QList<Label*> DatabaseQueries::getLabels(const QSqlDatabase& db, int account_id) {
QList<Label*> DatabaseQueries::getLabelsForAccount(const QSqlDatabase& db, int account_id) {
QList<Label*> labels;
QSqlQuery q(db);
@ -654,16 +676,16 @@ QList<Message> DatabaseQueries::getUndeletedMessagesWithLabel(const QSqlDatabase
QList<Message> messages;
QSqlQuery q(db);
q.prepare(QSL(
"SELECT Messages.id, Messages.is_read, Messages.is_deleted, Messages.is_important, Feeds.title, Messages.title, Messages.url, Messages.author, Messages.date_created, Messages.contents, Messages.is_pdeleted, Messages.enclosures, Messages.account_id, Messages.custom_id, Messages.custom_hash, Messages.feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
"FROM Messages "
"INNER JOIN Feeds "
"ON Messages.feed = Feeds.custom_id AND Messages.account_id = :account_id AND Messages.account_id = Feeds.account_id "
"INNER JOIN LabelsInMessages "
"ON "
" Messages.is_pdeleted = 0 AND Messages.is_deleted = 0 AND "
" LabelsInMessages.account_id = :account_id AND LabelsInMessages.account_id = Messages.account_id AND "
" LabelsInMessages.label = :label AND LabelsInMessages.message = Messages.custom_id;"));
q.prepare(QSL("SELECT %1 "
"FROM Messages "
"INNER JOIN Feeds "
"ON Messages.feed = Feeds.custom_id AND Messages.account_id = :account_id AND Messages.account_id = Feeds.account_id "
"INNER JOIN LabelsInMessages "
"ON "
" Messages.is_pdeleted = 0 AND Messages.is_deleted = 0 AND "
" LabelsInMessages.account_id = :account_id AND LabelsInMessages.account_id = Messages.account_id AND "
" LabelsInMessages.label = :label AND "
" LabelsInMessages.message = Messages.custom_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
q.bindValue(QSL(":account_id"), label->getParentServiceRoot()->accountId());
q.bindValue(QSL(":label"), label->customId());
@ -694,12 +716,14 @@ QList<Message> DatabaseQueries::getUndeletedLabelledMessages(const QSqlDatabase&
QList<Message> messages;
QSqlQuery q(db);
q.prepare(QSL(
"SELECT Messages.id, Messages.is_read, Messages.is_deleted, Messages.is_important, Feeds.title, Messages.title, Messages.url, Messages.author, Messages.date_created, Messages.contents, Messages.is_pdeleted, Messages.enclosures, Messages.account_id, Messages.custom_id, Messages.custom_hash, Messages.feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
"FROM Messages "
"LEFT JOIN Feeds "
"ON Messages.feed = Feeds.custom_id AND Messages.account_id = Feeds.account_id "
"WHERE Messages.is_deleted = 0 AND Messages.is_pdeleted = 0 AND Messages.account_id = :account_id AND (SELECT COUNT(*) FROM LabelsInMessages WHERE account_id = :account_id AND message = Messages.custom_id) > 0;"));
q.prepare(QSL("SELECT %1 "
"FROM Messages "
"LEFT JOIN Feeds "
"ON Messages.feed = Feeds.custom_id AND Messages.account_id = Feeds.account_id "
"WHERE Messages.is_deleted = 0 AND Messages.is_pdeleted = 0 AND Messages.account_id = :account_id AND "
" (SELECT COUNT(*) FROM LabelsInMessages "
" WHERE account_id = :account_id AND "
" message = Messages.custom_id) > 0;").arg(messageTableAttributes(true).values().join(QSL(", "))));
q.bindValue(QSL(":account_id"), account_id);
if (q.exec()) {
@ -730,9 +754,10 @@ QList<Message> DatabaseQueries::getUndeletedImportantMessages(const QSqlDatabase
QSqlQuery q(db);
q.setForwardOnly(true);
q.prepare("SELECT id, is_read, is_deleted, is_important, custom_id, title, url, author, date_created, contents, is_pdeleted, enclosures, account_id, custom_id, custom_hash, feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
"FROM Messages "
"WHERE is_important = 1 AND is_deleted = 0 AND is_pdeleted = 0 AND account_id = :account_id;");
q.prepare(QSL("SELECT %1 "
"FROM Messages "
"WHERE is_important = 1 AND is_deleted = 0 AND "
" is_pdeleted = 0 AND account_id = :account_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
q.bindValue(QSL(":account_id"), account_id);
if (q.exec()) {
@ -764,9 +789,10 @@ QList<Message> DatabaseQueries::getUndeletedMessagesForFeed(const QSqlDatabase&
QSqlQuery q(db);
q.setForwardOnly(true);
q.prepare("SELECT id, is_read, is_deleted, is_important, custom_id, title, url, author, date_created, contents, is_pdeleted, enclosures, account_id, custom_id, custom_hash, feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
"FROM Messages "
"WHERE is_deleted = 0 AND is_pdeleted = 0 AND feed = :feed AND account_id = :account_id;");
q.prepare(QSL("SELECT %1 "
"FROM Messages "
"WHERE is_deleted = 0 AND is_pdeleted = 0 AND "
" feed = :feed AND account_id = :account_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
q.bindValue(QSL(":feed"), feed_custom_id);
q.bindValue(QSL(":account_id"), account_id);
@ -798,9 +824,9 @@ QList<Message> DatabaseQueries::getUndeletedMessagesForBin(const QSqlDatabase& d
QSqlQuery q(db);
q.setForwardOnly(true);
q.prepare("SELECT id, is_read, is_deleted, is_important, custom_id, title, url, author, date_created, contents, is_pdeleted, enclosures, account_id, custom_id, custom_hash, feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
"FROM Messages "
"WHERE is_deleted = 1 AND is_pdeleted = 0 AND account_id = :account_id;");
q.prepare(QSL("SELECT %1 "
"FROM Messages "
"WHERE is_deleted = 1 AND is_pdeleted = 0 AND account_id = :account_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
q.bindValue(QSL(":account_id"), account_id);
if (q.exec()) {
@ -831,9 +857,9 @@ QList<Message> DatabaseQueries::getUndeletedMessagesForAccount(const QSqlDatabas
QSqlQuery q(db);
q.setForwardOnly(true);
q.prepare("SELECT id, is_read, is_deleted, is_important, custom_id, title, url, author, date_created, contents, is_pdeleted, enclosures, account_id, custom_id, custom_hash, feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
"FROM Messages "
"WHERE is_deleted = 0 AND is_pdeleted = 0 AND account_id = :account_id;");
q.prepare(QSL("SELECT %1 "
"FROM Messages "
"WHERE is_deleted = 0 AND is_pdeleted = 0 AND account_id = :account_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
q.bindValue(QSL(":account_id"), account_id);
if (q.exec()) {
@ -851,6 +877,8 @@ QList<Message> DatabaseQueries::getUndeletedMessagesForAccount(const QSqlDatabas
}
}
else {
auto aa = q.lastError().text();
if (ok != nullptr) {
*ok = false;
}

9
src/librssguard/miscellaneous/databasequeries.h
View File

@ -22,6 +22,7 @@
class DatabaseQueries {
public:
static QMap<int, QString> messageTableAttributes(bool only_msg_table);
// Custom data serializers.
static QString serializeCustomData(const QVariantHash& data);
@ -32,7 +33,7 @@ class DatabaseQueries {
static bool deassignLabelFromMessage(const QSqlDatabase& db, Label* label, const Message& msg);
static bool assignLabelToMessage(const QSqlDatabase& db, Label* label, const Message& msg);
static bool setLabelsForMessage(const QSqlDatabase& db, const QList<Label*>& labels, const Message& msg);
static QList<Label*> getLabels(const QSqlDatabase& db, int account_id);
static QList<Label*> getLabelsForAccount(const QSqlDatabase& db, int account_id);
static QList<Label*> getLabelsForMessage(const QSqlDatabase& db, const Message& msg, const QList<Label*> installed_labels);
static bool updateLabel(const QSqlDatabase& db, Label* label);
static bool deleteLabel(const QSqlDatabase& db, Label* label);
@ -61,8 +62,8 @@ class DatabaseQueries {
static bool purgeLeftoverMessages(const QSqlDatabase& db, int account_id);
// Purges message/label assignments where source message or label does not exist.
// If account ID smaller than 0 is passed, then do this for all accounts.
static bool purgeLeftoverLabelAssignments(const QSqlDatabase& db, int account_id = -1);
// If account ID smaller than 1 is passed, then do this for all accounts.
static bool purgeLeftoverLabelAssignments(const QSqlDatabase& db, int account_id = 0);
static bool purgeLabelsAndLabelAssignments(const QSqlDatabase& db, int account_id);
// Counts of unread/all messages.
@ -321,7 +322,7 @@ void DatabaseQueries::loadFromDatabase(ServiceRoot* root) {
QSqlDatabase database = qApp->database()->connection(root->metaObject()->className());
Assignment categories = DatabaseQueries::getCategories<Categ>(database, root->accountId());
Assignment feeds = DatabaseQueries::getFeeds<Fee>(database, qApp->feedReader()->messageFilters(), root->accountId());
auto labels = DatabaseQueries::getLabels(database, root->accountId());
auto labels = DatabaseQueries::getLabelsForAccount(database, root->accountId());
root->performInitialAssembly(categories, feeds, labels);
}