github releases scraper
This commit is contained in:
Martin Rotter
parent
db156ce4a7
commit
c67e0e3157
|
|
@ -30,7 +30,7 @@
|
|||
<url type="donation">https://martinrotter.github.io/donate/</url>
|
||||
<content_rating type="oars-1.1" />
|
||||
<releases>
|
||||
<release version="3.9.0" date="2021-03-02"/>
|
||||
<release version="3.9.0" date="2021-03-04"/>
|
||||
</releases>
|
||||
<content_rating type="oars-1.0">
|
||||
<content_attribute id="violence-cartoon">none</content_attribute>
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ Here is the reference of methods and properties of some types available in your
|
|||
| `String url` | URL of the message. |
|
||||
| `String author` | Author of the message. |
|
||||
| `String contents` | Contents of the message. |
|
||||
| `Number score` | Arbitrary number in range <0.0, 100.0>. You can use this number to order messages in a custom fashion as this attribute also has its own column in messages list. |
|
||||
| `Date created` | Date/time of the message. |
|
||||
| `Boolean isRead` | Is message read? |
|
||||
| `Boolean isImportant` | Is message important? |
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
# Sample file: https://api.github.com/repos/<user>/<repo>/releases
|
||||
#
|
||||
# File is downloaded and we expect its contents via stdin.
|
||||
# This script produces JSON feed.
|
||||
|
||||
import re
|
||||
import json
|
||||
import sys
|
||||
import distutils.util
|
||||
from datetime import datetime
|
||||
|
||||
leave_out_prereleases = distutils.util.strtobool(sys.argv[1])
|
||||
input_data = sys.stdin.read()
|
||||
json_data = json.loads(input_data)
|
||||
|
||||
json_feed = "{{\"title\": \"{title}\", \"items\": [{items}]}}"
|
||||
items = list()
|
||||
|
||||
for rel in json_data:
|
||||
if rel['prerelease'] and leave_out_prereleases:
|
||||
continue
|
||||
|
||||
article_url = json.dumps(rel['url'])
|
||||
article_title = json.dumps(rel['tag_name'])
|
||||
article_time = json.dumps(datetime.strptime(rel["published_at"], "%Y-%m-%dT%H:%M:%SZ").isoformat())
|
||||
items.append("{{\"title\": {title}, \"content_html\": {html}, \"url\": {url}, \"date_published\": {date}}}".format(title=article_title,
|
||||
html=article_title,
|
||||
url=article_url,
|
||||
date=article_time))
|
||||
|
||||
json_feed = json_feed.format(title = "Releases", items = ", ".join(items))
|
||||
print(json_feed)
|
||||
|
|
@ -10,7 +10,7 @@
|
|||
|
||||
MessagesForFiltersModel::MessagesForFiltersModel(QObject* parent) : QAbstractTableModel(parent) {
|
||||
m_headerData << tr("Read") << tr("Important") << tr("In recycle bin") << tr("Title")
|
||||
<< tr("URL") << tr("Author") << tr("Created on");
|
||||
<< tr("URL") << tr("Author") << tr("Created on") << tr("Score");
|
||||
}
|
||||
|
||||
void MessagesForFiltersModel::setMessages(const QList<Message>& messages) {
|
||||
|
|
@ -75,6 +75,9 @@ QVariant MessagesForFiltersModel::data(const QModelIndex& index, int role) const
|
|||
|
||||
case MFM_MODEL_CREATED:
|
||||
return msg.m_created;
|
||||
|
||||
case MFM_MODEL_SCORE:
|
||||
return msg.m_score;
|
||||
}
|
||||
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
#include "definitions/definitions.h"
|
||||
#include "miscellaneous/application.h"
|
||||
#include "miscellaneous/databasequeries.h"
|
||||
|
||||
MessagesModelSqlLayer::MessagesModelSqlLayer()
|
||||
: m_filter(QSL(DEFAULT_SQL_MESSAGES_FILTER)), m_fieldNames({}), m_orderByNames({}),
|
||||
|
|
@ -11,31 +12,14 @@ MessagesModelSqlLayer::MessagesModelSqlLayer()
|
|||
m_db = qApp->database()->connection(QSL("MessagesModel"));
|
||||
|
||||
// Used in <x>: SELECT <x1>, <x2> FROM ....;
|
||||
m_fieldNames[MSG_DB_ID_INDEX] = "Messages.id";
|
||||
m_fieldNames[MSG_DB_READ_INDEX] = "Messages.is_read";
|
||||
m_fieldNames[MSG_DB_IMPORTANT_INDEX] = "Messages.is_important";
|
||||
m_fieldNames[MSG_DB_DELETED_INDEX] = "Messages.is_deleted";
|
||||
m_fieldNames[MSG_DB_PDELETED_INDEX] = "Messages.is_pdeleted";
|
||||
m_fieldNames[MSG_DB_FEED_CUSTOM_ID_INDEX] = "Messages.feed";
|
||||
m_fieldNames[MSG_DB_TITLE_INDEX] = "Messages.title";
|
||||
m_fieldNames[MSG_DB_URL_INDEX] = "Messages.url";
|
||||
m_fieldNames[MSG_DB_AUTHOR_INDEX] = "Messages.author";
|
||||
m_fieldNames[MSG_DB_DCREATED_INDEX] = "Messages.date_created";
|
||||
m_fieldNames[MSG_DB_CONTENTS_INDEX] = "Messages.contents";
|
||||
m_fieldNames[MSG_DB_ENCLOSURES_INDEX] = "Messages.enclosures";
|
||||
m_fieldNames[MSG_DB_SCORE_INDEX] = "Messages.score";
|
||||
m_fieldNames[MSG_DB_ACCOUNT_ID_INDEX] = "Messages.account_id";
|
||||
m_fieldNames[MSG_DB_CUSTOM_ID_INDEX] = "Messages.custom_id";
|
||||
m_fieldNames[MSG_DB_CUSTOM_HASH_INDEX] = "Messages.custom_hash";
|
||||
m_fieldNames[MSG_DB_FEED_TITLE_INDEX] = "Feeds.title";
|
||||
m_fieldNames[MSG_DB_HAS_ENCLOSURES] = "CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures";
|
||||
m_fieldNames = DatabaseQueries::messageTableAttributes(false);
|
||||
|
||||
// Used in <x>: SELECT ... FROM ... ORDER BY <x1> DESC, <x2> ASC;
|
||||
m_orderByNames[MSG_DB_ID_INDEX] = "Messages.id";
|
||||
m_orderByNames[MSG_DB_READ_INDEX] = "Messages.is_read";
|
||||
m_orderByNames[MSG_DB_IMPORTANT_INDEX] = "Messages.is_important";
|
||||
m_orderByNames[MSG_DB_PDELETED_INDEX] = "Messages.is_pdeleted";
|
||||
m_orderByNames[MSG_DB_DELETED_INDEX] = "Messages.is_deleted";
|
||||
m_orderByNames[MSG_DB_PDELETED_INDEX] = "Messages.is_pdeleted";
|
||||
m_orderByNames[MSG_DB_FEED_CUSTOM_ID_INDEX] = "Messages.feed";
|
||||
m_orderByNames[MSG_DB_TITLE_INDEX] = "Messages.title";
|
||||
m_orderByNames[MSG_DB_URL_INDEX] = "Messages.url";
|
||||
|
|
|
|||
|
|
@ -229,6 +229,7 @@
|
|||
#define MFM_MODEL_URL 4
|
||||
#define MFM_MODEL_AUTHOR 5
|
||||
#define MFM_MODEL_CREATED 6
|
||||
#define MFM_MODEL_SCORE 7
|
||||
|
||||
#if defined(Q_OS_LINUX)
|
||||
#define OS_ID "Linux"
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ FormMessageFiltersManager::FormMessageFiltersManager(FeedReader* reader, const Q
|
|||
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_ISDELETED, QHeaderView::ResizeMode::ResizeToContents);
|
||||
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_AUTHOR, QHeaderView::ResizeMode::ResizeToContents);
|
||||
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_CREATED, QHeaderView::ResizeMode::ResizeToContents);
|
||||
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_SCORE, QHeaderView::ResizeMode::ResizeToContents);
|
||||
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_TITLE, QHeaderView::ResizeMode::Interactive);
|
||||
m_ui.m_treeExistingMessages->header()->setSectionResizeMode(MFM_MODEL_URL, QHeaderView::ResizeMode::Interactive);
|
||||
|
||||
|
|
|
|||
|
|
@ -8,14 +8,36 @@
|
|||
#include "miscellaneous/iconfactory.h"
|
||||
#include "network-web/oauth2service.h"
|
||||
#include "services/abstract/category.h"
|
||||
#include "services/standard/standardcategory.h"
|
||||
#include "services/standard/standardfeed.h"
|
||||
#include "services/standard/standardserviceroot.h"
|
||||
|
||||
#include <QSqlDriver>
|
||||
#include <QUrl>
|
||||
#include <QVariant>
|
||||
|
||||
QMap<int, QString> DatabaseQueries::messageTableAttributes(bool only_msg_table) {
|
||||
QMap<int, QString> field_names;
|
||||
|
||||
field_names[MSG_DB_ID_INDEX] = "Messages.id";
|
||||
field_names[MSG_DB_READ_INDEX] = "Messages.is_read";
|
||||
field_names[MSG_DB_IMPORTANT_INDEX] = "Messages.is_important";
|
||||
field_names[MSG_DB_DELETED_INDEX] = "Messages.is_deleted";
|
||||
field_names[MSG_DB_PDELETED_INDEX] = "Messages.is_pdeleted";
|
||||
field_names[MSG_DB_FEED_CUSTOM_ID_INDEX] = "Messages.feed";
|
||||
field_names[MSG_DB_TITLE_INDEX] = "Messages.title";
|
||||
field_names[MSG_DB_URL_INDEX] = "Messages.url";
|
||||
field_names[MSG_DB_AUTHOR_INDEX] = "Messages.author";
|
||||
field_names[MSG_DB_DCREATED_INDEX] = "Messages.date_created";
|
||||
field_names[MSG_DB_CONTENTS_INDEX] = "Messages.contents";
|
||||
field_names[MSG_DB_ENCLOSURES_INDEX] = "Messages.enclosures";
|
||||
field_names[MSG_DB_SCORE_INDEX] = "Messages.score";
|
||||
field_names[MSG_DB_ACCOUNT_ID_INDEX] = "Messages.account_id";
|
||||
field_names[MSG_DB_CUSTOM_ID_INDEX] = "Messages.custom_id";
|
||||
field_names[MSG_DB_CUSTOM_HASH_INDEX] = "Messages.custom_hash";
|
||||
field_names[MSG_DB_FEED_TITLE_INDEX] = only_msg_table ? "Messages.feed" : "Feeds.title";
|
||||
field_names[MSG_DB_HAS_ENCLOSURES] = "CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures";
|
||||
|
||||
return field_names;
|
||||
}
|
||||
|
||||
QString DatabaseQueries::serializeCustomData(const QVariantHash& data) {
|
||||
if (!data.isEmpty()) {
|
||||
return QString::fromUtf8(QJsonDocument::fromVariant(data).toJson(QJsonDocument::JsonFormat::Indented));
|
||||
|
|
@ -115,7 +137,7 @@ bool DatabaseQueries::setLabelsForMessage(const QSqlDatabase& db, const QList<La
|
|||
return true;
|
||||
}
|
||||
|
||||
QList<Label*> DatabaseQueries::getLabels(const QSqlDatabase& db, int account_id) {
|
||||
QList<Label*> DatabaseQueries::getLabelsForAccount(const QSqlDatabase& db, int account_id) {
|
||||
QList<Label*> labels;
|
||||
QSqlQuery q(db);
|
||||
|
||||
|
|
@ -654,16 +676,16 @@ QList<Message> DatabaseQueries::getUndeletedMessagesWithLabel(const QSqlDatabase
|
|||
QList<Message> messages;
|
||||
QSqlQuery q(db);
|
||||
|
||||
q.prepare(QSL(
|
||||
"SELECT Messages.id, Messages.is_read, Messages.is_deleted, Messages.is_important, Feeds.title, Messages.title, Messages.url, Messages.author, Messages.date_created, Messages.contents, Messages.is_pdeleted, Messages.enclosures, Messages.account_id, Messages.custom_id, Messages.custom_hash, Messages.feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
|
||||
"FROM Messages "
|
||||
"INNER JOIN Feeds "
|
||||
"ON Messages.feed = Feeds.custom_id AND Messages.account_id = :account_id AND Messages.account_id = Feeds.account_id "
|
||||
"INNER JOIN LabelsInMessages "
|
||||
"ON "
|
||||
" Messages.is_pdeleted = 0 AND Messages.is_deleted = 0 AND "
|
||||
" LabelsInMessages.account_id = :account_id AND LabelsInMessages.account_id = Messages.account_id AND "
|
||||
" LabelsInMessages.label = :label AND LabelsInMessages.message = Messages.custom_id;"));
|
||||
q.prepare(QSL("SELECT %1 "
|
||||
"FROM Messages "
|
||||
"INNER JOIN Feeds "
|
||||
"ON Messages.feed = Feeds.custom_id AND Messages.account_id = :account_id AND Messages.account_id = Feeds.account_id "
|
||||
"INNER JOIN LabelsInMessages "
|
||||
"ON "
|
||||
" Messages.is_pdeleted = 0 AND Messages.is_deleted = 0 AND "
|
||||
" LabelsInMessages.account_id = :account_id AND LabelsInMessages.account_id = Messages.account_id AND "
|
||||
" LabelsInMessages.label = :label AND "
|
||||
" LabelsInMessages.message = Messages.custom_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
|
||||
q.bindValue(QSL(":account_id"), label->getParentServiceRoot()->accountId());
|
||||
q.bindValue(QSL(":label"), label->customId());
|
||||
|
||||
|
|
@ -694,12 +716,14 @@ QList<Message> DatabaseQueries::getUndeletedLabelledMessages(const QSqlDatabase&
|
|||
QList<Message> messages;
|
||||
QSqlQuery q(db);
|
||||
|
||||
q.prepare(QSL(
|
||||
"SELECT Messages.id, Messages.is_read, Messages.is_deleted, Messages.is_important, Feeds.title, Messages.title, Messages.url, Messages.author, Messages.date_created, Messages.contents, Messages.is_pdeleted, Messages.enclosures, Messages.account_id, Messages.custom_id, Messages.custom_hash, Messages.feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
|
||||
"FROM Messages "
|
||||
"LEFT JOIN Feeds "
|
||||
"ON Messages.feed = Feeds.custom_id AND Messages.account_id = Feeds.account_id "
|
||||
"WHERE Messages.is_deleted = 0 AND Messages.is_pdeleted = 0 AND Messages.account_id = :account_id AND (SELECT COUNT(*) FROM LabelsInMessages WHERE account_id = :account_id AND message = Messages.custom_id) > 0;"));
|
||||
q.prepare(QSL("SELECT %1 "
|
||||
"FROM Messages "
|
||||
"LEFT JOIN Feeds "
|
||||
"ON Messages.feed = Feeds.custom_id AND Messages.account_id = Feeds.account_id "
|
||||
"WHERE Messages.is_deleted = 0 AND Messages.is_pdeleted = 0 AND Messages.account_id = :account_id AND "
|
||||
" (SELECT COUNT(*) FROM LabelsInMessages "
|
||||
" WHERE account_id = :account_id AND "
|
||||
" message = Messages.custom_id) > 0;").arg(messageTableAttributes(true).values().join(QSL(", "))));
|
||||
q.bindValue(QSL(":account_id"), account_id);
|
||||
|
||||
if (q.exec()) {
|
||||
|
|
@ -730,9 +754,10 @@ QList<Message> DatabaseQueries::getUndeletedImportantMessages(const QSqlDatabase
|
|||
QSqlQuery q(db);
|
||||
|
||||
q.setForwardOnly(true);
|
||||
q.prepare("SELECT id, is_read, is_deleted, is_important, custom_id, title, url, author, date_created, contents, is_pdeleted, enclosures, account_id, custom_id, custom_hash, feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
|
||||
"FROM Messages "
|
||||
"WHERE is_important = 1 AND is_deleted = 0 AND is_pdeleted = 0 AND account_id = :account_id;");
|
||||
q.prepare(QSL("SELECT %1 "
|
||||
"FROM Messages "
|
||||
"WHERE is_important = 1 AND is_deleted = 0 AND "
|
||||
" is_pdeleted = 0 AND account_id = :account_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
|
||||
q.bindValue(QSL(":account_id"), account_id);
|
||||
|
||||
if (q.exec()) {
|
||||
|
|
@ -764,9 +789,10 @@ QList<Message> DatabaseQueries::getUndeletedMessagesForFeed(const QSqlDatabase&
|
|||
QSqlQuery q(db);
|
||||
|
||||
q.setForwardOnly(true);
|
||||
q.prepare("SELECT id, is_read, is_deleted, is_important, custom_id, title, url, author, date_created, contents, is_pdeleted, enclosures, account_id, custom_id, custom_hash, feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
|
||||
"FROM Messages "
|
||||
"WHERE is_deleted = 0 AND is_pdeleted = 0 AND feed = :feed AND account_id = :account_id;");
|
||||
q.prepare(QSL("SELECT %1 "
|
||||
"FROM Messages "
|
||||
"WHERE is_deleted = 0 AND is_pdeleted = 0 AND "
|
||||
" feed = :feed AND account_id = :account_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
|
||||
q.bindValue(QSL(":feed"), feed_custom_id);
|
||||
q.bindValue(QSL(":account_id"), account_id);
|
||||
|
||||
|
|
@ -798,9 +824,9 @@ QList<Message> DatabaseQueries::getUndeletedMessagesForBin(const QSqlDatabase& d
|
|||
QSqlQuery q(db);
|
||||
|
||||
q.setForwardOnly(true);
|
||||
q.prepare("SELECT id, is_read, is_deleted, is_important, custom_id, title, url, author, date_created, contents, is_pdeleted, enclosures, account_id, custom_id, custom_hash, feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
|
||||
"FROM Messages "
|
||||
"WHERE is_deleted = 1 AND is_pdeleted = 0 AND account_id = :account_id;");
|
||||
q.prepare(QSL("SELECT %1 "
|
||||
"FROM Messages "
|
||||
"WHERE is_deleted = 1 AND is_pdeleted = 0 AND account_id = :account_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
|
||||
q.bindValue(QSL(":account_id"), account_id);
|
||||
|
||||
if (q.exec()) {
|
||||
|
|
@ -831,9 +857,9 @@ QList<Message> DatabaseQueries::getUndeletedMessagesForAccount(const QSqlDatabas
|
|||
QSqlQuery q(db);
|
||||
|
||||
q.setForwardOnly(true);
|
||||
q.prepare("SELECT id, is_read, is_deleted, is_important, custom_id, title, url, author, date_created, contents, is_pdeleted, enclosures, account_id, custom_id, custom_hash, feed, CASE WHEN length(Messages.enclosures) > 10 THEN 'true' ELSE 'false' END AS has_enclosures "
|
||||
"FROM Messages "
|
||||
"WHERE is_deleted = 0 AND is_pdeleted = 0 AND account_id = :account_id;");
|
||||
q.prepare(QSL("SELECT %1 "
|
||||
"FROM Messages "
|
||||
"WHERE is_deleted = 0 AND is_pdeleted = 0 AND account_id = :account_id;").arg(messageTableAttributes(true).values().join(QSL(", "))));
|
||||
q.bindValue(QSL(":account_id"), account_id);
|
||||
|
||||
if (q.exec()) {
|
||||
|
|
@ -851,6 +877,8 @@ QList<Message> DatabaseQueries::getUndeletedMessagesForAccount(const QSqlDatabas
|
|||
}
|
||||
}
|
||||
else {
|
||||
auto aa = q.lastError().text();
|
||||
|
||||
if (ok != nullptr) {
|
||||
*ok = false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
|
||||
class DatabaseQueries {
|
||||
public:
|
||||
static QMap<int, QString> messageTableAttributes(bool only_msg_table);
|
||||
|
||||
// Custom data serializers.
|
||||
static QString serializeCustomData(const QVariantHash& data);
|
||||
|
|
@ -32,7 +33,7 @@ class DatabaseQueries {
|
|||
static bool deassignLabelFromMessage(const QSqlDatabase& db, Label* label, const Message& msg);
|
||||
static bool assignLabelToMessage(const QSqlDatabase& db, Label* label, const Message& msg);
|
||||
static bool setLabelsForMessage(const QSqlDatabase& db, const QList<Label*>& labels, const Message& msg);
|
||||
static QList<Label*> getLabels(const QSqlDatabase& db, int account_id);
|
||||
static QList<Label*> getLabelsForAccount(const QSqlDatabase& db, int account_id);
|
||||
static QList<Label*> getLabelsForMessage(const QSqlDatabase& db, const Message& msg, const QList<Label*> installed_labels);
|
||||
static bool updateLabel(const QSqlDatabase& db, Label* label);
|
||||
static bool deleteLabel(const QSqlDatabase& db, Label* label);
|
||||
|
|
@ -61,8 +62,8 @@ class DatabaseQueries {
|
|||
static bool purgeLeftoverMessages(const QSqlDatabase& db, int account_id);
|
||||
|
||||
// Purges message/label assignments where source message or label does not exist.
|
||||
// If account ID smaller than 0 is passed, then do this for all accounts.
|
||||
static bool purgeLeftoverLabelAssignments(const QSqlDatabase& db, int account_id = -1);
|
||||
// If account ID smaller than 1 is passed, then do this for all accounts.
|
||||
static bool purgeLeftoverLabelAssignments(const QSqlDatabase& db, int account_id = 0);
|
||||
static bool purgeLabelsAndLabelAssignments(const QSqlDatabase& db, int account_id);
|
||||
|
||||
// Counts of unread/all messages.
|
||||
|
|
@ -321,7 +322,7 @@ void DatabaseQueries::loadFromDatabase(ServiceRoot* root) {
|
|||
QSqlDatabase database = qApp->database()->connection(root->metaObject()->className());
|
||||
Assignment categories = DatabaseQueries::getCategories<Categ>(database, root->accountId());
|
||||
Assignment feeds = DatabaseQueries::getFeeds<Fee>(database, qApp->feedReader()->messageFilters(), root->accountId());
|
||||
auto labels = DatabaseQueries::getLabels(database, root->accountId());
|
||||
auto labels = DatabaseQueries::getLabelsForAccount(database, root->accountId());
|
||||
|
||||
root->performInitialAssembly(categories, feeds, labels);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue