FreshRSS/rssguard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

JSON feed support added.

Browse Source
This commit is contained in:
Martin Rotter 2020-10-16 18:52:20 +02:00
parent b51f2e6dd5
commit e8ea98d3fa
13 changed files with 254 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/librssguard/core/message.h
View File

@ -36,6 +36,8 @@ class Message {
// Creates Message from given record, which contains
// row from query SELECT * FROM Messages WHERE ....;
static Message fromSqlRecord(const QSqlRecord& record, bool* result = nullptr);
public:
QString m_title;
QString m_url;
QString m_author;

2
src/librssguard/definitions/definitions.h
View File

@ -111,7 +111,7 @@
#define FEED_INITIAL_OPML_PATTERN "feeds-%1.opml"
#define FEED_REGEX_MATCHER "<link[^>]+type=\"application\\/(?:atom|rss)\\+xml\"[^>]*>"
#define FEED_REGEX_MATCHER "<link[^>]+type=\"application\\/(?:atom\\+xml|rss\\+xml|feed\\+json|json)\"[^>]*>"
#define FEED_HREF_REGEX_MATCHER "href=\"([^\"]+)\""
#define PLACEHOLDER_UNREAD_COUNTS "%unread"

10
src/librssguard/gui/discoverfeedsbutton.cpp
View File

@ -62,10 +62,10 @@ void DiscoverFeedsButton::fillMenu() {
menu()->clear();
for (const ServiceRoot* root : qApp->feedReader()->feedsModel()->serviceRoots()) {
QMenu* root_menu = menu()->addMenu(root->icon(), root->title());
if (root->supportsFeedAdding()) {
QMenu* root_menu = menu()->addMenu(root->icon(), root->title());
for (const QString& url : m_addresses) {
if (root->supportsFeedAdding()) {
for (const QString& url : m_addresses) {
QAction* url_action = root_menu->addAction(root->icon(), url);
url_action->setProperty("url", url);
@ -73,4 +73,8 @@ void DiscoverFeedsButton::fillMenu() {
}
}
}
if (menu()->isEmpty()) {
menu()->addAction(tr("Feeds were detected, but no suitable accounts are configured."))->setEnabled(false);
}
}

2
src/librssguard/librssguard.pro
View File

@ -158,6 +158,7 @@ HEADERS += core/feeddownloader.h \
services/standard/gui/formstandardcategorydetails.h \
services/standard/gui/formstandardfeeddetails.h \
services/standard/gui/formstandardimportexport.h \
services/standard/jsonparser.h \
services/standard/rdfparser.h \
services/standard/rssparser.h \
services/standard/standardcategory.h \
@ -302,6 +303,7 @@ SOURCES += core/feeddownloader.cpp \
services/standard/gui/formstandardcategorydetails.cpp \
services/standard/gui/formstandardfeeddetails.cpp \
services/standard/gui/formstandardimportexport.cpp \
services/standard/jsonparser.cpp \
services/standard/rdfparser.cpp \
services/standard/rssparser.cpp \
services/standard/standardcategory.cpp \

3
src/librssguard/miscellaneous/databasequeries.h
View File

@ -196,7 +196,8 @@ inline void DatabaseQueries::fillFeedData(StandardFeed* feed, const QSqlRecord&
case StandardFeed::Type::Atom10:
case StandardFeed::Type::Rdf:
case StandardFeed::Type::Rss0X:
case StandardFeed::Type::Rss2X: {
case StandardFeed::Type::Rss2X:
case StandardFeed::Type::Json: {
feed->setType(type);
break;
}

22
src/librssguard/network-web/networkfactory.cpp
View File

@ -143,6 +143,22 @@ QNetworkReply::NetworkError NetworkFactory::downloadIcon(const QList<QString>& u
QNetworkReply::NetworkError network_result = QNetworkReply::UnknownNetworkError;
for (const QString& url : urls) {
QByteArray icon_data;
network_result = performNetworkOperation(url, timeout, QByteArray(), icon_data,
QNetworkAccessManager::GetOperation).first;
if (network_result == QNetworkReply::NoError) {
QPixmap icon_pixmap;
icon_pixmap.loadFromData(icon_data);
output = QIcon(icon_pixmap);
if (!output.isNull()) {
break;
}
}
QString host = QUrl(url).host();
if (host.startsWith(QSL("www."))) {
@ -150,7 +166,6 @@ QNetworkReply::NetworkError NetworkFactory::downloadIcon(const QList<QString>& u
}
const QString google_s2_with_url = QString("http://www.google.com/s2/favicons?domain=%1").arg(host);
QByteArray icon_data;
network_result = performNetworkOperation(google_s2_with_url, timeout, QByteArray(), icon_data,
QNetworkAccessManager::GetOperation).first;
@ -160,7 +175,10 @@ QNetworkReply::NetworkError NetworkFactory::downloadIcon(const QList<QString>& u
icon_pixmap.loadFromData(icon_data);
output = QIcon(icon_pixmap);
break;
if (!output.isNull()) {
break;
}
}
}

1
src/librssguard/services/abstract/gui/formfeeddetails.cpp
View File

@ -315,6 +315,7 @@ void FormFeedDetails::initialize() {
m_ui->m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rdf), QVariant::fromValue(int(StandardFeed::Type::Rdf)));
m_ui->m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rss0X), QVariant::fromValue(int(StandardFeed::Type::Rss0X)));
m_ui->m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rss2X), QVariant::fromValue(int(StandardFeed::Type::Rss2X)));
m_ui->m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Json), QVariant::fromValue(int(StandardFeed::Type::Json)));
// Load available encodings.
const QList<QByteArray> encodings = QTextCodec::availableCodecs();

1
src/librssguard/services/standard/feedparser.h
View File

@ -8,6 +8,7 @@
#include "core/message.h"
// Base class for all XML-based feed parsers.
class FeedParser {
public:
explicit FeedParser(QString data);

67
src/librssguard/services/standard/jsonparser.cpp Normal file
View File

@ -0,0 +1,67 @@
// For license of this file, see <project-root-folder>/LICENSE.md.
#include "services/standard/jsonparser.h"
#include "miscellaneous/textfactory.h"
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
JsonParser::JsonParser(const QString& data) : m_jsonData(data) {}
QList<Message> JsonParser::messages() const {
QList<Message> msgs;
QJsonDocument json = QJsonDocument::fromJson(m_jsonData.toUtf8());
QString global_author = json.object()["author"].toObject()["name"].toString();
if (global_author.isEmpty()) {
global_author = json.object()["authors"].toArray().at(0).toObject()["name"].toString();
}
for (const QJsonValue& msg_val : json.object()["items"].toArray()) {
QJsonObject msg_obj = msg_val.toObject();
Message msg;
msg.m_title = msg_obj["title"].toString();
msg.m_url = msg_obj["url"].toString();
msg.m_contents = msg_obj.contains("content_html") ? msg_obj["content_html"].toString() : msg_obj["content_text"].toString();
msg.m_created = TextFactory::parseDateTime(msg_obj.contains("date_modified")
? msg_obj["date_modified"].toString()
: msg_obj["date_published"].toString());
if (!msg.m_created.isValid()) {
msg.m_created = QDateTime::currentDateTime();
msg.m_createdFromFeed = false;
}
else {
msg.m_createdFromFeed = true;
}
if (msg_obj.contains("author")) {
msg.m_author = msg_obj["author"].toObject()["name"].toString();
}
else if (msg_obj.contains("authors")) {
msg.m_author = msg_obj["authors"].toArray().at(0).toObject()["name"].toString();
}
else if (!global_author.isEmpty()) {
msg.m_author = global_author;
}
for (const QJsonValue& att : msg_obj["attachments"].toArray()) {
QJsonObject att_obj = att.toObject();
auto xx = att_obj["url"].toString();
msg.m_enclosures.append(Enclosure(att_obj["url"].toString(), att_obj["mime_type"].toString()));
}
if (msg.m_title.isEmpty() && !msg.m_url.isEmpty()) {
msg.m_title = msg.m_url;
}
msgs.append(msg);
}
return msgs;
}

18
src/librssguard/services/standard/jsonparser.h Normal file
View File

@ -0,0 +1,18 @@
// For license of this file, see <project-root-folder>/LICENSE.md.
#ifndef JSONPARSER_H
#define JSONPARSER_H
#include "core/message.h"
class JsonParser {
public:
explicit JsonParser(const QString& data);
QList<Message> messages() const;
private:
QString m_jsonData;
};
#endif // JSONPARSER_H

214
src/librssguard/services/standard/standardfeed.cpp
View File

@ -15,6 +15,7 @@
#include "services/abstract/recyclebin.h"
#include "services/standard/atomparser.h"
#include "services/standard/gui/formstandardfeeddetails.h"
#include "services/standard/jsonparser.h"
#include "services/standard/rdfparser.h"
#include "services/standard/rssparser.h"
#include "services/standard/standardserviceroot.h"
@ -22,6 +23,8 @@
#include <QDomDocument>
#include <QDomElement>
#include <QDomNode>
#include <QJsonDocument>
#include <QJsonObject>
#include <QPointer>
#include <QTextCodec>
#include <QVariant>
@ -104,6 +107,9 @@ QString StandardFeed::typeToString(StandardFeed::Type type) {
case Type::Rss0X:
return QSL("RSS 0.91/0.92/0.93");
case Type::Json:
return QSL("JSON 1.0/1.1");
case Type::Rss2X:
default:
return QSL("RSS 2.0/2.0.1");
@ -162,107 +168,130 @@ QPair<StandardFeed*, QNetworkReply::NetworkError> StandardFeed::guessFeed(const
result.first = new StandardFeed();
}
// Feed XML was obtained, now we need to try to guess
// its encoding before we can read further data.
QString xml_schema_encoding;
QString xml_contents_encoded;
QString enc = QRegularExpression(QSL("encoding=\"([A-Z0-9\\-]+)\""),
QRegularExpression::PatternOption::CaseInsensitiveOption).match(feed_contents).captured(1);
if (!enc.isEmpty()) {
// Some "encoding" attribute was found get the encoding
// out of it.
xml_schema_encoding = enc;
}
QTextCodec* custom_codec = QTextCodec::codecForName(xml_schema_encoding.toLocal8Bit());
if (custom_codec != nullptr) {
// Feed encoding was probably guessed.
xml_contents_encoded = custom_codec->toUnicode(feed_contents);
result.first->setEncoding(xml_schema_encoding);
}
else {
// Feed encoding probably not guessed, set it as
// default.
xml_contents_encoded = feed_contents;
result.first->setEncoding(DEFAULT_FEED_ENCODING);
}
// Feed XML was obtained, guess it now.
QDomDocument xml_document;
QString error_msg;
int error_line, error_column;
if (!xml_document.setContent(xml_contents_encoded,
&error_msg,
&error_line,
&error_column)) {
qDebugNN << LOGSEC_CORE
<< "XML of feed" << QUOTE_W_SPACE(url) << "is not valid and cannot be loaded. "
<< "Error:" << QUOTE_W_SPACE(error_msg) << "(line " << error_line
<< ", column " << error_column << ").";
result.second = QNetworkReply::UnknownContentError;
// XML is invalid, exit.
return result;
}
QDomElement root_element = xml_document.documentElement();
QString root_tag_name = root_element.tagName();
QList<QString> icon_possible_locations;
icon_possible_locations.append(url);
if (root_tag_name == QL1S("rdf:RDF")) {
// We found RDF feed.
QDomElement channel_element = root_element.namedItem(QSL("channel")).toElement();
if (network_result.second.toString().contains(QSL("json"), Qt::CaseSensitivity::CaseInsensitive)) {
// We have JSON feed.
result.first->setEncoding(DEFAULT_FEED_ENCODING);
result.first->setType(Type::Json);
result.first->setType(Type::Rdf);
result.first->setTitle(channel_element.namedItem(QSL("title")).toElement().text());
result.first->setDescription(channel_element.namedItem(QSL("description")).toElement().text());
QString source_link = channel_element.namedItem(QSL("link")).toElement().text();
QJsonDocument json = QJsonDocument::fromJson(feed_contents);
if (!source_link.isEmpty()) {
icon_possible_locations.prepend(source_link);
}
}
else if (root_tag_name == QL1S("rss")) {
// We found RSS 0.91/0.92/0.93/2.0/2.0.1 feed.
QString rss_type = root_element.attribute("version", "2.0");
result.first->setTitle(json.object()["title"].toString());
result.first->setDescription(json.object()["description"].toString());
if (rss_type == QL1S("0.91") || rss_type == QL1S("0.92") || rss_type == QL1S("0.93")) {
result.first->setType(Type::Rss0X);
}
else {
result.first->setType(Type::Rss2X);
auto icon = json.object()["icon"].toString();
if (icon.isEmpty()) {
icon = json.object()["favicon"].toString();
}
QDomElement channel_element = root_element.namedItem(QSL("channel")).toElement();
result.first->setTitle(channel_element.namedItem(QSL("title")).toElement().text());
result.first->setDescription(channel_element.namedItem(QSL("description")).toElement().text());
QString source_link = channel_element.namedItem(QSL("link")).toElement().text();
if (!source_link.isEmpty()) {
icon_possible_locations.prepend(source_link);
}
}
else if (root_tag_name == QL1S("feed")) {
// We found ATOM feed.
result.first->setType(Type::Atom10);
result.first->setTitle(root_element.namedItem(QSL("title")).toElement().text());
result.first->setDescription(root_element.namedItem(QSL("subtitle")).toElement().text());
QString source_link = root_element.namedItem(QSL("link")).toElement().text();
if (!source_link.isEmpty()) {
icon_possible_locations.prepend(source_link);
if (!icon.isEmpty()) {
icon_possible_locations.prepend(icon);
}
}
else {
// File was downloaded and it really was XML file
// but feed format was NOT recognized.
result.second = QNetworkReply::UnknownContentError;
// Feed XML was obtained, now we need to try to guess
// its encoding before we can read further data.
QString xml_schema_encoding;
QString xml_contents_encoded;
QString enc = QRegularExpression(QSL("encoding=\"([A-Z0-9\\-]+)\""),
QRegularExpression::PatternOption::CaseInsensitiveOption).match(feed_contents).captured(1);
if (!enc.isEmpty()) {
// Some "encoding" attribute was found get the encoding
// out of it.
xml_schema_encoding = enc;
}
QTextCodec* custom_codec = QTextCodec::codecForName(xml_schema_encoding.toLocal8Bit());
if (custom_codec != nullptr) {
// Feed encoding was probably guessed.
xml_contents_encoded = custom_codec->toUnicode(feed_contents);
result.first->setEncoding(xml_schema_encoding);
}
else {
// Feed encoding probably not guessed, set it as
// default.
xml_contents_encoded = feed_contents;
result.first->setEncoding(DEFAULT_FEED_ENCODING);
}
// Feed XML was obtained, guess it now.
QDomDocument xml_document;
QString error_msg;
int error_line, error_column;
if (!xml_document.setContent(xml_contents_encoded,
&error_msg,
&error_line,
&error_column)) {
qDebugNN << LOGSEC_CORE
<< "XML of feed" << QUOTE_W_SPACE(url) << "is not valid and cannot be loaded. "
<< "Error:" << QUOTE_W_SPACE(error_msg) << "(line " << error_line
<< ", column " << error_column << ").";
result.second = QNetworkReply::UnknownContentError;
// XML is invalid, exit.
return result;
}
QDomElement root_element = xml_document.documentElement();
QString root_tag_name = root_element.tagName();
if (root_tag_name == QL1S("rdf:RDF")) {
// We found RDF feed.
QDomElement channel_element = root_element.namedItem(QSL("channel")).toElement();
result.first->setType(Type::Rdf);
result.first->setTitle(channel_element.namedItem(QSL("title")).toElement().text());
result.first->setDescription(channel_element.namedItem(QSL("description")).toElement().text());
QString source_link = channel_element.namedItem(QSL("link")).toElement().text();
if (!source_link.isEmpty()) {
icon_possible_locations.prepend(source_link);
}
}
else if (root_tag_name == QL1S("rss")) {
// We found RSS 0.91/0.92/0.93/2.0/2.0.1 feed.
QString rss_type = root_element.attribute("version", "2.0");
if (rss_type == QL1S("0.91") || rss_type == QL1S("0.92") || rss_type == QL1S("0.93")) {
result.first->setType(Type::Rss0X);
}
else {
result.first->setType(Type::Rss2X);
}
QDomElement channel_element = root_element.namedItem(QSL("channel")).toElement();
result.first->setTitle(channel_element.namedItem(QSL("title")).toElement().text());
result.first->setDescription(channel_element.namedItem(QSL("description")).toElement().text());
QString source_link = channel_element.namedItem(QSL("link")).toElement().text();
if (!source_link.isEmpty()) {
icon_possible_locations.prepend(source_link);
}
}
else if (root_tag_name == QL1S("feed")) {
// We found ATOM feed.
result.first->setType(Type::Atom10);
result.first->setTitle(root_element.namedItem(QSL("title")).toElement().text());
result.first->setDescription(root_element.namedItem(QSL("subtitle")).toElement().text());
QString source_link = root_element.namedItem(QSL("link")).toElement().text();
if (!source_link.isEmpty()) {
icon_possible_locations.prepend(source_link);
}
}
else {
// File was downloaded and it really was XML file
// but feed format was NOT recognized.
result.second = QNetworkReply::UnknownContentError;
}
}
// Try to obtain icon.
@ -280,7 +309,7 @@ QPair<StandardFeed*, QNetworkReply::NetworkError> StandardFeed::guessFeed(const
}
Qt::ItemFlags StandardFeed::additionalFlags() const {
return Qt::ItemIsDragEnabled;
return Qt::ItemFlag::ItemIsDragEnabled;
}
bool StandardFeed::performDragDropChange(RootItem* target_item) {
@ -457,6 +486,11 @@ QList<Message> StandardFeed::obtainNewMessages(bool* error_during_obtaining) {
case StandardFeed::Type::Atom10:
messages = AtomParser(formatted_feed_contents).messages();
break;
case StandardFeed::Type::Json:
messages = JsonParser(formatted_feed_contents).messages();
break;
default:
break;

3
src/librssguard/services/standard/standardfeed.h
View File

@ -24,7 +24,8 @@ class StandardFeed : public Feed {
Rss0X = 0,
Rss2X = 1,
Rdf = 2, // Sometimes denoted as RSS 1.0.
Atom10 = 3
Atom10 = 3,
Json = 4
};
// Constructors and destructors.

7
src/librssguard/services/standard/standardfeedsimportexportmodel.cpp
View File

@ -108,6 +108,10 @@ bool FeedsImportExportModel::exportToOMPL20(QByteArray& result) {
outline_feed.setAttribute(QSL("version"), QSL("ATOM"));
break;
case StandardFeed::Type::Json:
outline_feed.setAttribute(QSL("version"), QSL("JSON"));
break;
default:
break;
}
@ -202,6 +206,9 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data, bool fetch_m
if (feed_type == QL1S("RSS1")) {
new_feed->setType(StandardFeed::Type::Rdf);
}
else if (feed_type == QL1S("JSON")) {
new_feed->setType(StandardFeed::Type::Json);
}
else if (feed_type == QL1S("ATOM")) {
new_feed->setType(StandardFeed::Type::Atom10);
}