diff --git a/src/librssguard/core/message.h b/src/librssguard/core/message.h index 462a35c92..91cc1be3f 100644 --- a/src/librssguard/core/message.h +++ b/src/librssguard/core/message.h @@ -36,6 +36,8 @@ class Message { // Creates Message from given record, which contains // row from query SELECT * FROM Messages WHERE ....; static Message fromSqlRecord(const QSqlRecord& record, bool* result = nullptr); + + public: QString m_title; QString m_url; QString m_author; diff --git a/src/librssguard/definitions/definitions.h b/src/librssguard/definitions/definitions.h index fb53ed06c..29f086c2f 100755 --- a/src/librssguard/definitions/definitions.h +++ b/src/librssguard/definitions/definitions.h @@ -111,7 +111,7 @@ #define FEED_INITIAL_OPML_PATTERN "feeds-%1.opml" -#define FEED_REGEX_MATCHER "]+type=\"application\\/(?:atom|rss)\\+xml\"[^>]*>" +#define FEED_REGEX_MATCHER "]+type=\"application\\/(?:atom\\+xml|rss\\+xml|feed\\+json|json)\"[^>]*>" #define FEED_HREF_REGEX_MATCHER "href=\"([^\"]+)\"" #define PLACEHOLDER_UNREAD_COUNTS "%unread" diff --git a/src/librssguard/gui/discoverfeedsbutton.cpp b/src/librssguard/gui/discoverfeedsbutton.cpp index e81c451eb..c44777719 100644 --- a/src/librssguard/gui/discoverfeedsbutton.cpp +++ b/src/librssguard/gui/discoverfeedsbutton.cpp @@ -62,10 +62,10 @@ void DiscoverFeedsButton::fillMenu() { menu()->clear(); for (const ServiceRoot* root : qApp->feedReader()->feedsModel()->serviceRoots()) { - QMenu* root_menu = menu()->addMenu(root->icon(), root->title()); + if (root->supportsFeedAdding()) { + QMenu* root_menu = menu()->addMenu(root->icon(), root->title()); - for (const QString& url : m_addresses) { - if (root->supportsFeedAdding()) { + for (const QString& url : m_addresses) { QAction* url_action = root_menu->addAction(root->icon(), url); url_action->setProperty("url", url); @@ -73,4 +73,8 @@ void DiscoverFeedsButton::fillMenu() { } } } + + if (menu()->isEmpty()) { + menu()->addAction(tr("Feeds were detected, but no suitable accounts are configured."))->setEnabled(false); + } } diff --git a/src/librssguard/librssguard.pro b/src/librssguard/librssguard.pro index e8e80d8f3..bd5e50b39 100644 --- a/src/librssguard/librssguard.pro +++ b/src/librssguard/librssguard.pro @@ -158,6 +158,7 @@ HEADERS += core/feeddownloader.h \ services/standard/gui/formstandardcategorydetails.h \ services/standard/gui/formstandardfeeddetails.h \ services/standard/gui/formstandardimportexport.h \ + services/standard/jsonparser.h \ services/standard/rdfparser.h \ services/standard/rssparser.h \ services/standard/standardcategory.h \ @@ -302,6 +303,7 @@ SOURCES += core/feeddownloader.cpp \ services/standard/gui/formstandardcategorydetails.cpp \ services/standard/gui/formstandardfeeddetails.cpp \ services/standard/gui/formstandardimportexport.cpp \ + services/standard/jsonparser.cpp \ services/standard/rdfparser.cpp \ services/standard/rssparser.cpp \ services/standard/standardcategory.cpp \ diff --git a/src/librssguard/miscellaneous/databasequeries.h b/src/librssguard/miscellaneous/databasequeries.h index 09195c0c8..134c3cfe1 100644 --- a/src/librssguard/miscellaneous/databasequeries.h +++ b/src/librssguard/miscellaneous/databasequeries.h @@ -196,7 +196,8 @@ inline void DatabaseQueries::fillFeedData(StandardFeed* feed, const QSqlRecord& case StandardFeed::Type::Atom10: case StandardFeed::Type::Rdf: case StandardFeed::Type::Rss0X: - case StandardFeed::Type::Rss2X: { + case StandardFeed::Type::Rss2X: + case StandardFeed::Type::Json: { feed->setType(type); break; } diff --git a/src/librssguard/network-web/networkfactory.cpp b/src/librssguard/network-web/networkfactory.cpp index f0b890c45..7513f423b 100644 --- a/src/librssguard/network-web/networkfactory.cpp +++ b/src/librssguard/network-web/networkfactory.cpp @@ -143,6 +143,22 @@ QNetworkReply::NetworkError NetworkFactory::downloadIcon(const QList& u QNetworkReply::NetworkError network_result = QNetworkReply::UnknownNetworkError; for (const QString& url : urls) { + QByteArray icon_data; + + network_result = performNetworkOperation(url, timeout, QByteArray(), icon_data, + QNetworkAccessManager::GetOperation).first; + + if (network_result == QNetworkReply::NoError) { + QPixmap icon_pixmap; + + icon_pixmap.loadFromData(icon_data); + output = QIcon(icon_pixmap); + + if (!output.isNull()) { + break; + } + } + QString host = QUrl(url).host(); if (host.startsWith(QSL("www."))) { @@ -150,7 +166,6 @@ QNetworkReply::NetworkError NetworkFactory::downloadIcon(const QList& u } const QString google_s2_with_url = QString("http://www.google.com/s2/favicons?domain=%1").arg(host); - QByteArray icon_data; network_result = performNetworkOperation(google_s2_with_url, timeout, QByteArray(), icon_data, QNetworkAccessManager::GetOperation).first; @@ -160,7 +175,10 @@ QNetworkReply::NetworkError NetworkFactory::downloadIcon(const QList& u icon_pixmap.loadFromData(icon_data); output = QIcon(icon_pixmap); - break; + + if (!output.isNull()) { + break; + } } } diff --git a/src/librssguard/services/abstract/gui/formfeeddetails.cpp b/src/librssguard/services/abstract/gui/formfeeddetails.cpp index fff5fc7ba..7599930f4 100644 --- a/src/librssguard/services/abstract/gui/formfeeddetails.cpp +++ b/src/librssguard/services/abstract/gui/formfeeddetails.cpp @@ -315,6 +315,7 @@ void FormFeedDetails::initialize() { m_ui->m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rdf), QVariant::fromValue(int(StandardFeed::Type::Rdf))); m_ui->m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rss0X), QVariant::fromValue(int(StandardFeed::Type::Rss0X))); m_ui->m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Rss2X), QVariant::fromValue(int(StandardFeed::Type::Rss2X))); + m_ui->m_cmbType->addItem(StandardFeed::typeToString(StandardFeed::Type::Json), QVariant::fromValue(int(StandardFeed::Type::Json))); // Load available encodings. const QList encodings = QTextCodec::availableCodecs(); diff --git a/src/librssguard/services/standard/feedparser.h b/src/librssguard/services/standard/feedparser.h index 84423fc35..fc750fad9 100755 --- a/src/librssguard/services/standard/feedparser.h +++ b/src/librssguard/services/standard/feedparser.h @@ -8,6 +8,7 @@ #include "core/message.h" +// Base class for all XML-based feed parsers. class FeedParser { public: explicit FeedParser(QString data); diff --git a/src/librssguard/services/standard/jsonparser.cpp b/src/librssguard/services/standard/jsonparser.cpp new file mode 100644 index 000000000..2ad6a5cbf --- /dev/null +++ b/src/librssguard/services/standard/jsonparser.cpp @@ -0,0 +1,67 @@ +// For license of this file, see /LICENSE.md. + +#include "services/standard/jsonparser.h" + +#include "miscellaneous/textfactory.h" + +#include +#include +#include + +JsonParser::JsonParser(const QString& data) : m_jsonData(data) {} + +QList JsonParser::messages() const { + QList msgs; + QJsonDocument json = QJsonDocument::fromJson(m_jsonData.toUtf8()); + QString global_author = json.object()["author"].toObject()["name"].toString(); + + if (global_author.isEmpty()) { + global_author = json.object()["authors"].toArray().at(0).toObject()["name"].toString(); + } + + for (const QJsonValue& msg_val : json.object()["items"].toArray()) { + QJsonObject msg_obj = msg_val.toObject(); + Message msg; + + msg.m_title = msg_obj["title"].toString(); + msg.m_url = msg_obj["url"].toString(); + msg.m_contents = msg_obj.contains("content_html") ? msg_obj["content_html"].toString() : msg_obj["content_text"].toString(); + + msg.m_created = TextFactory::parseDateTime(msg_obj.contains("date_modified") + ? msg_obj["date_modified"].toString() + : msg_obj["date_published"].toString()); + + if (!msg.m_created.isValid()) { + msg.m_created = QDateTime::currentDateTime(); + msg.m_createdFromFeed = false; + } + else { + msg.m_createdFromFeed = true; + } + + if (msg_obj.contains("author")) { + msg.m_author = msg_obj["author"].toObject()["name"].toString(); + } + else if (msg_obj.contains("authors")) { + msg.m_author = msg_obj["authors"].toArray().at(0).toObject()["name"].toString(); + } + else if (!global_author.isEmpty()) { + msg.m_author = global_author; + } + + for (const QJsonValue& att : msg_obj["attachments"].toArray()) { + QJsonObject att_obj = att.toObject(); + auto xx = att_obj["url"].toString(); + + msg.m_enclosures.append(Enclosure(att_obj["url"].toString(), att_obj["mime_type"].toString())); + } + + if (msg.m_title.isEmpty() && !msg.m_url.isEmpty()) { + msg.m_title = msg.m_url; + } + + msgs.append(msg); + } + + return msgs; +} diff --git a/src/librssguard/services/standard/jsonparser.h b/src/librssguard/services/standard/jsonparser.h new file mode 100644 index 000000000..5dd922fcf --- /dev/null +++ b/src/librssguard/services/standard/jsonparser.h @@ -0,0 +1,18 @@ +// For license of this file, see /LICENSE.md. + +#ifndef JSONPARSER_H +#define JSONPARSER_H + +#include "core/message.h" + +class JsonParser { + public: + explicit JsonParser(const QString& data); + + QList messages() const; + + private: + QString m_jsonData; +}; + +#endif // JSONPARSER_H diff --git a/src/librssguard/services/standard/standardfeed.cpp b/src/librssguard/services/standard/standardfeed.cpp index 897702107..432d38f81 100644 --- a/src/librssguard/services/standard/standardfeed.cpp +++ b/src/librssguard/services/standard/standardfeed.cpp @@ -15,6 +15,7 @@ #include "services/abstract/recyclebin.h" #include "services/standard/atomparser.h" #include "services/standard/gui/formstandardfeeddetails.h" +#include "services/standard/jsonparser.h" #include "services/standard/rdfparser.h" #include "services/standard/rssparser.h" #include "services/standard/standardserviceroot.h" @@ -22,6 +23,8 @@ #include #include #include +#include +#include #include #include #include @@ -104,6 +107,9 @@ QString StandardFeed::typeToString(StandardFeed::Type type) { case Type::Rss0X: return QSL("RSS 0.91/0.92/0.93"); + case Type::Json: + return QSL("JSON 1.0/1.1"); + case Type::Rss2X: default: return QSL("RSS 2.0/2.0.1"); @@ -162,107 +168,130 @@ QPair StandardFeed::guessFeed(const result.first = new StandardFeed(); } - // Feed XML was obtained, now we need to try to guess - // its encoding before we can read further data. - QString xml_schema_encoding; - QString xml_contents_encoded; - QString enc = QRegularExpression(QSL("encoding=\"([A-Z0-9\\-]+)\""), - QRegularExpression::PatternOption::CaseInsensitiveOption).match(feed_contents).captured(1); - - if (!enc.isEmpty()) { - // Some "encoding" attribute was found get the encoding - // out of it. - xml_schema_encoding = enc; - } - - QTextCodec* custom_codec = QTextCodec::codecForName(xml_schema_encoding.toLocal8Bit()); - - if (custom_codec != nullptr) { - // Feed encoding was probably guessed. - xml_contents_encoded = custom_codec->toUnicode(feed_contents); - result.first->setEncoding(xml_schema_encoding); - } - else { - // Feed encoding probably not guessed, set it as - // default. - xml_contents_encoded = feed_contents; - result.first->setEncoding(DEFAULT_FEED_ENCODING); - } - - // Feed XML was obtained, guess it now. - QDomDocument xml_document; - QString error_msg; - int error_line, error_column; - - if (!xml_document.setContent(xml_contents_encoded, - &error_msg, - &error_line, - &error_column)) { - qDebugNN << LOGSEC_CORE - << "XML of feed" << QUOTE_W_SPACE(url) << "is not valid and cannot be loaded. " - << "Error:" << QUOTE_W_SPACE(error_msg) << "(line " << error_line - << ", column " << error_column << ")."; - result.second = QNetworkReply::UnknownContentError; - - // XML is invalid, exit. - return result; - } - - QDomElement root_element = xml_document.documentElement(); - QString root_tag_name = root_element.tagName(); QList icon_possible_locations; icon_possible_locations.append(url); - if (root_tag_name == QL1S("rdf:RDF")) { - // We found RDF feed. - QDomElement channel_element = root_element.namedItem(QSL("channel")).toElement(); + if (network_result.second.toString().contains(QSL("json"), Qt::CaseSensitivity::CaseInsensitive)) { + // We have JSON feed. + result.first->setEncoding(DEFAULT_FEED_ENCODING); + result.first->setType(Type::Json); - result.first->setType(Type::Rdf); - result.first->setTitle(channel_element.namedItem(QSL("title")).toElement().text()); - result.first->setDescription(channel_element.namedItem(QSL("description")).toElement().text()); - QString source_link = channel_element.namedItem(QSL("link")).toElement().text(); + QJsonDocument json = QJsonDocument::fromJson(feed_contents); - if (!source_link.isEmpty()) { - icon_possible_locations.prepend(source_link); - } - } - else if (root_tag_name == QL1S("rss")) { - // We found RSS 0.91/0.92/0.93/2.0/2.0.1 feed. - QString rss_type = root_element.attribute("version", "2.0"); + result.first->setTitle(json.object()["title"].toString()); + result.first->setDescription(json.object()["description"].toString()); - if (rss_type == QL1S("0.91") || rss_type == QL1S("0.92") || rss_type == QL1S("0.93")) { - result.first->setType(Type::Rss0X); - } - else { - result.first->setType(Type::Rss2X); + auto icon = json.object()["icon"].toString(); + + if (icon.isEmpty()) { + icon = json.object()["favicon"].toString(); } - QDomElement channel_element = root_element.namedItem(QSL("channel")).toElement(); - - result.first->setTitle(channel_element.namedItem(QSL("title")).toElement().text()); - result.first->setDescription(channel_element.namedItem(QSL("description")).toElement().text()); - QString source_link = channel_element.namedItem(QSL("link")).toElement().text(); - - if (!source_link.isEmpty()) { - icon_possible_locations.prepend(source_link); - } - } - else if (root_tag_name == QL1S("feed")) { - // We found ATOM feed. - result.first->setType(Type::Atom10); - result.first->setTitle(root_element.namedItem(QSL("title")).toElement().text()); - result.first->setDescription(root_element.namedItem(QSL("subtitle")).toElement().text()); - QString source_link = root_element.namedItem(QSL("link")).toElement().text(); - - if (!source_link.isEmpty()) { - icon_possible_locations.prepend(source_link); + if (!icon.isEmpty()) { + icon_possible_locations.prepend(icon); } } else { - // File was downloaded and it really was XML file - // but feed format was NOT recognized. - result.second = QNetworkReply::UnknownContentError; + // Feed XML was obtained, now we need to try to guess + // its encoding before we can read further data. + QString xml_schema_encoding; + QString xml_contents_encoded; + QString enc = QRegularExpression(QSL("encoding=\"([A-Z0-9\\-]+)\""), + QRegularExpression::PatternOption::CaseInsensitiveOption).match(feed_contents).captured(1); + + if (!enc.isEmpty()) { + // Some "encoding" attribute was found get the encoding + // out of it. + xml_schema_encoding = enc; + } + + QTextCodec* custom_codec = QTextCodec::codecForName(xml_schema_encoding.toLocal8Bit()); + + if (custom_codec != nullptr) { + // Feed encoding was probably guessed. + xml_contents_encoded = custom_codec->toUnicode(feed_contents); + result.first->setEncoding(xml_schema_encoding); + } + else { + // Feed encoding probably not guessed, set it as + // default. + xml_contents_encoded = feed_contents; + result.first->setEncoding(DEFAULT_FEED_ENCODING); + } + + // Feed XML was obtained, guess it now. + QDomDocument xml_document; + QString error_msg; + int error_line, error_column; + + if (!xml_document.setContent(xml_contents_encoded, + &error_msg, + &error_line, + &error_column)) { + qDebugNN << LOGSEC_CORE + << "XML of feed" << QUOTE_W_SPACE(url) << "is not valid and cannot be loaded. " + << "Error:" << QUOTE_W_SPACE(error_msg) << "(line " << error_line + << ", column " << error_column << ")."; + result.second = QNetworkReply::UnknownContentError; + + // XML is invalid, exit. + return result; + } + + QDomElement root_element = xml_document.documentElement(); + QString root_tag_name = root_element.tagName(); + + if (root_tag_name == QL1S("rdf:RDF")) { + // We found RDF feed. + QDomElement channel_element = root_element.namedItem(QSL("channel")).toElement(); + + result.first->setType(Type::Rdf); + result.first->setTitle(channel_element.namedItem(QSL("title")).toElement().text()); + result.first->setDescription(channel_element.namedItem(QSL("description")).toElement().text()); + QString source_link = channel_element.namedItem(QSL("link")).toElement().text(); + + if (!source_link.isEmpty()) { + icon_possible_locations.prepend(source_link); + } + } + else if (root_tag_name == QL1S("rss")) { + // We found RSS 0.91/0.92/0.93/2.0/2.0.1 feed. + QString rss_type = root_element.attribute("version", "2.0"); + + if (rss_type == QL1S("0.91") || rss_type == QL1S("0.92") || rss_type == QL1S("0.93")) { + result.first->setType(Type::Rss0X); + } + else { + result.first->setType(Type::Rss2X); + } + + QDomElement channel_element = root_element.namedItem(QSL("channel")).toElement(); + + result.first->setTitle(channel_element.namedItem(QSL("title")).toElement().text()); + result.first->setDescription(channel_element.namedItem(QSL("description")).toElement().text()); + QString source_link = channel_element.namedItem(QSL("link")).toElement().text(); + + if (!source_link.isEmpty()) { + icon_possible_locations.prepend(source_link); + } + } + else if (root_tag_name == QL1S("feed")) { + // We found ATOM feed. + result.first->setType(Type::Atom10); + result.first->setTitle(root_element.namedItem(QSL("title")).toElement().text()); + result.first->setDescription(root_element.namedItem(QSL("subtitle")).toElement().text()); + QString source_link = root_element.namedItem(QSL("link")).toElement().text(); + + if (!source_link.isEmpty()) { + icon_possible_locations.prepend(source_link); + } + } + else { + // File was downloaded and it really was XML file + // but feed format was NOT recognized. + result.second = QNetworkReply::UnknownContentError; + } } // Try to obtain icon. @@ -280,7 +309,7 @@ QPair StandardFeed::guessFeed(const } Qt::ItemFlags StandardFeed::additionalFlags() const { - return Qt::ItemIsDragEnabled; + return Qt::ItemFlag::ItemIsDragEnabled; } bool StandardFeed::performDragDropChange(RootItem* target_item) { @@ -457,6 +486,11 @@ QList StandardFeed::obtainNewMessages(bool* error_during_obtaining) { case StandardFeed::Type::Atom10: messages = AtomParser(formatted_feed_contents).messages(); + break; + + case StandardFeed::Type::Json: + messages = JsonParser(formatted_feed_contents).messages(); + break; default: break; diff --git a/src/librssguard/services/standard/standardfeed.h b/src/librssguard/services/standard/standardfeed.h index d60a4ed6a..ddf5cb50a 100644 --- a/src/librssguard/services/standard/standardfeed.h +++ b/src/librssguard/services/standard/standardfeed.h @@ -24,7 +24,8 @@ class StandardFeed : public Feed { Rss0X = 0, Rss2X = 1, Rdf = 2, // Sometimes denoted as RSS 1.0. - Atom10 = 3 + Atom10 = 3, + Json = 4 }; // Constructors and destructors. diff --git a/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp b/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp index d9faf5b7b..6cdaec2f7 100644 --- a/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp +++ b/src/librssguard/services/standard/standardfeedsimportexportmodel.cpp @@ -108,6 +108,10 @@ bool FeedsImportExportModel::exportToOMPL20(QByteArray& result) { outline_feed.setAttribute(QSL("version"), QSL("ATOM")); break; + case StandardFeed::Type::Json: + outline_feed.setAttribute(QSL("version"), QSL("JSON")); + break; + default: break; } @@ -202,6 +206,9 @@ void FeedsImportExportModel::importAsOPML20(const QByteArray& data, bool fetch_m if (feed_type == QL1S("RSS1")) { new_feed->setType(StandardFeed::Type::Rdf); } + else if (feed_type == QL1S("JSON")) { + new_feed->setType(StandardFeed::Type::Json); + } else if (feed_type == QL1S("ATOM")) { new_feed->setType(StandardFeed::Type::Atom10); }