fix #1490 - add option to set per-feed HTTP headers

src/librssguard-standard/src/gui/formstandardfeeddetails.cpp

@@ -13,6 +13,7 @@
 #include <librssguard/network-web/webfactory.h>
 #include <librssguard/services/abstract/category.h>
 #include <librssguard/services/abstract/gui/authenticationdetails.h>
+#include <librssguard/services/abstract/gui/httpheadersdetails.h>
 #include <librssguard/services/abstract/serviceroot.h>
 
 #include <QComboBox>
@@ -27,9 +28,10 @@ FormStandardFeedDetails::FormStandardFeedDetails(ServiceRoot* service_root,
                                                  QWidget* parent)
   : FormFeedDetails(service_root, parent), m_standardFeedDetails(new StandardFeedDetails(this)),
     m_standardFeedExpDetails(new StandardFeedExpDetails(this)), m_authDetails(new AuthenticationDetails(false, this)),
-    m_parentToSelect(parent_to_select), m_urlToProcess(url) {
+    m_headersDetails(new HttpHeadersDetails(this)), m_parentToSelect(parent_to_select), m_urlToProcess(url) {
   insertCustomTab(m_standardFeedDetails, tr("General"), 0);
-  insertCustomTab(m_authDetails, tr("Network"), 2);
+  insertCustomTab(m_headersDetails, tr("HTTP headers"), 2);
+  insertCustomTab(m_authDetails, tr("Auth"), 2);
   insertCustomTab(m_standardFeedExpDetails, tr("Experimental"));
   activateTab(0);
 
@@ -53,6 +55,7 @@ void FormStandardFeedDetails::guessFeed() {
                                    m_authDetails->authenticationType(),
                                    m_authDetails->username(),
                                    m_authDetails->password(),
+                                   StandardFeed::httpHeadersToList(m_headersDetails->httpHeaders()),
                                    m_serviceRoot->networkProxy());
 }
 
@@ -63,6 +66,7 @@ void FormStandardFeedDetails::guessIconOnly() {
                                        m_authDetails->authenticationType(),
                                        m_authDetails->username(),
                                        m_authDetails->password(),
+                                       StandardFeed::httpHeadersToList(m_headersDetails->httpHeaders()),
                                        m_serviceRoot->networkProxy());
 }
 
@@ -120,15 +124,19 @@ void FormStandardFeedDetails::apply() {
       std_feed->setProtection(m_authDetails->authenticationType());
     }
 
-    if (isChangeAllowed(m_standardFeedExpDetails->m_ui.m_mcbDontUseRawXml)) {
-      std_feed->setDontUseRawXmlSaving(m_standardFeedExpDetails->m_ui.m_cbDontUseRawXml->isChecked());
-    }
-
     if (isChangeAllowed(m_authDetails->findChild<MultiFeedEditCheckBox*>(QSL("m_mcbAuthentication")))) {
       std_feed->setUsername(m_authDetails->username());
       std_feed->setPassword(m_authDetails->password());
     }
 
+    if (isChangeAllowed(m_headersDetails->findChild<MultiFeedEditCheckBox*>(QSL("m_mcbHttpHeaders")))) {
+      std_feed->setHttpHeaders(m_headersDetails->httpHeaders());
+    }
+
+    if (isChangeAllowed(m_standardFeedExpDetails->m_ui.m_mcbDontUseRawXml)) {
+      std_feed->setDontUseRawXmlSaving(m_standardFeedExpDetails->m_ui.m_cbDontUseRawXml->isChecked());
+    }
+
     std_feed->setCreationDate(QDateTime::currentDateTime());
     std_feed->setLastEtag({});
 
@@ -177,6 +185,9 @@ void FormStandardFeedDetails::loadFeedData() {
     m_authDetails->findChild<MultiFeedEditCheckBox*>(QSL("m_mcbAuthentication"))
       ->addActionWidget(m_authDetails->findChild<QGroupBox*>(QSL("m_gbAuthentication")));
 
+    m_headersDetails->findChild<MultiFeedEditCheckBox*>(QSL("m_mcbHttpHeaders"))
+      ->addActionWidget(m_headersDetails->findChild<QPlainTextEdit*>(QSL("m_txtHttpHeaders")));
+
     m_standardFeedDetails->m_ui.m_btnFetchMetadata->setEnabled(false);
 
     m_standardFeedExpDetails->m_ui.m_mcbDontUseRawXml
@@ -198,6 +209,8 @@ void FormStandardFeedDetails::loadFeedData() {
   m_authDetails->setUsername(std_feed->username());
   m_authDetails->setPassword(std_feed->password());
 
+  m_headersDetails->loadHttpHeaders(std_feed->httpHeaders());
+
   if (m_creatingNew) {
     // auto processed_url = qApp->web()->processFeedUriScheme(m_urlToProcess);
 

src/librssguard-standard/src/gui/formstandardfeeddetails.h

@@ -7,6 +7,7 @@
 
 class StandardFeedDetails;
 class StandardFeedExpDetails;
+class HttpHeadersDetails;
 class StandardServiceRoot;
 class AuthenticationDetails;
 class StandardFeed;
@@ -34,6 +35,7 @@ class FormStandardFeedDetails : public FormFeedDetails {
     StandardFeedDetails* m_standardFeedDetails;
     StandardFeedExpDetails* m_standardFeedExpDetails;
     AuthenticationDetails* m_authDetails;
+    HttpHeadersDetails* m_headersDetails;
     RootItem* m_parentToSelect;
     QString m_urlToProcess;
 };

src/librssguard-standard/src/gui/standardfeeddetails.cpp

@@ -143,6 +143,7 @@ void StandardFeedDetails::guessIconOnly(StandardFeed::SourceType source_type,
                                         NetworkFactory::NetworkAuthentication protection,
                                         const QString& username,
                                         const QString& password,
+                                        const QList<QPair<QByteArray, QByteArray>>& headers,
                                         const QNetworkProxy& custom_proxy) {
   try {
     StandardFeed* metadata = StandardFeed::guessFeed(source_type,
@@ -152,6 +153,7 @@ void StandardFeedDetails::guessIconOnly(StandardFeed::SourceType source_type,
                                                      true,
                                                      username,
                                                      password,
+                                                     {},
                                                      custom_proxy);
 
     // Icon or whole feed was guessed.
@@ -186,6 +188,7 @@ void StandardFeedDetails::guessFeed(StandardFeed::SourceType source_type,
                                     NetworkFactory::NetworkAuthentication protection,
                                     const QString& username,
                                     const QString& password,
+                                    const QList<QPair<QByteArray, QByteArray>>& headers,
                                     const QNetworkProxy& custom_proxy) {
   try {
     StandardFeed* metadata = StandardFeed::guessFeed(source_type,
@@ -195,6 +198,7 @@ void StandardFeedDetails::guessFeed(StandardFeed::SourceType source_type,
                                                      true,
                                                      username,
                                                      password,
+                                                     headers,
                                                      custom_proxy);
 
     // Icon or whole feed was guessed.

src/librssguard-standard/src/gui/standardfeeddetails.h

@@ -30,13 +30,16 @@ class StandardFeedDetails : public QWidget {
                        NetworkFactory::NetworkAuthentication protection,
                        const QString& username,
                        const QString& password,
+                       const QList<QPair<QByteArray, QByteArray>>& headers = {},
                        const QNetworkProxy& custom_proxy = QNetworkProxy::ProxyType::DefaultProxy);
+
     void guessFeed(StandardFeed::SourceType source_type,
                    const QString& source,
                    const QString& post_process_script,
                    NetworkFactory::NetworkAuthentication protection,
                    const QString& username,
                    const QString& password,
+                   const QList<QPair<QByteArray, QByteArray>>& headers = {},
                    const QNetworkProxy& custom_proxy = QNetworkProxy::ProxyType::DefaultProxy);
 
     void onTitleChanged(const QString& new_title);

src/librssguard-standard/src/standardfeed.cpp

@@ -52,6 +52,7 @@ StandardFeed::StandardFeed(RootItem* parent_item) : Feed(parent_item) {
   m_protection = NetworkFactory::NetworkAuthentication::NoAuthentication;
   m_username = QString();
   m_password = QString();
+  m_httpHeaders = {};
   m_dontUseRawXmlSaving = false;
 }
 
@@ -64,6 +65,7 @@ StandardFeed::StandardFeed(const StandardFeed& other) : Feed(other) {
   m_username = other.username();
   m_password = other.password();
   m_dontUseRawXmlSaving = other.dontUseRawXmlSaving();
+  m_httpHeaders = other.httpHeaders();
 }
 
 QList<QAction*> StandardFeed::contextMenuFeedsList() {
@@ -164,6 +166,7 @@ QVariantHash StandardFeed::customDatabaseData() const {
   data[QSL("username")] = username();
   data[QSL("password")] = TextFactory::encrypt(password());
   data[QSL("dont_use_raw_xml_saving")] = dontUseRawXmlSaving();
+  data[QSL("http_headers")] = httpHeaders();
 
   return data;
 }
@@ -177,6 +180,7 @@ void StandardFeed::setCustomDatabaseData(const QVariantHash& data) {
   setUsername(data[QSL("username")].toString());
   setPassword(TextFactory::decrypt(data[QSL("password")].toString()));
   setDontUseRawXmlSaving(data[QSL("dont_use_raw_xml_saving")].toBool());
+  setHttpHeaders(data[QSL("http_headers")].toHash());
 }
 
 QString StandardFeed::typeToString(StandardFeed::Type type) {
@@ -233,6 +237,7 @@ void StandardFeed::fetchMetadataForItself() {
                                        true,
                                        username(),
                                        password(),
+                                       {},
                                        getParentServiceRoot()->networkProxy());
 
     // Copy metadata to our object.
@@ -281,14 +286,16 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type,
                                       bool fetch_icons,
                                       const QString& username,
                                       const QString& password,
+                                      const QList<QPair<QByteArray, QByteArray>>& http_headers,
                                       const QNetworkProxy& custom_proxy) {
   auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
   QByteArray feed_contents;
   QString content_type;
 
   if (source_type == StandardFeed::SourceType::Url) {
-    QList<QPair<QByteArray, QByteArray>> headers = {
+    QList<QPair<QByteArray, QByteArray>> headers = http_headers;
-      NetworkFactory::generateBasicAuthHeader(protection, username, password)};
+    headers << NetworkFactory::generateBasicAuthHeader(protection, username, password);
+
     NetworkResult network_result =
       NetworkFactory::performNetworkOperation(source,
                                               timeout,
@@ -392,8 +399,11 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type,
     // Try to obtain icon.
     QPixmap icon_data;
 
-    if (NetworkFactory::downloadIcon(icon_possible_locations, DOWNLOAD_TIMEOUT, icon_data, {}, custom_proxy) ==
+    if (NetworkFactory::downloadIcon(icon_possible_locations,
-        QNetworkReply::NetworkError::NoError) {
+                                     DOWNLOAD_TIMEOUT,
+                                     icon_data,
+                                     http_headers,
+                                     custom_proxy) == QNetworkReply::NetworkError::NoError) {
       // Icon for feed was downloaded and is stored now in icon_data.
       feed->setIcon(icon_data);
     }
@@ -433,6 +443,14 @@ bool StandardFeed::removeItself() {
   return DatabaseQueries::deleteFeed(database, this, getParentServiceRoot()->accountId());
 }
 
+QVariantHash StandardFeed::httpHeaders() const {
+  return m_httpHeaders;
+}
+
+void StandardFeed::setHttpHeaders(const QVariantHash& http_headers) {
+  m_httpHeaders = http_headers;
+}
+
 bool StandardFeed::dontUseRawXmlSaving() const {
   return m_dontUseRawXmlSaving;
 }
@@ -547,6 +565,16 @@ QByteArray StandardFeed::runScriptProcess(const QStringList& cmd_args,
   }
 }
 
+QList<QPair<QByteArray, QByteArray>> StandardFeed::httpHeadersToList(const QVariantHash& headers) {
+  QList<QPair<QByteArray, QByteArray>> hdrs_list;
+
+  for (auto i = headers.cbegin(), end = headers.cend(); i != end; i++) {
+    hdrs_list.append({i.key().toLocal8Bit(), i.value().toString().toLocal8Bit()});
+  }
+
+  return hdrs_list;
+}
+
 QByteArray StandardFeed::generateFeedFileWithScript(const QString& execution_line, int run_timeout) {
   auto prepared_query = prepareExecutionLine(execution_line);
 

src/librssguard-standard/src/standardfeed.h

@@ -86,6 +86,7 @@ class StandardFeed : public Feed {
                                    bool fetch_icons = true,
                                    const QString& username = {},
                                    const QString& password = {},
+                                   const QList<QPair<QByteArray, QByteArray>>& http_headers = {},
                                    const QNetworkProxy& custom_proxy = QNetworkProxy::ProxyType::DefaultProxy);
 
     // Converts particular feed type to string.
@@ -104,6 +105,8 @@ class StandardFeed : public Feed {
                                        bool provide_input,
                                        const QString& input = {});
 
+    static QList<QPair<QByteArray, QByteArray>> httpHeadersToList(const QVariantHash& headers);
+
     QString lastEtag() const;
     void setLastEtag(const QString& etag);
 
@@ -113,6 +116,10 @@ class StandardFeed : public Feed {
     bool dontUseRawXmlSaving() const;
     void setDontUseRawXmlSaving(bool no_raw_xml_saving);
 
+    // NOTE: Contains hash table where key is name of HTTP header.
+    QVariantHash httpHeaders() const;
+    void setHttpHeaders(const QVariantHash& http_headers);
+
   public slots:
     void fetchMetadataForItself();
 
@@ -131,6 +138,7 @@ class StandardFeed : public Feed {
     QString m_password;
     QString m_lastEtag;
     bool m_dontUseRawXmlSaving;
+    QVariantHash m_httpHeaders;
 };
 
 Q_DECLARE_METATYPE(StandardFeed::SourceType)

src/librssguard-standard/src/standardfeedsimportexportmodel.cpp

@@ -203,6 +203,7 @@ bool FeedsImportExportModel::produceFeed(const FeedLookup& feed_lookup) {
                                            !feed_lookup.do_not_fetch_icons,
                                            {},
                                            {},
+                                           {},
                                            feed_lookup.custom_proxy);
 
         new_feed->setSourceType(source_type);

src/librssguard-standard/src/standardserviceroot.cpp

@@ -210,7 +210,7 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
   if (f->sourceType() == StandardFeed::SourceType::Url) {
     qDebugNN << LOGSEC_CORE << "Downloading URL" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data.";
 
-    QList<QPair<QByteArray, QByteArray>> headers;
+    QList<QPair<QByteArray, QByteArray>> headers = StandardFeed::httpHeadersToList(f->httpHeaders());
 
     headers << NetworkFactory::generateBasicAuthHeader(f->protection(), f->username(), f->password());
 

src/librssguard-ttrss/src/gui/formttrssfeeddetails.cpp

@@ -59,7 +59,7 @@ void FormTtRssFeedDetails::loadFeedData() {
 
   if (m_creatingNew) {
     insertCustomTab(m_feedDetails, tr("General"), 0);
-    insertCustomTab(m_authDetails, tr("Network"), 1);
+    insertCustomTab(m_authDetails, tr("Auth"), 1);
     activateTab(0);
 
     m_feedDetails->loadCategories(m_serviceRoot->getSubTreeCategories(), m_serviceRoot, m_parentToSelect);

src/librssguard/CMakeLists.txt

@@ -307,6 +307,8 @@ set(SOURCES
   services/abstract/gui/formcategorydetails.h
   services/abstract/gui/formfeeddetails.cpp
   services/abstract/gui/formfeeddetails.h
+  services/abstract/gui/httpheadersdetails.cpp
+  services/abstract/gui/httpheadersdetails.h
   services/abstract/gui/multifeededitcheckbox.cpp
   services/abstract/gui/multifeededitcheckbox.h
   services/abstract/importantnode.cpp
@@ -371,6 +373,7 @@ set(UI_FILES
   services/abstract/gui/formaddeditprobe.ui
   services/abstract/gui/formcategorydetails.ui
   services/abstract/gui/formfeeddetails.ui
+  services/abstract/gui/httpheadersdetails.ui
 )
 
 if(ENABLE_MEDIAPLAYER)

src/librssguard/network-web/basenetworkaccessmanager.cpp

@@ -73,20 +73,20 @@ QNetworkReply* BaseNetworkAccessManager::createRequest(QNetworkAccessManager::Op
   new_request.setRawHeader(HTTP_HEADERS_COOKIE, QSL("JSESSIONID= ").toLocal8Bit());
 
   auto custom_ua = qApp->web()->customUserAgent();
+  auto existing_ua = new_request.rawHeader(QSL(HTTP_HEADERS_USER_AGENT));
 
-  if (custom_ua.isEmpty()) {
+  if (existing_ua.isEmpty()) {
-    new_request.setRawHeader(HTTP_HEADERS_USER_AGENT, HTTP_COMPLETE_USERAGENT);
+    if (custom_ua.isEmpty()) {
-  }
+      new_request.setRawHeader(HTTP_HEADERS_USER_AGENT, HTTP_COMPLETE_USERAGENT);
-  else {
+    }
-    new_request.setRawHeader(HTTP_HEADERS_USER_AGENT, custom_ua.toLocal8Bit());
+    else {
+      new_request.setRawHeader(HTTP_HEADERS_USER_AGENT, custom_ua.toLocal8Bit());
+    }
   }
 
   auto reply = QNetworkAccessManager::createRequest(op, new_request, outgoingData);
-
   auto ssl_conf = reply->sslConfiguration();
 
-  auto aa = ssl_conf.backendConfiguration();
-
   ssl_conf.setPeerVerifyMode(QSslSocket::PeerVerifyMode::VerifyNone);
   ssl_conf.setSslOption(QSsl::SslOption::SslOptionDisableLegacyRenegotiation, false);
 

src/librssguard/services/abstract/gui/httpheadersdetails.cpp

@@ -0,0 +1,44 @@
+// For license of this file, see <project-root-folder>/LICENSE.md.
+
+#include "services/abstract/gui/httpheadersdetails.h"
+
+#include "definitions/definitions.h"
+
+#include "ui_httpheadersdetails.h"
+
+#include <QRegularExpression>
+
+HttpHeadersDetails::HttpHeadersDetails(QWidget* parent) : QWidget(parent), m_ui(new Ui::HttpHeadersDetails()) {
+  m_ui->setupUi(this);
+  m_ui->m_helpHttpHeaders
+    ->setHelpText(tr("Enter each key/value HTTP header pair on separate line. Note that all spaces are significant "
+                     "and that header names are case-sensitive. "
+                     "Also, make sure to separate key from value with '=', like the example below:") +
+                    QSL("<br/><br/><b>HeaderKey=HeaderValue</b>"),
+                  false);
+}
+
+void HttpHeadersDetails::loadHttpHeaders(const QVariantHash& headers) {
+  m_ui->m_txtHttpHeaders->clear();
+
+  for (auto i = headers.cbegin(), end = headers.cend(); i != end; i++) {
+    m_ui->m_txtHttpHeaders->appendPlainText(QSL("%1=%2").arg(i.key(), i.value().toString()));
+  }
+}
+
+QVariantHash HttpHeadersDetails::httpHeaders() const {
+  QVariantHash h;
+
+  QRegularExpression exp(QSL("^([^=]+)=(.+)$"), QRegularExpression::PatternOption::MultilineOption);
+  auto exp_match = exp.globalMatch(m_ui->m_txtHttpHeaders->toPlainText());
+
+  while (exp_match.hasNext()) {
+    auto match = exp_match.next();
+
+    h.insert(match.captured(1), match.captured(2));
+  }
+
+  return h;
+}
+
+HttpHeadersDetails::~HttpHeadersDetails() = default;

src/librssguard/services/abstract/gui/httpheadersdetails.h

@@ -0,0 +1,29 @@
+// For license of this file, see <project-root-folder>/LICENSE.md.
+
+#ifndef HTTPHEADERSDETAILS_H
+#define HTTPHEADERSDETAILS_H
+
+#include "network-web/networkfactory.h"
+#include "services/abstract/feed.h"
+
+#include <QWidget>
+
+namespace Ui {
+  class HttpHeadersDetails;
+}
+
+class RSSGUARD_DLLSPEC HttpHeadersDetails : public QWidget {
+    Q_OBJECT
+
+  public:
+    explicit HttpHeadersDetails(QWidget* parent = nullptr);
+    virtual ~HttpHeadersDetails();
+
+    void loadHttpHeaders(const QVariantHash& headers);
+    QVariantHash httpHeaders() const;
+
+  private:
+    QScopedPointer<Ui::HttpHeadersDetails> m_ui;
+};
+
+#endif // HTTPHEADERSDETAILS_H

src/librssguard/services/abstract/gui/httpheadersdetails.ui

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>HttpHeadersDetails</class>
+ <widget class="QWidget" name="HttpHeadersDetails">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>504</width>
+    <height>207</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QFormLayout" name="formLayout">
+   <item row="0" column="0">
+    <widget class="MultiFeedEditCheckBox" name="m_mcbHttpHeaders"/>
+   </item>
+   <item row="1" column="0" colspan="2">
+    <widget class="HelpSpoiler" name="m_helpHttpHeaders" native="true"/>
+   </item>
+   <item row="0" column="1">
+    <widget class="QPlainTextEdit" name="m_txtHttpHeaders"/>
+   </item>
+   <item row="2" column="0" colspan="2">
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Orientation::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>40</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+  </layout>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>MultiFeedEditCheckBox</class>
+   <extends>QCheckBox</extends>
+   <header>multifeededitcheckbox.h</header>
+  </customwidget>
+  <customwidget>
+   <class>HelpSpoiler</class>
+   <extends>QWidget</extends>
+   <header>helpspoiler.h</header>
+   <container>1</container>
+  </customwidget>
+ </customwidgets>
+ <resources/>
+ <connections/>
+</ui>