- Add cache awareness to CefURLRequest (issue #2283)

- Properly set referer values on server requests (issue #2278)

include/internal/cef_types.h

@@ -1222,15 +1222,26 @@ typedef enum {
   UR_FLAG_NONE = 0,
 
   ///
-  // If set the cache will be skipped when handling the request.
+  // If set the cache will be skipped when handling the request. Setting this
+  // value is equivalent to specifying the "Cache-Control: no-cache" request
+  // header. Setting this value in combination with UR_FLAG_ONLY_FROM_CACHE will
+  // cause the request to fail.
   ///
   UR_FLAG_SKIP_CACHE = 1 << 0,
 
+  ///
+  // If set the request will fail if it cannot be served from the cache (or some
+  // equivalent local store). Setting this value is equivalent to specifying the
+  // "Cache-Control: only-if-cached" request header. Setting this value in
+  // combination with UR_FLAG_SKIP_CACHE will cause the request to fail.
+  ///
+  UR_FLAG_ONLY_FROM_CACHE = 1 << 1,
+
   ///
   // If set user name, password, and cookies may be sent with the request, and
   // cookies may be saved from the response.
   ///
-  UR_FLAG_ALLOW_CACHED_CREDENTIALS = 1 << 1,
+  UR_FLAG_ALLOW_STORED_CREDENTIALS = 1 << 2,
 
   ///
   // If set upload progress events will be generated when a request has a body.
@@ -1240,14 +1251,14 @@ typedef enum {
   ///
   // If set the CefURLRequestClient::OnDownloadData method will not be called.
   ///
-  UR_FLAG_NO_DOWNLOAD_DATA = 1 << 6,
+  UR_FLAG_NO_DOWNLOAD_DATA = 1 << 4,
 
   ///
   // If set 5XX redirect errors will be propagated to the observer instead of
   // automatically re-tried. This currently only applies for requests
   // originated in the browser process.
   ///
-  UR_FLAG_NO_RETRY_ON_5XX = 1 << 7,
+  UR_FLAG_NO_RETRY_ON_5XX = 1 << 5,
 } cef_urlrequest_flags_t;
 
 ///

libcef/browser/server_impl.cc

@@ -40,6 +40,8 @@
 
 namespace {
 
+const char kReferrerLowerCase[] = "referer";
+
 // Wrap a string in a unique_ptr to avoid extra copies.
 std::unique_ptr<std::string> CreateUniqueString(const void* data,
                                                 size_t data_size) {
@@ -67,18 +69,27 @@ CefRefPtr<CefRequest> CreateRequest(const std::string& address,
     post_data->AddElement(post_element);
   }
 
+  std::string referer;
+
   CefRequest::HeaderMap header_map;
   if (!info.headers.empty()) {
     net::HttpServerRequestInfo::HeadersMap::const_iterator it =
         info.headers.begin();
     for (; it != info.headers.end(); ++it) {
-      header_map.insert(std::make_pair(it->first, it->second));
+      // Don't include Referer in the header map.
+      if (base::LowerCaseEqualsASCII(it->first, kReferrerLowerCase)) {
+        referer = it->second;
+      } else {
+        header_map.insert(std::make_pair(it->first, it->second));
+      }
     }
   }
 
   CefRefPtr<CefRequestImpl> request = new CefRequestImpl();
   request->Set((is_websocket ? "ws://" : "http://") + address + info.path,
                info.method, post_data, header_map);
+  if (!referer.empty())
+    request->SetReferrer(referer, REFERRER_POLICY_DEFAULT);
   request->SetReadOnly(true);
   return request;
 }

libcef/common/request_impl.cc

@@ -2,6 +2,7 @@
 // reserved. Use of this source code is governed by a BSD-style license that
 // can be found in the LICENSE file.
 
+#include <algorithm>
 #include <limits>
 #include <string>
 #include <utility>
@@ -44,8 +45,14 @@ namespace {
 
 const char kReferrerLowerCase[] = "referer";
 const char kContentTypeLowerCase[] = "content-type";
+const char kCacheControlLowerCase[] = "cache-control";
+const char kCacheControlDirectiveNoCacheLowerCase[] = "no-cache";
+const char kCacheControlDirectiveOnlyIfCachedLowerCase[] = "only-if-cached";
 const char kApplicationFormURLEncoded[] = "application/x-www-form-urlencoded";
 
+// Mask of values that configure the cache policy.
+const int kURCachePolicyMask = (UR_FLAG_SKIP_CACHE | UR_FLAG_ONLY_FROM_CACHE);
+
 // A subclass of net::UploadBytesElementReader that keeps the associated
 // UploadElement alive until the request completes.
 class BytesElementReader : public net::UploadBytesElementReader {
@@ -100,6 +107,71 @@ std::string GetURLRequestReferrer(const GURL& referrer_url) {
   return referrer_url.spec();
 }
 
+// Returns the cef_urlrequest_flags_t policy specified by the Cache-Control
+// request header directives, if any. The directives are case-insensitive and
+// some have an optional argument. Multiple directives are comma-separated.
+// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
+// for details.
+int GetCacheControlHeaderPolicy(CefRequest::HeaderMap headerMap) {
+  std::string line;
+
+  // Extract the Cache-Control header line.
+  {
+    CefRequest::HeaderMap::const_iterator it = headerMap.begin();
+    for (; it != headerMap.end(); ++it) {
+      if (base::LowerCaseEqualsASCII(it->first.ToString(),
+                                     kCacheControlLowerCase)) {
+        line = it->second;
+        break;
+      }
+    }
+  }
+
+  int flags = 0;
+
+  if (!line.empty()) {
+    std::transform(line.begin(), line.end(), line.begin(), ::tolower);
+
+    std::vector<base::StringPiece> pieces = base::SplitStringPiece(
+        line, ",", base::TRIM_WHITESPACE, base::SPLIT_WANT_NONEMPTY);
+    for (const auto& piece : pieces) {
+      if (base::LowerCaseEqualsASCII(piece,
+                                     kCacheControlDirectiveNoCacheLowerCase)) {
+        flags |= UR_FLAG_SKIP_CACHE;
+      } else if (base::LowerCaseEqualsASCII(
+                     piece, kCacheControlDirectiveOnlyIfCachedLowerCase)) {
+        flags |= UR_FLAG_ONLY_FROM_CACHE;
+      }
+    }
+  }
+
+  return flags;
+}
+
+// Convert cef_urlrequest_flags_t to blink::WebCachePolicy.
+blink::WebCachePolicy GetWebCachePolicy(int ur_flags) {
+  if ((ur_flags & kURCachePolicyMask) == kURCachePolicyMask) {
+    return blink::WebCachePolicy::kBypassCacheLoadOnlyFromCache;
+  } else if (ur_flags & UR_FLAG_SKIP_CACHE) {
+    return blink::WebCachePolicy::kBypassingCache;
+  } else if (ur_flags & UR_FLAG_ONLY_FROM_CACHE) {
+    return blink::WebCachePolicy::kReturnCacheDataDontLoad;
+  }
+  return blink::WebCachePolicy::kUseProtocolCachePolicy;
+}
+
+// Convert blink::WebCachePolicy to cef_urlrequest_flags_t.
+int GetURCachePolicy(blink::WebCachePolicy web_policy) {
+  if (web_policy == blink::WebCachePolicy::kBypassCacheLoadOnlyFromCache) {
+    return kURCachePolicyMask;
+  } else if (web_policy == blink::WebCachePolicy::kBypassingCache) {
+    return UR_FLAG_SKIP_CACHE;
+  } else if (web_policy == blink::WebCachePolicy::kReturnCacheDataDontLoad) {
+    return UR_FLAG_ONLY_FROM_CACHE;
+  }
+  return 0;
+}
+
 blink::WebString FilePathStringToWebString(
     const base::FilePath::StringType& str) {
 #if defined(OS_POSIX)
@@ -216,10 +288,9 @@ CefRefPtr<CefRequest> CefRequest::Create() {
 
 CefRequestImpl::CefRequestImpl() : read_only_(false), track_changes_(false) {
   // Verify that our enum matches Chromium's values.
-  static_assert(
+  static_assert(static_cast<int>(REFERRER_POLICY_LAST_VALUE) ==
-      static_cast<int>(REFERRER_POLICY_LAST_VALUE) ==
+                    static_cast<int>(net::URLRequest::MAX_REFERRER_POLICY),
-      static_cast<int>(net::URLRequest::MAX_REFERRER_POLICY),
+                "enum mismatch");
-      "enum mismatch");
 
   base::AutoLock lock_scope(lock_);
   Reset();
@@ -495,10 +566,9 @@ void CefRequestImpl::Set(const blink::WebURLRequest& request) {
 
   site_for_cookies_ = request.SiteForCookies();
 
-  if (request.GetCachePolicy() == blink::WebCachePolicy::kBypassingCache)
+  flags_ |= GetURCachePolicy(request.GetCachePolicy());
-    flags_ |= UR_FLAG_SKIP_CACHE;
   if (request.AllowStoredCredentials())
-    flags_ |= UR_FLAG_ALLOW_CACHED_CREDENTIALS;
+    flags_ |= UR_FLAG_ALLOW_STORED_CREDENTIALS;
   if (request.ReportUploadProgress())
     flags_ |= UR_FLAG_REPORT_UPLOAD_PROGRESS;
 }
@@ -545,12 +615,16 @@ void CefRequestImpl::Get(blink::WebURLRequest& request,
   if (!site_for_cookies_.is_empty())
     request.SetSiteForCookies(site_for_cookies_);
 
-  request.SetCachePolicy((flags_ & UR_FLAG_SKIP_CACHE)
+  int flags = flags_;
-                             ? blink::WebCachePolicy::kBypassingCache
+  if (!(flags & kURCachePolicyMask)) {
-                             : blink::WebCachePolicy::kUseProtocolCachePolicy);
+    // Only consider the Cache-Control directives when a cache policy is not
+    // explicitly set on the request.
+    flags |= GetCacheControlHeaderPolicy(headermap_);
+  }
+  request.SetCachePolicy(GetWebCachePolicy(flags));
 
   SETBOOLFLAG(request, flags_, SetAllowStoredCredentials,
-              UR_FLAG_ALLOW_CACHED_CREDENTIALS);
+              UR_FLAG_ALLOW_STORED_CREDENTIALS);
   SETBOOLFLAG(request, flags_, SetReportUploadProgress,
               UR_FLAG_REPORT_UPLOAD_PROGRESS);
 }
@@ -576,12 +650,14 @@ void CefRequestImpl::Get(const CefMsg_LoadRequest_Params& params,
     }
   }
 
+  CefRequest::HeaderMap headerMap;
   if (!params.headers.empty()) {
     for (net::HttpUtil::HeadersIterator i(params.headers.begin(),
                                           params.headers.end(), "\n");
          i.GetNext();) {
       request.AddHTTPHeaderField(blink::WebString::FromUTF8(i.name()),
                                  blink::WebString::FromUTF8(i.values()));
+      headerMap.insert(std::make_pair(i.name(), i.values()));
     }
   }
 
@@ -624,12 +700,16 @@ void CefRequestImpl::Get(const CefMsg_LoadRequest_Params& params,
   if (params.site_for_cookies.is_valid())
     request.SetSiteForCookies(params.site_for_cookies);
 
-  request.SetCachePolicy((params.load_flags & UR_FLAG_SKIP_CACHE)
+  int flags = params.load_flags;
-                             ? blink::WebCachePolicy::kBypassingCache
+  if (!(flags & kURCachePolicyMask)) {
-                             : blink::WebCachePolicy::kUseProtocolCachePolicy);
+    // Only consider the Cache-Control directives when a cache policy is not
+    // explicitly set on the request.
+    flags |= GetCacheControlHeaderPolicy(headerMap);
+  }
+  request.SetCachePolicy(GetWebCachePolicy(flags));
 
   SETBOOLFLAG(request, params.load_flags, SetAllowStoredCredentials,
-              UR_FLAG_ALLOW_CACHED_CREDENTIALS);
+              UR_FLAG_ALLOW_STORED_CREDENTIALS);
   SETBOOLFLAG(request, params.load_flags, SetReportUploadProgress,
               UR_FLAG_REPORT_UPLOAD_PROGRESS);
 }
@@ -726,21 +806,31 @@ void CefRequestImpl::Get(net::URLFetcher& fetcher,
   if (!site_for_cookies_.is_empty())
     fetcher.SetInitiator(url::Origin(site_for_cookies_));
 
-  if (flags_ & UR_FLAG_NO_RETRY_ON_5XX)
+  int flags = flags_;
-    fetcher.SetAutomaticallyRetryOn5xx(false);
+  if (!(flags & kURCachePolicyMask)) {
-
+    // Only consider the Cache-Control directives when a cache policy is not
-  int load_flags = 0;
+    // explicitly set on the request.
-
+    flags |= GetCacheControlHeaderPolicy(headerMap);
-  if (flags_ & UR_FLAG_SKIP_CACHE)
-    load_flags |= net::LOAD_BYPASS_CACHE;
-
-  if (!(flags_ & UR_FLAG_ALLOW_CACHED_CREDENTIALS)) {
-    load_flags |= net::LOAD_DO_NOT_SEND_AUTH_DATA;
-    load_flags |= net::LOAD_DO_NOT_SEND_COOKIES;
-    load_flags |= net::LOAD_DO_NOT_SAVE_COOKIES;
   }
 
-  fetcher.SetLoadFlags(load_flags);
+  if (flags & UR_FLAG_NO_RETRY_ON_5XX)
+    fetcher.SetAutomaticallyRetryOn5xx(false);
+
+  int net_flags = 0;
+
+  if (flags & UR_FLAG_SKIP_CACHE) {
+    net_flags |= net::LOAD_BYPASS_CACHE;
+  }
+  if (flags & UR_FLAG_ONLY_FROM_CACHE) {
+    net_flags |= net::LOAD_ONLY_FROM_CACHE | net::LOAD_SKIP_CACHE_VALIDATION;
+  }
+
+  if (!(flags & UR_FLAG_ALLOW_STORED_CREDENTIALS)) {
+    net_flags |= net::LOAD_DO_NOT_SEND_AUTH_DATA |
+                 net::LOAD_DO_NOT_SEND_COOKIES | net::LOAD_DO_NOT_SAVE_COOKIES;
+  }
+
+  fetcher.SetLoadFlags(net_flags);
 }
 
 void CefRequestImpl::SetReadOnly(bool read_only) {

tests/ceftests/server_unittest.cc

@@ -966,6 +966,7 @@ class StaticHttpRequestRunner : public HttpTestRunner::RequestRunner {
     CefRefPtr<CefRequest> request = CreateTestServerRequest(
         path, "POST", "foo=bar&choo=too", "application/x-www-form-urlencoded",
         request_headers);
+    request->SetReferrer("http://tests/referer.html", REFERRER_POLICY_DEFAULT);
 
     HttpServerResponse response(HttpServerResponse::TYPE_CUSTOM);
     response.response_code = 202;