Bug 822869 - Expand user options and limit default behavior for sending of HTTP referers. r=jduell
authorDan Auerbach <dan.t.auerbach@gmail.com>
Fri, 15 Nov 2013 21:46:37 -0500
changeset 154917 e544d028da74
parent 154916 444714c3820a
child 154918 8faa1855c483
push id25657
push userMs2ger@gmail.com
push date2013-11-17 13:24 +0000
treeherdermozilla-central@0e88f511e067 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjduell
bugs822869
milestone28.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 822869 - Expand user options and limit default behavior for sending of HTTP referers. r=jduell
modules/libpref/src/init/all.js
netwerk/protocol/http/HttpBaseChannel.cpp
netwerk/protocol/http/HttpBaseChannel.h
netwerk/protocol/http/nsHttpHandler.cpp
netwerk/protocol/http/nsHttpHandler.h
netwerk/test/unit/test_referrer.js
netwerk/test/unit/xpcshell.ini
--- a/modules/libpref/src/init/all.js
+++ b/modules/libpref/src/init/all.js
@@ -1030,17 +1030,26 @@ pref("network.http.max-persistent-connec
 // amount of time (in seconds) to suspend pending requests, before spawning a
 // new connection, once the limit on the number of persistent connections per
 // host has been reached.  however, a new connection will not be created if
 // max-connections or max-connections-per-server has also been reached.
 pref("network.http.request.max-start-delay", 10);
 
 // Headers
 pref("network.http.accept.default", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
-pref("network.http.sendRefererHeader",      2); // 0=don't send any, 1=send only on clicks, 2=send on image requests as well
+
+// Prefs allowing granular control of referers
+// 0=don't send any, 1=send only on clicks, 2=send on image requests as well
+pref("network.http.sendRefererHeader",      2); 
+// false=real referer, true=spoof referer (use target URI as referer)                                              
+pref("network.http.referer.spoofSource", false); 
+// 0=full URI, 1=scheme+host+port+path, 2=scheme+host+port
+pref("network.http.referer.trimmingPolicy", 0); 
+// 0=always send, 1=send iff base domains match, 2=send iff hosts match
+pref("network.http.referer.XOriginPolicy", 0); 
 
 // Controls whether we send HTTPS referres to other HTTPS sites.
 // By default this is enabled for compatibility (see bug 141641)
 pref("network.http.sendSecureXSiteReferrer", true);
 
 // Maximum number of consecutive redirects before aborting.
 pref("network.http.redirection-limit", 20);
 
--- a/netwerk/protocol/http/HttpBaseChannel.cpp
+++ b/netwerk/protocol/http/HttpBaseChannel.cpp
@@ -848,54 +848,73 @@ HttpBaseChannel::SetReferrer(nsIURI *ref
 
   // clear existing referrer, if any
   mReferrer = nullptr;
   mRequestHead.ClearHeader(nsHttp::Referer);
 
   if (!referrer)
       return NS_OK;
 
+  // 0: never send referer
+  // 1: send referer for direct user action
+  // 2: always send referer
+  uint32_t userReferrerLevel = gHttpHandler->ReferrerLevel();
+
+  // false: use real referrer
+  // true: spoof with URI of the current request
+  bool userSpoofReferrerSource = gHttpHandler->SpoofReferrerSource();
+
+  // 0: full URI
+  // 1: scheme+host+port+path
+  // 2: scheme+host+port
+  int userReferrerTrimmingPolicy = gHttpHandler->ReferrerTrimmingPolicy();
+
+  // 0: send referer no matter what
+  // 1: send referer ONLY when base domains match
+  // 2: send referer ONLY when hosts match
+  int userReferrerXOriginPolicy = gHttpHandler->ReferrerXOriginPolicy();
+
   // check referrer blocking pref
   uint32_t referrerLevel;
   if (mLoadFlags & LOAD_INITIAL_DOCUMENT_URI)
     referrerLevel = 1; // user action
   else
     referrerLevel = 2; // inline content
-  if (gHttpHandler->ReferrerLevel() < referrerLevel)
+  if (userReferrerLevel < referrerLevel)
     return NS_OK;
 
   nsCOMPtr<nsIURI> referrerGrip;
   nsresult rv;
   bool match;
 
   //
   // Strip off "wyciwyg://123/" from wyciwyg referrers.
   //
   // XXX this really belongs elsewhere since wyciwyg URLs aren't part of necko.
-  //     perhaps some sort of generic nsINestedURI could be used.  then, if an URI
-  //     fails the whitelist test, then we could check for an inner URI and try
-  //     that instead.  though, that might be too automatic.
+  //   perhaps some sort of generic nsINestedURI could be used.  then, if an URI
+  //   fails the whitelist test, then we could check for an inner URI and try
+  //   that instead.  though, that might be too automatic.
   //
   rv = referrer->SchemeIs("wyciwyg", &match);
   if (NS_FAILED(rv)) return rv;
   if (match) {
     nsAutoCString path;
     rv = referrer->GetPath(path);
     if (NS_FAILED(rv)) return rv;
 
     uint32_t pathLength = path.Length();
     if (pathLength <= 2) return NS_ERROR_FAILURE;
 
-    // Path is of the form "//123/http://foo/bar", with a variable number of digits.
-    // To figure out where the "real" URL starts, search path for a '/', starting at
-    // the third character.
+    // Path is of the form "//123/http://foo/bar", with a variable number of
+    // digits. To figure out where the "real" URL starts, search path for a
+    // '/', starting at the third character.
     int32_t slashIndex = path.FindChar('/', 2);
     if (slashIndex == kNotFound) return NS_ERROR_FAILURE;
 
-    // Get the charset of the original URI so we can pass it to our fixed up URI.
+    // Get charset of the original URI so we can pass it to our fixed up URI.
     nsAutoCString charset;
     referrer->GetOriginCharset(charset);
 
     // Replace |referrer| with a URI without wyciwyg://123/.
     rv = NS_NewURI(getter_AddRefs(referrerGrip),
                    Substring(path, slashIndex + 1, pathLength - slashIndex - 1),
                    charset.get());
     if (NS_FAILED(rv)) return rv;
@@ -905,17 +924,16 @@ HttpBaseChannel::SetReferrer(nsIURI *ref
 
   //
   // block referrer if not on our white list...
   //
   static const char *const referrerWhiteList[] = {
     "http",
     "https",
     "ftp",
-    "gopher",
     nullptr
   };
   match = false;
   const char *const *scheme = referrerWhiteList;
   for (; *scheme && !match; ++scheme) {
     rv = referrer->SchemeIs(*scheme, &match);
     if (NS_FAILED(rv)) return rv;
   }
@@ -957,23 +975,96 @@ HttpBaseChannel::SetReferrer(nsIURI *ref
   // we need to clone the referrer, so we can:
   //  (1) modify it
   //  (2) keep a reference to it after returning from this function
   //
   // Use CloneIgnoringRef to strip away any fragment per RFC 2616 section 14.36
   rv = referrer->CloneIgnoringRef(getter_AddRefs(clone));
   if (NS_FAILED(rv)) return rv;
 
+  nsAutoCString currentHost;
+  nsAutoCString referrerHost;
+
+  rv = mURI->GetAsciiHost(currentHost);
+  if (NS_FAILED(rv)) return rv;
+
+  rv = clone->GetAsciiHost(referrerHost);
+  if (NS_FAILED(rv)) return rv;
+
+  // check policy for sending ref only when hosts match
+  if (userReferrerXOriginPolicy == 2 && !currentHost.Equals(referrerHost))
+    return NS_OK;
+
+  if (userReferrerXOriginPolicy == 1) {
+    nsAutoCString currentDomain = currentHost;
+    nsAutoCString referrerDomain = referrerHost;
+    uint32_t extraDomains = 0;
+    nsCOMPtr<nsIEffectiveTLDService> eTLDService = do_GetService(
+      NS_EFFECTIVETLDSERVICE_CONTRACTID);
+    if (eTLDService) {
+      rv = eTLDService->GetBaseDomain(mURI, extraDomains, currentDomain);
+      if (NS_FAILED(rv)) return rv;
+      rv = eTLDService->GetBaseDomain(clone, extraDomains, referrerDomain); 
+      if (NS_FAILED(rv)) return rv;
+    }
+
+    // check policy for sending only when effective top level domain matches.
+    // this falls back on using host if eTLDService does not work
+    if (!currentDomain.Equals(referrerDomain))
+      return NS_OK;
+  }
+
+  // send spoofed referrer if desired
+  if (userSpoofReferrerSource) {
+    nsCOMPtr<nsIURI> mURIclone;
+    rv = mURI->CloneIgnoringRef(getter_AddRefs(mURIclone));
+    if (NS_FAILED(rv)) return rv;
+    clone = mURIclone;
+    currentHost = referrerHost;
+  }
+
   // strip away any userpass; we don't want to be giving out passwords ;-)
   rv = clone->SetUserPass(EmptyCString());
   if (NS_FAILED(rv)) return rv;
 
   nsAutoCString spec;
-  rv = clone->GetAsciiSpec(spec);
-  if (NS_FAILED(rv)) return rv;
+
+  // check how much referer to send
+  switch (userReferrerTrimmingPolicy) {
+
+  case 1: {
+    // scheme+host+port+path
+    nsAutoCString prepath, path;
+    rv = clone->GetPrePath(prepath);
+    if (NS_FAILED(rv)) return rv;
+
+    nsCOMPtr<nsIURL> url(do_QueryInterface(clone));
+    if (!url) {
+      // if this isn't a url, play it safe
+      // and just send the prepath
+      spec = prepath;
+      break;
+    }
+    rv = url->GetFilePath(path);
+    if (NS_FAILED(rv)) return rv;
+    spec = prepath + path;
+    break;
+  }
+  case 2:
+    // scheme+host+port
+    rv = clone->GetPrePath(spec);
+    if (NS_FAILED(rv)) return rv;
+    break;
+
+  default:
+    // full URI
+    rv = clone->GetAsciiSpec(spec);
+    if (NS_FAILED(rv)) return rv;
+    break;
+  }
 
   // finally, remember the referrer URI and set the Referer header.
   mReferrer = clone;
   mRequestHead.SetHeader(nsHttp::Referer, spec);
   return NS_OK;
 }
 
 NS_IMETHODIMP
--- a/netwerk/protocol/http/HttpBaseChannel.h
+++ b/netwerk/protocol/http/HttpBaseChannel.h
@@ -18,16 +18,17 @@
 #include "nsIEncodedChannel.h"
 #include "nsIHttpChannel.h"
 #include "nsHttpHandler.h"
 #include "nsIHttpChannelInternal.h"
 #include "nsIUploadChannel.h"
 #include "nsIUploadChannel2.h"
 #include "nsIProgressEventSink.h"
 #include "nsIURI.h"
+#include "nsIEffectiveTLDService.h"
 #include "nsIStringEnumerator.h"
 #include "nsISupportsPriority.h"
 #include "nsIApplicationCache.h"
 #include "nsIResumableChannel.h"
 #include "nsITraceableChannel.h"
 #include "nsILoadContext.h"
 #include "mozilla/net/NeckoCommon.h"
 #include "nsThreadUtils.h"
--- a/netwerk/protocol/http/nsHttpHandler.cpp
+++ b/netwerk/protocol/http/nsHttpHandler.cpp
@@ -140,16 +140,19 @@ NewURI(const nsACString &aSpec,
 nsHttpHandler *gHttpHandler = nullptr;
 
 nsHttpHandler::nsHttpHandler()
     : mConnMgr(nullptr)
     , mHttpVersion(NS_HTTP_VERSION_1_1)
     , mProxyHttpVersion(NS_HTTP_VERSION_1_1)
     , mCapabilities(NS_HTTP_ALLOW_KEEPALIVE)
     , mReferrerLevel(0xff) // by default we always send a referrer
+    , mSpoofReferrerSource(false)
+    , mReferrerTrimmingPolicy(0)
+    , mReferrerXOriginPolicy(0)
     , mFastFallbackToIPv4(false)
     , mProxyPipelining(true)
     , mIdleTimeout(PR_SecondsToInterval(10))
     , mSpdyTimeout(PR_SecondsToInterval(180))
     , mMaxRequestAttempts(10)
     , mMaxRequestDelay(10)
     , mIdleSynTimeout(250)
     , mPipeliningEnabled(false)
@@ -897,16 +900,34 @@ nsHttpHandler::PrefsChanged(nsIPrefBranc
     }
 
     if (PREF_CHANGED(HTTP_PREF("sendRefererHeader"))) {
         rv = prefs->GetIntPref(HTTP_PREF("sendRefererHeader"), &val);
         if (NS_SUCCEEDED(rv))
             mReferrerLevel = (uint8_t) clamped(val, 0, 0xff);
     }
 
+    if (PREF_CHANGED(HTTP_PREF("referer.spoofSource"))) {
+        rv = prefs->GetBoolPref(HTTP_PREF("referer.spoofSource"), &cVar);
+        if (NS_SUCCEEDED(rv))
+            mSpoofReferrerSource = cVar;
+    }
+
+    if (PREF_CHANGED(HTTP_PREF("referer.trimmingPolicy"))) {
+        rv = prefs->GetIntPref(HTTP_PREF("referer.trimmingPolicy"), &val);
+        if (NS_SUCCEEDED(rv))
+            mReferrerTrimmingPolicy = (uint8_t) clamped(val, 0, 0xff);
+    }
+
+    if (PREF_CHANGED(HTTP_PREF("referer.XOriginPolicy"))) {
+        rv = prefs->GetIntPref(HTTP_PREF("referer.XOriginPolicy"), &val);
+        if (NS_SUCCEEDED(rv))
+            mReferrerXOriginPolicy = (uint8_t) clamped(val, 0, 0xff);
+    }
+
     if (PREF_CHANGED(HTTP_PREF("redirection-limit"))) {
         rv = prefs->GetIntPref(HTTP_PREF("redirection-limit"), &val);
         if (NS_SUCCEEDED(rv))
             mRedirectionLimit = (uint8_t) clamped(val, 0, 0xff);
     }
 
     if (PREF_CHANGED(HTTP_PREF("connection-retry-timeout"))) {
         rv = prefs->GetIntPref(HTTP_PREF("connection-retry-timeout"), &val);
--- a/netwerk/protocol/http/nsHttpHandler.h
+++ b/netwerk/protocol/http/nsHttpHandler.h
@@ -71,16 +71,19 @@ public:
                                  uint32_t capabilities);
     bool     IsAcceptableEncoding(const char *encoding);
 
     const nsAFlatCString &UserAgent();
 
     nsHttpVersion  HttpVersion()             { return mHttpVersion; }
     nsHttpVersion  ProxyHttpVersion()        { return mProxyHttpVersion; }
     uint8_t        ReferrerLevel()           { return mReferrerLevel; }
+    bool           SpoofReferrerSource()     { return mSpoofReferrerSource; }
+    uint8_t        ReferrerTrimmingPolicy()  { return mReferrerTrimmingPolicy; }
+    uint8_t        ReferrerXOriginPolicy()   { return mReferrerXOriginPolicy; }
     bool           SendSecureXSiteReferrer() { return mSendSecureXSiteReferrer; }
     uint8_t        RedirectionLimit()        { return mRedirectionLimit; }
     PRIntervalTime IdleTimeout()             { return mIdleTimeout; }
     PRIntervalTime SpdyTimeout()             { return mSpdyTimeout; }
     uint16_t       MaxRequestAttempts()      { return mMaxRequestAttempts; }
     const char    *DefaultSocketType()       { return mDefaultSocketType.get(); /* ok to return null */ }
     uint32_t       PhishyUserPassLength()    { return mPhishyUserPassLength; }
     uint8_t        GetQoSBits()              { return mQoSBits; }
@@ -320,16 +323,19 @@ private:
     //
     // prefs
     //
 
     uint8_t  mHttpVersion;
     uint8_t  mProxyHttpVersion;
     uint32_t mCapabilities;
     uint8_t  mReferrerLevel;
+    uint8_t  mSpoofReferrerSource;
+    uint8_t  mReferrerTrimmingPolicy;
+    uint8_t  mReferrerXOriginPolicy;
 
     bool mFastFallbackToIPv4;
     bool mProxyPipelining;
     PRIntervalTime mIdleTimeout;
     PRIntervalTime mSpdyTimeout;
 
     uint16_t mMaxRequestAttempts;
     uint16_t mMaxRequestDelay;
new file mode 100644
--- /dev/null
+++ b/netwerk/test/unit/test_referrer.js
@@ -0,0 +1,76 @@
+var ios = Cc["@mozilla.org/network/io-service;1"].
+    getService(Ci.nsIIOService);
+
+function getTestReferrer(server_uri, referer_uri) {
+  var chan = ios.newChannel(server_uri, "", null);
+  chan.QueryInterface(Components.interfaces.nsIHttpChannel);
+  chan.referrer = ios.newURI(referer_uri, null, null);
+  var header = null;
+  try {
+    header = chan.getRequestHeader("Referer");
+  }
+  catch (NS_ERROR_NOT_AVAILABLE) {}
+  return header;
+}
+
+function run_test() {
+  var prefs = Cc["@mozilla.org/preferences-service;1"]
+                .getService(Components.interfaces.nsIPrefBranch);
+
+  var server_uri = "http://bar.examplesite.com/path2";
+  var server_uri_2 = "http://bar.example.com/anotherpath";
+  var referer_uri = "http://foo.example.com/path";
+  var referer_uri_2 = "http://bar.examplesite.com/path3?q=blah";
+  var referer_uri_2_anchor = "http://bar.examplesite.com/path3?q=blah#anchor";
+
+  // for https tests
+  var server_uri_https = "https://bar.example.com/anotherpath";
+  var referer_uri_https = "https://bar.example.com/path3?q=blah";
+
+  // tests for sendRefererHeader
+  prefs.setIntPref("network.http.sendRefererHeader", 0);
+  do_check_null(getTestReferrer(server_uri, referer_uri));
+  prefs.setIntPref("network.http.sendRefererHeader", 2);
+  do_check_eq(getTestReferrer(server_uri, referer_uri), referer_uri);
+
+  // test that https ref is not sent to http
+  do_check_null(getTestReferrer(server_uri_2, referer_uri_https));
+
+  // tests for referer.spoofSource
+  prefs.setBoolPref("network.http.referer.spoofSource", true);
+  do_check_eq(getTestReferrer(server_uri, referer_uri), server_uri);
+  prefs.setBoolPref("network.http.referer.spoofSource", false);
+  do_check_eq(getTestReferrer(server_uri, referer_uri), referer_uri);
+
+  // tests for referer.XOriginPolicy
+  prefs.setIntPref("network.http.referer.XOriginPolicy", 2);
+  do_check_null(getTestReferrer(server_uri_2, referer_uri));
+  do_check_eq(getTestReferrer(server_uri, referer_uri_2), referer_uri_2);
+  prefs.setIntPref("network.http.referer.XOriginPolicy", 1);
+  do_check_eq(getTestReferrer(server_uri_2, referer_uri), referer_uri);
+  do_check_null(getTestReferrer(server_uri, referer_uri));
+  // https test
+  do_check_eq(getTestReferrer(server_uri_https, referer_uri_https), referer_uri_https);
+  prefs.setIntPref("network.http.referer.XOriginPolicy", 0);
+  do_check_eq(getTestReferrer(server_uri, referer_uri), referer_uri);
+
+  // tests for referer.trimmingPolicy
+  prefs.setIntPref("network.http.referer.trimmingPolicy", 1);
+  do_check_eq(getTestReferrer(server_uri, referer_uri_2), "http://bar.examplesite.com/path3");
+  prefs.setIntPref("network.http.referer.trimmingPolicy", 2);
+  do_check_eq(getTestReferrer(server_uri, referer_uri_2), "http://bar.examplesite.com");
+  // https test
+  do_check_eq(getTestReferrer(server_uri_https, referer_uri_https), "https://bar.example.com");
+  prefs.setIntPref("network.http.referer.trimmingPolicy", 0);
+  // test that anchor is lopped off in ordinary case
+  do_check_eq(getTestReferrer(server_uri, referer_uri_2_anchor), referer_uri_2);
+
+  // combination test: send spoofed path-only when hosts match
+  var combo_referer_uri = "http://blah.foo.com/path?q=hot";
+  var dest_uri = "http://blah.foo.com:9999/spoofedpath?q=bad";
+  prefs.setIntPref("network.http.referer.trimmingPolicy", 1);
+  prefs.setBoolPref("network.http.referer.spoofSource", true);
+  prefs.setIntPref("network.http.referer.XOriginPolicy", 2);
+  do_check_eq(getTestReferrer(dest_uri, combo_referer_uri), "http://blah.foo.com:9999/spoofedpath");
+  do_check_null(getTestReferrer(dest_uri, "http://gah.foo.com/anotherpath"));
+}
--- a/netwerk/test/unit/xpcshell.ini
+++ b/netwerk/test/unit/xpcshell.ini
@@ -290,13 +290,14 @@ skip-if = os == "android"
 [test_bug856978.js]
 [test_unix_domain.js]
 # The xpcshell temp directory on Android doesn't seem to let us create
 # Unix domain sockets. (Perhaps it's a FAT filesystem?)
 skip-if = os == "android"
 [test_addr_in_use_error.js]
 [test_about_networking.js]
 [test_ping_aboutnetworking.js]
+[test_referrer.js]
 [test_seer.js]
 # Android version detection w/in gecko does not work right on infra, so we just
 # disable this test on all android versions, even though it's enabled on 2.3+ in
 # the wild.
 skip-if = os == "android"