Bug 822869 - Expand user options and limit default behavior for sending of HTTP referers. r=jduell
authorDan Auerbach <dan.t.auerbach@gmail.com>
Fri, 15 Nov 2013 21:46:37 -0500
changeset 154917 e544d028da74
parent 154916 444714c3820a
child 154918 8faa1855c483
push id25657
push userMs2ger@gmail.com
push date2013-11-17 13:24 +0000
treeherdermozilla-central@0e88f511e067 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjduell
bugs822869
milestone28.0a1
Bug 822869 - Expand user options and limit default behavior for sending of HTTP referers. r=jduell
modules/libpref/src/init/all.js
netwerk/protocol/http/HttpBaseChannel.cpp
netwerk/protocol/http/HttpBaseChannel.h
netwerk/protocol/http/nsHttpHandler.cpp
netwerk/protocol/http/nsHttpHandler.h
netwerk/test/unit/test_referrer.js
netwerk/test/unit/xpcshell.ini
--- a/modules/libpref/src/init/all.js
+++ b/modules/libpref/src/init/all.js
@@ -1030,17 +1030,26 @@ pref("network.http.max-persistent-connec
 // amount of time (in seconds) to suspend pending requests, before spawning a
 // new connection, once the limit on the number of persistent connections per
 // host has been reached.  however, a new connection will not be created if
 // max-connections or max-connections-per-server has also been reached.
 pref("network.http.request.max-start-delay", 10);
 
 // Headers
 pref("network.http.accept.default", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
-pref("network.http.sendRefererHeader",      2); // 0=don't send any, 1=send only on clicks, 2=send on image requests as well
+
+// Prefs allowing granular control of referers
+// 0=don't send any, 1=send only on clicks, 2=send on image requests as well
+pref("network.http.sendRefererHeader",      2); 
+// false=real referer, true=spoof referer (use target URI as referer)                                              
+pref("network.http.referer.spoofSource", false); 
+// 0=full URI, 1=scheme+host+port+path, 2=scheme+host+port
+pref("network.http.referer.trimmingPolicy", 0); 
+// 0=always send, 1=send iff base domains match, 2=send iff hosts match
+pref("network.http.referer.XOriginPolicy", 0); 
 
 // Controls whether we send HTTPS referres to other HTTPS sites.
 // By default this is enabled for compatibility (see bug 141641)
 pref("network.http.sendSecureXSiteReferrer", true);
 
 // Maximum number of consecutive redirects before aborting.
 pref("network.http.redirection-limit", 20);
 
--- a/netwerk/protocol/http/HttpBaseChannel.cpp
+++ b/netwerk/protocol/http/HttpBaseChannel.cpp
@@ -848,54 +848,73 @@ HttpBaseChannel::SetReferrer(nsIURI *ref
 
   // clear existing referrer, if any
   mReferrer = nullptr;
   mRequestHead.ClearHeader(nsHttp::Referer);
 
   if (!referrer)
       return NS_OK;
 
+  // 0: never send referer
+  // 1: send referer for direct user action
+  // 2: always send referer
+  uint32_t userReferrerLevel = gHttpHandler->ReferrerLevel();
+
+  // false: use real referrer
+  // true: spoof with URI of the current request
+  bool userSpoofReferrerSource = gHttpHandler->SpoofReferrerSource();
+
+  // 0: full URI
+  // 1: scheme+host+port+path
+  // 2: scheme+host+port
+  int userReferrerTrimmingPolicy = gHttpHandler->ReferrerTrimmingPolicy();
+
+  // 0: send referer no matter what
+  // 1: send referer ONLY when base domains match
+  // 2: send referer ONLY when hosts match
+  int userReferrerXOriginPolicy = gHttpHandler->ReferrerXOriginPolicy();
+
   // check referrer blocking pref
   uint32_t referrerLevel;
   if (mLoadFlags & LOAD_INITIAL_DOCUMENT_URI)
     referrerLevel = 1; // user action
   else
     referrerLevel = 2; // inline content
-  if (gHttpHandler->ReferrerLevel() < referrerLevel)
+  if (userReferrerLevel < referrerLevel)
     return NS_OK;
 
   nsCOMPtr<nsIURI> referrerGrip;
   nsresult rv;
   bool match;
 
   //
   // Strip off "wyciwyg://123/" from wyciwyg referrers.
   //
   // XXX this really belongs elsewhere since wyciwyg URLs aren't part of necko.
-  //     perhaps some sort of generic nsINestedURI could be used.  then, if an URI
-  //     fails the whitelist test, then we could check for an inner URI and try
-  //     that instead.  though, that might be too automatic.
+  //   perhaps some sort of generic nsINestedURI could be used.  then, if an URI
+  //   fails the whitelist test, then we could check for an inner URI and try
+  //   that instead.  though, that might be too automatic.
   //
   rv = referrer->SchemeIs("wyciwyg", &match);
   if (NS_FAILED(rv)) return rv;
   if (match) {
     nsAutoCString path;
     rv = referrer->GetPath(path);
     if (NS_FAILED(rv)) return rv;
 
     uint32_t pathLength = path.Length();
     if (pathLength <= 2) return NS_ERROR_FAILURE;
 
-    // Path is of the form "//123/http://foo/bar", with a variable number of digits.
-    // To figure out where the "real" URL starts, search path for a '/', starting at
-    // the third character.
+    // Path is of the form "//123/http://foo/bar", with a variable number of
+    // digits. To figure out where the "real" URL starts, search path for a
+    // '/', starting at the third character.
     int32_t slashIndex = path.FindChar('/', 2);
     if (slashIndex == kNotFound) return NS_ERROR_FAILURE;
 
-    // Get the charset of the original URI so we can pass it to our fixed up URI.
+    // Get charset of the original URI so we can pass it to our fixed up URI.
     nsAutoCString charset;
     referrer->GetOriginCharset(charset);
 
     // Replace |referrer| with a URI without wyciwyg://123/.
     rv = NS_NewURI(getter_AddRefs(referrerGrip),
                    Substring(path, slashIndex + 1, pathLength - slashIndex - 1),
                    charset.get());
     if (NS_FAILED(rv)) return rv;
@@ -905,17 +924,16 @@ HttpBaseChannel::SetReferrer(nsIURI *ref
 
   //
   // block referrer if not on our white list...
   //
   static const char *const referrerWhiteList[] = {
     "http",
     "https",
     "ftp",
-    "gopher",
     nullptr
   };
   match = false;
   const char *const *scheme = referrerWhiteList;
   for (; *scheme && !match; ++scheme) {
     rv = referrer->SchemeIs(*scheme, &match);
     if (NS_FAILED(rv)) return rv;
   }
@@ -957,23 +975,96 @@ HttpBaseChannel::SetReferrer(nsIURI *ref
   // we need to clone the referrer, so we can:
   //  (1) modify it
   //  (2) keep a reference to it after returning from this function
   //
   // Use CloneIgnoringRef to strip away any fragment per RFC 2616 section 14.36
   rv = referrer->CloneIgnoringRef(getter_AddRefs(clone));
   if (NS_FAILED(rv)) return rv;
 
+  nsAutoCString currentHost;
+  nsAutoCString referrerHost;
+
+  rv = mURI->GetAsciiHost(currentHost);
+  if (NS_FAILED(rv)) return rv;
+
+  rv = clone->GetAsciiHost(referrerHost);
+  if (NS_FAILED(rv)) return rv;
+
+  // check policy for sending ref only when hosts match
+  if (userReferrerXOriginPolicy == 2 && !currentHost.Equals(referrerHost))
+    return NS_OK;
+
+  if (userReferrerXOriginPolicy == 1) {
+    nsAutoCString currentDomain = currentHost;
+    nsAutoCString referrerDomain = referrerHost;
+    uint32_t extraDomains = 0;
+    nsCOMPtr<nsIEffectiveTLDService> eTLDService = do_GetService(
+      NS_EFFECTIVETLDSERVICE_CONTRACTID);
+    if (eTLDService) {
+      rv = eTLDService->GetBaseDomain(mURI, extraDomains, currentDomain);
+      if (NS_FAILED(rv)) return rv;
+      rv = eTLDService->GetBaseDomain(clone, extraDomains, referrerDomain); 
+      if (NS_FAILED(rv)) return rv;
+    }
+
+    // check policy for sending only when effective top level domain matches.
+    // this falls back on using host if eTLDService does not work
+    if (!currentDomain.Equals(referrerDomain))
+      return NS_OK;
+  }
+
+  // send spoofed referrer if desired
+  if (userSpoofReferrerSource) {
+    nsCOMPtr<nsIURI> mURIclone;
+    rv = mURI->CloneIgnoringRef(getter_AddRefs(mURIclone));
+    if (NS_FAILED(rv)) return rv;
+    clone = mURIclone;
+    currentHost = referrerHost;
+  }
+
   // strip away any userpass; we don't want to be giving out passwords ;-)
   rv = clone->SetUserPass(EmptyCString());
   if (NS_FAILED(rv)) return rv;
 
   nsAutoCString spec;
-  rv = clone->GetAsciiSpec(spec);
-  if (NS_FAILED(rv)) return rv;
+
+  // check how much referer to send
+  switch (userReferrerTrimmingPolicy) {
+
+  case 1: {
+    // scheme+host+port+path
+    nsAutoCString prepath, path;
+    rv = clone->GetPrePath(prepath);
+    if (NS_FAILED(rv)) return rv;
+
+    nsCOMPtr<nsIURL> url(do_QueryInterface(clone));
+    if (!url) {
+      // if this isn't a url, play it safe
+      // and just send the prepath
+      spec = prepath;
+      break;
+    }
+    rv = url->GetFilePath(path);
+    if (NS_FAILED(rv)) return rv;
+    spec = prepath + path;
+    break;
+  }
+  case 2:
+    // scheme+host+port
+    rv = clone->GetPrePath(spec);
+    if (NS_FAILED(rv)) return rv;
+    break;
+
+  default:
+    // full URI
+    rv = clone->GetAsciiSpec(spec);
+    if (NS_FAILED(rv)) return rv;
+    break;
+  }
 
   // finally, remember the referrer URI and set the Referer header.
   mReferrer = clone;
   mRequestHead.SetHeader(nsHttp::Referer, spec);
   return NS_OK;
 }
 
 NS_IMETHODIMP
--- a/netwerk/protocol/http/HttpBaseChannel.h
+++ b/netwerk/protocol/http/HttpBaseChannel.h
@@ -18,16 +18,17 @@
 #include "nsIEncodedChannel.h"
 #include "nsIHttpChannel.h"
 #include "nsHttpHandler.h"
 #include "nsIHttpChannelInternal.h"
 #include "nsIUploadChannel.h"
 #include "nsIUploadChannel2.h"
 #include "nsIProgressEventSink.h"
 #include "nsIURI.h"
+#include "nsIEffectiveTLDService.h"
 #include "nsIStringEnumerator.h"
 #include "nsISupportsPriority.h"
 #include "nsIApplicationCache.h"
 #include "nsIResumableChannel.h"
 #include "nsITraceableChannel.h"
 #include "nsILoadContext.h"
 #include "mozilla/net/NeckoCommon.h"
 #include "nsThreadUtils.h"
--- a/netwerk/protocol/http/nsHttpHandler.cpp
+++ b/netwerk/protocol/http/nsHttpHandler.cpp
@@ -140,16 +140,19 @@ NewURI(const nsACString &aSpec,
 nsHttpHandler *gHttpHandler = nullptr;
 
 nsHttpHandler::nsHttpHandler()
     : mConnMgr(nullptr)
     , mHttpVersion(NS_HTTP_VERSION_1_1)
     , mProxyHttpVersion(NS_HTTP_VERSION_1_1)
     , mCapabilities(NS_HTTP_ALLOW_KEEPALIVE)
     , mReferrerLevel(0xff) // by default we always send a referrer
+    , mSpoofReferrerSource(false)
+    , mReferrerTrimmingPolicy(0)
+    , mReferrerXOriginPolicy(0)
     , mFastFallbackToIPv4(false)
     , mProxyPipelining(true)
     , mIdleTimeout(PR_SecondsToInterval(10))
     , mSpdyTimeout(PR_SecondsToInterval(180))
     , mMaxRequestAttempts(10)
     , mMaxRequestDelay(10)
     , mIdleSynTimeout(250)
     , mPipeliningEnabled(false)
@@ -897,16 +900,34 @@ nsHttpHandler::PrefsChanged(nsIPrefBranc
     }
 
     if (PREF_CHANGED(HTTP_PREF("sendRefererHeader"))) {
         rv = prefs->GetIntPref(HTTP_PREF("sendRefererHeader"), &val);
         if (NS_SUCCEEDED(rv))
             mReferrerLevel = (uint8_t) clamped(val, 0, 0xff);
     }
 
+    if (PREF_CHANGED(HTTP_PREF("referer.spoofSource"))) {
+        rv = prefs->GetBoolPref(HTTP_PREF("referer.spoofSource"), &cVar);
+        if (NS_SUCCEEDED(rv))
+            mSpoofReferrerSource = cVar;
+    }
+
+    if (PREF_CHANGED(HTTP_PREF("referer.trimmingPolicy"))) {
+        rv = prefs->GetIntPref(HTTP_PREF("referer.trimmingPolicy"), &val);
+        if (NS_SUCCEEDED(rv))
+            mReferrerTrimmingPolicy = (uint8_t) clamped(val, 0, 0xff);
+    }
+
+    if (PREF_CHANGED(HTTP_PREF("referer.XOriginPolicy"))) {
+        rv = prefs->GetIntPref(HTTP_PREF("referer.XOriginPolicy"), &val);
+        if (NS_SUCCEEDED(rv))
+            mReferrerXOriginPolicy = (uint8_t) clamped(val, 0, 0xff);
+    }
+
     if (PREF_CHANGED(HTTP_PREF("redirection-limit"))) {
         rv = prefs->GetIntPref(HTTP_PREF("redirection-limit"), &val);
         if (NS_SUCCEEDED(rv))
             mRedirectionLimit = (uint8_t) clamped(val, 0, 0xff);
     }
 
     if (PREF_CHANGED(HTTP_PREF("connection-retry-timeout"))) {
         rv = prefs->GetIntPref(HTTP_PREF("connection-retry-timeout"), &val);
--- a/netwerk/protocol/http/nsHttpHandler.h
+++ b/netwerk/protocol/http/nsHttpHandler.h
@@ -71,16 +71,19 @@ public:
                                  uint32_t capabilities);
     bool     IsAcceptableEncoding(const char *encoding);
 
     const nsAFlatCString &UserAgent();
 
     nsHttpVersion  HttpVersion()             { return mHttpVersion; }
     nsHttpVersion  ProxyHttpVersion()        { return mProxyHttpVersion; }
     uint8_t        ReferrerLevel()           { return mReferrerLevel; }
+    bool           SpoofReferrerSource()     { return mSpoofReferrerSource; }
+    uint8_t        ReferrerTrimmingPolicy()  { return mReferrerTrimmingPolicy; }
+    uint8_t        ReferrerXOriginPolicy()   { return mReferrerXOriginPolicy; }
     bool           SendSecureXSiteReferrer() { return mSendSecureXSiteReferrer; }
     uint8_t        RedirectionLimit()        { return mRedirectionLimit; }
     PRIntervalTime IdleTimeout()             { return mIdleTimeout; }
     PRIntervalTime SpdyTimeout()             { return mSpdyTimeout; }
     uint16_t       MaxRequestAttempts()      { return mMaxRequestAttempts; }
     const char    *DefaultSocketType()       { return mDefaultSocketType.get(); /* ok to return null */ }
     uint32_t       PhishyUserPassLength()    { return mPhishyUserPassLength; }
     uint8_t        GetQoSBits()              { return mQoSBits; }
@@ -320,16 +323,19 @@ private:
     //
     // prefs
     //
 
     uint8_t  mHttpVersion;
     uint8_t  mProxyHttpVersion;
     uint32_t mCapabilities;
     uint8_t  mReferrerLevel;
+    uint8_t  mSpoofReferrerSource;
+    uint8_t  mReferrerTrimmingPolicy;
+    uint8_t  mReferrerXOriginPolicy;
 
     bool mFastFallbackToIPv4;
     bool mProxyPipelining;
     PRIntervalTime mIdleTimeout;
     PRIntervalTime mSpdyTimeout;
 
     uint16_t mMaxRequestAttempts;
     uint16_t mMaxRequestDelay;
new file mode 100644
--- /dev/null
+++ b/netwerk/test/unit/test_referrer.js
@@ -0,0 +1,76 @@
+var ios = Cc["@mozilla.org/network/io-service;1"].
+    getService(Ci.nsIIOService);
+
+function getTestReferrer(server_uri, referer_uri) {
+  var chan = ios.newChannel(server_uri, "", null);
+  chan.QueryInterface(Components.interfaces.nsIHttpChannel);
+  chan.referrer = ios.newURI(referer_uri, null, null);
+  var header = null;
+  try {
+    header = chan.getRequestHeader("Referer");
+  }
+  catch (NS_ERROR_NOT_AVAILABLE) {}
+  return header;
+}
+
+function run_test() {
+  var prefs = Cc["@mozilla.org/preferences-service;1"]
+                .getService(Components.interfaces.nsIPrefBranch);
+
+  var server_uri = "http://bar.examplesite.com/path2";
+  var server_uri_2 = "http://bar.example.com/anotherpath";
+  var referer_uri = "http://foo.example.com/path";
+  var referer_uri_2 = "http://bar.examplesite.com/path3?q=blah";
+  var referer_uri_2_anchor = "http://bar.examplesite.com/path3?q=blah#anchor";
+
+  // for https tests
+  var server_uri_https = "https://bar.example.com/anotherpath";
+  var referer_uri_https = "https://bar.example.com/path3?q=blah";
+
+  // tests for sendRefererHeader
+  prefs.setIntPref("network.http.sendRefererHeader", 0);
+  do_check_null(getTestReferrer(server_uri, referer_uri));
+  prefs.setIntPref("network.http.sendRefererHeader", 2);
+  do_check_eq(getTestReferrer(server_uri, referer_uri), referer_uri);
+
+  // test that https ref is not sent to http
+  do_check_null(getTestReferrer(server_uri_2, referer_uri_https));
+
+  // tests for referer.spoofSource
+  prefs.setBoolPref("network.http.referer.spoofSource", true);
+  do_check_eq(getTestReferrer(server_uri, referer_uri), server_uri);
+  prefs.setBoolPref("network.http.referer.spoofSource", false);
+  do_check_eq(getTestReferrer(server_uri, referer_uri), referer_uri);
+
+  // tests for referer.XOriginPolicy
+  prefs.setIntPref("network.http.referer.XOriginPolicy", 2);
+  do_check_null(getTestReferrer(server_uri_2, referer_uri));
+  do_check_eq(getTestReferrer(server_uri, referer_uri_2), referer_uri_2);
+  prefs.setIntPref("network.http.referer.XOriginPolicy", 1);
+  do_check_eq(getTestReferrer(server_uri_2, referer_uri), referer_uri);
+  do_check_null(getTestReferrer(server_uri, referer_uri));
+  // https test
+  do_check_eq(getTestReferrer(server_uri_https, referer_uri_https), referer_uri_https);
+  prefs.setIntPref("network.http.referer.XOriginPolicy", 0);
+  do_check_eq(getTestReferrer(server_uri, referer_uri), referer_uri);
+
+  // tests for referer.trimmingPolicy
+  prefs.setIntPref("network.http.referer.trimmingPolicy", 1);
+  do_check_eq(getTestReferrer(server_uri, referer_uri_2), "http://bar.examplesite.com/path3");
+  prefs.setIntPref("network.http.referer.trimmingPolicy", 2);
+  do_check_eq(getTestReferrer(server_uri, referer_uri_2), "http://bar.examplesite.com");
+  // https test
+  do_check_eq(getTestReferrer(server_uri_https, referer_uri_https), "https://bar.example.com");
+  prefs.setIntPref("network.http.referer.trimmingPolicy", 0);
+  // test that anchor is lopped off in ordinary case
+  do_check_eq(getTestReferrer(server_uri, referer_uri_2_anchor), referer_uri_2);
+
+  // combination test: send spoofed path-only when hosts match
+  var combo_referer_uri = "http://blah.foo.com/path?q=hot";
+  var dest_uri = "http://blah.foo.com:9999/spoofedpath?q=bad";
+  prefs.setIntPref("network.http.referer.trimmingPolicy", 1);
+  prefs.setBoolPref("network.http.referer.spoofSource", true);
+  prefs.setIntPref("network.http.referer.XOriginPolicy", 2);
+  do_check_eq(getTestReferrer(dest_uri, combo_referer_uri), "http://blah.foo.com:9999/spoofedpath");
+  do_check_null(getTestReferrer(dest_uri, "http://gah.foo.com/anotherpath"));
+}
--- a/netwerk/test/unit/xpcshell.ini
+++ b/netwerk/test/unit/xpcshell.ini
@@ -290,13 +290,14 @@ skip-if = os == "android"
 [test_bug856978.js]
 [test_unix_domain.js]
 # The xpcshell temp directory on Android doesn't seem to let us create
 # Unix domain sockets. (Perhaps it's a FAT filesystem?)
 skip-if = os == "android"
 [test_addr_in_use_error.js]
 [test_about_networking.js]
 [test_ping_aboutnetworking.js]
+[test_referrer.js]
 [test_seer.js]
 # Android version detection w/in gecko does not work right on infra, so we just
 # disable this test on all android versions, even though it's enabled on 2.3+ in
 # the wild.
 skip-if = os == "android"