Bug 1274917 - Add net original headers to cache.r=mayhemer
authorDragana Damjanovic dd.mozilla@gmail.com
Wed, 01 Jun 2016 06:25:00 +0200
changeset 338962 3c2ac62acf836714976ba5206a4fd171df8b0d73
parent 338961 ec9c63bd09fc8273d921bd87835285497a3a75b1
child 338963 f6cde20f6294f7ad98fc1372dade8557a80a935c
push id6249
push userjlund@mozilla.com
push dateMon, 01 Aug 2016 13:59:36 +0000
treeherdermozilla-beta@bad9d4f5bf7e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmayhemer
bugs1274917
milestone49.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1274917 - Add net original headers to cache.r=mayhemer
netwerk/protocol/http/nsHttpChannel.cpp
netwerk/protocol/http/nsHttpHeaderArray.cpp
netwerk/protocol/http/nsHttpHeaderArray.h
netwerk/protocol/http/nsHttpResponseHead.cpp
netwerk/protocol/http/nsHttpResponseHead.h
netwerk/protocol/http/nsHttpTransaction.cpp
netwerk/test/unit/test_original_sent_received_head.js
--- a/netwerk/protocol/http/nsHttpChannel.cpp
+++ b/netwerk/protocol/http/nsHttpChannel.cpp
@@ -3370,22 +3370,38 @@ nsHttpChannel::OnCacheEntryCheck(nsICach
     NS_ENSURE_SUCCESS(rv, rv);
 
     // Determine if this is the first time that this cache entry
     // has been accessed during this session.
     bool fromPreviousSession =
             (gHttpHandler->SessionStartTime() > lastModifiedTime);
 
     // Get the cached HTTP response headers
+    mCachedResponseHead = new nsHttpResponseHead();
+
+    // A "original-response-headers" metadata element holds network original headers,
+    // i.e. the headers in the form as they arrieved from the network.
+    // We need to get the network original headers first, because we need to keep them
+    // in order.
+    rv = entry->GetMetaDataElement("original-response-headers", getter_Copies(buf));
+    if (NS_SUCCEEDED(rv)) {
+        mCachedResponseHead->ParseCachedOriginalHeaders((char *) buf.get());
+    }
+
+    buf.Adopt(0);
+    // A "response-head" metadata element holds response head, e.g. response status
+    // line and headers in the form Firefox uses them internally (no dupicate
+    // headers, etc.).
     rv = entry->GetMetaDataElement("response-head", getter_Copies(buf));
     NS_ENSURE_SUCCESS(rv, rv);
 
-    // Parse the cached HTTP response headers
-    mCachedResponseHead = new nsHttpResponseHead();
-    rv = mCachedResponseHead->Parse((char *) buf.get());
+    // Parse string stored in a "response-head" metadata element.
+    // These response headers will be merged with the orignal headers (i.e. the
+    // headers stored in a "original-response-headers" metadata element).
+    rv = mCachedResponseHead->ParseCachedHead((char *) buf.get());
     NS_ENSURE_SUCCESS(rv, rv);
     buf.Adopt(0);
 
     bool isCachedRedirect = WillRedirect(mCachedResponseHead);
 
     // Do not return 304 responses from the cache, and also do not return
     // any other non-redirect 3xx responses from the cache (see bug 759043).
     NS_ENSURE_TRUE((mCachedResponseHead->Status() / 100 != 3) ||
@@ -4620,16 +4636,20 @@ DoAddCacheEntryHeaders(nsHttpChannel *se
     }
 
     // Store the received HTTP head with the cache entry as an element of
     // the meta data.
     nsAutoCString head;
     responseHead->Flatten(head, true);
     rv = entry->SetMetaDataElement("response-head", head.get());
     if (NS_FAILED(rv)) return rv;
+    head.Truncate();
+    responseHead->FlattenNetworkOriginalHeaders(head);
+    rv = entry->SetMetaDataElement("original-response-headers", head.get());
+    if (NS_FAILED(rv)) return rv;
 
     // Indicate we have successfully finished setting metadata on the cache entry.
     rv = entry->MetaDataReady();
 
     return rv;
 }
 
 nsresult
--- a/netwerk/protocol/http/nsHttpHeaderArray.cpp
+++ b/netwerk/protocol/http/nsHttpHeaderArray.cpp
@@ -167,16 +167,51 @@ nsHttpHeaderArray::SetHeaderFromNet(nsHt
             return SetHeader_internal(header, value,
                                       eVarietyResponseNetOriginal);
         }
     }
 
     return NS_OK;
 }
 
+nsresult
+nsHttpHeaderArray::SetResponseHeaderFromCache(nsHttpAtom header,
+                                              const nsACString &value,
+                                              nsHttpHeaderArray::HeaderVariety variety)
+{
+    MOZ_ASSERT((variety == eVarietyResponse) ||
+               (variety == eVarietyResponseNetOriginal),
+               "Headers from cache can only be eVarietyResponse and "
+               "eVarietyResponseNetOriginal");
+
+    if (variety == eVarietyResponseNetOriginal) {
+        return SetHeader_internal(header, value,
+                                  eVarietyResponseNetOriginal);
+    } else {
+        nsTArray<nsEntry>::index_type index = 0;
+        do {
+            index = mHeaders.IndexOf(header, index, nsEntry::MatchHeader());
+            if (index != mHeaders.NoIndex) {
+                nsEntry &entry = mHeaders[index];
+                if (value.Equals(entry.value)) {
+                    MOZ_ASSERT((entry.variety == eVarietyResponseNetOriginal) ||
+                               (entry.variety == eVarietyResponseNetOriginalAndResponse),
+                               "This array must contain only eVarietyResponseNetOriginal"
+                               " and eVarietyResponseNetOriginalAndRespons headers!");
+                    entry.variety = eVarietyResponseNetOriginalAndResponse;
+                    return NS_OK;
+                }
+                index++;
+            }
+        } while (index != mHeaders.NoIndex);
+        // If we are here, we have not found an entry so add a new one.
+        return SetHeader_internal(header, value, eVarietyResponse);
+    }
+}
+
 void
 nsHttpHeaderArray::ClearHeader(nsHttpAtom header)
 {
     nsEntry *entry = nullptr;
     int32_t index = LookupEntry(header, &entry);
     if (entry) {
         if (entry->variety == eVarietyResponseNetOriginalAndResponse) {
             entry->variety = eVarietyResponseNetOriginal;
--- a/netwerk/protocol/http/nsHttpHeaderArray.h
+++ b/netwerk/protocol/http/nsHttpHeaderArray.h
@@ -56,16 +56,19 @@ public:
     nsresult SetEmptyHeader(nsHttpAtom header, HeaderVariety variety);
 
     // Merges supported headers. For other duplicate values, determines if error
     // needs to be thrown or 1st value kept.
     // For the response header we keep the original headers as well.
     nsresult SetHeaderFromNet(nsHttpAtom header, const nsACString &value,
                               bool response);
 
+    nsresult SetResponseHeaderFromCache(nsHttpAtom header, const nsACString &value,
+                                        HeaderVariety variety);
+
     nsresult GetHeader(nsHttpAtom header, nsACString &value) const;
     nsresult GetOriginalHeader(nsHttpAtom aHeader,
                                nsIHttpHeaderVisitor *aVisitor);
     void     ClearHeader(nsHttpAtom h);
 
     // Find the location of the given header value, or null if none exists.
     const char *FindHeaderValue(nsHttpAtom header, const char *value) const
     {
--- a/netwerk/protocol/http/nsHttpResponseHead.cpp
+++ b/netwerk/protocol/http/nsHttpResponseHead.cpp
@@ -254,34 +254,31 @@ nsHttpResponseHead::Flatten(nsACString &
                mStatusText +
                NS_LITERAL_CSTRING("\r\n"));
 
 
     mHeaders.Flatten(buf, false, pruneTransients);
 }
 
 void
-nsHttpResponseHead::FlattenOriginalHeader(nsACString &buf)
+nsHttpResponseHead::FlattenNetworkOriginalHeaders(nsACString &buf)
 {
     ReentrantMonitorAutoEnter monitor(mReentrantMonitor);
     if (mVersion == NS_HTTP_VERSION_0_9) {
         return;
     }
 
-    buf.AppendLiteral("  OriginalHeaders");
-    buf.AppendLiteral("\r\n");
-
     mHeaders.FlattenOriginalHeader(buf);
 }
 
 nsresult
-nsHttpResponseHead::Parse(char *block)
+nsHttpResponseHead::ParseCachedHead(char *block)
 {
     ReentrantMonitorAutoEnter monitor(mReentrantMonitor);
-    LOG(("nsHttpResponseHead::Parse [this=%p]\n", this));
+    LOG(("nsHttpResponseHead::ParseCachedHead [this=%p]\n", this));
 
     // this command works on a buffer as prepared by Flatten, as such it is
     // not very forgiving ;-)
 
     char *p = PL_strstr(block, "\r\n");
     if (!p)
         return NS_ERROR_UNEXPECTED;
 
@@ -294,18 +291,65 @@ nsHttpResponseHead::Parse(char *block)
         if (*block == 0)
             break;
 
         p = PL_strstr(block, "\r\n");
         if (!p)
             return NS_ERROR_UNEXPECTED;
 
         *p = 0;
-        ParseHeaderLine_locked(block);
+        ParseHeaderLine_locked(block, false);
+
+    } while (1);
+
+    return NS_OK;
+}
+
+nsresult
+nsHttpResponseHead::ParseCachedOriginalHeaders(char *block)
+{
+    ReentrantMonitorAutoEnter monitor(mReentrantMonitor);
+    LOG(("nsHttpResponseHead::ParseCachedOriginalHeader [this=%p]\n", this));
+
+    // this command works on a buffer as prepared by FlattenOriginalHeader,
+    // as such it is not very forgiving ;-)
+
+    if (!block) {
+        return NS_ERROR_UNEXPECTED;
+    }
+
+    char *p = block;
+    nsHttpAtom hdr = {0};
+    char *val;
+    nsresult rv;
 
+    do {
+        block = p;
+
+        if (*block == 0)
+            break;
+
+        p = PL_strstr(block, "\r\n");
+        if (!p)
+            return NS_ERROR_UNEXPECTED;
+
+        *p = 0;
+        if (NS_FAILED(nsHttpHeaderArray::ParseHeaderLine(block, &hdr, &val))) {
+            return NS_OK;
+        }
+
+        rv = mHeaders.SetResponseHeaderFromCache(hdr,
+                                                 nsDependentCString(val),
+                                                 nsHttpHeaderArray::eVarietyResponseNetOriginal);
+
+        if (NS_FAILED(rv)) {
+            return rv;
+        }
+
+        p = p + 2;
     } while (1);
 
     return NS_OK;
 }
 
 void
 nsHttpResponseHead::AssignDefaultStatusText()
 {
@@ -500,31 +544,38 @@ nsHttpResponseHead::ParseStatusLine_lock
     LOG(("Have status line [version=%u status=%u statusText=%s]\n",
         unsigned(mVersion), unsigned(mStatus), mStatusText.get()));
 }
 
 nsresult
 nsHttpResponseHead::ParseHeaderLine(const char *line)
 {
     ReentrantMonitorAutoEnter monitor(mReentrantMonitor);
-    return ParseHeaderLine_locked(line);
+    return ParseHeaderLine_locked(line, true);
 }
 
 nsresult
-nsHttpResponseHead::ParseHeaderLine_locked(const char *line)
+nsHttpResponseHead::ParseHeaderLine_locked(const char *line, bool originalFromNetHeaders)
 {
     nsHttpAtom hdr = {0};
     char *val;
 
     if (NS_FAILED(nsHttpHeaderArray::ParseHeaderLine(line, &hdr, &val))) {
         return NS_OK;
-    } 
-    nsresult rv = mHeaders.SetHeaderFromNet(hdr,
-                                            nsDependentCString(val),
-                                            true);
+    }
+    nsresult rv;
+    if (originalFromNetHeaders) {
+        rv = mHeaders.SetHeaderFromNet(hdr,
+                                       nsDependentCString(val),
+                                       true);
+    } else {
+        rv = mHeaders.SetResponseHeaderFromCache(hdr,
+                                                 nsDependentCString(val),
+                                                 nsHttpHeaderArray::eVarietyResponse);
+    }
     if (NS_FAILED(rv)) {
         return rv;
     }
 
     // leading and trailing LWS has been removed from |val|
 
     // handle some special case headers...
     if (hdr == nsHttp::Content_Length) {
--- a/netwerk/protocol/http/nsHttpResponseHead.h
+++ b/netwerk/protocol/http/nsHttpResponseHead.h
@@ -76,21 +76,27 @@ public:
     void SetContentCharset(const nsACString &s);
     void SetContentLength(int64_t);
 
     // write out the response status line and headers as a single text block,
     // optionally pruning out transient headers (ie. headers that only make
     // sense the first time the response is handled).
     // Both functions append to the string supplied string.
     void Flatten(nsACString &, bool pruneTransients);
-    void FlattenOriginalHeader(nsACString &buf);
+    void FlattenNetworkOriginalHeaders(nsACString &buf);
 
-    // parse flattened response head. block must be null terminated. parsing is
-    // destructive.
-    nsresult Parse(char *block);
+    // The next 2 functions parse flattened response head and original net headers.
+    // They are used when we are reading an entry from the cache.
+    //
+    // To keep proper order of the original headers we MUST call
+    // ParseCachedOriginalHeaders FIRST and then ParseCachedHead.
+    //
+    // block must be null terminated. parsing is destructive.
+    nsresult ParseCachedHead(char *block);
+    nsresult ParseCachedOriginalHeaders(char *block);
 
     // parse the status line. line must be null terminated.
     void ParseStatusLine(const char *line);
 
     // parse a header line. line must be null terminated. parsing is destructive.
     nsresult ParseHeaderLine(const char *line);
 
     // cache validation support methods
@@ -134,17 +140,17 @@ private:
     nsresult SetHeader_locked(nsHttpAtom h, const nsACString &v,
                               bool m=false);
     void AssignDefaultStatusText();
     void ParseVersion(const char *);
     void ParseCacheControl(const char *);
     void ParsePragma(const char *);
 
     void ParseStatusLine_locked(const char *line);
-    nsresult ParseHeaderLine_locked(const char *line);
+    nsresult ParseHeaderLine_locked(const char *line, bool originalFromNetHeaders);
 
     // these return failure if the header does not exist.
     nsresult ParseDateHeader(nsHttpAtom header, uint32_t *result) const;
 
     bool ExpiresInPast_locked() const;
     nsresult GetAgeValue_locked(uint32_t *result) const;
     nsresult GetExpiresValue_locked(uint32_t *result) const;
     nsresult GetMaxAgeValue_locked(uint32_t *result) const;
--- a/netwerk/protocol/http/nsHttpTransaction.cpp
+++ b/netwerk/protocol/http/nsHttpTransaction.cpp
@@ -1559,17 +1559,19 @@ nsHttpTransaction::HandleContentStart()
     LOG(("nsHttpTransaction::HandleContentStart [this=%p]\n", this));
     MOZ_ASSERT(PR_GetCurrentThread() == gSocketThread);
 
     if (mResponseHead) {
         if (LOG3_ENABLED()) {
             LOG3(("http response [\n"));
             nsAutoCString headers;
             mResponseHead->Flatten(headers, false);
-            mResponseHead->FlattenOriginalHeader(headers);
+            headers.AppendLiteral("  OriginalHeaders");
+            headers.AppendLiteral("\r\n");
+            mResponseHead->FlattenNetworkOriginalHeaders(headers);
             LogHeaders(headers.get());
             LOG3(("]\n"));
         }
 
         // Save http version, mResponseHead isn't available anymore after
         // TakeResponseHead() is called
         mHttpVersion = mResponseHead->Version();
         mHttpResponseCode = mResponseHead->Status();
--- a/netwerk/test/unit/test_original_sent_received_head.js
+++ b/netwerk/test/unit/test_original_sent_received_head.js
@@ -3,91 +3,121 @@
 //  Response headers can be changed after they have been received, e.g. empty
 //  headers are deleted, some duplicate header are merged (if no error is
 //  thrown), etc.
 //
 //  The "original header" is introduced to hold the header array in the order
 //  and the form as they have been received from the network.
 //  Here, the "original headers" are tested.
 //
+//  Original headers will be stored in the cache as well. This test checks
+//  that too.
 
 // Note: sets Cc and Ci variables
 
 Cu.import("resource://testing-common/httpd.js");
 Cu.import("resource://gre/modules/NetUtil.jsm");
 
 XPCOMUtils.defineLazyGetter(this, "URL", function() {
   return "http://localhost:" + httpserver.identity.primaryPort;
 });
 
 var httpserver = new HttpServer();
 var testpath = "/simple";
 var httpbody = "0123456789";
-var channel;
-var ios;
 
 var dbg=1
-if (dbg) { print("============== START =========="); }
 
 function run_test() {
-  setup_test();
-  do_test_pending();
-}
 
-function setup_test() {
-  if (dbg) { print("============== setup_test: in"); }
+  if (dbg) { print("============== START =========="); }
 
   httpserver.registerPathHandler(testpath, serverHandler);
   httpserver.start(-1);
+  run_next_test();
+}
 
-  channel = setupChannel(testpath);
+add_test(function test_headerChange() {
+  if (dbg) { print("============== test_headerChange setup: in"); }
+
+  var channel1 = setupChannel(testpath);
+  channel1.loadFlags = Components.interfaces.nsIRequest.LOAD_BYPASS_CACHE;
 
   // ChannelListener defined in head_channels.js
-  channel.asyncOpen2(new ChannelListener(checkResponse, channel));
+  channel1.asyncOpen2(new ChannelListener(checkResponse, null));
+
+  if (dbg) { print("============== test_headerChange setup: out"); }
+});
+
+add_test(function test_fromCache() {
+  if (dbg) { print("============== test_fromCache setup: in"); }
+
+  var channel2 = setupChannel(testpath);
+  channel2.loadFlags = Components.interfaces.nsIRequest.LOAD_FROM_CACHE;
 
-  if (dbg) { print("============== setup_test: out"); }
-}
+  // ChannelListener defined in head_channels.js
+  channel2.asyncOpen2(new ChannelListener(checkResponse, null));
+
+  if (dbg) { print("============== test_fromCache setup: out"); }
+});
+
+add_test(function finish() {
+  if (dbg) { print("============== STOP =========="); }
+  httpserver.stop(do_test_finished);
+});
 
 function setupChannel(path) {
   var chan = NetUtil.newChannel ({
     uri: URL + path,
     loadUsingSystemPrincipal: true
   }).QueryInterface(Components.interfaces.nsIHttpChannel);
   chan.requestMethod = "GET";
   return chan;
 }
 
 function serverHandler(metadata, response) {
   if (dbg) { print("============== serverHandler: in"); }
 
-  response.setHeader("Content-Type", "text/plain", false);
-  response.setStatusLine("1.1", 200, "OK");
+  try {
+    var etag = metadata.getHeader("If-None-Match");
+  } catch(ex) {
+    var etag = "";
+  }
+  if (etag == "testtag") {
+    if (dbg) { print("============== 304 answerr: in"); }
+    response.setStatusLine("1.1", 304, "Not Modified");
+  } else {
+    response.setHeader("Content-Type", "text/plain", false);
+    response.setStatusLine("1.1", 200, "OK");
 
-  // Set a empty header. A empty link header will not appear in header list,
-  // but in the "original headers", it will be still exactly as received.
-  response.setHeaderNoCheck("Link", "", true);
-  response.setHeaderNoCheck("Link", "value1");
-  response.setHeaderNoCheck("Link", "value2");
-  response.setHeaderNoCheck("Location", "loc");
-  response.bodyOutputStream.write(httpbody, httpbody.length);
-
+    // Set a empty header. A empty link header will not appear in header list,
+    // but in the "original headers", it will be still exactly as received.
+    response.setHeaderNoCheck("Link", "", true);
+    response.setHeaderNoCheck("Link", "value1");
+    response.setHeaderNoCheck("Link", "value2");
+    response.setHeaderNoCheck("Location", "loc");
+    response.setHeader("Cache-Control", "max-age=10000", false);
+    response.setHeader("ETag", "testtag", false);
+    response.bodyOutputStream.write(httpbody, httpbody.length);
+  }
   if (dbg) { print("============== serverHandler: out"); }
 }
 
 function checkResponse(request, data, context) {
   if (dbg) { print("============== checkResponse: in"); }
 
-  do_check_eq(channel.responseStatus, 200);
-  do_check_eq(channel.responseStatusText, "OK");
-  do_check_true(channel.requestSucceeded);
+  request.QueryInterface(Components.interfaces.nsIHttpChannel);
+  do_check_eq(request.responseStatus, 200);
+  do_check_eq(request.responseStatusText, "OK");
+  do_check_true(request.requestSucceeded);
 
   // Response header have only one link header.
   var linkHeaderFound = 0;
   var locationHeaderFound = 0;
-  channel.visitResponseHeaders({
+  request.visitResponseHeaders({
     visitHeader: function visit(aName, aValue) {
       if (aName == "Link") {
         linkHeaderFound++;
         do_check_eq(aValue, "value1, value2");
       }
       if (aName == "Location") {
         locationHeaderFound++;
         do_check_eq(aValue, "loc");
@@ -95,17 +125,17 @@ function checkResponse(request, data, co
     }
   });
   do_check_eq(linkHeaderFound, 1);
   do_check_eq(locationHeaderFound, 1);
 
   // The "original header" still contains 3 link headers.
   var linkOrgHeaderFound = 0;
   var locationOrgHeaderFound = 0;
-  channel.visitOriginalResponseHeaders({
+  request.visitOriginalResponseHeaders({
     visitHeader: function visitOrg(aName, aValue) {
       if (aName == "Link") {
         if (linkOrgHeaderFound == 0) {
           do_check_eq(aValue, "");
         } else if (linkOrgHeaderFound == 1 ) {
           do_check_eq(aValue, "value1");
         } else {
           do_check_eq(aValue, "value2");
@@ -118,38 +148,38 @@ function checkResponse(request, data, co
       }
     }
   });
   do_check_eq(linkOrgHeaderFound, 3);
   do_check_eq(locationOrgHeaderFound, 1);
 
   if (dbg) { print("============== Remove headers"); }
   // Remove header.
-  channel.setResponseHeader("Link", "", false);
-  channel.setResponseHeader("Location", "", false);
+  request.setResponseHeader("Link", "", false);
+  request.setResponseHeader("Location", "", false);
 
   var linkHeaderFound2 = false;
   var locationHeaderFound2 = 0;
-  channel.visitResponseHeaders({
+  request.visitResponseHeaders({
     visitHeader: function visit(aName, aValue) {
       if (aName == "Link") {
         linkHeaderFound2 = true;
       }
       if (aName == "Location") {
         locationHeaderFound2 = true;
       }
     }
   });
   do_check_false(linkHeaderFound2, "There should be no link header");
   do_check_false(locationHeaderFound2, "There should be no location headers.");
 
   // The "original header" still contains the empty header.
   var linkOrgHeaderFound2 = 0;
   var locationOrgHeaderFound2 = 0;
-  channel.visitOriginalResponseHeaders({
+  request.visitOriginalResponseHeaders({
     visitHeader: function visitOrg(aName, aValue) {
       if (aName == "Link") {
         if (linkOrgHeaderFound2 == 0) {
           do_check_eq(aValue, "");
         } else if (linkOrgHeaderFound2 == 1 ) {
           do_check_eq(aValue, "value1");
         } else {
           do_check_eq(aValue, "value2");
@@ -164,26 +194,27 @@ function checkResponse(request, data, co
   });
   do_check_true(linkOrgHeaderFound2 == 3,
                 "Original link header still here.");
   do_check_true(locationOrgHeaderFound2 == 1,
                 "Original location header still here.");
 
   if (dbg) { print("============== Test GetResponseHeader"); }
   var linkOrgHeaderFound3 = 0;
-  channel.getOriginalResponseHeader("Link",{
+  request.getOriginalResponseHeader("Link",{
     visitHeader: function visitOrg(aName, aValue) {
       if (linkOrgHeaderFound3 == 0) {
         do_check_eq(aValue, "");
       } else if (linkOrgHeaderFound3 == 1 ) {
         do_check_eq(aValue, "value1");
       } else {
         do_check_eq(aValue, "value2");
       }
       linkOrgHeaderFound3++;
     }
   });
   do_check_true(linkOrgHeaderFound2 == 3, 
                 "Original link header still here.");
 
-  httpserver.stop(do_test_finished);
   if (dbg) { print("============== checkResponse: out"); }
+
+  run_next_test();
 }