bug 366559 - patch 7, content-encoding brotli for https r=bagder
authorPatrick McManus <mcmanus@ducksong.com>
Fri, 18 Sep 2015 18:04:28 -0400
changeset 263754 4d45c4323570
parent 263753 5e0a3850571f
child 263755 d08142841263
push id65417
push usermcmanus@ducksong.com
push date2015-09-22 16:12 +0000
treeherdermozilla-inbound@4d45c4323570 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbagder
bugs366559
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 366559 - patch 7, content-encoding brotli for https r=bagder
modules/libpref/init/all.js
netwerk/build/nsNetModule.cpp
netwerk/mime/nsMimeTypes.h
netwerk/protocol/http/HttpBaseChannel.cpp
netwerk/protocol/http/nsHttpHandler.cpp
netwerk/streamconv/converters/nsHTTPCompressConv.cpp
netwerk/streamconv/converters/nsHTTPCompressConv.h
netwerk/test/unit/test_content_encoding_gzip.js
toolkit/components/telemetry/Histograms.json
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -1317,17 +1317,17 @@ pref("network.http.enablePerElementRefer
 
 // Maximum number of consecutive redirects before aborting.
 pref("network.http.redirection-limit", 20);
 
 // Enable http compression: comment this out in case of problems with 1.1
 // NOTE: support for "compress" has been disabled per bug 196406.
 // NOTE: separate values with comma+space (", "): see bug 576033
 pref("network.http.accept-encoding", "gzip, deflate");
-pref("network.http.accept-encoding.secure", "gzip, deflate");
+pref("network.http.accept-encoding.secure", "gzip, deflate, brotli");
 
 pref("network.http.pipelining"      , false);
 pref("network.http.pipelining.ssl"  , false); // disable pipelining over SSL
 pref("network.http.pipelining.abtest", false);
 pref("network.http.proxy.pipelining", false);
 
 // Max number of requests in the pipeline
 pref("network.http.pipelining.maxrequests" , 32);
--- a/netwerk/build/nsNetModule.cpp
+++ b/netwerk/build/nsNetModule.cpp
@@ -438,16 +438,17 @@ nsresult NS_NewStreamConv(nsStreamConver
 #define INDEX_TO_HTML                "?from=application/http-index-format&to=text/html"
 #define MULTI_MIXED_X                "?from=multipart/x-mixed-replace&to=*/*"
 #define MULTI_MIXED                  "?from=multipart/mixed&to=*/*"
 #define APPLICATION_PACKAGE_CONV     "?from=" APPLICATION_PACKAGE "&to=*/*"
 #define MULTI_BYTERANGES             "?from=multipart/byteranges&to=*/*"
 #define UNKNOWN_CONTENT              "?from=" UNKNOWN_CONTENT_TYPE "&to=*/*"
 #define GZIP_TO_UNCOMPRESSED         "?from=gzip&to=uncompressed"
 #define XGZIP_TO_UNCOMPRESSED        "?from=x-gzip&to=uncompressed"
+#define BROTLI_TO_UNCOMPRESSED       "?from=brotli&to=uncompressed"
 #define COMPRESS_TO_UNCOMPRESSED     "?from=compress&to=uncompressed"
 #define XCOMPRESS_TO_UNCOMPRESSED    "?from=x-compress&to=uncompressed"
 #define DEFLATE_TO_UNCOMPRESSED      "?from=deflate&to=uncompressed"
 #define PLAIN_TO_HTML                "?from=text/plain&to=text/html"
 
 #ifdef BUILD_BINHEX_DECODER
 #define BINHEX_TO_WILD               "?from=application/mac-binhex40&to=*/*"
 #endif
@@ -457,16 +458,17 @@ static const mozilla::Module::CategoryEn
     { NS_ISTREAMCONVERTER_KEY, INDEX_TO_HTML, "" },
     { NS_ISTREAMCONVERTER_KEY, MULTI_MIXED_X, "" },
     { NS_ISTREAMCONVERTER_KEY, MULTI_MIXED, "" },
     { NS_ISTREAMCONVERTER_KEY, APPLICATION_PACKAGE_CONV, "" },
     { NS_ISTREAMCONVERTER_KEY, MULTI_BYTERANGES, "" },
     { NS_ISTREAMCONVERTER_KEY, UNKNOWN_CONTENT, "" },
     { NS_ISTREAMCONVERTER_KEY, GZIP_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, XGZIP_TO_UNCOMPRESSED, "" },
+    { NS_ISTREAMCONVERTER_KEY, BROTLI_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, COMPRESS_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, XCOMPRESS_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, DEFLATE_TO_UNCOMPRESSED, "" },
 #ifdef BUILD_BINHEX_DECODER
     { NS_ISTREAMCONVERTER_KEY, BINHEX_TO_WILD, "" },
 #endif
     { NS_ISTREAMCONVERTER_KEY, PLAIN_TO_HTML, "" },
     NS_BINARYDETECTOR_CATEGORYENTRY,
@@ -1045,16 +1047,17 @@ static const mozilla::Module::ContractID
     { NS_ISTREAMCONVERTER_KEY MULTI_BYTERANGES, &kNS_MULTIMIXEDCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY MULTI_MIXED, &kNS_MULTIMIXEDCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY APPLICATION_PACKAGE_CONV, &kNS_MULTIMIXEDCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY UNKNOWN_CONTENT, &kNS_UNKNOWNDECODER_CID },
     { NS_GENERIC_CONTENT_SNIFFER, &kNS_UNKNOWNDECODER_CID },
     { NS_BINARYDETECTOR_CONTRACTID, &kNS_BINARYDETECTOR_CID },
     { NS_ISTREAMCONVERTER_KEY GZIP_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY XGZIP_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
+    { NS_ISTREAMCONVERTER_KEY BROTLI_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY COMPRESS_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY XCOMPRESS_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY DEFLATE_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY PLAIN_TO_HTML, &kNS_NSTXTTOHTMLCONVERTER_CID },
 #ifdef BUILD_BINHEX_DECODER
     { NS_ISTREAMCONVERTER_KEY BINHEX_TO_WILD, &kNS_BINHEXDECODER_CID },
 #endif
     { MOZ_TXTTOHTMLCONV_CONTRACTID, &kMOZITXTTOHTMLCONV_CID },
--- a/netwerk/mime/nsMimeTypes.h
+++ b/netwerk/mime/nsMimeTypes.h
@@ -25,16 +25,17 @@
 #define APPLICATION_MACBINARY               "application/x-macbinary"
 #define APPLICATION_COMPRESS                "application/x-compress"
 #define APPLICATION_COMPRESS2               "application/compress"
 #define APPLICATION_FORTEZZA_CKL            "application/x-fortezza-ckl"
 #define APPLICATION_FORTEZZA_KRL            "application/x-fortezza-krl"
 #define APPLICATION_GZIP                    "application/x-gzip"
 #define APPLICATION_GZIP2                   "application/gzip"
 #define APPLICATION_GZIP3                   "application/x-gunzip"
+#define APPLICATION_BROTLI                  "application/brotli"
 #define APPLICATION_ZIP                     "application/zip"
 #define APPLICATION_HTTP_INDEX_FORMAT       "application/http-index-format"
 #define APPLICATION_ECMASCRIPT              "application/ecmascript"
 #define APPLICATION_JAVASCRIPT              "application/javascript"
 #define APPLICATION_XJAVASCRIPT             "application/x-javascript"
 #define APPLICATION_JSON                    "application/json"
 #define APPLICATION_NETSCAPE_REVOCATION     "application/x-netscape-revocation"
 #define APPLICATION_NS_PROXY_AUTOCONFIG     "application/x-ns-proxy-autoconfig"
--- a/netwerk/protocol/http/HttpBaseChannel.cpp
+++ b/netwerk/protocol/http/HttpBaseChannel.cpp
@@ -831,16 +831,27 @@ HttpBaseChannel::DoApplyContentConversio
                                   aCtxt,
                                   getter_AddRefs(converter));
       if (NS_FAILED(rv)) {
         LOG(("Unexpected failure of AsyncConvertData %s\n", val));
         return rv;
       }
 
       LOG(("converter removed '%s' content-encoding\n", val));
+      if (gHttpHandler->IsTelemetryEnabled()) {
+        int mode = 0;
+        if (from.Equals("gzip") || from.Equals("x-gzip")) {
+          mode = 1;
+        } else if (from.Equals("deflate") || from.Equals("x-deflate")) {
+          mode = 2;
+        } else if (from.Equals("brotli")) {
+          mode = 3;
+        }
+        Telemetry::Accumulate(Telemetry::HTTP_CONTENT_ENCODING, mode);
+      }
       nextListener = converter;
     }
     else {
       if (val)
         LOG(("Unknown content encoding '%s', ignoring\n", val));
     }
   }
   *aNewNextListener = nextListener;
@@ -935,16 +946,24 @@ HttpBaseChannel::nsContentEncodings::Get
   if (!haveType) {
     encoding.BeginReading(start);
     if (CaseInsensitiveFindInReadable(NS_LITERAL_CSTRING("deflate"), start, end)) {
       aNextEncoding.AssignLiteral(APPLICATION_ZIP);
       haveType = true;
     }
   }
 
+  if (!haveType) {
+    encoding.BeginReading(start);
+    if (CaseInsensitiveFindInReadable(NS_LITERAL_CSTRING("brotli"), start, end)) {
+      aNextEncoding.AssignLiteral(APPLICATION_BROTLI);
+      haveType = true;
+    }
+  }
+
   // Prepare to fetch the next encoding
   mCurEnd = mCurStart;
   mReady = false;
 
   if (haveType)
     return NS_OK;
 
   NS_WARNING("Unknown encoding type");
--- a/netwerk/protocol/http/nsHttpHandler.cpp
+++ b/netwerk/protocol/http/nsHttpHandler.cpp
@@ -488,32 +488,34 @@ nsHttpHandler::AddConnectionHeader(nsHtt
 }
 
 bool
 nsHttpHandler::IsAcceptableEncoding(const char *enc, bool isSecure)
 {
     if (!enc)
         return false;
 
-    // HTTP 1.1 allows servers to send x-gzip and x-compress instead
-    // of gzip and compress, for example.  So, we'll always strip off
-    // an "x-" prefix before matching the encoding to one we claim
-    // to accept.
-    if (!PL_strncasecmp(enc, "x-", 2))
-        enc += 2;
-
+    // we used to accept x-foo anytime foo was acceptable, but that's just
+    // continuing bad behavior.. so limit it to known x-* patterns
+    bool rv;
+    if (isSecure) {
+        rv = nsHttp::FindToken(mHttpsAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    } else {
+        rv = nsHttp::FindToken(mHttpAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    }
     // gzip and deflate are inherently acceptable in modern HTTP - always
     // process them if a stream converter can also be found.
-    if (!PL_strcasecmp(enc, "gzip") || !PL_strcasecmp(enc, "deflate"))
-        return true;
-
-    if (isSecure) {
-        return nsHttp::FindToken(mHttpsAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    if (!rv &&
+        (!PL_strcasecmp(enc, "gzip") || !PL_strcasecmp(enc, "deflate") ||
+         !PL_strcasecmp(enc, "x-gzip") || !PL_strcasecmp(enc, "x-deflate"))) {
+        rv = true;
     }
-    return nsHttp::FindToken(mHttpAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    LOG(("nsHttpHandler::IsAceptableEncoding %s https=%d %d\n",
+         enc, isSecure, rv));
+    return rv;
 }
 
 nsresult
 nsHttpHandler::GetStreamConverterService(nsIStreamConverterService **result)
 {
     if (!mStreamConvSvc) {
         nsresult rv;
         nsCOMPtr<nsIStreamConverterService> service =
--- a/netwerk/streamconv/converters/nsHTTPCompressConv.cpp
+++ b/netwerk/streamconv/converters/nsHTTPCompressConv.cpp
@@ -9,16 +9,24 @@
 #include "plstr.h"
 #include "nsCOMPtr.h"
 #include "nsError.h"
 #include "nsStreamUtils.h"
 #include "nsStringStream.h"
 #include "nsComponentManagerUtils.h"
 #include "nsThreadUtils.h"
 #include "mozilla/Preferences.h"
+#include "nsIForcePendingChannel.h"
+
+// brotli headers
+#include "state.h"
+#include "decode.h"
+
+extern PRLogModuleInfo *gHttpLog;
+#define LOG(args) MOZ_LOG(gHttpLog, mozilla::LogLevel::Debug, args)
 
 namespace mozilla {
 namespace net {
 
 // nsISupports implementation
 NS_IMPL_ISUPPORTS(nsHTTPCompressConv,
                   nsIStreamConverter,
                   nsIStreamListener,
@@ -34,26 +42,28 @@ nsHTTPCompressConv::nsHTTPCompressConv()
   , mCheckHeaderDone(false)
   , mStreamEnded(false)
   , mStreamInitialized(false)
   , mLen(0)
   , hMode(0)
   , mSkipCount(0)
   , mFlags(0)
 {
+  LOG(("nsHttpCompresssConv %p ctor\n", this));
   if (NS_IsMainThread()) {
     mFailUncleanStops =
       Preferences::GetBool("network.http.enforce-framing.http", false);
   } else {
     mFailUncleanStops = false;
   }
 }
 
 nsHTTPCompressConv::~nsHTTPCompressConv()
 {
+  LOG(("nsHttpCompresssConv %p dtor\n", this));
   if (mInpBuffer) {
     free(mInpBuffer);
   }
 
   if (mOutBuffer) {
     free(mOutBuffer);
   }
 
@@ -73,55 +83,153 @@ nsHTTPCompressConv::AsyncConvertData(con
   if (!PL_strncasecmp(aFromType, HTTP_COMPRESS_TYPE, sizeof(HTTP_COMPRESS_TYPE)-1) ||
       !PL_strncasecmp(aFromType, HTTP_X_COMPRESS_TYPE, sizeof(HTTP_X_COMPRESS_TYPE)-1)) {
     mMode = HTTP_COMPRESS_COMPRESS;
   } else if (!PL_strncasecmp(aFromType, HTTP_GZIP_TYPE, sizeof(HTTP_GZIP_TYPE)-1) ||
              !PL_strncasecmp(aFromType, HTTP_X_GZIP_TYPE, sizeof(HTTP_X_GZIP_TYPE)-1)) {
     mMode = HTTP_COMPRESS_GZIP;
   } else if (!PL_strncasecmp(aFromType, HTTP_DEFLATE_TYPE, sizeof(HTTP_DEFLATE_TYPE)-1)) {
     mMode = HTTP_COMPRESS_DEFLATE;
+  } else if (!PL_strncasecmp(aFromType, HTTP_BROTLI_TYPE, sizeof(HTTP_BROTLI_TYPE)-1)) {
+    mMode = HTTP_COMPRESS_BROTLI;
   }
+  LOG(("nsHttpCompresssConv %p AsyncConvertData %s %s mode %d\n",
+       this, aFromType, aToType, mMode));
 
   // hook ourself up with the receiving listener.
   mListener = aListener;
 
   mAsyncConvContext = aCtxt;
   return NS_OK;
 }
 
 NS_IMETHODIMP
 nsHTTPCompressConv::OnStartRequest(nsIRequest* request, nsISupports *aContext)
 {
+  LOG(("nsHttpCompresssConv %p onstart\n", this));
   return mListener->OnStartRequest(request, aContext);
 }
 
 NS_IMETHODIMP
 nsHTTPCompressConv::OnStopRequest(nsIRequest* request, nsISupports *aContext,
                                   nsresult aStatus)
 {
+  nsresult status = aStatus;
+  LOG(("nsHttpCompresssConv %p onstop %x\n", this, aStatus));
+  
   // Framing integrity is enforced for content-encoding: gzip, but not for
   // content-encoding: deflate. Note that gzip vs deflate is NOT determined
   // by content sniffing but only via header.
-  if (!mStreamEnded && NS_SUCCEEDED(aStatus) &&
+  if (!mStreamEnded && NS_SUCCEEDED(status) &&
       (mFailUncleanStops && (mMode == HTTP_COMPRESS_GZIP)) ) {
     // This is not a clean end of gzip stream: the transfer is incomplete.
-    aStatus = NS_ERROR_NET_PARTIAL_TRANSFER;
+    status = NS_ERROR_NET_PARTIAL_TRANSFER;
+    LOG(("nsHttpCompresssConv %p onstop partial gzip\n", this));
+  }
+  if (NS_SUCCEEDED(status) && mMode == HTTP_COMPRESS_BROTLI) {
+    uint32_t waste;
+    nsCOMPtr<nsIForcePendingChannel> fpChannel = do_QueryInterface(request);
+    bool isPending = false;
+    if (request) {
+      request->IsPending(&isPending);
+    }
+    if (fpChannel && !isPending) {
+      fpChannel->ForcePending(true);
+    }
+    status = BrotliHandler(nullptr, this, nullptr, 0, 0, &waste);
+    LOG(("nsHttpCompresssConv %p onstop brotlihandler rv %x\n", this, status));
+    if (fpChannel && !isPending) {
+      fpChannel->ForcePending(false);
+    }
+  }
+  if (NS_FAILED(status) && status != aStatus) {
+    LOG(("nsHttpCompresssConv %p onstop calling cancel %x\n", this, status));
+    request->Cancel(status);
   }
-  return mListener->OnStopRequest(request, aContext, aStatus);
+  return mListener->OnStopRequest(request, aContext, status);
+}
+
+
+// static
+NS_METHOD
+nsHTTPCompressConv::BrotliHandler(nsIInputStream *stream, void *closure, const char *dataIn,
+                                  uint32_t, uint32_t aAvail, uint32_t *countRead)
+{
+  nsHTTPCompressConv *self = static_cast<nsHTTPCompressConv *>(closure);
+  *countRead = 0;
+
+  const uint32_t kOutSize = 128 * 1024; // just a chunk size, we call in a loop
+  unsigned char outBuffer[kOutSize];
+  unsigned char *outPtr;
+  size_t outSize;
+  size_t avail = aAvail;
+  BrotliResult res;
+
+  do {
+    outSize = kOutSize;
+    outPtr = outBuffer;
+
+    // brotli api is documented in brotli/dec/decode.h
+    LOG(("nsHttpCompresssConv %p brotlihandler decompress %d finish %d\n",
+         self, avail, !stream));
+    res = ::BrotliDecompressBufferStreaming(
+      &avail, reinterpret_cast<const unsigned char **>(&dataIn), stream ? 0 : 1,
+      &outSize, &outPtr, &self->mBrotli->mTotalOut, &self->mBrotli->mState);
+    outSize = kOutSize - outSize;
+    LOG(("nsHttpCompresssConv %p brotlihandler decompress rv=%x out=%d\n",
+         self, res, outSize));
+
+    if (res == BROTLI_RESULT_ERROR) {
+      LOG(("nsHttpCompressConv %p marking invalid encoding", self));
+      self->mBrotli->mStatus = NS_ERROR_INVALID_CONTENT_ENCODING;
+      return self->mBrotli->mStatus;
+    }
+
+    // in 'the current implementation' brotli consumes all input on success
+    MOZ_ASSERT(!avail);
+    if (avail) {
+      LOG(("nsHttpCompressConv %p did not consume all input", self));
+      self->mBrotli->mStatus = NS_ERROR_UNEXPECTED;
+      return self->mBrotli->mStatus;
+    }
+    if (outSize > 0) {
+      nsresult rv = self->do_OnDataAvailable(self->mBrotli->mRequest,
+                                             self->mBrotli->mContext,
+                                             self->mBrotli->mSourceOffset,
+                                             reinterpret_cast<const char *>(outBuffer),
+                                             outSize);
+      LOG(("nsHttpCompressConv %p BrotliHandler ODA rv=%x", self, rv));
+      if (NS_FAILED(rv)) {
+        self->mBrotli->mStatus = rv;
+        return self->mBrotli->mStatus;
+      }
+    }
+
+    if (res == BROTLI_RESULT_SUCCESS ||
+        res == BROTLI_RESULT_NEEDS_MORE_INPUT) {
+      *countRead = aAvail;
+      return NS_OK;
+    }
+    MOZ_ASSERT (res == BROTLI_RESULT_NEEDS_MORE_OUTPUT);
+  } while (res == BROTLI_RESULT_NEEDS_MORE_OUTPUT);
+
+  self->mBrotli->mStatus = NS_ERROR_UNEXPECTED;
+  return self->mBrotli->mStatus;
 }
 
 NS_IMETHODIMP
 nsHTTPCompressConv::OnDataAvailable(nsIRequest* request,
                                     nsISupports *aContext,
                                     nsIInputStream *iStr,
                                     uint64_t aSourceOffset,
                                     uint32_t aCount)
 {
   nsresult rv = NS_ERROR_INVALID_CONTENT_ENCODING;
   uint32_t streamLen = aCount;
+  LOG(("nsHttpCompressConv %p OnDataAvailable %d", this, aCount));
 
   if (streamLen == 0) {
     NS_ERROR("count of zero passed to OnDataAvailable");
     return NS_ERROR_UNEXPECTED;
   }
 
   if (mStreamEnded) {
     // Hmm... this may just indicate that the data stream is done and that
@@ -301,27 +409,47 @@ nsHTTPCompressConv::OnDataAvailable(nsIR
           break;
         } else {
           return NS_ERROR_INVALID_CONTENT_ENCODING;
         }
       } /* for */
     } /* gzip */
     break;
 
+  case HTTP_COMPRESS_BROTLI:
+  {
+    if (!mBrotli) {
+      mBrotli = new BrotliWrapper();
+    }
+
+    mBrotli->mRequest = request;
+    mBrotli->mContext = aContext;
+    mBrotli->mSourceOffset = aSourceOffset;
+
+    uint32_t countRead;
+    rv = iStr->ReadSegments(BrotliHandler, this, streamLen, &countRead);
+    if (NS_SUCCEEDED(rv)) {
+      rv = mBrotli->mStatus;
+    }
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
+  }
+    break;
+
   default:
     rv = mListener->OnDataAvailable(request, aContext, iStr, aSourceOffset, aCount);
     if (NS_FAILED (rv)) {
       return rv;
     }
   } /* switch */
 
   return NS_OK;
 } /* OnDataAvailable */
 
-
 // XXX/ruslan: need to implement this too
 
 NS_IMETHODIMP
 nsHTTPCompressConv::Convert(nsIInputStream *aFromStream,
                             const char *aFromType,
                             const char *aToType,
                             nsISupports *aCtxt,
                             nsIInputStream **_retval)
--- a/netwerk/streamconv/converters/nsHTTPCompressConv.h
+++ b/netwerk/streamconv/converters/nsHTTPCompressConv.h
@@ -6,16 +6,17 @@
 
 #if !defined (__nsHTTPCompressConv__h__)
 #define	__nsHTTPCompressConv__h__	1
 
 #include "nsIStreamConverter.h"
 #include "nsCOMPtr.h"
 
 #include "zlib.h"
+#include "state.h"
 
 class nsIStringInputStream;
 
 #define NS_HTTPCOMPRESSCONVERTER_CID                    \
   {                                                     \
     /* 66230b2b-17fa-4bd3-abf4-07986151022d */          \
     0x66230b2b,                                         \
       0x17fa,                                           \
@@ -24,29 +25,54 @@ class nsIStringInputStream;
   }
 
 
 #define	HTTP_DEFLATE_TYPE		"deflate"
 #define	HTTP_GZIP_TYPE	        "gzip"
 #define	HTTP_X_GZIP_TYPE	    "x-gzip"
 #define	HTTP_COMPRESS_TYPE	    "compress"
 #define	HTTP_X_COMPRESS_TYPE	"x-compress"
+#define	HTTP_BROTLI_TYPE        "brotli"
 #define	HTTP_IDENTITY_TYPE	    "identity"
 #define	HTTP_UNCOMPRESSED_TYPE	"uncompressed"
 
 namespace mozilla {
 namespace net {
 
 typedef enum    {
   HTTP_COMPRESS_GZIP,
   HTTP_COMPRESS_DEFLATE,
   HTTP_COMPRESS_COMPRESS,
+  HTTP_COMPRESS_BROTLI,
   HTTP_COMPRESS_IDENTITY
 } CompressMode;
 
+class BrotliWrapper
+{
+public:
+  BrotliWrapper()
+    : mTotalOut(0)
+    , mStatus(NS_OK)
+  {
+    BrotliStateInit(&mState);
+  }
+  ~BrotliWrapper()
+  {
+    BrotliStateCleanup(&mState);
+  }
+
+  BrotliState mState;
+  size_t       mTotalOut;
+  nsresult     mStatus;
+
+  nsIRequest  *mRequest;
+  nsISupports *mContext;
+  uint64_t     mSourceOffset;
+};
+
 class nsHTTPCompressConv : public nsIStreamConverter	{
   public:
   // nsISupports methods
     NS_DECL_THREADSAFE_ISUPPORTS
     NS_DECL_NSIREQUESTOBSERVER
     NS_DECL_NSISTREAMLISTENER
 
   // nsIStreamConverter methods
@@ -61,19 +87,25 @@ private:
     CompressMode        mMode;
 
     unsigned char *mOutBuffer;
     unsigned char *mInpBuffer;
 
     uint32_t	mOutBufferLen;
     uint32_t	mInpBufferLen;
 
+    nsAutoPtr<BrotliWrapper> mBrotli;
+
     nsCOMPtr<nsISupports>   mAsyncConvContext;
     nsCOMPtr<nsIStringInputStream>  mStream;
 
+    static NS_METHOD
+    BrotliHandler(nsIInputStream *stream, void *closure, const char *dataIn,
+                  uint32_t, uint32_t avail, uint32_t *countRead);
+
     nsresult do_OnDataAvailable (nsIRequest *request, nsISupports *aContext,
                                  uint64_t aSourceOffset, const char *buffer,
                                  uint32_t aCount);
 
     bool        mCheckHeaderDone;
     bool        mStreamEnded;
     bool        mStreamInitialized;
     bool        mDummyStreamInitialised;
--- a/netwerk/test/unit/test_content_encoding_gzip.js
+++ b/netwerk/test/unit/test_content_encoding_gzip.js
@@ -19,16 +19,41 @@ var tests = [
      ce: "gzip, gzip",
      body: [
 	 0x1f, 0x8b, 0x08, 0x00, 0x72, 0xa1, 0x31, 0x4f, 0x00, 0x03, 0x93, 0xef, 0xe6, 0xe0, 0x88, 0x5a, 
 	 0x60, 0xe8, 0xcf, 0xc0, 0x5c, 0x52, 0x51, 0xc2, 0xa0, 0x7d, 0xf2, 0x84, 0x4e, 0x18, 0xc3, 0xa2, 
 	 0x49, 0x57, 0x1e, 0x09, 0x39, 0xeb, 0x31, 0xec, 0x54, 0xbe, 0x6e, 0xcd, 0xc7, 0xc0, 0xc0, 0x00, 
 	 0x00, 0x6e, 0x90, 0x7a, 0x85, 0x24, 0x00, 0x00, 0x00],
      datalen: 14 // the data length of the uncompressed document
     },
+
+    {url: "/test/cebrotli1",
+     flags: CL_EXPECT_GZIP,
+     ce: "brotli",
+     body: [0x0B, 0x02, 0x80, 0x74, 0x65, 0x73, 0x74, 0x0A, 0x03],
+
+     datalen: 5 // the data length of the uncompressed document
+    },
+
+    // this is not a brotli document
+    {url: "/test/cebrotli2",
+     flags: CL_EXPECT_GZIP | CL_EXPECT_FAILURE,
+     ce: "brotli",
+     body: [0x0B, 0x0A, 0x09],
+     datalen: 3
+    },
+
+    // this is brotli but should come through as identity due to prefs
+    {url: "/test/cebrotli3",
+     flags: 0,
+     ce: "brotli",
+     body: [0x0B, 0x02, 0x80, 0x74, 0x65, 0x73, 0x74, 0x0A, 0x03],
+
+     datalen: 9
+    },
 ];
 
 function setupChannel(url) {
     var ios = Components.classes["@mozilla.org/network/io-service;1"].
                          getService(Ci.nsIIOService);
     var chan = ios.newChannel2("http://localhost:" +
                                httpserver.identity.primaryPort + url,
                                "",
@@ -37,32 +62,48 @@ function setupChannel(url) {
                                Services.scriptSecurityManager.getSystemPrincipal(),
                                null,      // aTriggeringPrincipal
                                Ci.nsILoadInfo.SEC_NORMAL,
                                Ci.nsIContentPolicy.TYPE_OTHER);
     return chan;
 }
 
 function startIter() {
+    if (tests[index].url === "/test/cebrotli3") {
+      // this test wants to make sure we don't do brotli when not in a-e
+      prefs.setCharPref("network.http.accept-encoding", "gzip, deflate");
+    }
     var channel = setupChannel(tests[index].url);
     channel.asyncOpen(new ChannelListener(completeIter, channel, tests[index].flags), null);
 }
 
 function completeIter(request, data, ctx) {
-    do_check_true(data.length == tests[index].datalen);
+    if (!(tests[index].flags & CL_EXPECT_FAILURE)) {
+	do_check_eq(data.length, tests[index].datalen);
+    }
     if (++index < tests.length) {
 	startIter();
     } else {
         httpserver.stop(do_test_finished);
+	prefs.setCharPref("network.http.accept-encoding", cePref);
     }
 }
 
+var prefs;
+var cePref;
 function run_test() {
+    prefs = Cc["@mozilla.org/preferences-service;1"].getService(Ci.nsIPrefBranch);
+    cePref = prefs.getCharPref("network.http.accept-encoding");
+    prefs.setCharPref("network.http.accept-encoding", "gzip, deflate, brotli");
+
     httpserver.registerPathHandler("/test/cegzip1", handler);
     httpserver.registerPathHandler("/test/cegzip2", handler);
+    httpserver.registerPathHandler("/test/cebrotli1", handler);
+    httpserver.registerPathHandler("/test/cebrotli2", handler);
+    httpserver.registerPathHandler("/test/cebrotli3", handler);
     httpserver.start(-1);
 
     startIter();
     do_test_pending();
 }
 
 function handler(metadata, response) {
     response.setStatusLine(metadata.httpVersion, 200, "OK");
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -1689,17 +1689,23 @@
     "extended_statistics_ok": true,
     "description": "Time from submission to dispatch of SPDY transaction (ms)"
   },
   "HTTP_SAW_QUIC_ALT_PROTOCOL": {
     "expires_in_version": "never",
     "kind": "boolean",
     "description": "Fraction of responses with a quic alt-protocol advertisement."
   },
- "HTTP_DISK_CACHE_OVERHEAD": {
+  "HTTP_CONTENT_ENCODING": {
+    "expires_in_version": "never",
+    "kind": "enumerated",
+    "n_values": 6,
+    "description": "encoding removed: 0=unknown, 1=gzip, 2=deflate, 3=brotli"
+  },
+  "HTTP_DISK_CACHE_OVERHEAD": {
     "expires_in_version": "default",
     "kind": "exponential",
     "high": "32000000",
     "n_buckets": 100,
     "extended_statistics_ok": true,
     "description": "HTTP Disk cache memory overhead (bytes)"
   },
   "CACHE_LM_INCONSISTENT": {