bug 366559 - patch 7, content-encoding brotli for https r=bagder
authorPatrick McManus <mcmanus@ducksong.com>
Fri, 18 Sep 2015 18:04:28 -0400
changeset 263754 4d45c4323570
parent 263753 5e0a3850571f
child 263755 d08142841263
push id65417
push usermcmanus@ducksong.com
push date2015-09-22 16:12 +0000
treeherdermozilla-inbound@4d45c4323570 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbagder
bugs366559
milestone44.0a1
bug 366559 - patch 7, content-encoding brotli for https r=bagder
modules/libpref/init/all.js
netwerk/build/nsNetModule.cpp
netwerk/mime/nsMimeTypes.h
netwerk/protocol/http/HttpBaseChannel.cpp
netwerk/protocol/http/nsHttpHandler.cpp
netwerk/streamconv/converters/nsHTTPCompressConv.cpp
netwerk/streamconv/converters/nsHTTPCompressConv.h
netwerk/test/unit/test_content_encoding_gzip.js
toolkit/components/telemetry/Histograms.json
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -1317,17 +1317,17 @@ pref("network.http.enablePerElementRefer
 
 // Maximum number of consecutive redirects before aborting.
 pref("network.http.redirection-limit", 20);
 
 // Enable http compression: comment this out in case of problems with 1.1
 // NOTE: support for "compress" has been disabled per bug 196406.
 // NOTE: separate values with comma+space (", "): see bug 576033
 pref("network.http.accept-encoding", "gzip, deflate");
-pref("network.http.accept-encoding.secure", "gzip, deflate");
+pref("network.http.accept-encoding.secure", "gzip, deflate, brotli");
 
 pref("network.http.pipelining"      , false);
 pref("network.http.pipelining.ssl"  , false); // disable pipelining over SSL
 pref("network.http.pipelining.abtest", false);
 pref("network.http.proxy.pipelining", false);
 
 // Max number of requests in the pipeline
 pref("network.http.pipelining.maxrequests" , 32);
--- a/netwerk/build/nsNetModule.cpp
+++ b/netwerk/build/nsNetModule.cpp
@@ -438,16 +438,17 @@ nsresult NS_NewStreamConv(nsStreamConver
 #define INDEX_TO_HTML                "?from=application/http-index-format&to=text/html"
 #define MULTI_MIXED_X                "?from=multipart/x-mixed-replace&to=*/*"
 #define MULTI_MIXED                  "?from=multipart/mixed&to=*/*"
 #define APPLICATION_PACKAGE_CONV     "?from=" APPLICATION_PACKAGE "&to=*/*"
 #define MULTI_BYTERANGES             "?from=multipart/byteranges&to=*/*"
 #define UNKNOWN_CONTENT              "?from=" UNKNOWN_CONTENT_TYPE "&to=*/*"
 #define GZIP_TO_UNCOMPRESSED         "?from=gzip&to=uncompressed"
 #define XGZIP_TO_UNCOMPRESSED        "?from=x-gzip&to=uncompressed"
+#define BROTLI_TO_UNCOMPRESSED       "?from=brotli&to=uncompressed"
 #define COMPRESS_TO_UNCOMPRESSED     "?from=compress&to=uncompressed"
 #define XCOMPRESS_TO_UNCOMPRESSED    "?from=x-compress&to=uncompressed"
 #define DEFLATE_TO_UNCOMPRESSED      "?from=deflate&to=uncompressed"
 #define PLAIN_TO_HTML                "?from=text/plain&to=text/html"
 
 #ifdef BUILD_BINHEX_DECODER
 #define BINHEX_TO_WILD               "?from=application/mac-binhex40&to=*/*"
 #endif
@@ -457,16 +458,17 @@ static const mozilla::Module::CategoryEn
     { NS_ISTREAMCONVERTER_KEY, INDEX_TO_HTML, "" },
     { NS_ISTREAMCONVERTER_KEY, MULTI_MIXED_X, "" },
     { NS_ISTREAMCONVERTER_KEY, MULTI_MIXED, "" },
     { NS_ISTREAMCONVERTER_KEY, APPLICATION_PACKAGE_CONV, "" },
     { NS_ISTREAMCONVERTER_KEY, MULTI_BYTERANGES, "" },
     { NS_ISTREAMCONVERTER_KEY, UNKNOWN_CONTENT, "" },
     { NS_ISTREAMCONVERTER_KEY, GZIP_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, XGZIP_TO_UNCOMPRESSED, "" },
+    { NS_ISTREAMCONVERTER_KEY, BROTLI_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, COMPRESS_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, XCOMPRESS_TO_UNCOMPRESSED, "" },
     { NS_ISTREAMCONVERTER_KEY, DEFLATE_TO_UNCOMPRESSED, "" },
 #ifdef BUILD_BINHEX_DECODER
     { NS_ISTREAMCONVERTER_KEY, BINHEX_TO_WILD, "" },
 #endif
     { NS_ISTREAMCONVERTER_KEY, PLAIN_TO_HTML, "" },
     NS_BINARYDETECTOR_CATEGORYENTRY,
@@ -1045,16 +1047,17 @@ static const mozilla::Module::ContractID
     { NS_ISTREAMCONVERTER_KEY MULTI_BYTERANGES, &kNS_MULTIMIXEDCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY MULTI_MIXED, &kNS_MULTIMIXEDCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY APPLICATION_PACKAGE_CONV, &kNS_MULTIMIXEDCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY UNKNOWN_CONTENT, &kNS_UNKNOWNDECODER_CID },
     { NS_GENERIC_CONTENT_SNIFFER, &kNS_UNKNOWNDECODER_CID },
     { NS_BINARYDETECTOR_CONTRACTID, &kNS_BINARYDETECTOR_CID },
     { NS_ISTREAMCONVERTER_KEY GZIP_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY XGZIP_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
+    { NS_ISTREAMCONVERTER_KEY BROTLI_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY COMPRESS_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY XCOMPRESS_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY DEFLATE_TO_UNCOMPRESSED, &kNS_HTTPCOMPRESSCONVERTER_CID },
     { NS_ISTREAMCONVERTER_KEY PLAIN_TO_HTML, &kNS_NSTXTTOHTMLCONVERTER_CID },
 #ifdef BUILD_BINHEX_DECODER
     { NS_ISTREAMCONVERTER_KEY BINHEX_TO_WILD, &kNS_BINHEXDECODER_CID },
 #endif
     { MOZ_TXTTOHTMLCONV_CONTRACTID, &kMOZITXTTOHTMLCONV_CID },
--- a/netwerk/mime/nsMimeTypes.h
+++ b/netwerk/mime/nsMimeTypes.h
@@ -25,16 +25,17 @@
 #define APPLICATION_MACBINARY               "application/x-macbinary"
 #define APPLICATION_COMPRESS                "application/x-compress"
 #define APPLICATION_COMPRESS2               "application/compress"
 #define APPLICATION_FORTEZZA_CKL            "application/x-fortezza-ckl"
 #define APPLICATION_FORTEZZA_KRL            "application/x-fortezza-krl"
 #define APPLICATION_GZIP                    "application/x-gzip"
 #define APPLICATION_GZIP2                   "application/gzip"
 #define APPLICATION_GZIP3                   "application/x-gunzip"
+#define APPLICATION_BROTLI                  "application/brotli"
 #define APPLICATION_ZIP                     "application/zip"
 #define APPLICATION_HTTP_INDEX_FORMAT       "application/http-index-format"
 #define APPLICATION_ECMASCRIPT              "application/ecmascript"
 #define APPLICATION_JAVASCRIPT              "application/javascript"
 #define APPLICATION_XJAVASCRIPT             "application/x-javascript"
 #define APPLICATION_JSON                    "application/json"
 #define APPLICATION_NETSCAPE_REVOCATION     "application/x-netscape-revocation"
 #define APPLICATION_NS_PROXY_AUTOCONFIG     "application/x-ns-proxy-autoconfig"
--- a/netwerk/protocol/http/HttpBaseChannel.cpp
+++ b/netwerk/protocol/http/HttpBaseChannel.cpp
@@ -831,16 +831,27 @@ HttpBaseChannel::DoApplyContentConversio
                                   aCtxt,
                                   getter_AddRefs(converter));
       if (NS_FAILED(rv)) {
         LOG(("Unexpected failure of AsyncConvertData %s\n", val));
         return rv;
       }
 
       LOG(("converter removed '%s' content-encoding\n", val));
+      if (gHttpHandler->IsTelemetryEnabled()) {
+        int mode = 0;
+        if (from.Equals("gzip") || from.Equals("x-gzip")) {
+          mode = 1;
+        } else if (from.Equals("deflate") || from.Equals("x-deflate")) {
+          mode = 2;
+        } else if (from.Equals("brotli")) {
+          mode = 3;
+        }
+        Telemetry::Accumulate(Telemetry::HTTP_CONTENT_ENCODING, mode);
+      }
       nextListener = converter;
     }
     else {
       if (val)
         LOG(("Unknown content encoding '%s', ignoring\n", val));
     }
   }
   *aNewNextListener = nextListener;
@@ -935,16 +946,24 @@ HttpBaseChannel::nsContentEncodings::Get
   if (!haveType) {
     encoding.BeginReading(start);
     if (CaseInsensitiveFindInReadable(NS_LITERAL_CSTRING("deflate"), start, end)) {
       aNextEncoding.AssignLiteral(APPLICATION_ZIP);
       haveType = true;
     }
   }
 
+  if (!haveType) {
+    encoding.BeginReading(start);
+    if (CaseInsensitiveFindInReadable(NS_LITERAL_CSTRING("brotli"), start, end)) {
+      aNextEncoding.AssignLiteral(APPLICATION_BROTLI);
+      haveType = true;
+    }
+  }
+
   // Prepare to fetch the next encoding
   mCurEnd = mCurStart;
   mReady = false;
 
   if (haveType)
     return NS_OK;
 
   NS_WARNING("Unknown encoding type");
--- a/netwerk/protocol/http/nsHttpHandler.cpp
+++ b/netwerk/protocol/http/nsHttpHandler.cpp
@@ -488,32 +488,34 @@ nsHttpHandler::AddConnectionHeader(nsHtt
 }
 
 bool
 nsHttpHandler::IsAcceptableEncoding(const char *enc, bool isSecure)
 {
     if (!enc)
         return false;
 
-    // HTTP 1.1 allows servers to send x-gzip and x-compress instead
-    // of gzip and compress, for example.  So, we'll always strip off
-    // an "x-" prefix before matching the encoding to one we claim
-    // to accept.
-    if (!PL_strncasecmp(enc, "x-", 2))
-        enc += 2;
-
+    // we used to accept x-foo anytime foo was acceptable, but that's just
+    // continuing bad behavior.. so limit it to known x-* patterns
+    bool rv;
+    if (isSecure) {
+        rv = nsHttp::FindToken(mHttpsAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    } else {
+        rv = nsHttp::FindToken(mHttpAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    }
     // gzip and deflate are inherently acceptable in modern HTTP - always
     // process them if a stream converter can also be found.
-    if (!PL_strcasecmp(enc, "gzip") || !PL_strcasecmp(enc, "deflate"))
-        return true;
-
-    if (isSecure) {
-        return nsHttp::FindToken(mHttpsAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    if (!rv &&
+        (!PL_strcasecmp(enc, "gzip") || !PL_strcasecmp(enc, "deflate") ||
+         !PL_strcasecmp(enc, "x-gzip") || !PL_strcasecmp(enc, "x-deflate"))) {
+        rv = true;
     }
-    return nsHttp::FindToken(mHttpAcceptEncodings.get(), enc, HTTP_LWS ",") != nullptr;
+    LOG(("nsHttpHandler::IsAceptableEncoding %s https=%d %d\n",
+         enc, isSecure, rv));
+    return rv;
 }
 
 nsresult
 nsHttpHandler::GetStreamConverterService(nsIStreamConverterService **result)
 {
     if (!mStreamConvSvc) {
         nsresult rv;
         nsCOMPtr<nsIStreamConverterService> service =
--- a/netwerk/streamconv/converters/nsHTTPCompressConv.cpp
+++ b/netwerk/streamconv/converters/nsHTTPCompressConv.cpp
@@ -9,16 +9,24 @@
 #include "plstr.h"
 #include "nsCOMPtr.h"
 #include "nsError.h"
 #include "nsStreamUtils.h"
 #include "nsStringStream.h"
 #include "nsComponentManagerUtils.h"
 #include "nsThreadUtils.h"
 #include "mozilla/Preferences.h"
+#include "nsIForcePendingChannel.h"
+
+// brotli headers
+#include "state.h"
+#include "decode.h"
+
+extern PRLogModuleInfo *gHttpLog;
+#define LOG(args) MOZ_LOG(gHttpLog, mozilla::LogLevel::Debug, args)
 
 namespace mozilla {
 namespace net {
 
 // nsISupports implementation
 NS_IMPL_ISUPPORTS(nsHTTPCompressConv,
                   nsIStreamConverter,
                   nsIStreamListener,
@@ -34,26 +42,28 @@ nsHTTPCompressConv::nsHTTPCompressConv()
   , mCheckHeaderDone(false)
   , mStreamEnded(false)
   , mStreamInitialized(false)
   , mLen(0)
   , hMode(0)
   , mSkipCount(0)
   , mFlags(0)
 {
+  LOG(("nsHttpCompresssConv %p ctor\n", this));
   if (NS_IsMainThread()) {
     mFailUncleanStops =
       Preferences::GetBool("network.http.enforce-framing.http", false);
   } else {
     mFailUncleanStops = false;
   }
 }
 
 nsHTTPCompressConv::~nsHTTPCompressConv()
 {
+  LOG(("nsHttpCompresssConv %p dtor\n", this));
   if (mInpBuffer) {
     free(mInpBuffer);
   }
 
   if (mOutBuffer) {
     free(mOutBuffer);
   }
 
@@ -73,55 +83,153 @@ nsHTTPCompressConv::AsyncConvertData(con
   if (!PL_strncasecmp(aFromType, HTTP_COMPRESS_TYPE, sizeof(HTTP_COMPRESS_TYPE)-1) ||
       !PL_strncasecmp(aFromType, HTTP_X_COMPRESS_TYPE, sizeof(HTTP_X_COMPRESS_TYPE)-1)) {
     mMode = HTTP_COMPRESS_COMPRESS;
   } else if (!PL_strncasecmp(aFromType, HTTP_GZIP_TYPE, sizeof(HTTP_GZIP_TYPE)-1) ||
              !PL_strncasecmp(aFromType, HTTP_X_GZIP_TYPE, sizeof(HTTP_X_GZIP_TYPE)-1)) {
     mMode = HTTP_COMPRESS_GZIP;
   } else if (!PL_strncasecmp(aFromType, HTTP_DEFLATE_TYPE, sizeof(HTTP_DEFLATE_TYPE)-1)) {
     mMode = HTTP_COMPRESS_DEFLATE;
+  } else if (!PL_strncasecmp(aFromType, HTTP_BROTLI_TYPE, sizeof(HTTP_BROTLI_TYPE)-1)) {
+    mMode = HTTP_COMPRESS_BROTLI;
   }
+  LOG(("nsHttpCompresssConv %p AsyncConvertData %s %s mode %d\n",
+       this, aFromType, aToType, mMode));
 
   // hook ourself up with the receiving listener.
   mListener = aListener;
 
   mAsyncConvContext = aCtxt;
   return NS_OK;
 }
 
 NS_IMETHODIMP
 nsHTTPCompressConv::OnStartRequest(nsIRequest* request, nsISupports *aContext)
 {
+  LOG(("nsHttpCompresssConv %p onstart\n", this));
   return mListener->OnStartRequest(request, aContext);
 }
 
 NS_IMETHODIMP
 nsHTTPCompressConv::OnStopRequest(nsIRequest* request, nsISupports *aContext,
                                   nsresult aStatus)
 {
+  nsresult status = aStatus;
+  LOG(("nsHttpCompresssConv %p onstop %x\n", this, aStatus));
+  
   // Framing integrity is enforced for content-encoding: gzip, but not for
   // content-encoding: deflate. Note that gzip vs deflate is NOT determined
   // by content sniffing but only via header.
-  if (!mStreamEnded && NS_SUCCEEDED(aStatus) &&
+  if (!mStreamEnded && NS_SUCCEEDED(status) &&
       (mFailUncleanStops && (mMode == HTTP_COMPRESS_GZIP)) ) {
     // This is not a clean end of gzip stream: the transfer is incomplete.
-    aStatus = NS_ERROR_NET_PARTIAL_TRANSFER;
+    status = NS_ERROR_NET_PARTIAL_TRANSFER;
+    LOG(("nsHttpCompresssConv %p onstop partial gzip\n", this));
   }
-  return mListener->OnStopRequest(request, aContext, aStatus);
+  if (NS_SUCCEEDED(status) && mMode == HTTP_COMPRESS_BROTLI) {
+    uint32_t waste;
+    nsCOMPtr<nsIForcePendingChannel> fpChannel = do_QueryInterface(request);
+    bool isPending = false;
+    if (request) {
+      request->IsPending(&isPending);
+    }
+    if (fpChannel && !isPending) {
+      fpChannel->ForcePending(true);
+    }
+    status = BrotliHandler(nullptr, this, nullptr, 0, 0, &waste);
+    LOG(("nsHttpCompresssConv %p onstop brotlihandler rv %x\n", this, status));
+    if (fpChannel && !isPending) {
+      fpChannel->ForcePending(false);
+    }
+  }
+  if (NS_FAILED(status) && status != aStatus) {
+    LOG(("nsHttpCompresssConv %p onstop calling cancel %x\n", this, status));
+    request->Cancel(status);
+  }
+  return mListener->OnStopRequest(request, aContext, status);
+}
+
+
+// static
+NS_METHOD
+nsHTTPCompressConv::BrotliHandler(nsIInputStream *stream, void *closure, const char *dataIn,
+                                  uint32_t, uint32_t aAvail, uint32_t *countRead)
+{
+  nsHTTPCompressConv *self = static_cast<nsHTTPCompressConv *>(closure);
+  *countRead = 0;
+
+  const uint32_t kOutSize = 128 * 1024; // just a chunk size, we call in a loop
+  unsigned char outBuffer[kOutSize];
+  unsigned char *outPtr;
+  size_t outSize;
+  size_t avail = aAvail;
+  BrotliResult res;
+
+  do {
+    outSize = kOutSize;
+    outPtr = outBuffer;
+
+    // brotli api is documented in brotli/dec/decode.h
+    LOG(("nsHttpCompresssConv %p brotlihandler decompress %d finish %d\n",
+         self, avail, !stream));
+    res = ::BrotliDecompressBufferStreaming(
+      &avail, reinterpret_cast<const unsigned char **>(&dataIn), stream ? 0 : 1,
+      &outSize, &outPtr, &self->mBrotli->mTotalOut, &self->mBrotli->mState);
+    outSize = kOutSize - outSize;
+    LOG(("nsHttpCompresssConv %p brotlihandler decompress rv=%x out=%d\n",
+         self, res, outSize));
+
+    if (res == BROTLI_RESULT_ERROR) {
+      LOG(("nsHttpCompressConv %p marking invalid encoding", self));
+      self->mBrotli->mStatus = NS_ERROR_INVALID_CONTENT_ENCODING;
+      return self->mBrotli->mStatus;
+    }
+
+    // in 'the current implementation' brotli consumes all input on success
+    MOZ_ASSERT(!avail);
+    if (avail) {
+      LOG(("nsHttpCompressConv %p did not consume all input", self));
+      self->mBrotli->mStatus = NS_ERROR_UNEXPECTED;
+      return self->mBrotli->mStatus;
+    }
+    if (outSize > 0) {
+      nsresult rv = self->do_OnDataAvailable(self->mBrotli->mRequest,
+                                             self->mBrotli->mContext,
+                                             self->mBrotli->mSourceOffset,
+                                             reinterpret_cast<const char *>(outBuffer),
+                                             outSize);
+      LOG(("nsHttpCompressConv %p BrotliHandler ODA rv=%x", self, rv));
+      if (NS_FAILED(rv)) {
+        self->mBrotli->mStatus = rv;
+        return self->mBrotli->mStatus;
+      }
+    }
+
+    if (res == BROTLI_RESULT_SUCCESS ||
+        res == BROTLI_RESULT_NEEDS_MORE_INPUT) {
+      *countRead = aAvail;
+      return NS_OK;
+    }
+    MOZ_ASSERT (res == BROTLI_RESULT_NEEDS_MORE_OUTPUT);
+  } while (res == BROTLI_RESULT_NEEDS_MORE_OUTPUT);
+
+  self->mBrotli->mStatus = NS_ERROR_UNEXPECTED;
+  return self->mBrotli->mStatus;
 }
 
 NS_IMETHODIMP
 nsHTTPCompressConv::OnDataAvailable(nsIRequest* request,
                                     nsISupports *aContext,
                                     nsIInputStream *iStr,
                                     uint64_t aSourceOffset,
                                     uint32_t aCount)
 {
   nsresult rv = NS_ERROR_INVALID_CONTENT_ENCODING;
   uint32_t streamLen = aCount;
+  LOG(("nsHttpCompressConv %p OnDataAvailable %d", this, aCount));
 
   if (streamLen == 0) {
     NS_ERROR("count of zero passed to OnDataAvailable");
     return NS_ERROR_UNEXPECTED;
   }
 
   if (mStreamEnded) {
     // Hmm... this may just indicate that the data stream is done and that
@@ -301,27 +409,47 @@ nsHTTPCompressConv::OnDataAvailable(nsIR
           break;
         } else {
           return NS_ERROR_INVALID_CONTENT_ENCODING;
         }
       } /* for */
     } /* gzip */
     break;
 
+  case HTTP_COMPRESS_BROTLI:
+  {
+    if (!mBrotli) {
+      mBrotli = new BrotliWrapper();
+    }
+
+    mBrotli->mRequest = request;
+    mBrotli->mContext = aContext;
+    mBrotli->mSourceOffset = aSourceOffset;
+
+    uint32_t countRead;
+    rv = iStr->ReadSegments(BrotliHandler, this, streamLen, &countRead);
+    if (NS_SUCCEEDED(rv)) {
+      rv = mBrotli->mStatus;
+    }
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
+  }
+    break;
+
   default:
     rv = mListener->OnDataAvailable(request, aContext, iStr, aSourceOffset, aCount);
     if (NS_FAILED (rv)) {
       return rv;
     }
   } /* switch */
 
   return NS_OK;
 } /* OnDataAvailable */
 
-
 // XXX/ruslan: need to implement this too
 
 NS_IMETHODIMP
 nsHTTPCompressConv::Convert(nsIInputStream *aFromStream,
                             const char *aFromType,
                             const char *aToType,
                             nsISupports *aCtxt,
                             nsIInputStream **_retval)
--- a/netwerk/streamconv/converters/nsHTTPCompressConv.h
+++ b/netwerk/streamconv/converters/nsHTTPCompressConv.h
@@ -6,16 +6,17 @@
 
 #if !defined (__nsHTTPCompressConv__h__)
 #define	__nsHTTPCompressConv__h__	1
 
 #include "nsIStreamConverter.h"
 #include "nsCOMPtr.h"
 
 #include "zlib.h"
+#include "state.h"
 
 class nsIStringInputStream;
 
 #define NS_HTTPCOMPRESSCONVERTER_CID                    \
   {                                                     \
     /* 66230b2b-17fa-4bd3-abf4-07986151022d */          \
     0x66230b2b,                                         \
       0x17fa,                                           \
@@ -24,29 +25,54 @@ class nsIStringInputStream;
   }
 
 
 #define	HTTP_DEFLATE_TYPE		"deflate"
 #define	HTTP_GZIP_TYPE	        "gzip"
 #define	HTTP_X_GZIP_TYPE	    "x-gzip"
 #define	HTTP_COMPRESS_TYPE	    "compress"
 #define	HTTP_X_COMPRESS_TYPE	"x-compress"
+#define	HTTP_BROTLI_TYPE        "brotli"
 #define	HTTP_IDENTITY_TYPE	    "identity"
 #define	HTTP_UNCOMPRESSED_TYPE	"uncompressed"
 
 namespace mozilla {
 namespace net {
 
 typedef enum    {
   HTTP_COMPRESS_GZIP,
   HTTP_COMPRESS_DEFLATE,
   HTTP_COMPRESS_COMPRESS,
+  HTTP_COMPRESS_BROTLI,
   HTTP_COMPRESS_IDENTITY
 } CompressMode;
 
+class BrotliWrapper
+{
+public:
+  BrotliWrapper()
+    : mTotalOut(0)
+    , mStatus(NS_OK)
+  {
+    BrotliStateInit(&mState);
+  }
+  ~BrotliWrapper()
+  {
+    BrotliStateCleanup(&mState);
+  }
+
+  BrotliState mState;
+  size_t       mTotalOut;
+  nsresult     mStatus;
+
+  nsIRequest  *mRequest;
+  nsISupports *mContext;
+  uint64_t     mSourceOffset;
+};
+
 class nsHTTPCompressConv : public nsIStreamConverter	{
   public:
   // nsISupports methods
     NS_DECL_THREADSAFE_ISUPPORTS
     NS_DECL_NSIREQUESTOBSERVER
     NS_DECL_NSISTREAMLISTENER
 
   // nsIStreamConverter methods
@@ -61,19 +87,25 @@ private:
     CompressMode        mMode;
 
     unsigned char *mOutBuffer;
     unsigned char *mInpBuffer;
 
     uint32_t	mOutBufferLen;
     uint32_t	mInpBufferLen;
 
+    nsAutoPtr<BrotliWrapper> mBrotli;
+
     nsCOMPtr<nsISupports>   mAsyncConvContext;
     nsCOMPtr<nsIStringInputStream>  mStream;
 
+    static NS_METHOD
+    BrotliHandler(nsIInputStream *stream, void *closure, const char *dataIn,
+                  uint32_t, uint32_t avail, uint32_t *countRead);
+
     nsresult do_OnDataAvailable (nsIRequest *request, nsISupports *aContext,
                                  uint64_t aSourceOffset, const char *buffer,
                                  uint32_t aCount);
 
     bool        mCheckHeaderDone;
     bool        mStreamEnded;
     bool        mStreamInitialized;
     bool        mDummyStreamInitialised;
--- a/netwerk/test/unit/test_content_encoding_gzip.js
+++ b/netwerk/test/unit/test_content_encoding_gzip.js
@@ -19,16 +19,41 @@ var tests = [
      ce: "gzip, gzip",
      body: [
 	 0x1f, 0x8b, 0x08, 0x00, 0x72, 0xa1, 0x31, 0x4f, 0x00, 0x03, 0x93, 0xef, 0xe6, 0xe0, 0x88, 0x5a, 
 	 0x60, 0xe8, 0xcf, 0xc0, 0x5c, 0x52, 0x51, 0xc2, 0xa0, 0x7d, 0xf2, 0x84, 0x4e, 0x18, 0xc3, 0xa2, 
 	 0x49, 0x57, 0x1e, 0x09, 0x39, 0xeb, 0x31, 0xec, 0x54, 0xbe, 0x6e, 0xcd, 0xc7, 0xc0, 0xc0, 0x00, 
 	 0x00, 0x6e, 0x90, 0x7a, 0x85, 0x24, 0x00, 0x00, 0x00],
      datalen: 14 // the data length of the uncompressed document
     },
+
+    {url: "/test/cebrotli1",
+     flags: CL_EXPECT_GZIP,
+     ce: "brotli",
+     body: [0x0B, 0x02, 0x80, 0x74, 0x65, 0x73, 0x74, 0x0A, 0x03],
+
+     datalen: 5 // the data length of the uncompressed document
+    },
+
+    // this is not a brotli document
+    {url: "/test/cebrotli2",
+     flags: CL_EXPECT_GZIP | CL_EXPECT_FAILURE,
+     ce: "brotli",
+     body: [0x0B, 0x0A, 0x09],
+     datalen: 3
+    },
+
+    // this is brotli but should come through as identity due to prefs
+    {url: "/test/cebrotli3",
+     flags: 0,
+     ce: "brotli",
+     body: [0x0B, 0x02, 0x80, 0x74, 0x65, 0x73, 0x74, 0x0A, 0x03],
+
+     datalen: 9
+    },
 ];
 
 function setupChannel(url) {
     var ios = Components.classes["@mozilla.org/network/io-service;1"].
                          getService(Ci.nsIIOService);
     var chan = ios.newChannel2("http://localhost:" +
                                httpserver.identity.primaryPort + url,
                                "",
@@ -37,32 +62,48 @@ function setupChannel(url) {
                                Services.scriptSecurityManager.getSystemPrincipal(),
                                null,      // aTriggeringPrincipal
                                Ci.nsILoadInfo.SEC_NORMAL,
                                Ci.nsIContentPolicy.TYPE_OTHER);
     return chan;
 }
 
 function startIter() {
+    if (tests[index].url === "/test/cebrotli3") {
+      // this test wants to make sure we don't do brotli when not in a-e
+      prefs.setCharPref("network.http.accept-encoding", "gzip, deflate");
+    }
     var channel = setupChannel(tests[index].url);
     channel.asyncOpen(new ChannelListener(completeIter, channel, tests[index].flags), null);
 }
 
 function completeIter(request, data, ctx) {
-    do_check_true(data.length == tests[index].datalen);
+    if (!(tests[index].flags & CL_EXPECT_FAILURE)) {
+	do_check_eq(data.length, tests[index].datalen);
+    }
     if (++index < tests.length) {
 	startIter();
     } else {
         httpserver.stop(do_test_finished);
+	prefs.setCharPref("network.http.accept-encoding", cePref);
     }
 }
 
+var prefs;
+var cePref;
 function run_test() {
+    prefs = Cc["@mozilla.org/preferences-service;1"].getService(Ci.nsIPrefBranch);
+    cePref = prefs.getCharPref("network.http.accept-encoding");
+    prefs.setCharPref("network.http.accept-encoding", "gzip, deflate, brotli");
+
     httpserver.registerPathHandler("/test/cegzip1", handler);
     httpserver.registerPathHandler("/test/cegzip2", handler);
+    httpserver.registerPathHandler("/test/cebrotli1", handler);
+    httpserver.registerPathHandler("/test/cebrotli2", handler);
+    httpserver.registerPathHandler("/test/cebrotli3", handler);
     httpserver.start(-1);
 
     startIter();
     do_test_pending();
 }
 
 function handler(metadata, response) {
     response.setStatusLine(metadata.httpVersion, 200, "OK");
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -1689,17 +1689,23 @@
     "extended_statistics_ok": true,
     "description": "Time from submission to dispatch of SPDY transaction (ms)"
   },
   "HTTP_SAW_QUIC_ALT_PROTOCOL": {
     "expires_in_version": "never",
     "kind": "boolean",
     "description": "Fraction of responses with a quic alt-protocol advertisement."
   },
- "HTTP_DISK_CACHE_OVERHEAD": {
+  "HTTP_CONTENT_ENCODING": {
+    "expires_in_version": "never",
+    "kind": "enumerated",
+    "n_values": 6,
+    "description": "encoding removed: 0=unknown, 1=gzip, 2=deflate, 3=brotli"
+  },
+  "HTTP_DISK_CACHE_OVERHEAD": {
     "expires_in_version": "default",
     "kind": "exponential",
     "high": "32000000",
     "n_buckets": 100,
     "extended_statistics_ok": true,
     "description": "HTTP Disk cache memory overhead (bytes)"
   },
   "CACHE_LM_INCONSISTENT": {