Bug 648429: HTTP cache: compress all compressible files; r=michal.novotny
authorGeoff Brown <gbrown@mozilla.com>
Sat, 17 Dec 2011 11:30:29 +0100
changeset 82854 dfc239e7dcf478dda03f503b5b5cd2e40a635cef
parent 82853 d9e8512aa9d4d72655ad184578e566cae290cce9
child 82855 ec4f2192e5337933010cda3ec7bd41f3828f1c3d
push id21705
push usermbrubeck@mozilla.com
push dateSat, 17 Dec 2011 17:01:50 +0000
treeherdermozilla-central@d4aad9645f77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmichal.novotny
bugs648429
milestone11.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 648429: HTTP cache: compress all compressible files; r=michal.novotny
mobile/xul/app/mobile.js
modules/libpref/src/init/all.js
netwerk/cache/nsCacheEntryDescriptor.cpp
netwerk/cache/nsCacheEntryDescriptor.h
netwerk/cache/nsCacheService.cpp
netwerk/cache/nsCacheService.h
netwerk/cache/nsICacheEntryDescriptor.idl
netwerk/protocol/about/nsAboutCacheEntry.cpp
netwerk/protocol/http/nsHttpChannel.cpp
--- a/mobile/xul/app/mobile.js
+++ b/mobile/xul/app/mobile.js
@@ -103,16 +103,19 @@ pref("image.cache.size", 1048576); // by
 
 /* offline cache prefs */
 pref("browser.offline-apps.notify", true);
 pref("browser.cache.offline.enable", true);
 pref("browser.cache.offline.capacity", 5120); // kilobytes
 pref("offline-apps.quota.max", 2048); // kilobytes
 pref("offline-apps.quota.warn", 1024); // kilobytes
 
+/* zlib compression level used for cache compression */
+pref("browser.cache.compression_level", 1);
+
 /* protocol warning prefs */
 pref("network.protocol-handler.warn-external.tel", false);
 pref("network.protocol-handler.warn-external.mailto", false);
 pref("network.protocol-handler.warn-external.vnd.youtube", false);
 
 /* http prefs */
 pref("network.http.pipelining", true);
 pref("network.http.pipelining.ssl", true);
--- a/modules/libpref/src/init/all.js
+++ b/modules/libpref/src/init/all.js
@@ -87,16 +87,22 @@ pref("browser.cache.offline.capacity",  
 // offline apps should be limited to this much data in global storage
 // (in kilobytes)
 pref("offline-apps.quota.max",        204800);
 
 // the user should be warned if offline app disk usage exceeds this amount
 // (in kilobytes)
 pref("offline-apps.quota.warn",        51200);
 
+// zlib compression level used for cache compression:
+// 0 => disable compression
+// 1 => best speed
+// 9 => best compression
+pref("browser.cache.compression_level", 5);
+
 // Whether or not indexedDB is enabled.
 pref("dom.indexedDB.enabled", true);
 // Space to allow indexedDB databases before prompting (in MB).
 pref("dom.indexedDB.warningQuota", 50);
 
 // Whether or not Web Workers are enabled.
 pref("dom.workers.enabled", true);
 // The number of workers per domain allowed to run concurrently.
--- a/netwerk/cache/nsCacheEntryDescriptor.cpp
+++ b/netwerk/cache/nsCacheEntryDescriptor.cpp
@@ -42,16 +42,19 @@
 #include "nsCache.h"
 #include "nsCacheService.h"
 #include "nsCacheEntryDescriptor.h"
 #include "nsCacheEntry.h"
 #include "nsReadableUtils.h"
 #include "nsIOutputStream.h"
 #include "nsCRT.h"
 
+#define kMinDecompressReadBufLen 1024
+#define kMinCompressWriteBufLen  1024
+
 NS_IMPL_THREADSAFE_ISUPPORTS2(nsCacheEntryDescriptor,
                               nsICacheEntryDescriptor,
                               nsICacheEntryInfo)
 
 nsCacheEntryDescriptor::nsCacheEntryDescriptor(nsCacheEntry * entry,
                                                nsCacheAccessMode accessGranted)
     : mCacheEntry(entry),
       mAccessGranted(accessGranted),
@@ -208,17 +211,35 @@ NS_IMETHODIMP nsCacheEntryDescriptor::Se
 }
 
 NS_IMETHODIMP nsCacheEntryDescriptor::GetDataSize(PRUint32 *result)
 {
     NS_ENSURE_ARG_POINTER(result);
     nsCacheServiceAutoLock lock;
     if (!mCacheEntry)  return NS_ERROR_NOT_AVAILABLE;
 
+    const char* val = mCacheEntry->GetMetaDataElement("uncompressed-len");
+    if (!val) {
+        *result = mCacheEntry->DataSize();
+    } else {
+        *result = atol(val);
+    }
+
+    return NS_OK;
+}
+
+
+NS_IMETHODIMP nsCacheEntryDescriptor::GetStorageDataSize(PRUint32 *result)
+{
+    NS_ENSURE_ARG_POINTER(result);
+    nsCacheServiceAutoLock lock;
+    if (!mCacheEntry)  return NS_ERROR_NOT_AVAILABLE;
+
     *result = mCacheEntry->DataSize();
+
     return NS_OK;
 }
 
 
 nsresult
 nsCacheEntryDescriptor::RequestDataSizeChange(PRInt32 deltaSize)
 {
     nsCacheServiceAutoLock lock;
@@ -271,18 +292,24 @@ nsCacheEntryDescriptor::OpenInputStream(
         if (!mCacheEntry)                  return NS_ERROR_NOT_AVAILABLE;
         if (!mCacheEntry->IsStreamData())  return NS_ERROR_CACHE_DATA_IS_NOT_STREAM;
 
         // ensure valid permissions
         if (!(mAccessGranted & nsICache::ACCESS_READ))
             return NS_ERROR_CACHE_READ_ACCESS_DENIED;
     }
 
-    nsInputStreamWrapper* cacheInput =
-        new nsInputStreamWrapper(this, offset);
+    nsInputStreamWrapper* cacheInput = nsnull;
+    const char *val;
+    val = mCacheEntry->GetMetaDataElement("uncompressed-len");
+    if (val) {
+        cacheInput = new nsDecompressInputStreamWrapper(this, offset);
+    } else {
+        cacheInput = new nsInputStreamWrapper(this, offset);
+    }
     if (!cacheInput) return NS_ERROR_OUT_OF_MEMORY;
 
     NS_ADDREF(*result = cacheInput);
     return NS_OK;
 }
 
 NS_IMETHODIMP
 nsCacheEntryDescriptor::OpenOutputStream(PRUint32 offset, nsIOutputStream ** result)
@@ -294,18 +321,25 @@ nsCacheEntryDescriptor::OpenOutputStream
         if (!mCacheEntry)                  return NS_ERROR_NOT_AVAILABLE;
         if (!mCacheEntry->IsStreamData())  return NS_ERROR_CACHE_DATA_IS_NOT_STREAM;
 
         // ensure valid permissions
         if (!(mAccessGranted & nsICache::ACCESS_WRITE))
             return NS_ERROR_CACHE_WRITE_ACCESS_DENIED;
     }
 
-    nsOutputStreamWrapper* cacheOutput =
-        new nsOutputStreamWrapper(this, offset);
+    nsOutputStreamWrapper* cacheOutput = nsnull;
+    PRInt32 compressionLevel = nsCacheService::CacheCompressionLevel();
+    const char *val;
+    val = mCacheEntry->GetMetaDataElement("uncompressed-len");
+    if ((compressionLevel > 0) && val) {
+        cacheOutput = new nsCompressOutputStreamWrapper(this, offset);
+    } else {
+        cacheOutput = new nsOutputStreamWrapper(this, offset);
+    }
     if (!cacheOutput) return NS_ERROR_OUT_OF_MEMORY;
 
     NS_ADDREF(*result = cacheOutput);
     return NS_OK;
 }
 
 
 NS_IMETHODIMP
@@ -507,17 +541,17 @@ nsCacheEntryDescriptor::VisitMetaData(ns
     NS_ENSURE_ARG_POINTER(visitor);
     if (!mCacheEntry)  return NS_ERROR_NOT_AVAILABLE;
 
     return mCacheEntry->VisitMetaDataElements(visitor);
 }
 
 
 /******************************************************************************
- * nsCacheInputStream - a wrapper for nsIInputstream keeps the cache entry
+ * nsCacheInputStream - a wrapper for nsIInputStream keeps the cache entry
  *                      open while referenced.
  ******************************************************************************/
 
 NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheEntryDescriptor::nsInputStreamWrapper,
                               nsIInputStream)
 
 nsresult nsCacheEntryDescriptor::
 nsInputStreamWrapper::LazyInit()
@@ -582,19 +616,146 @@ nsInputStreamWrapper::IsNonBlocking(bool
 {
     // cache streams will never return NS_BASE_STREAM_WOULD_BLOCK
     *result = false;
     return NS_OK;
 }
 
 
 /******************************************************************************
- * nsCacheOutputStream - a wrapper for nsIOutputstream to track the amount of
- *                       data written to a cache entry.
- *                     - also keeps the cache entry open while referenced.
+ * nsDecompressInputStreamWrapper - an input stream wrapper that decompresses
+ ******************************************************************************/
+
+NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheEntryDescriptor::nsDecompressInputStreamWrapper,
+                              nsIInputStream)
+
+NS_IMETHODIMP nsCacheEntryDescriptor::
+nsDecompressInputStreamWrapper::Read(char *    buf, 
+                                     PRUint32  count, 
+                                     PRUint32 *countRead)
+{
+    int zerr = Z_OK;
+    nsresult rv = NS_OK;
+
+    if (!mStreamInitialized) {
+        rv = InitZstream();
+        if (NS_FAILED(rv)) {
+            return rv;
+        }
+    }
+
+    mZstream.next_out = (Bytef*)buf;
+    mZstream.avail_out = count;
+
+    if (mReadBufferLen < count) {
+        // Allocate a buffer for reading from the input stream. This will
+        // determine the max number of compressed bytes read from the
+        // input stream at one time. Making the buffer size proportional
+        // to the request size is not necessary, but helps minimize the
+        // number of read requests to the input stream.
+        PRUint32 newBufLen = NS_MAX(count, (PRUint32)kMinDecompressReadBufLen);
+        unsigned char* newBuf;
+        newBuf = (unsigned char*)nsMemory::Realloc(mReadBuffer, 
+            newBufLen);
+        if (newBuf) {
+            mReadBuffer = newBuf;
+            mReadBufferLen = newBufLen;
+        }
+        if (!mReadBuffer) {
+            mReadBufferLen = 0;
+            return NS_ERROR_OUT_OF_MEMORY;
+        }
+    }
+
+    // read and inflate data until the output buffer is full, or
+    // there is no more data to read
+    while (NS_SUCCEEDED(rv) &&
+           zerr == Z_OK && 
+           mZstream.avail_out > 0 &&
+           count > 0) {
+        if (mZstream.avail_in == 0) {
+            rv = nsInputStreamWrapper::Read((char*)mReadBuffer, 
+                                            mReadBufferLen, 
+                                            &mZstream.avail_in);
+            if (NS_FAILED(rv) || !mZstream.avail_in) {
+                break;
+            }
+            mZstream.next_in = mReadBuffer;
+        }
+        zerr = inflate(&mZstream, Z_NO_FLUSH);
+        if (zerr == Z_STREAM_END) {
+            // The compressed data may have been stored in multiple
+            // chunks/streams. To allow for this case, re-initialize 
+            // the inflate stream and continue decompressing from 
+            // the next byte.
+            Bytef * saveNextIn = mZstream.next_in;
+            unsigned int saveAvailIn = mZstream.avail_in;
+            Bytef * saveNextOut = mZstream.next_out;
+            unsigned int saveAvailOut = mZstream.avail_out;
+            inflateReset(&mZstream);
+            mZstream.next_in = saveNextIn;
+            mZstream.avail_in = saveAvailIn;
+            mZstream.next_out = saveNextOut;
+            mZstream.avail_out = saveAvailOut;
+            zerr = Z_OK;
+        } else if (zerr != Z_OK) {
+            rv = NS_ERROR_INVALID_CONTENT_ENCODING;
+        }
+    }
+    if (NS_SUCCEEDED(rv)) {
+        *countRead = count - mZstream.avail_out;
+    }
+    return rv;
+}
+
+nsresult nsCacheEntryDescriptor::
+nsDecompressInputStreamWrapper::Close()
+{
+    EndZstream();
+    if (mReadBuffer) {
+        nsMemory::Free(mReadBuffer);
+        mReadBuffer = 0;
+        mReadBufferLen = 0;
+    }
+    return nsInputStreamWrapper::Close();
+}
+
+nsresult nsCacheEntryDescriptor::
+nsDecompressInputStreamWrapper::InitZstream()
+{
+    // Initialize zlib inflate stream
+    mZstream.zalloc = Z_NULL;
+    mZstream.zfree = Z_NULL;
+    mZstream.opaque = Z_NULL;
+    mZstream.next_out = Z_NULL;
+    mZstream.avail_out = 0;
+    mZstream.next_in = Z_NULL;
+    mZstream.avail_in = 0;
+    if (inflateInit(&mZstream) != Z_OK) {
+        return NS_ERROR_FAILURE;
+    }
+    mStreamInitialized = PR_TRUE;
+    return NS_OK;
+}
+
+nsresult nsCacheEntryDescriptor::
+nsDecompressInputStreamWrapper::EndZstream()
+{
+    if (mStreamInitialized && !mStreamEnded) {
+        inflateEnd(&mZstream);
+        mStreamEnded = PR_TRUE;
+    }
+    return NS_OK;
+}
+
+
+/******************************************************************************
+ * nsOutputStreamWrapper - a wrapper for nsIOutputstream to track the amount of
+ *                         data written to a cache entry.
+ *                       - also keeps the cache entry open while referenced.
  ******************************************************************************/
 
 NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheEntryDescriptor::nsOutputStreamWrapper,
                               nsIOutputStream)
 
 nsresult nsCacheEntryDescriptor::
 nsOutputStreamWrapper::LazyInit()
 {
@@ -699,8 +860,149 @@ nsOutputStreamWrapper::WriteSegments(nsR
 
 NS_IMETHODIMP nsCacheEntryDescriptor::
 nsOutputStreamWrapper::IsNonBlocking(bool *result)
 {
     // cache streams will never return NS_BASE_STREAM_WOULD_BLOCK
     *result = false;
     return NS_OK;
 }
+
+
+/******************************************************************************
+ * nsCompressOutputStreamWrapper - an output stream wrapper that compresses
+ *   data before it is written
+ ******************************************************************************/
+
+NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheEntryDescriptor::nsCompressOutputStreamWrapper,
+                              nsIOutputStream)
+
+NS_IMETHODIMP nsCacheEntryDescriptor::
+nsCompressOutputStreamWrapper::Write(const char * buf,
+                                     PRUint32     count,
+                                     PRUint32 *   result)
+{
+    int zerr = Z_OK;
+    nsresult rv = NS_OK;
+
+    if (!mStreamInitialized) {
+        rv = InitZstream();
+        if (NS_FAILED(rv)) {
+            return rv;
+        }
+    }
+
+    if (!mWriteBuffer) {
+        // Once allocated, this buffer is referenced by the zlib stream and
+        // cannot be grown. We use 2x(initial write request) to approximate
+        // a stream buffer size proportional to request buffers.
+        mWriteBufferLen = NS_MAX(count*2, (PRUint32)kMinCompressWriteBufLen);
+        mWriteBuffer = (unsigned char*)nsMemory::Alloc(mWriteBufferLen);
+        if (!mWriteBuffer) {
+            mWriteBufferLen = 0;
+            return NS_ERROR_OUT_OF_MEMORY;
+        }
+        mZstream.next_out = mWriteBuffer;
+        mZstream.avail_out = mWriteBufferLen;
+    }
+
+    // Compress (deflate) the requested buffer. Keep going
+    // until the entire buffer has been deflated.
+    mZstream.avail_in = count;
+    mZstream.next_in = (Bytef*)buf;
+    while (mZstream.avail_in > 0) {
+        zerr = deflate(&mZstream, Z_NO_FLUSH);
+        if (zerr == Z_STREAM_ERROR) {
+            return NS_ERROR_FAILURE;
+        }
+        // Note: Z_BUF_ERROR is non-fatal and sometimes expected here.
+
+        // If the compression stream output buffer is filled, write
+        // it out to the underlying stream wrapper.
+        if (mZstream.avail_out == 0) {
+            rv = WriteBuffer();
+            if (NS_FAILED(rv)) {
+                return rv;
+            }
+        }
+    }
+    *result = count;
+    mUncompressedCount += *result;
+    return NS_OK;
+}
+
+NS_IMETHODIMP nsCacheEntryDescriptor::
+nsCompressOutputStreamWrapper::Close()
+{
+    nsresult rv = NS_OK;
+    int zerr = 0;
+
+    if (mStreamInitialized) {
+        // complete compression of any data remaining in the zlib stream
+        do {
+            zerr = deflate(&mZstream, Z_FINISH);
+            rv = WriteBuffer();
+        } while (zerr == Z_OK && rv == NS_OK);
+        deflateEnd(&mZstream);
+    }
+
+    if (mDescriptor->CacheEntry()) {
+        nsCAutoString uncompressedLenStr;
+        rv = mDescriptor->GetMetaDataElement("uncompressed-len",
+                                             getter_Copies(uncompressedLenStr));
+        if (NS_SUCCEEDED(rv)) {
+            PRInt32 oldCount = uncompressedLenStr.ToInteger(&rv);
+            if (NS_SUCCEEDED(rv)) {
+                mUncompressedCount += oldCount;
+            }
+        }
+        uncompressedLenStr.Adopt(0);
+        uncompressedLenStr.AppendInt(mUncompressedCount);
+        rv = mDescriptor->SetMetaDataElement("uncompressed-len",
+            uncompressedLenStr.get());
+    }
+
+    if (mWriteBuffer) {
+        nsMemory::Free(mWriteBuffer);
+        mWriteBuffer = 0;
+        mWriteBufferLen = 0;
+    }
+
+    return nsOutputStreamWrapper::Close();
+}
+
+nsresult nsCacheEntryDescriptor::
+nsCompressOutputStreamWrapper::InitZstream()
+{
+    // Determine compression level: Aggressive compression
+    // may impact performance on mobile devices, while a
+    // lower compression level still provides substantial
+    // space savings for many text streams.
+    PRInt32 compressionLevel = nsCacheService::CacheCompressionLevel();
+
+    // Initialize zlib deflate stream
+    mZstream.zalloc = Z_NULL;
+    mZstream.zfree = Z_NULL;
+    mZstream.opaque = Z_NULL;
+    if (deflateInit2(&mZstream, compressionLevel, Z_DEFLATED,
+                     MAX_WBITS, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
+        return NS_ERROR_FAILURE;
+    }
+    mZstream.next_in = Z_NULL;
+    mZstream.avail_in = 0;
+
+    mStreamInitialized = PR_TRUE;
+
+    return NS_OK;
+}
+
+nsresult nsCacheEntryDescriptor::
+nsCompressOutputStreamWrapper::WriteBuffer()
+{
+    PRUint32 bytesToWrite = mWriteBufferLen - mZstream.avail_out;
+    PRUint32 result = 0;
+    nsresult rv = nsCacheEntryDescriptor::nsOutputStreamWrapper::Write(
+        (const char *)mWriteBuffer, bytesToWrite, &result);
+    mZstream.next_out = mWriteBuffer;
+    mZstream.avail_out = mWriteBufferLen;
+    return rv;
+}
+
--- a/netwerk/cache/nsCacheEntryDescriptor.h
+++ b/netwerk/cache/nsCacheEntryDescriptor.h
@@ -43,16 +43,17 @@
 #define _nsCacheEntryDescriptor_h_
 
 #include "nsICacheEntryDescriptor.h"
 #include "nsCacheEntry.h"
 #include "nsIInputStream.h"
 #include "nsIOutputStream.h"
 #include "nsCacheService.h"
 #include "nsIDiskCacheStreamInternal.h"
+#include "zlib.h"
 
 /******************************************************************************
 * nsCacheEntryDescriptor
 *******************************************************************************/
 class nsCacheEntryDescriptor :
     public PRCList,
     public nsICacheEntryDescriptor
 {
@@ -124,24 +125,55 @@ private:
 
      private:
          nsresult LazyInit();
          nsresult EnsureInit() { return mInitialized ? NS_OK : LazyInit(); }
      };
      friend class nsInputStreamWrapper;
 
 
+     class nsDecompressInputStreamWrapper : public nsInputStreamWrapper {
+     private:
+         unsigned char* mReadBuffer;
+         PRUint32 mReadBufferLen;
+         z_stream mZstream;
+         PRBool mStreamInitialized;
+         PRBool mStreamEnded;
+     public:
+         NS_DECL_ISUPPORTS
+
+         nsDecompressInputStreamWrapper(nsCacheEntryDescriptor * desc,
+                                      PRUint32 off)
+          : nsInputStreamWrapper(desc, off)
+          , mReadBuffer(0)
+          , mReadBufferLen(0)
+          , mStreamInitialized(PR_FALSE)
+          , mStreamEnded(PR_FALSE)
+         {
+         }
+         virtual ~nsDecompressInputStreamWrapper()
+         {
+             Close();
+         }
+         NS_IMETHOD Read(char* buf, PRUint32 count, PRUint32 * result);
+         NS_IMETHOD Close();
+     private:
+         nsresult InitZstream();
+         nsresult EndZstream();
+     };
+
+
      /*************************************************************************
       * output stream wrapper class -
       *
       * The output stream wrapper references the descriptor, but the descriptor
       * doesn't need any references to the stream wrapper.
       *************************************************************************/
      class nsOutputStreamWrapper : public nsIOutputStream {
-     private:
+     protected:
          nsCacheEntryDescriptor *    mDescriptor;
          nsCOMPtr<nsIOutputStream>   mOutput;
          PRUint32                    mStartOffset;
          bool                        mInitialized;
      public:
          NS_DECL_ISUPPORTS
          NS_DECL_NSIOUTPUTSTREAM
 
@@ -165,16 +197,46 @@ private:
 
      private:
          nsresult LazyInit();
          nsresult EnsureInit() { return mInitialized ? NS_OK : LazyInit(); }
          nsresult OnWrite(PRUint32 count);
      };
      friend class nsOutputStreamWrapper;
 
+     class nsCompressOutputStreamWrapper : public nsOutputStreamWrapper {
+     private:
+         unsigned char* mWriteBuffer;
+         PRUint32 mWriteBufferLen;
+         z_stream mZstream;
+         PRBool mStreamInitialized;
+         PRUint32 mUncompressedCount;
+     public:
+         NS_DECL_ISUPPORTS
+
+         nsCompressOutputStreamWrapper(nsCacheEntryDescriptor * desc, 
+                                       PRUint32 off)
+          : nsOutputStreamWrapper(desc, off)
+          , mWriteBuffer(0)
+          , mWriteBufferLen(0)
+          , mStreamInitialized(PR_FALSE)
+          , mUncompressedCount(0)
+         {
+         }
+         virtual ~nsCompressOutputStreamWrapper()
+         { 
+             Close();
+         }
+         NS_IMETHOD Write(const char* buf, PRUint32 count, PRUint32 * result);
+         NS_IMETHOD Close();
+     private:
+         nsresult InitZstream();
+         nsresult WriteBuffer();
+     };
+
  private:
      /**
       * nsCacheEntryDescriptor data members
       */
      nsCacheEntry          * mCacheEntry; // we are a child of the entry
      nsCacheAccessMode       mAccessGranted;
      nsIOutputStream       * mOutput;
 };
--- a/netwerk/cache/nsCacheService.cpp
+++ b/netwerk/cache/nsCacheService.cpp
@@ -100,16 +100,19 @@ using namespace mozilla;
 #define OFFLINE_CACHE_DIR_PREF      "browser.cache.offline.parent_directory"
 #define OFFLINE_CACHE_CAPACITY_PREF "browser.cache.offline.capacity"
 #define OFFLINE_CACHE_CAPACITY      512000
 
 #define MEMORY_CACHE_ENABLE_PREF    "browser.cache.memory.enable"
 #define MEMORY_CACHE_CAPACITY_PREF  "browser.cache.memory.capacity"
 #define MEMORY_CACHE_MAX_ENTRY_SIZE_PREF "browser.cache.memory.max_entry_size"
 
+#define CACHE_COMPRESSION_LEVEL_PREF "browser.cache.compression_level"
+#define CACHE_COMPRESSION_LEVEL     1
+
 static const char * observerList[] = { 
     "profile-before-change",
     "profile-do-change",
     NS_XPCOM_SHUTDOWN_OBSERVER_ID,
     NS_PRIVATE_BROWSING_SWITCH_TOPIC
 };
 static const char * prefList[] = { 
     DISK_CACHE_ENABLE_PREF,
@@ -117,17 +120,18 @@ static const char * prefList[] = {
     DISK_CACHE_CAPACITY_PREF,
     DISK_CACHE_DIR_PREF,
     DISK_CACHE_MAX_ENTRY_SIZE_PREF,
     OFFLINE_CACHE_ENABLE_PREF,
     OFFLINE_CACHE_CAPACITY_PREF,
     OFFLINE_CACHE_DIR_PREF,
     MEMORY_CACHE_ENABLE_PREF,
     MEMORY_CACHE_CAPACITY_PREF,
-    MEMORY_CACHE_MAX_ENTRY_SIZE_PREF
+    MEMORY_CACHE_MAX_ENTRY_SIZE_PREF,
+    CACHE_COMPRESSION_LEVEL_PREF
 };
 
 // Cache sizes, in KB
 const PRInt32 DEFAULT_CACHE_SIZE = 250 * 1024;  // 250 MB
 const PRInt32 MIN_CACHE_SIZE = 50 * 1024;       //  50 MB
 const PRInt32 MAX_CACHE_SIZE = 1024 * 1024;     //   1 GB
 // Default cache size was 50 MB for many years until FF 4:
 const PRInt32 PRE_GECKO_2_0_DEFAULT_CACHE_SIZE = 50 * 1024;
@@ -144,16 +148,17 @@ public:
         , mDiskCacheCapacity(0)
         , mDiskCacheMaxEntrySize(-1) // -1 means "no limit"
         , mOfflineCacheEnabled(false)
         , mOfflineCacheCapacity(0)
         , mMemoryCacheEnabled(true)
         , mMemoryCacheCapacity(-1)
         , mMemoryCacheMaxEntrySize(-1) // -1 means "no limit"
         , mInPrivateBrowsing(false)
+        , mCacheCompressionLevel(CACHE_COMPRESSION_LEVEL)
     {
     }
 
     virtual ~nsCacheProfilePrefObserver() {}
     
     nsresult        Install();
     void            Remove();
     nsresult        ReadPrefs(nsIPrefBranch* branch);
@@ -167,16 +172,18 @@ public:
     bool            OfflineCacheEnabled();
     PRInt32         OfflineCacheCapacity()         { return mOfflineCacheCapacity; }
     nsILocalFile *  OfflineCacheParentDirectory()  { return mOfflineCacheParentDirectory; }
     
     bool            MemoryCacheEnabled();
     PRInt32         MemoryCacheCapacity();
     PRInt32         MemoryCacheMaxEntrySize()     { return mMemoryCacheMaxEntrySize; }
 
+    PRInt32         CacheCompressionLevel();
+
     static PRUint32 GetSmartCacheSize(const nsAString& cachePath,
                                       PRUint32 currentSize);
 
 private:
     bool                    PermittedToSmartSize(nsIPrefBranch*, bool firstRun);
     bool                    mHaveProfile;
     
     bool                    mDiskCacheEnabled;
@@ -188,16 +195,18 @@ private:
     PRInt32                 mOfflineCacheCapacity; // in kilobytes
     nsCOMPtr<nsILocalFile>  mOfflineCacheParentDirectory;
     
     bool                    mMemoryCacheEnabled;
     PRInt32                 mMemoryCacheCapacity; // in kilobytes
     PRInt32                 mMemoryCacheMaxEntrySize; // in kilobytes
 
     bool                    mInPrivateBrowsing;
+
+    PRInt32                 mCacheCompressionLevel;
 };
 
 NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheProfilePrefObserver, nsIObserver)
 
 // Runnable sent to main thread after the cache IO thread calculates available
 // disk space, so that there is no race in setting mDiskCacheCapacity.
 class nsSetSmartSizeEvent: public nsRunnable 
 {
@@ -503,16 +512,22 @@ nsCacheProfilePrefObserver::Observe(nsIS
             PRInt32 newMaxSize;
             rv = branch->GetIntPref(MEMORY_CACHE_MAX_ENTRY_SIZE_PREF,
                                      &newMaxSize);
             if (NS_FAILED(rv)) 
                 return rv;
             
             mMemoryCacheMaxEntrySize = NS_MAX(-1, newMaxSize);
             nsCacheService::SetMemoryCacheMaxEntrySize(mMemoryCacheMaxEntrySize);
+        } else if (!strcmp(CACHE_COMPRESSION_LEVEL_PREF, data.get())) {
+            mCacheCompressionLevel = CACHE_COMPRESSION_LEVEL;
+            (void)branch->GetIntPref(CACHE_COMPRESSION_LEVEL_PREF,
+                                     &mCacheCompressionLevel);
+            mCacheCompressionLevel = NS_MAX(0, mCacheCompressionLevel);
+            mCacheCompressionLevel = NS_MIN(9, mCacheCompressionLevel);
         }
     } else if (!strcmp(NS_PRIVATE_BROWSING_SWITCH_TOPIC, topic)) {
         if (!strcmp(NS_PRIVATE_BROWSING_ENTER, data.get())) {
             mInPrivateBrowsing = true;
 
             nsCacheService::OnEnterExitPrivateBrowsing();
 
             mDiskCacheEnabled = false;
@@ -792,16 +807,24 @@ nsCacheProfilePrefObserver::ReadPrefs(ns
 
     mMemoryCacheCapacity = -1;
     (void) branch->GetIntPref(MEMORY_CACHE_CAPACITY_PREF,
                               &mMemoryCacheCapacity);
 
     (void) branch->GetIntPref(MEMORY_CACHE_MAX_ENTRY_SIZE_PREF,
                               &mMemoryCacheMaxEntrySize);
     mMemoryCacheMaxEntrySize = NS_MAX(-1, mMemoryCacheMaxEntrySize);
+
+    // read cache compression level pref
+    mCacheCompressionLevel = CACHE_COMPRESSION_LEVEL;
+    (void)branch->GetIntPref(CACHE_COMPRESSION_LEVEL_PREF,
+                             &mCacheCompressionLevel);
+    mCacheCompressionLevel = NS_MAX(0, mCacheCompressionLevel);
+    mCacheCompressionLevel = NS_MIN(9, mCacheCompressionLevel);
+
     return rv;
 }
 
 nsresult
 nsCacheService::DispatchToCacheIOThread(nsIRunnable* event)
 {
     if (!gService->mCacheIOThread) return NS_ERROR_NOT_AVAILABLE;
     return gService->mCacheIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
@@ -925,16 +948,21 @@ nsCacheProfilePrefObserver::MemoryCacheC
         capacity   *= 1024;
     } else {
         capacity    = 0;
     }
 
     return capacity;
 }
 
+PRInt32
+nsCacheProfilePrefObserver::CacheCompressionLevel()
+{
+    return mCacheCompressionLevel;
+}
 
 /******************************************************************************
  * nsProcessRequestEvent
  *****************************************************************************/
 
 class nsProcessRequestEvent : public nsRunnable {
 public:
     nsProcessRequestEvent(nsCacheRequest *aRequest)
@@ -2286,16 +2314,24 @@ nsCacheService::ValidateEntry(nsCacheEnt
     nsresult rv = gService->ProcessPendingRequests(entry);
     NS_ASSERTION(rv == NS_OK, "ProcessPendingRequests failed.");
     // XXX what else should be done?
 
     return rv;
 }
 
 
+PRInt32
+nsCacheService::CacheCompressionLevel()
+{
+    PRInt32 level = gService->mObserver->CacheCompressionLevel();
+    return level;
+}
+
+
 void
 nsCacheService::DeactivateEntry(nsCacheEntry * entry)
 {
     CACHE_LOG_DEBUG(("Deactivating entry %p\n", entry));
     nsresult  rv = NS_OK;
     NS_ASSERTION(entry->IsNotInUse(), "### deactivating an entry while in use!");
     nsCacheDevice * device = nsnull;
 
--- a/netwerk/cache/nsCacheService.h
+++ b/netwerk/cache/nsCacheService.h
@@ -118,16 +118,17 @@ public:
                                               nsIOutputStream ** result);
 
     static nsresult  OnDataSizeChange(nsCacheEntry * entry, PRInt32 deltaSize);
 
     static nsresult  SetCacheElement(nsCacheEntry * entry, nsISupports * element);
 
     static nsresult  ValidateEntry(nsCacheEntry * entry);
 
+    static PRInt32   CacheCompressionLevel();
 
     /**
      * Methods called by any cache classes
      */
 
     static
     nsCacheService * GlobalInstance()   { return gService; }
 
@@ -171,16 +172,18 @@ public:
     static void      SetMemoryCacheMaxEntrySize(PRInt32  maxSize);
 
     static void      SetOfflineCacheEnabled(bool    enabled);
     // Sets the offline cache capacity (in kilobytes)
     static void      SetOfflineCacheCapacity(PRInt32  capacity);
 
     static void      SetMemoryCache();
 
+    static void      SetCacheCompressionLevel(PRInt32 level);
+
     static void      OnEnterExitPrivateBrowsing();
 
     // Starts smart cache size computation if disk device is available
     static nsresult  SetDiskSmartSize();
 
     nsresult         Init();
     void             Shutdown();
 
--- a/netwerk/cache/nsICacheEntryDescriptor.idl
+++ b/netwerk/cache/nsICacheEntryDescriptor.idl
@@ -132,16 +132,22 @@ interface nsICacheEntryDescriptor : nsIC
 
     /**
      * Get/set security info on the cache entry for this descriptor.  This fails
      * if the storage policy is not STORE_IN_MEMORY.
      */
     attribute nsISupports securityInfo;
     
     /**
+     * Get the size of the cache entry data, as stored. This may differ
+     * from the entry's dataSize, if the entry is compressed.
+     */
+    readonly attribute unsigned long storageDataSize;
+
+    /**
      * Doom the cache entry this descriptor references in order to slate it for 
      * removal.  Once doomed a cache entry cannot be undoomed.
      *
      * A descriptor with WRITE access can doom the cache entry and choose to
      * fail pending requests.  This means that pending requests will not get
      * a cache descriptor.  This is meant as a tool for clients that wish to
      * instruct pending requests to skip the cache.
      */
--- a/netwerk/protocol/about/nsAboutCacheEntry.cpp
+++ b/netwerk/protocol/about/nsAboutCacheEntry.cpp
@@ -335,17 +335,17 @@ nsAboutCacheEntry::WriteCacheEntryDescri
         APPEND_ROW("expires", timeBuf);
     } else {
         APPEND_ROW("expires", "No expiration time");
     }
 
     // Data Size
     s.Truncate();
     PRUint32 dataSize;
-    descriptor->GetDataSize(&dataSize);
+    descriptor->GetStorageDataSize(&dataSize);
     s.AppendInt((PRInt32)dataSize);     // XXX nsICacheEntryInfo interfaces should be fixed.
     APPEND_ROW("Data size", s);
 
     // Storage Policy
 
     // XXX Stream Based?
 
     // XXX Cache Device
@@ -399,21 +399,18 @@ nsAboutCacheEntry::WriteCacheEntryDescri
     if (dataSize) { // don't draw an <hr> if the Data Size is 0.
         nsCOMPtr<nsIInputStream> stream;
         descriptor->OpenInputStream(0, getter_AddRefs(stream));
         if (stream) {
             buffer.AssignLiteral("<hr/>\n"
                                  "<pre>");
             PRUint32 hexDumpState = 0;
             char chunk[4096];
-            while (dataSize) {
-                PRUint32 count = NS_MIN<PRUint32>(dataSize, sizeof(chunk));
-                if (NS_FAILED(stream->Read(chunk, count, &n)) || n == 0)
-                    break;
-                dataSize -= n;
+            while(NS_SUCCEEDED(stream->Read(chunk, sizeof(chunk), &n)) && 
+                  n > 0) {
                 HexDump(&hexDumpState, chunk, n, buffer);
                 outputStream->Write(buffer.get(), buffer.Length(), &n);
                 buffer.Truncate();
             }
             buffer.AssignLiteral("</pre>\n");
             outputStream->Write(buffer.get(), buffer.Length(), &n);
       }
     }
--- a/netwerk/protocol/http/nsHttpChannel.cpp
+++ b/netwerk/protocol/http/nsHttpChannel.cpp
@@ -3198,16 +3198,42 @@ nsHttpChannel::InstallCacheListener(PRUi
 {
     nsresult rv;
 
     LOG(("Preparing to write data into the cache [uri=%s]\n", mSpec.get()));
 
     NS_ASSERTION(mCacheEntry, "no cache entry");
     NS_ASSERTION(mListener, "no listener");
 
+    nsCacheStoragePolicy policy;
+    rv = mCacheEntry->GetStoragePolicy(&policy);
+    if (NS_FAILED(rv)) {
+        policy = nsICache::STORE_ON_DISK_AS_FILE;
+    }
+
+    // If the content is compressible and the server has not compressed it,
+    // mark the cache entry for compression.
+    if ((mResponseHead->PeekHeader(nsHttp::Content_Encoding) == nsnull) && (
+         policy != nsICache::STORE_ON_DISK_AS_FILE) && (
+         mResponseHead->ContentType().EqualsLiteral(TEXT_HTML) ||
+         mResponseHead->ContentType().EqualsLiteral(TEXT_PLAIN) ||
+         mResponseHead->ContentType().EqualsLiteral(TEXT_CSS) ||
+         mResponseHead->ContentType().EqualsLiteral(TEXT_JAVASCRIPT) ||
+         mResponseHead->ContentType().EqualsLiteral(TEXT_ECMASCRIPT) ||
+         mResponseHead->ContentType().EqualsLiteral(TEXT_XML) ||
+         mResponseHead->ContentType().EqualsLiteral(APPLICATION_JAVASCRIPT) ||
+         mResponseHead->ContentType().EqualsLiteral(APPLICATION_ECMASCRIPT) ||
+         mResponseHead->ContentType().EqualsLiteral(APPLICATION_XJAVASCRIPT) ||
+         mResponseHead->ContentType().EqualsLiteral(APPLICATION_XHTML_XML))) {
+        rv = mCacheEntry->SetMetaDataElement("uncompressed-len", "0"); 
+        if (NS_FAILED(rv)) {
+            LOG(("unable to mark cache entry for compression"));
+        }
+    } 
+      
     nsCOMPtr<nsIOutputStream> out;
     rv = mCacheEntry->OpenOutputStream(offset, getter_AddRefs(out));
     if (NS_FAILED(rv)) return rv;
 
     // XXX disk cache does not support overlapped i/o yet
 #if 0
     // Mark entry valid inorder to allow simultaneous reading...
     rv = mCacheEntry->MarkValid();
@@ -3220,20 +3246,17 @@ nsHttpChannel::InstallCacheListener(PRUi
 
     nsCOMPtr<nsICacheService> serv =
         do_GetService(NS_CACHESERVICE_CONTRACTID, &rv);
     NS_ENSURE_SUCCESS(rv, rv);
 
     nsCOMPtr<nsIEventTarget> cacheIOTarget;
     serv->GetCacheIOTarget(getter_AddRefs(cacheIOTarget));
 
-    nsCacheStoragePolicy policy;
-    rv = mCacheEntry->GetStoragePolicy(&policy);
-
-    if (NS_FAILED(rv) || policy == nsICache::STORE_ON_DISK_AS_FILE ||
+    if (policy == nsICache::STORE_ON_DISK_AS_FILE ||
         !cacheIOTarget) {
         LOG(("nsHttpChannel::InstallCacheListener sync tee %p rv=%x policy=%d "
              "cacheIOTarget=%p", tee.get(), rv, policy, cacheIOTarget.get()));
         rv = tee->Init(mListener, out, nsnull);
     } else {
         LOG(("nsHttpChannel::InstallCacheListener async tee %p", tee.get()));
         rv = tee->InitAsync(mListener, cacheIOTarget, out, nsnull);
     }