Bug 804857 - Start with a small compression buffer. r=njn
authorBenjamin Peterson <benjamin@python.org>
Wed, 24 Oct 2012 16:15:48 -0700
changeset 111302 1c27fd77dbaa
parent 111301 dfb516a4afc2
child 111303 ea436c6f7d2d
push id23740
push userryanvm@gmail.com
push date2012-10-25 12:13 +0000
treeherdermozilla-central@5374fb480634 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnjn
bugs804857
milestone19.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 804857 - Start with a small compression buffer. r=njn
js/src/jit-test/tests/basic/function-tosource-bug779694.js
js/src/jsscript.cpp
js/src/jsutil.cpp
js/src/jsutil.h
--- a/js/src/jit-test/tests/basic/function-tosource-bug779694.js
+++ b/js/src/jit-test/tests/basic/function-tosource-bug779694.js
@@ -1,6 +1,8 @@
+// This test creates poorly compressible input, which tests certain paths in
+// source code compression.
 var x = "";
 for (var i=0; i<400; ++i) {
     x += String.fromCharCode(i * 289);
 }
 var s = "'" + x + "'";
 assertEq(Function("evt", s).toString(), "function anonymous(evt) {\n" + s + "\n}");
--- a/js/src/jsscript.cpp
+++ b/js/src/jsscript.cpp
@@ -927,52 +927,90 @@ SourceCompressorThread::finish()
 bool
 SourceCompressorThread::internalCompress()
 {
     JS_ASSERT(state == COMPRESSING);
     JS_ASSERT(tok);
 
     ScriptSource *ss = tok->ss;
     JS_ASSERT(!ss->ready());
-    const size_t COMPRESS_THRESHOLD = 512;
     size_t compressedLength = 0;
     size_t nbytes = sizeof(jschar) * ss->length_;
 
     // Memory allocation functions on JSRuntime and JSContext are not
     // threadsafe. We have to use the js_* variants.
-    ss->data.compressed = static_cast<unsigned char *>(js_malloc(nbytes));
-    if (!ss->data.compressed)
-        return false;
 
 #ifdef USE_ZLIB
+    const size_t COMPRESS_THRESHOLD = 512;
     if (nbytes >= COMPRESS_THRESHOLD) {
-        Compressor comp(reinterpret_cast<const unsigned char *>(tok->chars),
-                        nbytes, ss->data.compressed);
-        if (comp.init()) {
-            while (!stop && comp.compressMore())
-                ;
-            compressedLength = comp.finish();
-            if (stop || compressedLength == nbytes)
-                compressedLength = 0;
+        // Try to keep the maximum memory usage down by only allocating half the
+        // size of the string, first.
+        size_t firstSize = nbytes / 2;
+        ss->data.compressed = static_cast<unsigned char *>(js_malloc(firstSize));
+        if (!ss->data.compressed)
+            return false;
+        Compressor comp(reinterpret_cast<const unsigned char *>(tok->chars), nbytes);
+        if (!comp.init())
+            return false;
+        comp.setOutput(ss->data.compressed, firstSize);
+        bool cont = !stop;
+        while (cont) {
+            switch (comp.compressMore()) {
+              case Compressor::CONTINUE:
+                break;
+              case Compressor::MOREOUTPUT: {
+                if (comp.outWritten() == nbytes) {
+                    cont = false;
+                    break;
+                }
+
+                // The compressed output is greater than half the size of the
+                // original string. Reallocate to the full size.
+                void *newmem = js_realloc(ss->data.compressed, nbytes);
+                if (!newmem) {
+                    js_free(ss->data.compressed);
+                    ss->data.compressed = NULL;
+                    return false;
+                }
+                ss->data.compressed = static_cast<unsigned char *>(newmem);
+                comp.setOutput(ss->data.compressed, nbytes);
+                break;
+              }
+              case Compressor::DONE:
+                cont = false;
+                break;
+              case Compressor::OOM:
+                return false;
+            }
+            cont = cont && !stop;
         }
+        compressedLength = comp.outWritten();
+        if (stop || compressedLength == nbytes)
+            compressedLength = 0;
     }
 #endif
-    ss->compressedLength_ = compressedLength;
     if (compressedLength == 0) {
+        // Note ss->data.source might be NULL.
+        jschar *buf = static_cast<jschar *>(js_realloc(ss->data.source, nbytes));
+        if (!buf) {
+            if (ss->data.source) {
+                js_free(ss->data.source);
+                ss->data.source = NULL;
+            }
+            return false;
+        }
+        ss->data.source = buf;
         PodCopy(ss->data.source, tok->chars, ss->length());
     } else {
-        // Shrink the buffer to the size of the compressed data.
+        // Shrink the buffer to the size of the compressed data. Shouldn't fail.
         void *newmem = js_realloc(ss->data.compressed, compressedLength);
-        if (!newmem) {
-            js_free(ss->data.compressed);
-            ss->data.compressed = NULL;
-            return false;
-        }
+        JS_ASSERT(newmem);
         ss->data.compressed = static_cast<unsigned char *>(newmem);
     }
+    ss->compressedLength_ = compressedLength;
     return true;
 }
 
 void
 SourceCompressorThread::threadLoop()
 {
     PR_Lock(lock);
     while (true) {
@@ -1167,17 +1205,17 @@ ScriptSource::setSourceCopy(JSContext *c
         ready_ = false;
 #endif
         tok->ss = this;
         tok->chars = src.get();
         cx->runtime->sourceCompressorThread.compress(tok);
     } else
 #endif
     {
-        data.source = cx->pod_malloc<jschar>(length);
+        data.source = cx->runtime->pod_malloc<jschar>(length);
         if (!data.source)
             return false;
         PodCopy(data.source, src.get(), length_);
     }
 
     return true;
 }
 
--- a/js/src/jsutil.cpp
+++ b/js/src/jsutil.cpp
@@ -37,65 +37,87 @@ zlib_alloc(void *cx, uInt items, uInt si
 }
 
 static void
 zlib_free(void *cx, void *addr)
 {
     js_free(addr);
 }
 
+Compressor::Compressor(const unsigned char *inp, size_t inplen)
+    : inp(inp),
+      inplen(inplen),
+      outbytes(0)
+{
+    JS_ASSERT(inplen > 0);
+    zs.opaque = NULL;
+    zs.next_in = (Bytef *)inp;
+    zs.avail_in = 0;
+    zs.next_out = NULL;
+    zs.avail_out = 0;
+    zs.zalloc = zlib_alloc;
+    zs.zfree = zlib_free;
+}
+
+
+Compressor::~Compressor()
+{
+    int ret = deflateEnd(&zs);
+    if (ret != Z_OK) {
+        // If we finished early, we can get a Z_DATA_ERROR.
+        JS_ASSERT(ret == Z_DATA_ERROR);
+        JS_ASSERT(uInt(zs.next_in - inp) < inplen || !zs.avail_out);
+    }
+}
+
 bool
 Compressor::init()
 {
     if (inplen >= UINT32_MAX)
         return false;
-    zs.zalloc = zlib_alloc;
-    zs.zfree = zlib_free;
     int ret = deflateInit(&zs, Z_DEFAULT_COMPRESSION);
     if (ret != Z_OK) {
         JS_ASSERT(ret == Z_MEM_ERROR);
         return false;
     }
     return true;
 }
 
-bool
+void
+Compressor::setOutput(unsigned char *out, size_t outlen)
+{
+    JS_ASSERT(outlen > outbytes);
+    zs.next_out = out + outbytes;
+    zs.avail_out = outlen - outbytes;
+}
+
+Compressor::Status
 Compressor::compressMore()
 {
+    JS_ASSERT(zs.next_out);
     uInt left = inplen - (zs.next_in - inp);
     bool done = left <= CHUNKSIZE;
     if (done)
         zs.avail_in = left;
     else if (zs.avail_in == 0)
         zs.avail_in = CHUNKSIZE;
+    Bytef *oldout = zs.next_out;
     int ret = deflate(&zs, done ? Z_FINISH : Z_NO_FLUSH);
+    outbytes += zs.next_out - oldout;
     if (ret == Z_MEM_ERROR) {
         zs.avail_out = 0;
-        return false;
+        return OOM;
     }
     if (ret == Z_BUF_ERROR || (done && ret == Z_OK)) {
         JS_ASSERT(zs.avail_out == 0);
-        return false;
+        return MOREOUTPUT;
     }
     JS_ASSERT_IF(!done, ret == Z_OK);
     JS_ASSERT_IF(done, ret == Z_STREAM_END);
-    return !done;
-}
-
-size_t
-Compressor::finish()
-{
-    size_t outlen = inplen - zs.avail_out;
-    int ret = deflateEnd(&zs);
-    if (ret != Z_OK) {
-        // If we finished early, we can get a Z_DATA_ERROR.
-        JS_ASSERT(ret == Z_DATA_ERROR);
-        JS_ASSERT(uInt(zs.next_in - inp) < inplen || !zs.avail_out);
-    }
-    return outlen;
+    return done ? DONE : CONTINUE;
 }
 
 bool
 js::DecompressString(const unsigned char *inp, size_t inplen, unsigned char *out, size_t outlen)
 {
     JS_ASSERT(inplen <= UINT32_MAX);
     z_stream zs;
     zs.zalloc = zlib_alloc;
--- a/js/src/jsutil.h
+++ b/js/src/jsutil.h
@@ -364,33 +364,33 @@ ClearAllBitArrayElements(size_t *array, 
 #ifdef USE_ZLIB
 class Compressor
 {
     /* Number of bytes we should hand to zlib each compressMore() call. */
     static const size_t CHUNKSIZE = 2048;
     z_stream zs;
     const unsigned char *inp;
     size_t inplen;
+    size_t outbytes;
+
   public:
-    Compressor(const unsigned char *inp, size_t inplen, unsigned char *out)
-        : inp(inp),
-        inplen(inplen)
-    {
-        JS_ASSERT(inplen > 0);
-        zs.opaque = NULL;
-        zs.next_in = (Bytef *)inp;
-        zs.avail_in = 0;
-        zs.next_out = out;
-        zs.avail_out = inplen;
-    }
+    enum Status {
+        MOREOUTPUT,
+        DONE,
+        CONTINUE,
+        OOM
+    };
+
+    Compressor(const unsigned char *inp, size_t inplen);
+    ~Compressor();
     bool init();
+    void setOutput(unsigned char *out, size_t outlen);
+    size_t outWritten() const { return outbytes; }
     /* Compress some of the input. Return true if it should be called again. */
-    bool compressMore();
-    /* Finalize compression. Return the length of the compressed input. */
-    size_t finish();
+    Status compressMore();
 };
 
 /*
  * Decompress a string. The caller must know the length of the output and
  * allocate |out| to a string of that length.
  */
 bool DecompressString(const unsigned char *inp, size_t inplen,
                       unsigned char *out, size_t outlen);