Bug 804857 - Start with a small compression buffer. r=njn
authorBenjamin Peterson <benjamin@python.org>
Wed, 24 Oct 2012 16:15:48 -0700
changeset 111434 1c27fd77dbaa2c71ff5d5985cabb883ee285601e
parent 111433 dfb516a4afc21e4adcd8055bdbd6aca01cff9752
child 111435 ea436c6f7d2de0ac0ccf4709efb63513ac9387b0
push id93
push usernmatsakis@mozilla.com
push dateWed, 31 Oct 2012 21:26:57 +0000
reviewersnjn
bugs804857
milestone19.0a1
Bug 804857 - Start with a small compression buffer. r=njn
js/src/jit-test/tests/basic/function-tosource-bug779694.js
js/src/jsscript.cpp
js/src/jsutil.cpp
js/src/jsutil.h
--- a/js/src/jit-test/tests/basic/function-tosource-bug779694.js
+++ b/js/src/jit-test/tests/basic/function-tosource-bug779694.js
@@ -1,6 +1,8 @@
+// This test creates poorly compressible input, which tests certain paths in
+// source code compression.
 var x = "";
 for (var i=0; i<400; ++i) {
     x += String.fromCharCode(i * 289);
 }
 var s = "'" + x + "'";
 assertEq(Function("evt", s).toString(), "function anonymous(evt) {\n" + s + "\n}");
--- a/js/src/jsscript.cpp
+++ b/js/src/jsscript.cpp
@@ -927,52 +927,90 @@ SourceCompressorThread::finish()
 bool
 SourceCompressorThread::internalCompress()
 {
     JS_ASSERT(state == COMPRESSING);
     JS_ASSERT(tok);
 
     ScriptSource *ss = tok->ss;
     JS_ASSERT(!ss->ready());
-    const size_t COMPRESS_THRESHOLD = 512;
     size_t compressedLength = 0;
     size_t nbytes = sizeof(jschar) * ss->length_;
 
     // Memory allocation functions on JSRuntime and JSContext are not
     // threadsafe. We have to use the js_* variants.
-    ss->data.compressed = static_cast<unsigned char *>(js_malloc(nbytes));
-    if (!ss->data.compressed)
-        return false;
 
 #ifdef USE_ZLIB
+    const size_t COMPRESS_THRESHOLD = 512;
     if (nbytes >= COMPRESS_THRESHOLD) {
-        Compressor comp(reinterpret_cast<const unsigned char *>(tok->chars),
-                        nbytes, ss->data.compressed);
-        if (comp.init()) {
-            while (!stop && comp.compressMore())
-                ;
-            compressedLength = comp.finish();
-            if (stop || compressedLength == nbytes)
-                compressedLength = 0;
+        // Try to keep the maximum memory usage down by only allocating half the
+        // size of the string, first.
+        size_t firstSize = nbytes / 2;
+        ss->data.compressed = static_cast<unsigned char *>(js_malloc(firstSize));
+        if (!ss->data.compressed)
+            return false;
+        Compressor comp(reinterpret_cast<const unsigned char *>(tok->chars), nbytes);
+        if (!comp.init())
+            return false;
+        comp.setOutput(ss->data.compressed, firstSize);
+        bool cont = !stop;
+        while (cont) {
+            switch (comp.compressMore()) {
+              case Compressor::CONTINUE:
+                break;
+              case Compressor::MOREOUTPUT: {
+                if (comp.outWritten() == nbytes) {
+                    cont = false;
+                    break;
+                }
+
+                // The compressed output is greater than half the size of the
+                // original string. Reallocate to the full size.
+                void *newmem = js_realloc(ss->data.compressed, nbytes);
+                if (!newmem) {
+                    js_free(ss->data.compressed);
+                    ss->data.compressed = NULL;
+                    return false;
+                }
+                ss->data.compressed = static_cast<unsigned char *>(newmem);
+                comp.setOutput(ss->data.compressed, nbytes);
+                break;
+              }
+              case Compressor::DONE:
+                cont = false;
+                break;
+              case Compressor::OOM:
+                return false;
+            }
+            cont = cont && !stop;
         }
+        compressedLength = comp.outWritten();
+        if (stop || compressedLength == nbytes)
+            compressedLength = 0;
     }
 #endif
-    ss->compressedLength_ = compressedLength;
     if (compressedLength == 0) {
+        // Note ss->data.source might be NULL.
+        jschar *buf = static_cast<jschar *>(js_realloc(ss->data.source, nbytes));
+        if (!buf) {
+            if (ss->data.source) {
+                js_free(ss->data.source);
+                ss->data.source = NULL;
+            }
+            return false;
+        }
+        ss->data.source = buf;
         PodCopy(ss->data.source, tok->chars, ss->length());
     } else {
-        // Shrink the buffer to the size of the compressed data.
+        // Shrink the buffer to the size of the compressed data. Shouldn't fail.
         void *newmem = js_realloc(ss->data.compressed, compressedLength);
-        if (!newmem) {
-            js_free(ss->data.compressed);
-            ss->data.compressed = NULL;
-            return false;
-        }
+        JS_ASSERT(newmem);
         ss->data.compressed = static_cast<unsigned char *>(newmem);
     }
+    ss->compressedLength_ = compressedLength;
     return true;
 }
 
 void
 SourceCompressorThread::threadLoop()
 {
     PR_Lock(lock);
     while (true) {
@@ -1167,17 +1205,17 @@ ScriptSource::setSourceCopy(JSContext *c
         ready_ = false;
 #endif
         tok->ss = this;
         tok->chars = src.get();
         cx->runtime->sourceCompressorThread.compress(tok);
     } else
 #endif
     {
-        data.source = cx->pod_malloc<jschar>(length);
+        data.source = cx->runtime->pod_malloc<jschar>(length);
         if (!data.source)
             return false;
         PodCopy(data.source, src.get(), length_);
     }
 
     return true;
 }
 
--- a/js/src/jsutil.cpp
+++ b/js/src/jsutil.cpp
@@ -37,65 +37,87 @@ zlib_alloc(void *cx, uInt items, uInt si
 }
 
 static void
 zlib_free(void *cx, void *addr)
 {
     js_free(addr);
 }
 
+Compressor::Compressor(const unsigned char *inp, size_t inplen)
+    : inp(inp),
+      inplen(inplen),
+      outbytes(0)
+{
+    JS_ASSERT(inplen > 0);
+    zs.opaque = NULL;
+    zs.next_in = (Bytef *)inp;
+    zs.avail_in = 0;
+    zs.next_out = NULL;
+    zs.avail_out = 0;
+    zs.zalloc = zlib_alloc;
+    zs.zfree = zlib_free;
+}
+
+
+Compressor::~Compressor()
+{
+    int ret = deflateEnd(&zs);
+    if (ret != Z_OK) {
+        // If we finished early, we can get a Z_DATA_ERROR.
+        JS_ASSERT(ret == Z_DATA_ERROR);
+        JS_ASSERT(uInt(zs.next_in - inp) < inplen || !zs.avail_out);
+    }
+}
+
 bool
 Compressor::init()
 {
     if (inplen >= UINT32_MAX)
         return false;
-    zs.zalloc = zlib_alloc;
-    zs.zfree = zlib_free;
     int ret = deflateInit(&zs, Z_DEFAULT_COMPRESSION);
     if (ret != Z_OK) {
         JS_ASSERT(ret == Z_MEM_ERROR);
         return false;
     }
     return true;
 }
 
-bool
+void
+Compressor::setOutput(unsigned char *out, size_t outlen)
+{
+    JS_ASSERT(outlen > outbytes);
+    zs.next_out = out + outbytes;
+    zs.avail_out = outlen - outbytes;
+}
+
+Compressor::Status
 Compressor::compressMore()
 {
+    JS_ASSERT(zs.next_out);
     uInt left = inplen - (zs.next_in - inp);
     bool done = left <= CHUNKSIZE;
     if (done)
         zs.avail_in = left;
     else if (zs.avail_in == 0)
         zs.avail_in = CHUNKSIZE;
+    Bytef *oldout = zs.next_out;
     int ret = deflate(&zs, done ? Z_FINISH : Z_NO_FLUSH);
+    outbytes += zs.next_out - oldout;
     if (ret == Z_MEM_ERROR) {
         zs.avail_out = 0;
-        return false;
+        return OOM;
     }
     if (ret == Z_BUF_ERROR || (done && ret == Z_OK)) {
         JS_ASSERT(zs.avail_out == 0);
-        return false;
+        return MOREOUTPUT;
     }
     JS_ASSERT_IF(!done, ret == Z_OK);
     JS_ASSERT_IF(done, ret == Z_STREAM_END);
-    return !done;
-}
-
-size_t
-Compressor::finish()
-{
-    size_t outlen = inplen - zs.avail_out;
-    int ret = deflateEnd(&zs);
-    if (ret != Z_OK) {
-        // If we finished early, we can get a Z_DATA_ERROR.
-        JS_ASSERT(ret == Z_DATA_ERROR);
-        JS_ASSERT(uInt(zs.next_in - inp) < inplen || !zs.avail_out);
-    }
-    return outlen;
+    return done ? DONE : CONTINUE;
 }
 
 bool
 js::DecompressString(const unsigned char *inp, size_t inplen, unsigned char *out, size_t outlen)
 {
     JS_ASSERT(inplen <= UINT32_MAX);
     z_stream zs;
     zs.zalloc = zlib_alloc;
--- a/js/src/jsutil.h
+++ b/js/src/jsutil.h
@@ -364,33 +364,33 @@ ClearAllBitArrayElements(size_t *array, 
 #ifdef USE_ZLIB
 class Compressor
 {
     /* Number of bytes we should hand to zlib each compressMore() call. */
     static const size_t CHUNKSIZE = 2048;
     z_stream zs;
     const unsigned char *inp;
     size_t inplen;
+    size_t outbytes;
+
   public:
-    Compressor(const unsigned char *inp, size_t inplen, unsigned char *out)
-        : inp(inp),
-        inplen(inplen)
-    {
-        JS_ASSERT(inplen > 0);
-        zs.opaque = NULL;
-        zs.next_in = (Bytef *)inp;
-        zs.avail_in = 0;
-        zs.next_out = out;
-        zs.avail_out = inplen;
-    }
+    enum Status {
+        MOREOUTPUT,
+        DONE,
+        CONTINUE,
+        OOM
+    };
+
+    Compressor(const unsigned char *inp, size_t inplen);
+    ~Compressor();
     bool init();
+    void setOutput(unsigned char *out, size_t outlen);
+    size_t outWritten() const { return outbytes; }
     /* Compress some of the input. Return true if it should be called again. */
-    bool compressMore();
-    /* Finalize compression. Return the length of the compressed input. */
-    size_t finish();
+    Status compressMore();
 };
 
 /*
  * Decompress a string. The caller must know the length of the output and
  * allocate |out| to a string of that length.
  */
 bool DecompressString(const unsigned char *inp, size_t inplen,
                       unsigned char *out, size_t outlen);