Bug 1015917 part 1 - Support string concatenation for Latin1 strings. r=luke
authorJan de Mooij <jdemooij@mozilla.com>
Sat, 31 May 2014 10:44:32 +0200
changeset 205210 ca48add6d1540dbf0baf34646d8df4edbe922d41
parent 205209 062236a88303c42fac21a33f8cbb57635a4785ae
child 205211 4dd20813fe3b9c3c89eebd283c962ca07d4421b5
push id3741
push userasasaki@mozilla.com
push dateMon, 21 Jul 2014 20:25:18 +0000
treeherdermozilla-beta@4d6f46f5af68 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersluke
bugs1015917
milestone32.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1015917 part 1 - Support string concatenation for Latin1 strings. r=luke
js/src/gc/Statistics.cpp
js/src/jit/CodeGenerator.cpp
js/src/jit/ParallelFunctions.cpp
js/src/jit/arm/MacroAssembler-arm.cpp
js/src/jit/arm/MacroAssembler-arm.h
js/src/jit/shared/MacroAssembler-x86-shared.h
js/src/jsapi.cpp
js/src/jsatom.cpp
js/src/jsnum.cpp
js/src/jsstr.cpp
js/src/jsstr.h
js/src/vm/CharacterEncoding.cpp
js/src/vm/SelfHosting.cpp
js/src/vm/String-inl.h
js/src/vm/String.cpp
js/src/vm/String.h
js/src/vm/StringBuffer.h
--- a/js/src/gc/Statistics.cpp
+++ b/js/src/gc/Statistics.cpp
@@ -132,17 +132,17 @@ class gcstats::StatisticsSerializer
         size_t nchars = strlen(buf);
         jschar *out = js_pod_malloc<jschar>(nchars + 1);
         if (!out) {
             oom_ = true;
             js_free(buf);
             return nullptr;
         }
 
-        InflateStringToBuffer(buf, nchars, out);
+        CopyAndInflateChars(out, buf, nchars);
         js_free(buf);
 
         out[nchars] = 0;
         return out;
     }
 
     char *finishCString() {
         if (oom_)
--- a/js/src/jit/CodeGenerator.cpp
+++ b/js/src/jit/CodeGenerator.cpp
@@ -5149,39 +5149,148 @@ CodeGenerator::visitConcatPar(LConcatPar
     JS_ASSERT(ToRegister(lir->temp3()) == CallTempReg2);
     JS_ASSERT(ToRegister(lir->temp4()) == CallTempReg3);
     JS_ASSERT(output == CallTempReg5);
 
     return emitConcat(lir, lhs, rhs, output);
 }
 
 static void
-CopyStringChars(MacroAssembler &masm, Register to, Register from, Register len, Register scratch)
+CopyStringChars(MacroAssembler &masm, Register to, Register from, Register len, Register scratch,
+                size_t fromWidth, size_t toWidth)
 {
     // Copy |len| jschars from |from| to |to|. Assumes len > 0 (checked below in
     // debug builds), and when done |to| must point to the next available char.
 
 #ifdef DEBUG
     Label ok;
     masm.branch32(Assembler::GreaterThan, len, Imm32(0), &ok);
     masm.assumeUnreachable("Length should be greater than 0.");
     masm.bind(&ok);
 #endif
 
-    JS_STATIC_ASSERT(sizeof(jschar) == 2);
+    MOZ_ASSERT(fromWidth == 1 || fromWidth == 2);
+    MOZ_ASSERT(toWidth == 1 || toWidth == 2);
+    MOZ_ASSERT_IF(toWidth == 1, fromWidth == 1);
 
     Label start;
     masm.bind(&start);
-    masm.load16ZeroExtend(Address(from, 0), scratch);
-    masm.store16(scratch, Address(to, 0));
-    masm.addPtr(Imm32(2), from);
-    masm.addPtr(Imm32(2), to);
+    if (fromWidth == 2)
+        masm.load16ZeroExtend(Address(from, 0), scratch);
+    else
+        masm.load8ZeroExtend(Address(from, 0), scratch);
+    if (toWidth == 2)
+        masm.store16(scratch, Address(to, 0));
+    else
+        masm.store8(scratch, Address(to, 0));
+    masm.addPtr(Imm32(fromWidth), from);
+    masm.addPtr(Imm32(toWidth), to);
     masm.branchSub32(Assembler::NonZero, Imm32(1), len, &start);
 }
 
+static void
+CopyStringCharsMaybeInflate(MacroAssembler &masm, Register input, Register destChars,
+                            Register temp1, Register temp2)
+{
+    // destChars is TwoByte and input is a Latin1 or TwoByte string, so we may
+    // have to inflate.
+
+    Label isLatin1, done;
+    masm.loadStringLength(input, temp1);
+    masm.branchTest32(Assembler::NonZero, Address(input, JSString::offsetOfFlags()),
+                      Imm32(JSString::LATIN1_CHARS_BIT), &isLatin1);
+    {
+        masm.loadStringChars(input, input);
+        CopyStringChars(masm, destChars, input, temp1, temp2, sizeof(jschar), sizeof(jschar));
+        masm.jump(&done);
+    }
+    masm.bind(&isLatin1);
+    {
+        masm.loadStringChars(input, input);
+        CopyStringChars(masm, destChars, input, temp1, temp2, sizeof(char), sizeof(jschar));
+    }
+    masm.bind(&done);
+}
+
+static void
+ConcatFatInlineString(MacroAssembler &masm, Register lhs, Register rhs, Register output,
+                      Register temp1, Register temp2, Register temp3, Register forkJoinContext,
+                      ExecutionMode mode, Label *failure, Label *failurePopTemps, bool isTwoByte)
+{
+    // State: result length in temp2.
+
+    // Ensure both strings are linear.
+    masm.branchIfRope(lhs, failure);
+    masm.branchIfRope(rhs, failure);
+
+    // Allocate a JSFatInlineString.
+    switch (mode) {
+      case SequentialExecution:
+        masm.newGCFatInlineString(output, temp1, failure);
+        break;
+      case ParallelExecution:
+        masm.push(temp1);
+        masm.push(temp2);
+        masm.newGCFatInlineStringPar(output, forkJoinContext, temp1, temp2, failurePopTemps);
+        masm.pop(temp2);
+        masm.pop(temp1);
+        break;
+      default:
+        MOZ_ASSUME_UNREACHABLE("No such execution mode");
+    }
+
+    // Store length and flags.
+    uint32_t flags = JSString::INIT_FAT_INLINE_FLAGS;
+    if (!isTwoByte)
+        flags |= JSString::LATIN1_CHARS_BIT;
+    masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
+    masm.store32(temp2, Address(output, JSString::offsetOfLength()));
+
+    // Load chars pointer in temp2.
+    masm.computeEffectiveAddress(Address(output, JSInlineString::offsetOfInlineStorage()), temp2);
+
+    {
+        // We use temp3 in this block, which in parallel execution also holds
+        // a live ForkJoinContext pointer. If we are compiling for parallel
+        // execution, be sure to save and restore the ForkJoinContext.
+        if (mode == ParallelExecution)
+            masm.push(temp3);
+
+        // Copy lhs chars. Note that this advances temp2 to point to the next
+        // char. This also clobbers the lhs register.
+        if (isTwoByte) {
+            CopyStringCharsMaybeInflate(masm, lhs, temp2, temp1, temp3);
+        } else {
+            masm.loadStringLength(lhs, temp3);
+            masm.loadStringChars(lhs, lhs);
+            CopyStringChars(masm, temp2, lhs, temp3, temp1, sizeof(char), sizeof(char));
+        }
+
+        // Copy rhs chars. Clobbers the rhs register.
+        if (isTwoByte) {
+            CopyStringCharsMaybeInflate(masm, rhs, temp2, temp1, temp3);
+        } else {
+            masm.loadStringLength(rhs, temp3);
+            masm.loadStringChars(rhs, rhs);
+            CopyStringChars(masm, temp2, rhs, temp3, temp1, sizeof(char), sizeof(char));
+        }
+
+        // Null-terminate.
+        if (isTwoByte)
+            masm.store16(Imm32(0), Address(temp2, 0));
+        else
+            masm.store8(Imm32(0), Address(temp2, 0));
+
+        if (mode == ParallelExecution)
+            masm.pop(temp3);
+    }
+
+    masm.ret();
+}
+
 JitCode *
 JitCompartment::generateStringConcatStub(JSContext *cx, ExecutionMode mode)
 {
     MacroAssembler masm(cx);
 
     Register lhs = CallTempReg0;
     Register rhs = CallTempReg1;
     Register temp1 = CallTempReg2;
@@ -5203,20 +5312,37 @@ JitCompartment::generateStringConcatStub
 
     // If rhs is empty, return lhs.
     Label rightEmpty;
     masm.loadStringLength(rhs, temp2);
     masm.branchTest32(Assembler::Zero, temp2, temp2, &rightEmpty);
 
     masm.add32(temp1, temp2);
 
-    // Check if we can use a JSFatInlineString.
-    Label isFatInline;
-    masm.branch32(Assembler::BelowOrEqual, temp2, Imm32(JSFatInlineString::MAX_LENGTH_TWO_BYTE),
-                  &isFatInline);
+    // Check if we can use a JSFatInlineString. The result is a Latin1 string if
+    // lhs and rhs are both Latin1, so we AND the flags.
+    Label isFatInlineTwoByte, isFatInlineLatin1;
+    masm.load32(Address(lhs, JSString::offsetOfFlags()), temp1);
+    masm.and32(Address(rhs, JSString::offsetOfFlags()), temp1);
+
+    Label isLatin1, notInline;
+    masm.branchTest32(Assembler::NonZero, temp1, Imm32(JSString::LATIN1_CHARS_BIT), &isLatin1);
+    {
+        masm.branch32(Assembler::BelowOrEqual, temp2, Imm32(JSFatInlineString::MAX_LENGTH_TWO_BYTE),
+                      &isFatInlineTwoByte);
+        masm.jump(&notInline);
+    }
+    masm.bind(&isLatin1);
+    {
+        masm.branch32(Assembler::BelowOrEqual, temp2, Imm32(JSFatInlineString::MAX_LENGTH_LATIN1),
+                      &isFatInlineLatin1);
+    }
+    masm.bind(&notInline);
+
+    // Keep AND'ed flags in temp1.
 
     // Ensure result length <= JSString::MAX_LENGTH.
     masm.branch32(Assembler::Above, temp2, Imm32(JSString::MAX_LENGTH), &failure);
 
     // Allocate a new rope.
     switch (mode) {
       case SequentialExecution:
         masm.newGCString(output, temp3, &failure);
@@ -5227,89 +5353,44 @@ JitCompartment::generateStringConcatStub
         masm.newGCStringPar(output, forkJoinContext, temp1, temp2, &failurePopTemps);
         masm.pop(temp2);
         masm.pop(temp1);
         break;
       default:
         MOZ_ASSUME_UNREACHABLE("No such execution mode");
     }
 
-    // Store lengthAndFlags.
-    masm.store32(Imm32(JSString::ROPE_FLAGS), Address(output, JSString::offsetOfFlags()));
+    // Store rope length and flags. temp1 still holds the result of AND'ing the
+    // lhs and rhs flags, so we just have to clear the other flags to get our
+    // rope flags (Latin1 if both lhs and rhs are Latin1).
+    static_assert(JSString::ROPE_FLAGS == 0, "Rope flags must be 0");
+    masm.and32(Imm32(JSString::LATIN1_CHARS_BIT), temp1);
+    masm.store32(temp1, Address(output, JSString::offsetOfFlags()));
     masm.store32(temp2, Address(output, JSString::offsetOfLength()));
 
     // Store left and right nodes.
     masm.storePtr(lhs, Address(output, JSRope::offsetOfLeft()));
     masm.storePtr(rhs, Address(output, JSRope::offsetOfRight()));
     masm.ret();
 
     masm.bind(&leftEmpty);
     masm.mov(rhs, output);
     masm.ret();
 
     masm.bind(&rightEmpty);
     masm.mov(lhs, output);
     masm.ret();
 
-    masm.bind(&isFatInline);
-
-    // State: lhs length in temp1, result length in temp2.
-
-    // Ensure both strings are linear.
-    masm.branchIfRope(lhs, &failure);
-    masm.branchIfRope(rhs, &failure);
-
-    // Allocate a JSFatInlineString.
-    switch (mode) {
-      case SequentialExecution:
-        masm.newGCFatInlineString(output, temp3, &failure);
-        break;
-      case ParallelExecution:
-        masm.push(temp1);
-        masm.push(temp2);
-        masm.newGCFatInlineStringPar(output, forkJoinContext, temp1, temp2, &failurePopTemps);
-        masm.pop(temp2);
-        masm.pop(temp1);
-        break;
-      default:
-        MOZ_ASSUME_UNREACHABLE("No such execution mode");
-    }
-
-    // Set length and flags.
-    masm.store32(Imm32(JSString::INIT_FAT_INLINE_FLAGS), Address(output, JSString::offsetOfFlags()));
-    masm.store32(temp2, Address(output, JSString::offsetOfLength()));
-
-    // Load chars pointer in temp2.
-    masm.computeEffectiveAddress(Address(output, JSInlineString::offsetOfInlineStorage()), temp2);
-
-    {
-        // We use temp3 in this block, which in parallel execution also holds
-        // a live ForkJoinContext pointer. If we are compiling for parallel
-        // execution, be sure to save and restore the ForkJoinContext.
-        if (mode == ParallelExecution)
-            masm.push(temp3);
-
-        // Copy lhs chars. Temp1 still holds the lhs length. Note that this
-        // advances temp2 to point to the next char. Note that this also
-        // repurposes the lhs register.
-        masm.loadStringChars(lhs, lhs);
-        CopyStringChars(masm, temp2, lhs, temp1, temp3);
-
-        // Copy rhs chars.
-        masm.loadStringLength(rhs, temp1);
-        masm.loadStringChars(rhs, rhs);
-        CopyStringChars(masm, temp2, rhs, temp1, temp3);
-
-        if (mode == ParallelExecution)
-            masm.pop(temp3);
-    }
-
-    // Null-terminate.
-    masm.store16(Imm32(0), Address(temp2, 0));
-    masm.ret();
+    masm.bind(&isFatInlineTwoByte);
+    ConcatFatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3, forkJoinContext,
+                          mode, &failure, &failurePopTemps, true);
+
+    masm.bind(&isFatInlineLatin1);
+    ConcatFatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3, forkJoinContext,
+                          mode, &failure, &failurePopTemps, false);
 
     masm.bind(&failurePopTemps);
     masm.pop(temp2);
     masm.pop(temp1);
 
     masm.bind(&failure);
     masm.movePtr(ImmPtr(nullptr), output);
     masm.ret();
--- a/js/src/jit/ParallelFunctions.cpp
+++ b/js/src/jit/ParallelFunctions.cpp
@@ -352,21 +352,22 @@ do {                                    
     return true;                                                                \
 } while(0)
 
 static bool
 CompareStringsPar(ForkJoinContext *cx, JSString *left, JSString *right, int32_t *res)
 {
     ScopedThreadSafeStringInspector leftInspector(left);
     ScopedThreadSafeStringInspector rightInspector(right);
-    if (!leftInspector.ensureChars(cx) || !rightInspector.ensureChars(cx))
+    AutoCheckCannotGC nogc;
+    if (!leftInspector.ensureChars(cx, nogc) || !rightInspector.ensureChars(cx, nogc))
         return false;
 
-    *res = CompareChars(leftInspector.chars(), left->length(),
-                        rightInspector.chars(), right->length());
+    *res = CompareChars(leftInspector.twoByteChars(), left->length(),
+                        rightInspector.twoByteChars(), right->length());
     return true;
 }
 
 static bool
 CompareMaybeStringsPar(ForkJoinContext *cx, HandleValue v1, HandleValue v2, int32_t *res)
 {
     if (!v1.isString())
         return false;
--- a/js/src/jit/arm/MacroAssembler-arm.cpp
+++ b/js/src/jit/arm/MacroAssembler-arm.cpp
@@ -1932,16 +1932,23 @@ MacroAssemblerARMCompat::and32(Register 
 
 void
 MacroAssemblerARMCompat::and32(Imm32 imm, Register dest)
 {
     ma_and(imm, dest, SetCond);
 }
 
 void
+MacroAssemblerARMCompat::and32(const Address &src, Register dest)
+{
+    load32(src, ScratchRegister);
+    ma_and(ScratchRegister, dest, SetCond);
+}
+
+void
 MacroAssemblerARMCompat::addPtr(Register src, Register dest)
 {
     ma_add(src, dest);
 }
 
 void
 MacroAssemblerARMCompat::addPtr(const Address &src, Register dest)
 {
--- a/js/src/jit/arm/MacroAssembler-arm.h
+++ b/js/src/jit/arm/MacroAssembler-arm.h
@@ -1294,16 +1294,17 @@ class MacroAssemblerARMCompat : public M
         sub32(src, dest);
         j(cond, label);
     }
     void xor32(Imm32 imm, Register dest);
 
     void and32(Register src, Register dest);
     void and32(Imm32 imm, Register dest);
     void and32(Imm32 imm, const Address &dest);
+    void and32(const Address &src, Register dest);
     void or32(Imm32 imm, const Address &dest);
     void xorPtr(Imm32 imm, Register dest);
     void xorPtr(Register src, Register dest);
     void orPtr(Imm32 imm, Register dest);
     void orPtr(Register src, Register dest);
     void andPtr(Imm32 imm, Register dest);
     void andPtr(Register src, Register dest);
     void addPtr(Register src, Register dest);
--- a/js/src/jit/shared/MacroAssembler-x86-shared.h
+++ b/js/src/jit/shared/MacroAssembler-x86-shared.h
@@ -106,16 +106,19 @@ class MacroAssemblerX86Shared : public A
         movl(src, dest);
     }
     void move32(Register src, const Operand &dest) {
         movl(src, dest);
     }
     void and32(Register src, Register dest) {
         andl(src, dest);
     }
+    void and32(const Address &src, Register dest) {
+        andl(Operand(src), dest);
+    }
     void and32(Imm32 imm, Register dest) {
         andl(imm, dest);
     }
     void and32(Imm32 imm, const Address &dest) {
         andl(imm, Operand(dest));
     }
     void or32(Register src, Register dest) {
         orl(src, dest);
--- a/js/src/jsapi.cpp
+++ b/js/src/jsapi.cpp
@@ -5592,24 +5592,24 @@ JS_DecodeBytes(JSContext *cx, const char
     if (!dst) {
         *dstlenp = srclen;
         return true;
     }
 
     size_t dstlen = *dstlenp;
 
     if (srclen > dstlen) {
-        InflateStringToBuffer(src, dstlen, dst);
+        CopyAndInflateChars(dst, src, dstlen);
 
         AutoSuppressGC suppress(cx);
         JS_ReportErrorNumber(cx, js_GetErrorMessage, nullptr, JSMSG_BUFFER_TOO_SMALL);
         return false;
     }
 
-    InflateStringToBuffer(src, srclen, dst);
+    CopyAndInflateChars(dst, src, srclen);
     *dstlenp = srclen;
     return true;
 }
 
 JS_PUBLIC_API(char *)
 JS_EncodeString(JSContext *cx, JSString *str)
 {
     AssertHeapIsIdle(cx);
--- a/js/src/jsatom.cpp
+++ b/js/src/jsatom.cpp
@@ -447,17 +447,17 @@ js::Atomize(ExclusiveContext *cx, const 
         /*
          * Avoiding the malloc in InflateString on shorter strings saves us
          * over 20,000 malloc calls on mozilla browser startup. This compares to
          * only 131 calls where the string is longer than a 31 char (net) buffer.
          * The vast majority of atomized strings are already in the hashtable. So
          * js::AtomizeString rarely has to copy the temp string we make.
          */
         jschar inflated[ATOMIZE_BUF_MAX];
-        InflateStringToBuffer(bytes, length, inflated);
+        CopyAndInflateChars(inflated, bytes, length);
         return AtomizeAndCopyChars(cx, inflated, length, ib);
     }
 
     jschar *tbcharsZ = InflateString(cx, bytes, &length);
     if (!tbcharsZ)
         return nullptr;
     return AtomizeAndtake(cx, tbcharsZ, length, ib);
 }
--- a/js/src/jsnum.cpp
+++ b/js/src/jsnum.cpp
@@ -600,17 +600,17 @@ js::Int32ToString(ThreadSafeContext *cx,
     if (!str)
         return nullptr;
 
     jschar buffer[JSFatInlineString::MAX_LENGTH_TWO_BYTE + 1];
     size_t length;
     jschar *start = BackfillInt32InBuffer(si, buffer,
                                           JSFatInlineString::MAX_LENGTH_TWO_BYTE + 1, &length);
 
-    PodCopy(str->init(length), start, length + 1);
+    PodCopy(str->initTwoByte(length), start, length + 1);
 
     CacheNumber(cx, si, str);
     return str;
 }
 
 template JSFlatString *
 js::Int32ToString<CanGC>(ThreadSafeContext *cx, int32_t si);
 
@@ -1432,17 +1432,17 @@ js::IndexToString(JSContext *cx, uint32_
         return nullptr;
 
     jschar buffer[JSFatInlineString::MAX_LENGTH_TWO_BYTE + 1];
     RangedPtr<jschar> end(buffer + JSFatInlineString::MAX_LENGTH_TWO_BYTE,
                           buffer, JSFatInlineString::MAX_LENGTH_TWO_BYTE + 1);
     *end = '\0';
     RangedPtr<jschar> start = BackfillIndexInCharBuffer(index, end);
 
-    jschar *dst = str->init(end - start);
+    jschar *dst = str->initTwoByte(end - start);
     PodCopy(dst, start.get(), end - start + 1);
 
     c->dtoaCache.cache(10, index, str);
     return str;
 }
 
 bool JS_FASTCALL
 js::NumberValueToStringBuffer(JSContext *cx, const Value &v, StringBuffer &sb)
@@ -1528,21 +1528,22 @@ CharsToNumber(ThreadSafeContext *cx, con
         *result = d;
 
     return true;
 }
 
 bool
 js::StringToNumber(ThreadSafeContext *cx, JSString *str, double *result)
 {
+    AutoCheckCannotGC nogc;
     ScopedThreadSafeStringInspector inspector(str);
-    if (!inspector.ensureChars(cx))
+    if (!inspector.ensureChars(cx, nogc))
         return false;
 
-    return CharsToNumber(cx, inspector.chars(), str->length(), result);
+    return CharsToNumber(cx, inspector.twoByteChars(), str->length(), result);
 }
 
 bool
 js::NonObjectToNumberSlow(ThreadSafeContext *cx, Value v, double *out)
 {
     JS_ASSERT(!v.isNumber());
     JS_ASSERT(!v.isObject());
 
--- a/js/src/jsstr.cpp
+++ b/js/src/jsstr.cpp
@@ -2775,18 +2775,18 @@ static inline JSFatInlineString *
 FlattenSubstrings(JSContext *cx, const jschar *chars,
                   const StringRange *ranges, size_t rangesLen, size_t outputLen)
 {
     JS_ASSERT(JSFatInlineString::twoByteLengthFits(outputLen));
 
     JSFatInlineString *str = js_NewGCFatInlineString<CanGC>(cx);
     if (!str)
         return nullptr;
-    jschar *buf = str->init(outputLen);
-
+
+    jschar *buf = str->initTwoByte(outputLen);
     size_t pos = 0;
     for (size_t i = 0; i < rangesLen; i++) {
         PodCopy(buf + pos, chars + ranges[i].start, ranges[i].length);
         pos += ranges[i].length;
     }
     JS_ASSERT(pos == outputLen);
 
     buf[outputLen] = 0;
--- a/js/src/jsstr.h
+++ b/js/src/jsstr.h
@@ -251,17 +251,17 @@ namespace js {
 extern jschar *
 InflateString(ThreadSafeContext *cx, const char *bytes, size_t *length);
 
 /*
  * Inflate bytes to JS chars in an existing buffer. 'dst' must be large
  * enough for 'srclen' jschars. The buffer is NOT null-terminated.
  */
 inline void
-InflateStringToBuffer(const char *src, size_t srclen, jschar *dst)
+CopyAndInflateChars(jschar *dst, const char *src, size_t srclen)
 {
     for (size_t i = 0; i < srclen; i++)
         dst[i] = (unsigned char) src[i];
 }
 
 /*
  * Deflate JS chars to bytes into a buffer. 'bytes' must be large enough for
  * 'length chars. The buffer is NOT null-terminated. The destination length
--- a/js/src/vm/CharacterEncoding.cpp
+++ b/js/src/vm/CharacterEncoding.cpp
@@ -127,19 +127,21 @@ DeflateStringToUTF8Buffer(js::ThreadSafe
         }
         dstlen -= utf8Len;
     }
     *dstlenp = (origDstlen - dstlen);
     return true;
 
 bufferTooSmall:
     *dstlenp = (origDstlen - dstlen);
-    if (cx->isJSContext())
+    if (cx->isJSContext()) {
+        js::gc::AutoSuppressGC suppress(cx->asJSContext());
         JS_ReportErrorNumber(cx->asJSContext(), js_GetErrorMessage, nullptr,
                              JSMSG_BUFFER_TOO_SMALL);
+    }
     return false;
 }
 
 
 UTF8CharsZ
 JS::TwoByteCharsToNewUTF8CharsZ(js::ThreadSafeContext *cx, TwoByteChars tbchars)
 {
     JS_ASSERT(cx);
--- a/js/src/vm/SelfHosting.cpp
+++ b/js/src/vm/SelfHosting.cpp
@@ -274,21 +274,22 @@ JS_JITINFO_NATIVE_PARALLEL_THREADSAFE(in
 
 bool
 intrinsic_ParallelSpew(ThreadSafeContext *cx, unsigned argc, Value *vp)
 {
     CallArgs args = CallArgsFromVp(argc, vp);
     JS_ASSERT(args.length() == 1);
     JS_ASSERT(args[0].isString());
 
+    AutoCheckCannotGC nogc;
     ScopedThreadSafeStringInspector inspector(args[0].toString());
-    if (!inspector.ensureChars(cx))
+    if (!inspector.ensureChars(cx, nogc))
         return false;
 
-    ScopedJSFreePtr<char> bytes(TwoByteCharsToNewUTF8CharsZ(cx, inspector.range()).c_str());
+    ScopedJSFreePtr<char> bytes(TwoByteCharsToNewUTF8CharsZ(cx, inspector.twoByteRange()).c_str());
     parallel::Spew(parallel::SpewOps, bytes);
 
     args.rval().setUndefined();
     return true;
 }
 
 JS_JITINFO_NATIVE_PARALLEL_THREADSAFE(intrinsic_ParallelSpew_jitInfo, intrinsic_ParallelSpew_jitInfo,
                                       intrinsic_ParallelSpew);
--- a/js/src/vm/String-inl.h
+++ b/js/src/vm/String-inl.h
@@ -27,22 +27,22 @@ NewFatInlineString(ThreadSafeContext *cx
     JS_ASSERT(JSFatInlineString::twoByteLengthFits(len));
 
     JSInlineString *str;
     jschar *p;
     if (JSInlineString::twoByteLengthFits(len)) {
         str = JSInlineString::new_<allowGC>(cx);
         if (!str)
             return nullptr;
-        p = str->init(len);
+        p = str->initTwoByte(len);
     } else {
         JSFatInlineString *fatstr = JSFatInlineString::new_<allowGC>(cx);
         if (!fatstr)
             return nullptr;
-        p = fatstr->init(len);
+        p = fatstr->initTwoByte(len);
         str = fatstr;
     }
 
     for (size_t i = 0; i < len; ++i)
         p[i] = static_cast<jschar>(chars[i]);
     p[len] = '\0';
     return str;
 }
@@ -60,22 +60,22 @@ NewFatInlineString(ExclusiveContext *cx,
     JS_ASSERT(JSFatInlineString::twoByteLengthFits(len));
 
     JSInlineString *str;
     jschar *storage;
     if (JSInlineString::twoByteLengthFits(len)) {
         str = JSInlineString::new_<allowGC>(cx);
         if (!str)
             return nullptr;
-        storage = str->init(len);
+        storage = str->initTwoByte(len);
     } else {
         JSFatInlineString *fatstr = JSFatInlineString::new_<allowGC>(cx);
         if (!fatstr)
             return nullptr;
-        storage = fatstr->init(len);
+        storage = fatstr->initTwoByte(len);
         str = fatstr;
     }
 
     mozilla::PodCopy(storage, chars.start().get(), len);
     storage[len] = 0;
     return str;
 }
 
@@ -102,16 +102,18 @@ JSString::validateLength(js::ThreadSafeC
     return true;
 }
 
 MOZ_ALWAYS_INLINE void
 JSRope::init(js::ThreadSafeContext *cx, JSString *left, JSString *right, size_t length)
 {
     d.u1.length = length;
     d.u1.flags = ROPE_FLAGS;
+    if (left->hasLatin1Chars() && right->hasLatin1Chars())
+        d.u1.flags |= LATIN1_CHARS_BIT;
     d.s.u2.left = left;
     d.s.u3.right = right;
     js::StringWriteBarrierPost(cx, &d.s.u2.left);
     js::StringWriteBarrierPost(cx, &d.s.u3.right);
 }
 
 template <js::AllowGC allowGC>
 MOZ_ALWAYS_INLINE JSRope *
@@ -243,33 +245,51 @@ JSFlatString::toPropertyName(JSContext *
 template <js::AllowGC allowGC>
 MOZ_ALWAYS_INLINE JSInlineString *
 JSInlineString::new_(js::ThreadSafeContext *cx)
 {
     return (JSInlineString *)js_NewGCString<allowGC>(cx);
 }
 
 MOZ_ALWAYS_INLINE jschar *
-JSInlineString::init(size_t length)
+JSInlineString::initTwoByte(size_t length)
 {
     JS_ASSERT(twoByteLengthFits(length));
     d.u1.length = length;
     d.u1.flags = INIT_INLINE_FLAGS;
     return d.inlineStorageTwoByte;
 }
 
+MOZ_ALWAYS_INLINE char *
+JSInlineString::initLatin1(size_t length)
+{
+    JS_ASSERT(latin1LengthFits(length));
+    d.u1.length = length;
+    d.u1.flags = INIT_INLINE_FLAGS | LATIN1_CHARS_BIT;
+    return d.inlineStorageLatin1;
+}
+
 MOZ_ALWAYS_INLINE jschar *
-JSFatInlineString::init(size_t length)
+JSFatInlineString::initTwoByte(size_t length)
 {
     JS_ASSERT(twoByteLengthFits(length));
     d.u1.length = length;
     d.u1.flags = INIT_FAT_INLINE_FLAGS;
     return d.inlineStorageTwoByte;
 }
 
+MOZ_ALWAYS_INLINE char *
+JSFatInlineString::initLatin1(size_t length)
+{
+    JS_ASSERT(latin1LengthFits(length));
+    d.u1.length = length;
+    d.u1.flags = INIT_FAT_INLINE_FLAGS | LATIN1_CHARS_BIT;
+    return d.inlineStorageLatin1;
+}
+
 template <js::AllowGC allowGC>
 MOZ_ALWAYS_INLINE JSFatInlineString *
 JSFatInlineString::new_(js::ThreadSafeContext *cx)
 {
     return js_NewGCFatInlineString<allowGC>(cx);
 }
 
 MOZ_ALWAYS_INLINE void
@@ -334,36 +354,36 @@ JSString::finalize(js::FreeOp *fop)
 }
 
 inline void
 JSFlatString::finalize(js::FreeOp *fop)
 {
     JS_ASSERT(getAllocKind() != js::gc::FINALIZE_FAT_INLINE_STRING);
 
     if (!isInline())
-        fop->free_(const_cast<jschar *>(nonInlineChars()));
+        fop->free_(nonInlineCharsRaw());
 }
 
 inline void
 JSFatInlineString::finalize(js::FreeOp *fop)
 {
     JS_ASSERT(getAllocKind() == js::gc::FINALIZE_FAT_INLINE_STRING);
 
     if (!isInline())
-        fop->free_(const_cast<jschar *>(nonInlineChars()));
+        fop->free_(nonInlineCharsRaw());
 }
 
 inline void
 JSAtom::finalize(js::FreeOp *fop)
 {
     JS_ASSERT(JSString::isAtom());
     JS_ASSERT(JSString::isFlat());
 
     if (!isInline())
-        fop->free_(const_cast<jschar *>(nonInlineChars()));
+        fop->free_(nonInlineCharsRaw());
 }
 
 inline void
 JSExternalString::finalize(js::FreeOp *fop)
 {
     const JSStringFinalizer *fin = externalFinalizer();
     fin->finalize(fin, const_cast<jschar *>(nonInlineChars()));
 }
--- a/js/src/vm/String.cpp
+++ b/js/src/vm/String.cpp
@@ -400,31 +400,49 @@ js::ConcatStrings(ThreadSafeContext *cx,
     size_t rightLen = right->length();
     if (rightLen == 0)
         return left;
 
     size_t wholeLength = leftLen + rightLen;
     if (!JSString::validateLength(cx, wholeLength))
         return nullptr;
 
-    if (JSFatInlineString::twoByteLengthFits(wholeLength) && cx->isJSContext()) {
+    bool isLatin1 = left->hasLatin1Chars() && right->hasLatin1Chars();
+    bool canUseFatInline = isLatin1
+                           ? JSFatInlineString::latin1LengthFits(wholeLength)
+                           : JSFatInlineString::twoByteLengthFits(wholeLength);
+    if (canUseFatInline && cx->isJSContext()) {
         JSFatInlineString *str = js_NewGCFatInlineString<allowGC>(cx);
         if (!str)
             return nullptr;
 
+        AutoCheckCannotGC nogc;
         ScopedThreadSafeStringInspector leftInspector(left);
         ScopedThreadSafeStringInspector rightInspector(right);
-        if (!leftInspector.ensureChars(cx) || !rightInspector.ensureChars(cx))
+        if (!leftInspector.ensureChars(cx, nogc) || !rightInspector.ensureChars(cx, nogc))
             return nullptr;
 
-        jschar *buf = str->init(wholeLength);
-        PodCopy(buf, leftInspector.chars(), leftLen);
-        PodCopy(buf + leftLen, rightInspector.chars(), rightLen);
+        if (isLatin1) {
+            char *buf = str->initLatin1(wholeLength);
+            PodCopy(buf, leftInspector.latin1Chars(), leftLen);
+            PodCopy(buf + leftLen, rightInspector.latin1Chars(), rightLen);
+            buf[wholeLength] = 0;
+        } else {
+            jschar *buf = str->initTwoByte(wholeLength);
+            if (leftInspector.hasTwoByteChars())
+                PodCopy(buf, leftInspector.twoByteChars(), leftLen);
+            else
+                CopyAndInflateChars(buf, leftInspector.latin1Chars(), leftLen);
+            if (rightInspector.hasTwoByteChars())
+                PodCopy(buf + leftLen, rightInspector.twoByteChars(), rightLen);
+            else
+                CopyAndInflateChars(buf + leftLen, rightInspector.latin1Chars(), rightLen);
+            buf[wholeLength] = 0;
+        }
 
-        buf[wholeLength] = 0;
         return str;
     }
 
     return JSRope::new_<allowGC>(cx, left, right, wholeLength);
 }
 
 template JSString *
 js::ConcatStrings<CanGC>(ThreadSafeContext *cx, HandleString left, HandleString right);
@@ -525,37 +543,45 @@ JSFlatString::isIndexSlow(uint32_t *inde
         *indexp = index;
         return true;
     }
 
     return false;
 }
 
 bool
-ScopedThreadSafeStringInspector::ensureChars(ThreadSafeContext *cx)
+ScopedThreadSafeStringInspector::ensureChars(ThreadSafeContext *cx, const AutoCheckCannotGC &nogc)
 {
-    if (chars_)
+    if (state_ != Uninitialized)
         return true;
 
     if (cx->isExclusiveContext()) {
         JSLinearString *linear = str_->ensureLinear(cx->asExclusiveContext());
         if (!linear)
             return false;
-        chars_ = linear->chars();
+        if (linear->hasTwoByteChars()) {
+            state_ = TwoByte;
+            twoByteChars_ = linear->twoByteChars(nogc);
+        } else {
+            state_ = Latin1;
+            latin1Chars_ = linear->latin1Chars(nogc);
+        }
     } else {
         if (str_->hasPureChars()) {
-            chars_ = str_->pureChars();
+            state_ = TwoByte;
+            twoByteChars_ = str_->pureChars();
         } else {
             if (!str_->copyNonPureChars(cx, scopedChars_))
                 return false;
-            chars_ = scopedChars_;
+            state_ = TwoByte;
+            twoByteChars_ = scopedChars_;
         }
     }
 
-    JS_ASSERT(chars_);
+    MOZ_ASSERT(state_ != Uninitialized);
     return true;
 }
 
 /*
  * Set up some tools to make it easier to generate large tables. After constant
  * folding, for each n, Rn(0) is the comma-separated list R(0), R(1), ..., R(2^n-1).
  * Similary, Rn(k) (for any k and n) generates the list R(k), R(k+1), ..., R(k+2^n-1).
  * To use this, define R appropriately, then use Rn(0) (for some value of n), then
--- a/js/src/vm/String.h
+++ b/js/src/vm/String.h
@@ -572,16 +572,27 @@ class JSLinearString : public JSString
 {
     friend class JSString;
 
     /* Vacuous and therefore unimplemented. */
     JSLinearString *ensureLinear(JSContext *cx) MOZ_DELETE;
     bool isLinear() const MOZ_DELETE;
     JSLinearString &asLinear() const MOZ_DELETE;
 
+  protected:
+    /* Returns void pointer to latin1/twoByte chars, for finalizers. */
+    MOZ_ALWAYS_INLINE
+    void *nonInlineCharsRaw() const {
+        JS_ASSERT(!isInline());
+        static_assert(offsetof(JSLinearString, d.s.u2.nonInlineCharsTwoByte) ==
+                      offsetof(JSLinearString, d.s.u2.nonInlineCharsLatin1),
+                      "nonInlineCharsTwoByte and nonInlineCharsLatin1 must have same offset");
+        return (void *)d.s.u2.nonInlineCharsTwoByte;
+    }
+
   public:
     MOZ_ALWAYS_INLINE
     const jschar *nonInlineChars() const {
         JS_ASSERT(!isInline());
         JS_ASSERT(hasTwoByteChars());
         return d.s.u2.nonInlineCharsTwoByte;
     }
 
@@ -729,17 +740,18 @@ class JSInlineString : public JSFlatStri
 {
     static const size_t MAX_LENGTH_LATIN1 = NUM_INLINE_CHARS_LATIN1 - 1;
     static const size_t MAX_LENGTH_TWO_BYTE = NUM_INLINE_CHARS_TWO_BYTE - 1;
 
   public:
     template <js::AllowGC allowGC>
     static inline JSInlineString *new_(js::ThreadSafeContext *cx);
 
-    inline jschar *init(size_t length);
+    inline jschar *initTwoByte(size_t length);
+    inline char *initLatin1(size_t length);
 
     inline void resetLength(size_t length);
 
     MOZ_ALWAYS_INLINE
     const jschar *chars() const {
         JS_ASSERT(JSString::isInline());
         JS_ASSERT(hasTwoByteChars());
         return d.inlineStorageTwoByte;
@@ -817,17 +829,18 @@ class JSFatInlineString : public JSInlin
     static const size_t MAX_LENGTH_LATIN1 = JSString::NUM_INLINE_CHARS_LATIN1 +
                                             INLINE_EXTENSION_CHARS_LATIN1
                                             -1 /* null terminator */;
 
     static const size_t MAX_LENGTH_TWO_BYTE = JSString::NUM_INLINE_CHARS_TWO_BYTE +
                                               INLINE_EXTENSION_CHARS_TWO_BYTE
                                               -1 /* null terminator */;
 
-    inline jschar *init(size_t length);
+    inline jschar *initTwoByte(size_t length);
+    inline char *initLatin1(size_t length);
 
     static bool latin1LengthFits(size_t length) {
         return length <= MAX_LENGTH_LATIN1;
     }
     static bool twoByteLengthFits(size_t length) {
         return length <= MAX_LENGTH_TWO_BYTE;
     }
 
@@ -919,34 +932,46 @@ namespace js {
  * range() may not be valid after the inspector goes out of scope.
  */
 
 class ScopedThreadSafeStringInspector
 {
   private:
     JSString *str_;
     ScopedJSFreePtr<jschar> scopedChars_;
-    const jschar *chars_;
+    union {
+        const jschar *twoByteChars_;
+        const char *latin1Chars_;
+    };
+    enum State { Uninitialized, Latin1, TwoByte };
+    State state_;
 
   public:
     explicit ScopedThreadSafeStringInspector(JSString *str)
       : str_(str),
-        chars_(nullptr)
+        state_(Uninitialized)
     { }
 
-    bool ensureChars(ThreadSafeContext *cx);
+    bool ensureChars(ThreadSafeContext *cx, const JS::AutoCheckCannotGC &nogc);
+
+    bool hasTwoByteChars() const { return state_ == TwoByte; }
+    bool hasLatin1Chars() const { return state_ == Latin1; }
 
-    const jschar *chars() {
-        JS_ASSERT(chars_);
-        return chars_;
+    const jschar *twoByteChars() const {
+        MOZ_ASSERT(state_ == TwoByte);
+        return twoByteChars_;
+    }
+    const char *latin1Chars() const {
+        MOZ_ASSERT(state_ == Latin1);
+        return latin1Chars_;
     }
 
-    JS::TwoByteChars range() {
-        JS_ASSERT(chars_);
-        return JS::TwoByteChars(chars_, str_->length());
+    JS::TwoByteChars twoByteRange() const {
+        MOZ_ASSERT(state_ == TwoByte);
+        return JS::TwoByteChars(twoByteChars_, str_->length());
     }
 };
 
 class StaticStrings
 {
   private:
     /* Bigger chars cannot be in a length-2 string. */
     static const size_t SMALL_CHAR_LIMIT    = 128U;
--- a/js/src/vm/StringBuffer.h
+++ b/js/src/vm/StringBuffer.h
@@ -115,17 +115,17 @@ StringBuffer::append(JSString *str)
 }
 
 inline bool
 StringBuffer::appendInflated(const char *cstr, size_t cstrlen)
 {
     size_t lengthBefore = length();
     if (!cb.growByUninitialized(cstrlen))
         return false;
-    InflateStringToBuffer(cstr, cstrlen, begin() + lengthBefore);
+    CopyAndInflateChars(begin() + lengthBefore, cstr, cstrlen);
     return true;
 }
 
 /* ES5 9.8 ToString, appending the result to the string buffer. */
 extern bool
 ValueToStringBufferSlow(JSContext *cx, const Value &v, StringBuffer &sb);
 
 inline bool