Bug 1461481 - Move filling up a CharBuffer with the characters in a sequence of template literal characters into TokenStreamCharsBase. r=arai
authorJeff Walden <jwalden@mit.edu>
Thu, 10 May 2018 20:17:38 -0700
changeset 418605 43b6fedcb9af91bdfefc19af28331c985665c75e
parent 418604 70d1589b5df5b97a3ce480d8cc1f15994aa22f22
child 418606 c68fadb2d59673ae59641e00a77bf7ceb5301f4a
push id103351
push userjwalden@mit.edu
push dateThu, 17 May 2018 07:17:38 +0000
treeherdermozilla-inbound@e016aa76775e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersarai
bugs1461481
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1461481 - Move filling up a CharBuffer with the characters in a sequence of template literal characters into TokenStreamCharsBase. r=arai
js/src/frontend/TokenStream.h
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -1022,16 +1022,40 @@ class TokenStreamCharsBase
     MOZ_MUST_USE bool copyTokenbufTo(JSContext* cx,
                                      UniquePtr<char16_t[], JS::FreePolicy>* destination);
 
     using SourceUnits = frontend::SourceUnits<CharT>;
 
     MOZ_MUST_USE bool appendCodePointToTokenbuf(uint32_t codePoint);
 
   protected:
+    MOZ_MUST_USE bool
+    fillWithTemplateStringContents(CharBuffer& charbuf, const CharT* cur, const CharT* end) {
+        while (cur < end) {
+            // U+2028 LINE SEPARATOR and U+2029 PARAGRAPH SEPARATOR are
+            // interpreted literally inside template literal contents; only
+            // literal CRLF sequences are normalized to '\n'.  See
+            // <https://tc39.github.io/ecma262/#sec-static-semantics-tv-and-trv>.
+            CharT ch = *cur;
+            if (ch == '\r') {
+                ch = '\n';
+                if ((cur + 1 < end) && (*(cur + 1) == '\n'))
+                    cur++;
+            }
+
+            if (!charbuf.append(ch))
+                return false;
+
+            cur++;
+        }
+
+        return true;
+    }
+
+  protected:
     /** Code units in the source code being tokenized. */
     SourceUnits sourceUnits;
 
     /** Current token string buffer. */
     CharBuffer tokenbuf;
 };
 
 template<>
@@ -1305,16 +1329,17 @@ class MOZ_STACK_CLASS TokenStreamSpecifi
     using typename CharsSharedBase::SourceUnits;
 
   private:
     using CharsSharedBase::appendCodePointToTokenbuf;
     using CharsSharedBase::atomizeChars;
     using GeneralCharsBase::badToken;
     using GeneralCharsBase::consumeRestOfSingleLineComment;
     using CharsSharedBase::copyTokenbufTo;
+    using CharsSharedBase::fillWithTemplateStringContents;
     using CharsBase::getChar;
     using GeneralCharsBase::getCharIgnoreEOL;
     using CharsBase::matchMultiUnitCodePoint;
     using GeneralCharsBase::newAtomToken;
     using GeneralCharsBase::newNameToken;
     using GeneralCharsBase::newNumberToken;
     using GeneralCharsBase::newRegExpToken;
     using GeneralCharsBase::newSimpleToken;
@@ -1421,27 +1446,19 @@ class MOZ_STACK_CLASS TokenStreamSpecifi
             // Of the form    |`...${|   or   |}...${|
             end = sourceUnits.codeUnitPtrAt(anyChars.currentToken().pos.end - 2);
         } else {
             // NO_SUBS_TEMPLATE is of the form   |`...`|   or   |}...`|
             end = sourceUnits.codeUnitPtrAt(anyChars.currentToken().pos.end - 1);
         }
 
         CharBuffer charbuf(anyChars.cx);
-        while (cur < end) {
-            CharT ch = *cur;
-            if (ch == '\r') {
-                ch = '\n';
-                if ((cur + 1 < end) && (*(cur + 1) == '\n'))
-                    cur++;
-            }
-            if (!charbuf.append(ch))
-                return nullptr;
-            cur++;
-        }
+        if (!fillWithTemplateStringContents(charbuf, cur, end))
+            return nullptr;
+
         return atomizeChars(anyChars.cx, charbuf.begin(), charbuf.length());
     }
 
   private:
     // This is private because it should only be called by the tokenizer while
     // tokenizing not by, for example, BytecodeEmitter.
     bool reportStrictModeError(unsigned errorNumber, ...);