Bug 1478045 - Use SourceUnits::peekCodePoint when examining a non-ASCII code point that might terminate an IdentifierName. r=arai
authorJeff Walden <jwalden@mit.edu>
Wed, 25 Jul 2018 14:24:35 -0700
changeset 428541 466d4573b39a20544f7166d0c10a6b931373b8b9
parent 428540 51e2ba3c67d66b80d655c897a266bbb2f9e0f511
child 428542 d1d2c7b998efa62f6d199df72a1db07cf5bbbd5b
push id34337
push userncsoregi@mozilla.com
push dateThu, 26 Jul 2018 21:58:45 +0000
treeherdermozilla-central@8f2f847b2f9d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersarai
bugs1478045
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1478045 - Use SourceUnits::peekCodePoint when examining a non-ASCII code point that might terminate an IdentifierName. r=arai
js/src/frontend/TokenStream.cpp
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@@ -1473,41 +1473,46 @@ TokenStreamSpecific<CharT, AnyCharsAcces
         this->badToken();
     });
 
     // We've already consumed an initial code point in the identifer, to *know*
     // that this is an identifier.  So no need to worry about not consuming any
     // code points in the loop below.
     int32_t unit;
     while (true) {
-        unit = getCodeUnit();
+        unit = peekCodeUnit();
         if (unit == EOF)
             break;
 
         if (MOZ_LIKELY(isAsciiCodePoint(unit))) {
+            consumeKnownCodeUnit(unit);
+
             if (MOZ_UNLIKELY(!unicode::IsIdentifierPart(static_cast<char16_t>(unit)))) {
                 // Handle a Unicode escape -- otherwise it's not part of the
                 // identifier.
                 uint32_t codePoint;
                 if (unit != '\\' || !matchUnicodeEscapeIdent(&codePoint)) {
                     ungetCodeUnit(unit);
                     break;
                 }
 
                 escaping = IdentifierEscapes::SawUnicodeEscape;
             }
         } else {
-            int32_t codePoint;
-            if (!getNonAsciiCodePoint(unit, &codePoint))
-                return false;
-
-            if (!unicode::IsIdentifierPart(uint32_t(codePoint))) {
-                ungetNonAsciiNormalizedCodePoint(codePoint);
+            // This ignores encoding errors: subsequent caller-side code to
+            // handle source text after the IdentifierName will do so.
+            PeekedCodePoint<CharT> peeked = this->sourceUnits.peekCodePoint();
+            if (peeked.isNone() || !unicode::IsIdentifierPart(peeked.codePoint()))
                 break;
-            }
+
+            MOZ_ASSERT(!IsLineTerminator(peeked.codePoint()),
+                       "IdentifierPart must guarantee !IsLineTerminator or "
+                       "else we'll fail to maintain line-info/flags for EOL");
+
+            this->sourceUnits.consumeKnownCodePoint(peeked);
         }
     }
 
     JSAtom* atom;
     if (MOZ_UNLIKELY(escaping == IdentifierEscapes::SawUnicodeEscape)) {
         // Identifiers containing Unicode escapes have to be converted into
         // tokenbuf before atomizing.
         if (!putIdentInCharBuffer(identStart))