Bug 1478170 - Move TokenStreamChars::getCodePoint to a generalized TokenStreamSpecific::getCodePoint that works for any CharT. r=arai
authorJeff Walden <jwalden@mit.edu>
Tue, 17 Jul 2018 21:13:06 -0700
changeset 428555 38216fdba3ddea982cc4d3178c3b2c75637f71a1
parent 428554 107211d728e60e9b069c2d0107241f74b575d576
child 428556 dad6f1d062719f26f6f934c2e8a5225ab2491351
push id34337
push userncsoregi@mozilla.com
push dateThu, 26 Jul 2018 21:58:45 +0000
treeherdermozilla-central@8f2f847b2f9d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersarai
bugs1478170
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1478170 - Move TokenStreamChars::getCodePoint to a generalized TokenStreamSpecific::getCodePoint that works for any CharT. r=arai
js/src/frontend/TokenStream.cpp
js/src/frontend/TokenStream.h
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@@ -786,54 +786,16 @@ TokenStreamChars<Utf8Unit, AnyCharsAcces
         return false;
 
     *codePoint = maybeCodePoint.value();
     return true;
 }
 
 template<class AnyCharsAccess>
 bool
-TokenStreamChars<char16_t, AnyCharsAccess>::getCodePoint(int32_t* cp)
-{
-    TokenStreamAnyChars& anyChars = anyCharsAccess();
-
-    if (MOZ_UNLIKELY(this->sourceUnits.atEnd())) {
-        anyChars.flags.isEOF = true;
-        *cp = EOF;
-        return true;
-    }
-
-    int32_t c = this->sourceUnits.getCodeUnit();
-
-    do {
-        // Normalize the char16_t if it was a newline.
-        if (MOZ_UNLIKELY(c == '\n'))
-            break;
-
-        if (MOZ_UNLIKELY(c == '\r')) {
-            matchLineTerminator('\n');
-            break;
-        }
-
-        if (MOZ_UNLIKELY(c == unicode::LINE_SEPARATOR || c == unicode::PARA_SEPARATOR))
-            break;
-
-        *cp = c;
-        return true;
-    } while (false);
-
-    if (!updateLineInfoForEOL())
-        return false;
-
-    *cp = '\n';
-    return true;
-}
-
-template<class AnyCharsAccess>
-bool
 TokenStreamChars<char16_t, AnyCharsAccess>::getNonAsciiCodePoint(int32_t lead, int32_t* codePoint)
 {
     MOZ_ASSERT(lead != EOF);
     MOZ_ASSERT(!isAsciiCodePoint(lead),
                "ASCII code unit/point must be handled separately");
     MOZ_ASSERT(lead == this->sourceUnits.previousCodeUnit(),
                "getNonAsciiCodePoint called incorrectly");
 
@@ -876,16 +838,34 @@ TokenStreamChars<char16_t, AnyCharsAcces
     }
 
     // Otherwise we have a multi-unit code point.
     *codePoint = unicode::UTF16Decode(lead, this->sourceUnits.getCodeUnit());
     MOZ_ASSERT(!IsLineTerminator(AssertedCast<char32_t>(*codePoint)));
     return true;
 }
 
+template<typename CharT, class AnyCharsAccess>
+bool
+TokenStreamSpecific<CharT, AnyCharsAccess>::getCodePoint(int32_t* cp)
+{
+    int32_t unit = getCodeUnit();
+    if (unit == EOF) {
+        MOZ_ASSERT(anyCharsAccess().flags.isEOF,
+                   "flags.isEOF should have been set by getCodeUnit()");
+        *cp = EOF;
+        return true;
+    }
+
+    if (isAsciiCodePoint(unit))
+        return getFullAsciiCodePoint(unit, cp);
+
+    return getNonAsciiCodePoint(unit, cp);
+}
+
 template<class AnyCharsAccess>
 bool
 TokenStreamChars<Utf8Unit, AnyCharsAccess>::getNonAsciiCodePoint(int32_t unit, int32_t* codePoint)
 {
     MOZ_ASSERT(unit != EOF);
     MOZ_ASSERT(!isAsciiCodePoint(unit),
                "ASCII code unit/point must be handled separately");
 
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -2029,24 +2029,16 @@ class TokenStreamChars<char16_t, AnyChar
     MOZ_MUST_USE bool getNonAsciiCodePointDontNormalize(char16_t lead, char32_t* codePoint) {
         // There are no encoding errors in 16-bit JS, so implement this so that
         // the compiler knows it, too.
         *codePoint = infallibleGetNonAsciiCodePointDontNormalize(lead);
         return true;
     }
 
     /**
-     * Get the next code point, converting LineTerminatorSequences to '\n' and
-     * updating internal line-counter state if needed.  Return true on success
-     * and store the code point in |*c|.  Return false and leave |*c| undefined
-     * on failure.
-     */
-    MOZ_MUST_USE bool getCodePoint(int32_t* cp);
-
-    /**
      * Given a just-consumed non-ASCII code unit |lead| (which may also be a
      * full code point, for UTF-16), consume a full code point or
      * LineTerminatorSequence (normalizing it to '\n') and store it in
      * |*codePoint|.  Return true on success, otherwise return false and leave
      * |*codePoint| undefined on failure.
      *
      * If a LineTerminatorSequence was consumed, also update line/column info.
      *
@@ -2271,17 +2263,16 @@ class MOZ_STACK_CLASS TokenStreamSpecifi
     using TokenStreamCharsShared::appendCodePointToCharBuffer;
     using CharsBase::atomizeSourceChars;
     using GeneralCharsBase::badToken;
     // Deliberately don't |using| |charBuffer| because of bug 1472569.  :-(
     using CharsBase::consumeKnownCodeUnit;
     using TokenStreamCharsShared::copyCharBufferTo;
     using TokenStreamCharsShared::drainCharBufferIntoAtom;
     using CharsBase::fillCharBufferFromSourceNormalizingAsciiLineBreaks;
-    using SpecializedChars::getCodePoint;
     using GeneralCharsBase::getCodeUnit;
     using GeneralCharsBase::getFullAsciiCodePoint;
     using SpecializedChars::getNonAsciiCodePoint;
     using SpecializedChars::getNonAsciiCodePointDontNormalize;
     using GeneralCharsBase::internalComputeLineOfContext;
     using TokenStreamCharsShared::isAsciiCodePoint;
     using CharsBase::matchCodeUnit;
     using CharsBase::matchLineTerminator;
@@ -2298,16 +2289,24 @@ class MOZ_STACK_CLASS TokenStreamSpecifi
     using GeneralCharsBase::updateLineInfoForEOL;
 
     template<typename CharU> friend class TokenStreamPosition;
 
   public:
     TokenStreamSpecific(JSContext* cx, const ReadOnlyCompileOptions& options,
                         const CharT* base, size_t length);
 
+    /**
+     * Get the next code point, converting LineTerminatorSequences to '\n' and
+     * updating internal line-counter state if needed.  Return true on success
+     * and store the code point in |*cp|.  Return false and leave |*cp|
+     * undefined on failure.
+     */
+    MOZ_MUST_USE bool getCodePoint(int32_t* cp);
+
     // If there is an invalid escape in a template, report it and return false,
     // otherwise return true.
     bool checkForInvalidTemplateEscapeError() {
         if (anyCharsAccess().invalidTemplateEscapeType == InvalidEscapeType::None)
             return true;
 
         reportInvalidEscapeError(anyCharsAccess().invalidTemplateEscapeOffset,
                                  anyCharsAccess().invalidTemplateEscapeType);