Bug 1539877 - Remove Modifier::TemplateTail. r=jwalden
authorJason Orendorff <jorendorff@mozilla.com>
Thu, 11 Apr 2019 13:25:36 +0000
changeset 469000 a7adb4cb7f66a2b98f8a674458b43f10a4d7bf14
parent 468999 81253689d39c5a08134cf122dcc0e092e38db643
child 469001 a5fe792348ed187dd06b3047d18e731f4dd4b227
push id35856
push usercsabou@mozilla.com
push dateFri, 12 Apr 2019 03:19:48 +0000
treeherdermozilla-central@940684cd1065 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjwalden
bugs1539877
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1539877 - Remove Modifier::TemplateTail. r=jwalden Differential Revision: https://phabricator.services.mozilla.com/D25330
js/src/frontend/Parser.cpp
js/src/frontend/TokenStream.cpp
js/src/frontend/TokenStream.h
--- a/js/src/frontend/Parser.cpp
+++ b/js/src/frontend/Parser.cpp
@@ -2484,17 +2484,17 @@ bool GeneralParser<ParseHandler, Unit>::
   if (!tokenStream.getToken(&tt, TokenStream::SlashIsRegExp)) {
     return false;
   }
   if (tt != TokenKind::RightCurly) {
     error(JSMSG_TEMPLSTR_UNTERM_EXPR);
     return false;
   }
 
-  return tokenStream.getToken(ttp, TokenStream::TemplateTail);
+  return tokenStream.getTemplateToken(ttp);
 }
 
 template <class ParseHandler, typename Unit>
 bool GeneralParser<ParseHandler, Unit>::taggedTemplate(
     YieldHandling yieldHandling, ListNodeType tagArgsList, TokenKind tt) {
   CallSiteNodeType callSiteObjNode = handler_.newCallSiteObject(pos().begin);
   if (!callSiteObjNode) {
     return false;
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@@ -2429,22 +2429,16 @@ template <typename Unit, class AnyCharsA
 MOZ_MUST_USE bool TokenStreamSpecific<Unit, AnyCharsAccess>::getTokenInternal(
     TokenKind* const ttp, const Modifier modifier) {
   // Assume we'll fail: success cases will overwrite this.
 #ifdef DEBUG
   *ttp = TokenKind::Limit;
 #endif
   MOZ_MAKE_MEM_UNDEFINED(ttp, sizeof(*ttp));
 
-  // Check if in the middle of a template string. Have to get this out of
-  // the way first.
-  if (MOZ_UNLIKELY(modifier == TemplateTail)) {
-    return getStringOrTemplateToken('`', modifier, ttp);
-  }
-
   // This loop runs more than once only when whitespace or comments are
   // encountered.
   do {
     int32_t unit = peekCodeUnit();
     if (MOZ_UNLIKELY(unit == EOF)) {
       MOZ_ASSERT(this->sourceUnits.atEnd());
       anyCharsAccess().flags.isEOF = true;
       TokenStart start(this->sourceUnits, 0);
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -336,26 +336,16 @@ struct Token {
     // the parser calls `getToken(SlashIsInvalid)`, it must be prepared to see
     // either one (and throw a SyntaxError either way).
     //
     // It's OK to use SlashIsInvalid to get a token that was originally scanned
     // with SlashIsDiv or SlashIsRegExp. The reverse--peeking with
     // SlashIsInvalid, then getting with another mode--is not OK. If either Div
     // or RegExp is syntactically valid here, use the appropriate modifier.
     SlashIsInvalid,
-
-    // Treat subsequent code units as the tail of a template literal, after
-    // a template substitution, beginning with a "}", continuing with zero
-    // or more template literal code units, and ending with either "${" or
-    // the end of the template literal.  For example:
-    //
-    //   var entity = "world";
-    //   var s = `Hello ${entity}!`;
-    //                          ^ TemplateTail context
-    TemplateTail,
   };
   friend class TokenStreamShared;
 
  public:
   // WARNING: TokenStreamPosition assumes that the only GC things a Token
   //          includes are atoms.  DON'T ADD NON-ATOM GC THING POINTERS HERE
   //          UNLESS YOU ADD ADDITIONAL ROOTING TO THAT CLASS.
 
@@ -481,37 +471,24 @@ class TokenStreamShared {
 
  public:
   static constexpr unsigned maxLookahead = 2;
 
   using Modifier = Token::Modifier;
   static constexpr Modifier SlashIsDiv = Token::SlashIsDiv;
   static constexpr Modifier SlashIsRegExp = Token::SlashIsRegExp;
   static constexpr Modifier SlashIsInvalid = Token::SlashIsInvalid;
-  static constexpr Modifier TemplateTail = Token::TemplateTail;
 
   static void verifyConsistentModifier(Modifier modifier,
-                                       Token lookaheadToken) {
-#ifdef DEBUG
-    // Easy case: modifiers match.
-    if (modifier == lookaheadToken.modifier) {
-      return;
-    }
-
-    if (modifier == SlashIsInvalid &&
-        lookaheadToken.modifier != TemplateTail) {
-      // "Don't care" mode is fine after either SlashIsDiv or SlashIsRegExp.
-      return;
-    }
-
-    MOZ_ASSERT_UNREACHABLE(
-        "This token was scanned with both SlashIsRegExp and SlashIsDiv, indicating "
-        "the parser is confused about which one is allowed here. See comment "
-        "at Token::Modifier.");
-#endif
+                                       const Token& nextToken) {
+    MOZ_ASSERT(
+        modifier == nextToken.modifier || modifier == SlashIsInvalid,
+        "This token was scanned with both SlashIsRegExp and SlashIsDiv, "
+        "indicating the parser is confused about how to handle a slash here. "
+        "See comment at Token::Modifier.");
   }
 };
 
 static_assert(mozilla::IsEmpty<TokenStreamShared>::value,
               "TokenStreamShared shouldn't bloat classes that inherit from it");
 
 template <typename Unit, class AnyCharsAccess>
 class TokenStreamSpecific;
@@ -2765,16 +2742,39 @@ class MOZ_STACK_CLASS TokenStreamSpecifi
   MOZ_MUST_USE bool matchIdentifierStart(IdentifierEscapes* sawEscape);
 
   MOZ_MUST_USE bool getTokenInternal(TokenKind* const ttp,
                                      const Modifier modifier);
 
   MOZ_MUST_USE bool getStringOrTemplateToken(char untilChar, Modifier modifier,
                                              TokenKind* out);
 
+  // Parse a TemplateMiddle or TemplateTail token (one of the string-like parts
+  // of a template string) after already consuming the leading `RightCurly`.
+  // (The spec says the `}` is the first character of the TemplateMiddle/
+  // TemplateTail, but we treat it as a separate token because that's much
+  // easier to implement in both TokenStream and the parser.)
+  //
+  // This consumes a token and sets the current token, like `getToken()`.  It
+  // doesn't take a Modifier because there's no risk of encountering a division
+  // operator or RegExp literal.
+  //
+  // On success, `*ttp` is either `TokenKind::TemplateHead` (if we got a
+  // TemplateMiddle token) or `TokenKind::NoSubsTemplate` (if we got a
+  // TemplateTail). That may seem strange; there are four different template
+  // token types in the spec, but we only use two. We use `TemplateHead` for
+  // TemplateMiddle because both end with `...${`, and `NoSubsTemplate` for
+  // TemplateTail because both contain the end of the template, including the
+  // closing quote mark. They're not treated differently, either in the parser
+  // or in the tokenizer.
+  MOZ_MUST_USE bool getTemplateToken(TokenKind* ttp) {
+    MOZ_ASSERT(anyCharsAccess().currentToken().type == TokenKind::RightCurly);
+    return getStringOrTemplateToken('`', SlashIsInvalid, ttp);
+  }
+
   MOZ_MUST_USE bool getDirectives(bool isMultiline, bool shouldWarnDeprecated);
   MOZ_MUST_USE bool getDirective(
       bool isMultiline, bool shouldWarnDeprecated, const char* directive,
       uint8_t directiveLength, const char* errorMsgPragma,
       UniquePtr<char16_t[], JS::FreePolicy>* destination);
   MOZ_MUST_USE bool getDisplayURL(bool isMultiline, bool shouldWarnDeprecated);
   MOZ_MUST_USE bool getSourceMappingURL(bool isMultiline,
                                         bool shouldWarnDeprecated);