Bug 1194022 - Add another exception for a token after yield expression without operand and followed by EOL. r=Waldo
authorTooru Fujisawa <arai_a@mac.com>
Thu, 13 Aug 2015 09:40:05 +0900
changeset 257721 a466d341647f4b25758e56c674e7a6e4ff746bdf
parent 257720 8a50c71ddedacab0e700e429d8dbd25dea73d380
child 257722 c6c91bdf11c01a7bdb47a8ece5b6afa7201e15b6
push id29226
push userryanvm@gmail.com
push dateFri, 14 Aug 2015 13:01:14 +0000
treeherdermozilla-central@1b2402247429 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersWaldo
bugs1194022
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1194022 - Add another exception for a token after yield expression without operand and followed by EOL. r=Waldo
js/src/frontend/Parser.cpp
js/src/frontend/TokenStream.cpp
js/src/frontend/TokenStream.h
js/src/jit-test/tests/parser/yield-without-operand.js
--- a/js/src/frontend/Parser.cpp
+++ b/js/src/frontend/Parser.cpp
@@ -5700,17 +5700,19 @@ Parser<ParseHandler>::yieldExpression(In
           case TOK_SEMI:
           case TOK_RC:
           case TOK_RB:
           case TOK_RP:
           case TOK_COLON:
           case TOK_COMMA:
             // No value.
             exprNode = null();
-            tokenStream.addModifierException(TokenStream::NoneIsOperand);
+            tokenStream.addModifierException((tt == TOK_EOL || tt == TOK_EOF)
+                                             ? TokenStream::NoneIsOperandYieldEOL
+                                             : TokenStream::NoneIsOperand);
             break;
           case TOK_MUL:
             kind = PNK_YIELD_STAR;
             tokenStream.consumeKnownToken(TOK_MUL, TokenStream::Operand);
             // Fall through.
           default:
             exprNode = assignExpr(inHandling, YieldIsKeyword);
             if (!exprNode)
@@ -5763,17 +5765,19 @@ Parser<ParseHandler>::yieldExpression(In
           case TOK_SEMI:
           case TOK_RC:
           case TOK_RB:
           case TOK_RP:
           case TOK_COLON:
           case TOK_COMMA:
             // No value.
             exprNode = null();
-            tokenStream.addModifierException(TokenStream::NoneIsOperand);
+            tokenStream.addModifierException((tt == TOK_EOL || tt == TOK_EOF)
+                                             ? TokenStream::NoneIsOperandYieldEOL
+                                             : TokenStream::NoneIsOperand);
             break;
           default:
             exprNode = assignExpr(inHandling, YieldIsKeyword);
             if (!exprNode)
                 return null();
         }
 
         return newYieldExpression(begin, exprNode);
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@@ -1645,17 +1645,17 @@ TokenStream::getTokenInternal(TokenKind*
 
     flags.isDirtyLine = true;
     tp->pos.end = userbuf.offset();
 #ifdef DEBUG
     // Save the modifier used to get this token, so that if an ungetToken()
     // occurs and then the token is re-gotten (or peeked, etc.), we can assert
     // that both gets have used the same modifiers.
     tp->modifier = modifier;
-    tp->modifierExceptions = NoException;
+    tp->modifierException = NoException;
 #endif
     MOZ_ASSERT(IsTokenSane(tp));
     *ttp = tp->type;
     return true;
 
   error:
     if (flags.hitOOM)
         return reportError(JSMSG_OUT_OF_MEMORY);
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -112,16 +112,39 @@ struct Token
         // or more template literal characters, and ending with either "${" or
         // the end of the template literal.  For example:
         //
         //   var entity = "world";
         //   var s = `Hello ${entity}!`;
         //                          ^ TemplateTail context
         TemplateTail,
     };
+    enum ModifierException
+    {
+        NoException,
+
+        // If an yield expression operand is omitted and yield expression is
+        // followed by non-EOL, the next token is already gotten with Operand,
+        // but we expect operator (None).
+        NoneIsOperand,
+
+        // If an yield expression operand is omitted and yield expression is
+        // followed by EOL, the next token is already gotten with Operand, and
+        // we expect Operand in next statement, but MatchOrInsertSemicolon
+        // after expression statement expects operator (None).
+        NoneIsOperandYieldEOL,
+
+        // If a semicolon is inserted automatically, the next token is already
+        // gotten with None, but we expect Operand.
+        OperandIsNone,
+
+        // If name of method definition is `get` or `set`, the next token is
+        // already gotten with KeywordIsName, but we expect None.
+        NoneIsKeywordIsName,
+    };
     friend class TokenStream;
 
   public:
     TokenKind           type;           // char value or above enumerator
     TokenPos            pos;            // token position in file
     union {
       private:
         friend struct Token;
@@ -131,17 +154,17 @@ struct Token
             double      value;          // floating point number
             DecimalPoint decimalPoint;  // literal contains '.'
         } number;
         RegExpFlag      reflags;        // regexp flags; use tokenbuf to access
                                         //   regexp chars
     } u;
 #ifdef DEBUG
     Modifier modifier;                  // Modifier used to get this token
-    uint8_t modifierExceptions;         // Bitwise OR of modifier exceptions
+    ModifierException modifierException; // Exception for this modifier
 #endif
 
     // This constructor is necessary only for MSVC 2013 and how it compiles the
     // initialization of TokenStream::tokens.  That field is initialized as
     // tokens() in the constructor init-list.  This *should* zero the entire
     // array, then (because Token has a non-trivial constructor, because
     // TokenPos has a user-provided constructor) call the implicit Token
     // constructor on each element, which would call the TokenPos constructor
@@ -406,58 +429,68 @@ class MOZ_STACK_CLASS TokenStream
 
   public:
     typedef Token::Modifier Modifier;
     static MOZ_CONSTEXPR_VAR Modifier None = Token::None;
     static MOZ_CONSTEXPR_VAR Modifier Operand = Token::Operand;
     static MOZ_CONSTEXPR_VAR Modifier KeywordIsName = Token::KeywordIsName;
     static MOZ_CONSTEXPR_VAR Modifier TemplateTail = Token::TemplateTail;
 
-    enum ModifierException
-    {
-        NoException = 0x00,
-
-        // If a semicolon is inserted automatically, the next token is already
-        // gotten with None, but we expect Operand.
-        NoneIsOperand = 0x01,
-
-        // If an yield expression operand is omitted, the next token is already
-        // gotten with Operand, but we expect operator (None).
-        OperandIsNone = 0x02,
-
-        // If name of method definition is `get` or `set`, the next token is
-        // already gotten with KeywordIsName, but we expect None.
-        NoneIsKeywordIsName = 0x04,
-    };
+    typedef Token::ModifierException ModifierException;
+    static MOZ_CONSTEXPR_VAR ModifierException NoException = Token::NoException;
+    static MOZ_CONSTEXPR_VAR ModifierException NoneIsOperand = Token::NoneIsOperand;
+    static MOZ_CONSTEXPR_VAR ModifierException NoneIsOperandYieldEOL = Token::NoneIsOperandYieldEOL;
+    static MOZ_CONSTEXPR_VAR ModifierException OperandIsNone = Token::OperandIsNone;
+    static MOZ_CONSTEXPR_VAR ModifierException NoneIsKeywordIsName = Token::NoneIsKeywordIsName;
 
     void addModifierException(ModifierException modifierException) {
 #ifdef DEBUG
         const Token& next = nextToken();
+        if (next.modifierException == NoneIsOperand ||
+            next.modifierException == NoneIsOperandYieldEOL)
+        {
+            // Token after yield expression without operand already has
+            // NoneIsOperand or NoneIsOperandYieldEOL exception.
+            MOZ_ASSERT(modifierException == OperandIsNone);
+            if (next.modifierException == NoneIsOperand)
+                MOZ_ASSERT(next.type != TOK_DIV && next.type != TOK_REGEXP,
+                           "next token requires contextual specifier to be parsed unambiguously");
+            else
+                MOZ_ASSERT(next.type != TOK_DIV,
+                           "next token requires contextual specifier to be parsed unambiguously");
+
+            // Do not update modifierException.
+            return;
+        }
+
+        MOZ_ASSERT(next.modifierException == NoException);
         switch (modifierException) {
           case NoneIsOperand:
             MOZ_ASSERT(next.modifier == Operand);
             MOZ_ASSERT(next.type != TOK_DIV && next.type != TOK_REGEXP,
                        "next token requires contextual specifier to be parsed unambiguously");
             break;
+          case NoneIsOperandYieldEOL:
+            MOZ_ASSERT(next.modifier == Operand);
+            MOZ_ASSERT(next.type != TOK_DIV,
+                       "next token requires contextual specifier to be parsed unambiguously");
+            break;
           case OperandIsNone:
-            // Non-Operand token after yield/continue/break already has
-            // NoneIsOperand exception.
-            MOZ_ASSERT(next.modifier == None ||
-                       ((next.modifierExceptions & NoneIsOperand) && next.modifier == Operand));
+            MOZ_ASSERT(next.modifier == None);
             MOZ_ASSERT(next.type != TOK_DIV && next.type != TOK_REGEXP,
                        "next token requires contextual specifier to be parsed unambiguously");
             break;
           case NoneIsKeywordIsName:
             MOZ_ASSERT(next.modifier == KeywordIsName);
             MOZ_ASSERT(next.type != TOK_NAME);
             break;
           default:
             MOZ_CRASH("unexpected modifier exception");
         }
-        tokens[(cursor + 1) & ntokensMask].modifierExceptions |= modifierException;
+        tokens[(cursor + 1) & ntokensMask].modifierException = modifierException;
 #endif
     }
 
     bool hasLookahead() { return lookahead > 0; }
 
     Modifier getLookaheadModifier() {
 #ifdef DEBUG
         return nextToken().modifier;
@@ -468,29 +501,31 @@ class MOZ_STACK_CLASS TokenStream
 
     void
     verifyConsistentModifier(Modifier modifier, Token lookaheadToken) {
 #ifdef DEBUG
         // Easy case: modifiers match.
         if (modifier == lookaheadToken.modifier)
             return;
 
-        if (lookaheadToken.modifierExceptions & OperandIsNone) {
+        if (lookaheadToken.modifierException == OperandIsNone) {
             // getToken(Operand) permissibly following getToken().
             if (modifier == Operand && lookaheadToken.modifier == None)
                 return;
         }
 
-        if (lookaheadToken.modifierExceptions & NoneIsOperand) {
+        if (lookaheadToken.modifierException == NoneIsOperand ||
+            lookaheadToken.modifierException == NoneIsOperandYieldEOL)
+        {
             // getToken() permissibly following getToken(Operand).
             if (modifier == None && lookaheadToken.modifier == Operand)
                 return;
         }
 
-        if (lookaheadToken.modifierExceptions & NoneIsKeywordIsName) {
+        if (lookaheadToken.modifierException == NoneIsKeywordIsName) {
             // getToken() permissibly following getToken(KeywordIsName).
             if (modifier == None && lookaheadToken.modifier == KeywordIsName)
                 return;
         }
 
         MOZ_ASSERT_UNREACHABLE("this token was previously looked up with a "
                                "different modifier, potentially making "
                                "tokenization non-deterministic");
--- a/js/src/jit-test/tests/parser/yield-without-operand.js
+++ b/js/src/jit-test/tests/parser/yield-without-operand.js
@@ -1,18 +1,24 @@
 // yield without an operand is fine and dandy.
 
 load(libdir + "asserts.js");
 
 assertNoWarning(() => Function("yield"), SyntaxError,
                 "yield followed by EOF is fine");
 assertNoWarning(() => Function("yield;"), SyntaxError,
                 "yield followed by semicolon is fine");
+assertNoWarning(() => Function("yield\n"), SyntaxError,
+                "yield followed by newline is fine");
 assertNoWarning(() => Function("yield\n  print('ok');"), SyntaxError,
-                "yield followed by newline is fine");
+                "yield followed by newline and statement is fine");
+assertNoWarning(() => Function("yield\n  /x/;"), SyntaxError,
+                "yield followed by newline and regexp is fine");
+assertThrowsInstanceOf(() => Function("yield\n  /"), SyntaxError,
+                       "yield followed by newline and slash is fine");
 
 assertNoWarning(() => eval("(function () { yield; })"), SyntaxError,
                 "yield followed by semicolon in eval code is fine");
 assertNoWarning(() => eval("(function () { yield })"), SyntaxError,
                 "yield followed by } in eval code is fine");
 
 assertNoWarning(() => Function("yield 0;"),
                 "yield with an operand should be fine");