Bug 1089045 - Part 1: Supply consistent modifiers to TokenStream. r=Waldo
authorTooru Fujisawa <arai_a@mac.com>
Fri, 07 Aug 2015 04:11:59 +0900
changeset 257459 05f838caf0761e9f9d0beecdd49d1c93d98d8e75
parent 257458 9e979108667ed6ffae0586239b292e02813b4ea6
child 257460 4f87d8225c7adefbe74126ed61bb79e85a2e9355
push id29219
push userkwierso@gmail.com
push dateWed, 12 Aug 2015 22:16:22 +0000
treeherdermozilla-central@7649ffe28b67 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersWaldo
bugs1089045
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1089045 - Part 1: Supply consistent modifiers to TokenStream. r=Waldo
js/src/asmjs/AsmJSModule.cpp
js/src/asmjs/AsmJSValidate.cpp
js/src/frontend/Parser.cpp
js/src/frontend/TokenStream.cpp
js/src/frontend/TokenStream.h
--- a/js/src/asmjs/AsmJSModule.cpp
+++ b/js/src/asmjs/AsmJSModule.cpp
@@ -284,17 +284,17 @@ AsmJSModule::lookupHeapAccess(void* pc) 
 bool
 AsmJSModule::finish(ExclusiveContext* cx, TokenStream& tokenStream, MacroAssembler& masm,
                     const Label& interruptLabel, const Label& outOfBoundsLabel)
 {
     MOZ_ASSERT(isFinishedWithFunctionBodies() && !isFinished());
 
     uint32_t endBeforeCurly = tokenStream.currentToken().pos.end;
     TokenPos pos;
-    if (!tokenStream.peekTokenPos(&pos))
+    if (!tokenStream.peekTokenPos(&pos, TokenStream::Operand))
         return false;
     uint32_t endAfterCurly = pos.end;
     MOZ_ASSERT(endBeforeCurly >= srcBodyStart_);
     MOZ_ASSERT(endAfterCurly >= srcBodyStart_);
     pod.srcLength_ = endBeforeCurly - srcStart_;
     pod.srcLengthWithRightBrace_ = endAfterCurly - srcStart_;
 
     // Start global data on a new page so JIT code may be given independent
@@ -2041,17 +2041,17 @@ class ModuleChars
 
   public:
     static uint32_t beginOffset(AsmJSParser& parser) {
         return parser.pc->maybeFunction->pn_pos.begin;
     }
 
     static uint32_t endOffset(AsmJSParser& parser) {
         TokenPos pos(0, 0);  // initialize to silence GCC warning
-        MOZ_ALWAYS_TRUE(parser.tokenStream.peekTokenPos(&pos));
+        MOZ_ALWAYS_TRUE(parser.tokenStream.peekTokenPos(&pos, TokenStream::Operand));
         return pos.end;
     }
 };
 
 class ModuleCharsForStore : ModuleChars
 {
     uint32_t uncompressedSize_;
     uint32_t compressedSize_;
--- a/js/src/asmjs/AsmJSValidate.cpp
+++ b/js/src/asmjs/AsmJSValidate.cpp
@@ -480,17 +480,17 @@ PeekToken(AsmJSParser& parser, TokenKind
 {
     TokenStream& ts = parser.tokenStream;
     TokenKind tk;
     while (true) {
         if (!ts.peekToken(&tk, TokenStream::Operand))
             return false;
         if (tk != TOK_SEMI)
             break;
-        ts.consumeKnownToken(TOK_SEMI);
+        ts.consumeKnownToken(TOK_SEMI, TokenStream::Operand);
     }
     *tkp = tk;
     return true;
 }
 
 static bool
 ParseVarOrConstStatement(AsmJSParser& parser, ParseNode** var)
 {
@@ -1600,17 +1600,18 @@ class MOZ_STACK_CLASS ModuleCompiler
             return failOffset(pn->pn_pos.begin, str);
 
         // The exact rooting static analysis does not perform dataflow analysis, so it believes
         // that unrooted things on the stack during compilation may still be accessed after this.
         // Since pn is typically only null under OOM, this suppression simply forces any GC to be
         // delayed until the compilation is off the stack and more memory can be freed.
         gc::AutoSuppressGC nogc(cx_);
         TokenPos pos;
-        if (!tokenStream().peekTokenPos(&pos))
+        TokenStream::Modifier modifier = tokenStream().hasLookahead() ? tokenStream().getLookaheadModifier() : TokenStream::None;
+        if (!tokenStream().peekTokenPos(&pos, modifier))
             return false;
         return failOffset(pos.begin, str);
     }
 
     bool failfVA(ParseNode* pn, const char* fmt, va_list ap) {
         MOZ_ASSERT(!errorString_);
         MOZ_ASSERT(errorOffset_ == UINT32_MAX);
         MOZ_ASSERT(fmt);
@@ -4906,17 +4907,17 @@ CheckModuleGlobal(ModuleCompiler& m, Par
 }
 
 static bool
 CheckModuleProcessingDirectives(ModuleCompiler& m)
 {
     TokenStream& ts = m.parser().tokenStream;
     while (true) {
         bool matched;
-        if (!ts.matchToken(&matched, TOK_STRING))
+        if (!ts.matchToken(&matched, TOK_STRING, TokenStream::Operand))
             return false;
         if (!matched)
             return true;
 
         if (!IsIgnoredDirectiveName(m.cx(), ts.currentToken().atom()))
             return m.fail(nullptr, "unsupported processing directive");
 
         if (!ts.matchToken(&matched, TOK_SEMI))
@@ -9772,22 +9773,22 @@ CheckChangeHeap(ModuleCompiler& m, Parse
     return m.addChangeHeap(changeHeapName, fn, mask, min, max);
 }
 
 static bool
 ParseFunction(ModuleCompiler& m, ParseNode** fnOut)
 {
     TokenStream& tokenStream = m.tokenStream();
 
-    tokenStream.consumeKnownToken(TOK_FUNCTION);
+    tokenStream.consumeKnownToken(TOK_FUNCTION, TokenStream::Operand);
 
     RootedPropertyName name(m.cx());
 
     TokenKind tk;
-    if (!tokenStream.getToken(&tk))
+    if (!tokenStream.getToken(&tk, TokenStream::Operand))
         return false;
     if (tk == TOK_NAME) {
         name = tokenStream.currentName();
     } else if (tk == TOK_YIELD) {
         if (!m.parser().checkYieldNameValidity())
             return false;
         name = m.cx()->names().yield;
     } else {
--- a/js/src/frontend/Parser.cpp
+++ b/js/src/frontend/Parser.cpp
@@ -52,27 +52,28 @@ namespace js {
 namespace frontend {
 
 typedef Rooted<StaticBlockObject*> RootedStaticBlockObject;
 typedef Handle<StaticBlockObject*> HandleStaticBlockObject;
 typedef Rooted<NestedScopeObject*> RootedNestedScopeObject;
 typedef Handle<NestedScopeObject*> HandleNestedScopeObject;
 
 /* Read a token. Report an error and return null() if that token isn't of type tt. */
-#define MUST_MATCH_TOKEN(tt, errno)                                                         \
+#define MUST_MATCH_TOKEN_MOD(tt, modifier, errno)                                           \
     JS_BEGIN_MACRO                                                                          \
         TokenKind token;                                                                    \
-        if (!tokenStream.getToken(&token))                                                  \
+        if (!tokenStream.getToken(&token, modifier))                                        \
             return null();                                                                  \
         if (token != tt) {                                                                  \
             report(ParseError, false, null(), errno);                                       \
             return null();                                                                  \
         }                                                                                   \
     JS_END_MACRO
 
+#define MUST_MATCH_TOKEN(tt, errno) MUST_MATCH_TOKEN_MOD(tt, TokenStream::None, errno)
 
 template <>
 bool
 ParseContext<FullParseHandler>::checkLocalsOverflow(TokenStream& ts)
 {
     if (vars_.length() + bodyLevelLexicals_.length() >= LOCALNO_LIMIT) {
         ts.reportError(JSMSG_TOO_MANY_LOCALS);
         return false;
@@ -737,17 +738,17 @@ Parser<ParseHandler>::parse()
                                         &globalsc, /* newDirectives = */ nullptr,
                                         /* blockScopeDepth = */ 0);
     if (!globalpc.init(*this))
         return null();
 
     Node pn = statements(YieldIsName);
     if (pn) {
         TokenKind tt;
-        if (!tokenStream.getToken(&tt))
+        if (!tokenStream.getToken(&tt, TokenStream::Operand))
             return null();
         if (tt != TOK_EOF) {
             report(ParseError, false, null(), JSMSG_GARBAGE_AFTER_INPUT,
                    "script", TokenKindToDesc(tt));
             return null();
         }
         if (foldConstants) {
             if (!FoldConstants(context, &pn, this))
@@ -835,17 +836,17 @@ Parser<FullParseHandler>::standaloneFunc
     }
 
     YieldHandling yieldHandling = generatorKind != NotGenerator ? YieldIsKeyword : YieldIsName;
     ParseNode* pn = functionBody(InAllowed, yieldHandling, Statement, StatementListBody);
     if (!pn)
         return null();
 
     TokenKind tt;
-    if (!tokenStream.getToken(&tt))
+    if (!tokenStream.getToken(&tt, TokenStream::Operand))
         return null();
     if (tt != TOK_EOF) {
         report(ParseError, false, null(), JSMSG_GARBAGE_AFTER_INPUT,
                "function body", TokenKindToDesc(tt));
         return null();
     }
 
     if (!FoldConstants(context, &pn, this))
@@ -1342,29 +1343,33 @@ Parser<ParseHandler>::newFunction(Handle
     if (!fun)
         return nullptr;
     if (options().selfHostingMode)
         fun->setIsSelfHostedBuiltin();
     return fun;
 }
 
 static bool
-MatchOrInsertSemicolon(TokenStream& ts)
+MatchOrInsertSemicolon(TokenStream& ts, TokenStream::Modifier modifier = TokenStream::None)
 {
     TokenKind tt;
-    if (!ts.peekTokenSameLine(&tt, TokenStream::Operand))
+    if (!ts.peekTokenSameLine(&tt, modifier))
         return false;
     if (tt != TOK_EOF && tt != TOK_EOL && tt != TOK_SEMI && tt != TOK_RC) {
         /* Advance the scanner for proper error location reporting. */
-        ts.consumeKnownToken(tt);
+        ts.consumeKnownToken(tt, modifier);
         ts.reportError(JSMSG_SEMI_BEFORE_STMNT);
         return false;
     }
-    bool ignored;
-    return ts.matchToken(&ignored, TOK_SEMI);
+    bool matched;
+    if (!ts.matchToken(&matched, TOK_SEMI, modifier))
+        return false;
+    if (!matched && modifier == TokenStream::None)
+        ts.addModifierException(TokenStream::OperandIsNone);
+    return true;
 }
 
 /*
  * The function LexicalLookup searches a static binding for the given name in
  * the stack of statements enclosing the statement currently being parsed. Each
  * statement that introduces a new scope has a corresponding scope object, on
  * which the bindings for that scope are stored. LexicalLookup either returns
  * the innermost statement which has a scope object containing a binding with
@@ -1699,26 +1704,29 @@ bool
 Parser<ParseHandler>::functionArguments(YieldHandling yieldHandling, FunctionSyntaxKind kind,
                                         Node funcpn, bool* hasRest)
 {
     FunctionBox* funbox = pc->sc->asFunctionBox();
 
     *hasRest = false;
 
     bool parenFreeArrow = false;
+    TokenStream::Modifier modifier = TokenStream::None;
     if (kind == Arrow) {
         TokenKind tt;
-        if (!tokenStream.peekToken(&tt))
+        if (!tokenStream.peekToken(&tt, TokenStream::Operand))
             return false;
         if (tt == TOK_NAME)
             parenFreeArrow = true;
+        else
+            modifier = TokenStream::Operand;
     }
     if (!parenFreeArrow) {
         TokenKind tt;
-        if (!tokenStream.getToken(&tt))
+        if (!tokenStream.getToken(&tt, modifier))
             return false;
         if (tt != TOK_LP) {
             report(ParseError, false, null(),
                    kind == Arrow ? JSMSG_BAD_ARROW_ARGS : JSMSG_PAREN_BEFORE_FORMAL);
             return false;
         }
 
         // Record the start of function source (for FunctionToString). If we
@@ -1731,17 +1739,17 @@ Parser<ParseHandler>::functionArguments(
         return false;
     handler.setFunctionBody(funcpn, argsbody);
 
     bool hasArguments = false;
     if (parenFreeArrow) {
         hasArguments = true;
     } else {
         bool matched;
-        if (!tokenStream.matchToken(&matched, TOK_RP))
+        if (!tokenStream.matchToken(&matched, TOK_RP, TokenStream::Operand))
             return false;
         if (!matched)
             hasArguments = true;
     }
     if (hasArguments) {
         bool hasDefaults = false;
         Node duplicatedArg = null();
         bool disallowDuplicateArgs = kind == Arrow || kind == Method || kind == ClassConstructor;
@@ -1753,17 +1761,17 @@ Parser<ParseHandler>::functionArguments(
 
         while (true) {
             if (*hasRest) {
                 report(ParseError, false, null(), JSMSG_PARAMETER_AFTER_REST);
                 return false;
             }
 
             TokenKind tt;
-            if (!tokenStream.getToken(&tt))
+            if (!tokenStream.getToken(&tt, TokenStream::Operand))
                 return false;
             MOZ_ASSERT_IF(parenFreeArrow, tt == TOK_NAME);
             switch (tt) {
               case TOK_LB:
               case TOK_LC:
               {
                 /* See comment below in the TOK_NAME case. */
                 disallowDuplicateArgs = true;
@@ -2690,17 +2698,17 @@ Parser<ParseHandler>::functionArgsAndBod
     if ((kind != Method && !IsConstructorKind(kind)) && fun->name() &&
         !checkStrictBinding(fun->name(), pn))
     {
         return false;
     }
 
     if (bodyType == StatementListBody) {
         bool matched;
-        if (!tokenStream.matchToken(&matched, TOK_RC))
+        if (!tokenStream.matchToken(&matched, TOK_RC, TokenStream::Operand))
             return false;
         if (!matched) {
             report(ParseError, false, null(), JSMSG_CURLY_AFTER_BODY);
             return false;
         }
         funbox->bufEnd = pos().begin + 1;
     } else {
 #if !JS_HAS_EXPR_CLOSURES
@@ -3042,25 +3050,25 @@ template <typename ParseHandler>
 bool
 Parser<ParseHandler>::matchLabel(YieldHandling yieldHandling, MutableHandle<PropertyName*> label)
 {
     TokenKind tt;
     if (!tokenStream.peekTokenSameLine(&tt, TokenStream::Operand))
         return false;
 
     if (tt == TOK_NAME) {
-        tokenStream.consumeKnownToken(TOK_NAME);
+        tokenStream.consumeKnownToken(TOK_NAME, TokenStream::Operand);
         MOZ_ASSERT_IF(tokenStream.currentName() == context->names().yield,
                       yieldHandling == YieldIsName);
         label.set(tokenStream.currentName());
     } else if (tt == TOK_YIELD) {
         // We might still consider |yield| to be valid here, contrary to ES6.
         // Fix bug 1104014, then stop shipping legacy generators in chrome
         // code, then remove this check!
-        tokenStream.consumeKnownToken(TOK_YIELD);
+        tokenStream.consumeKnownToken(TOK_YIELD, TokenStream::Operand);
         if (!checkYieldNameValidity())
             return false;
         label.set(tokenStream.currentName());
     } else {
         label.set(nullptr);
     }
     return true;
 }
@@ -3854,17 +3862,17 @@ Parser<ParseHandler>::deprecatedLetBlock
     if (!block)
         return null();
 
     MUST_MATCH_TOKEN(TOK_LC, JSMSG_CURLY_BEFORE_LET);
 
     Node expr = statements(yieldHandling);
     if (!expr)
         return null();
-    MUST_MATCH_TOKEN(TOK_RC, JSMSG_CURLY_AFTER_LET);
+    MUST_MATCH_TOKEN_MOD(TOK_RC, TokenStream::Operand, JSMSG_CURLY_AFTER_LET);
 
     addTelemetry(JSCompartment::DeprecatedLetBlock);
     if (!report(ParseWarning, pc->sc->strict(), expr, JSMSG_DEPRECATED_LET_BLOCK))
         return null();
 
     handler.setLexicalScopeBody(block, expr);
 
     TokenPos letPos(begin, pos().end);
@@ -3881,17 +3889,17 @@ Parser<ParseHandler>::blockStatement(Yie
     AutoPushStmtInfoPC stmtInfo(*this, StmtType::BLOCK);
     if (!stmtInfo.generateBlockId())
         return null();
 
     Node list = statements(yieldHandling);
     if (!list)
         return null();
 
-    MUST_MATCH_TOKEN(TOK_RC, JSMSG_CURLY_IN_COMPOUND);
+    MUST_MATCH_TOKEN_MOD(TOK_RC, TokenStream::Operand, JSMSG_CURLY_IN_COMPOUND);
     return list;
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::newBindingNode(PropertyName* name, bool functionScope, VarContext varContext)
 {
     /*
@@ -4338,30 +4346,31 @@ Parser<SyntaxParseHandler>::letDeclarati
     return SyntaxParseHandler::NodeFailure;
 }
 
 template<>
 bool
 Parser<FullParseHandler>::namedImportsOrNamespaceImport(TokenKind tt, Node importSpecSet)
 {
     if (tt == TOK_LC) {
+        TokenStream::Modifier modifier = TokenStream::KeywordIsName;
         while (true) {
             // Handle the forms |import {} from 'a'| and
             // |import { ..., } from 'a'| (where ... is non empty), by
             // escaping the loop early if the next token is }.
             if (!tokenStream.peekToken(&tt, TokenStream::KeywordIsName))
                 return false;
 
             if (tt == TOK_RC)
                 break;
 
             // If the next token is a keyword, the previous call to
             // peekToken matched it as a TOK_NAME, and put it in the
             // lookahead buffer, so this call will match keywords as well.
-            MUST_MATCH_TOKEN(TOK_NAME, JSMSG_NO_IMPORT_NAME);
+            MUST_MATCH_TOKEN_MOD(TOK_NAME, TokenStream::KeywordIsName, JSMSG_NO_IMPORT_NAME);
             Node importName = newName(tokenStream.currentName());
             if (!importName)
                 return false;
 
             if (!tokenStream.getToken(&tt))
                 return false;
 
             if (tt == TOK_NAME && tokenStream.currentName() == context->names().as) {
@@ -4389,21 +4398,23 @@ Parser<FullParseHandler>::namedImportsOr
                 return false;
 
             handler.addList(importSpecSet, importSpec);
 
             bool matched;
             if (!tokenStream.matchToken(&matched, TOK_COMMA))
                 return false;
 
-            if (!matched)
+            if (!matched) {
+                modifier = TokenStream::None;
                 break;
-        }
-
-        MUST_MATCH_TOKEN(TOK_RC, JSMSG_RC_AFTER_IMPORT_SPEC_LIST);
+            }
+        }
+
+        MUST_MATCH_TOKEN_MOD(TOK_RC, modifier, JSMSG_RC_AFTER_IMPORT_SPEC_LIST);
     } else {
         MOZ_ASSERT(tt == TOK_MUL);
         if (!tokenStream.getToken(&tt))
             return false;
 
         if (tt != TOK_NAME || tokenStream.currentName() != context->names().as) {
             report(ParseError, false, null(), JSMSG_AS_AFTER_IMPORT_STAR);
             return false;
@@ -4514,17 +4525,17 @@ Parser<ParseHandler>::importDeclaration(
         report(ParseError, false, null(), JSMSG_DECLARATION_AFTER_IMPORT);
         return null();
     }
 
     Node moduleSpec = stringLiteral();
     if (!moduleSpec)
         return null();
 
-    if (!MatchOrInsertSemicolon(tokenStream))
+    if (!MatchOrInsertSemicolon(tokenStream, TokenStream::Operand))
         return null();
 
     return handler.newImportDeclaration(importSpecSet, moduleSpec, TokenPos(begin, pos().end));
 }
 
 template<>
 SyntaxParseHandler::Node
 Parser<SyntaxParseHandler>::importDeclaration()
@@ -4661,17 +4672,17 @@ Parser<FullParseHandler>::exportDeclarat
         kid->pn_xflags = PNX_POPVAR;
 
         kid = MatchOrInsertSemicolon(tokenStream) ? kid : nullptr;
         if (!kid)
             return null();
         break;
 
       case TOK_DEFAULT: {
-        if (!tokenStream.getToken(&tt))
+        if (!tokenStream.getToken(&tt, TokenStream::Operand))
             return null();
 
         switch (tt) {
           case TOK_FUNCTION:
             kid = functionStmt(YieldIsKeyword, AllowDefaultName);
             break;
           case TOK_CLASS:
             kid = classDefinition(YieldIsKeyword, ClassStatement, AllowDefaultName);
@@ -4790,29 +4801,31 @@ template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::doWhileStatement(YieldHandling yieldHandling)
 {
     uint32_t begin = pos().begin;
     AutoPushStmtInfoPC stmtInfo(*this, StmtType::DO_LOOP);
     Node body = statement(yieldHandling);
     if (!body)
         return null();
-    MUST_MATCH_TOKEN(TOK_WHILE, JSMSG_WHILE_AFTER_DO);
+    MUST_MATCH_TOKEN_MOD(TOK_WHILE, TokenStream::Operand, JSMSG_WHILE_AFTER_DO);
     Node cond = condition(InAllowed, yieldHandling);
     if (!cond)
         return null();
 
     // The semicolon after do-while is even more optional than most
     // semicolons in JS.  Web compat required this by 2004:
     //   http://bugzilla.mozilla.org/show_bug.cgi?id=238945
     // ES3 and ES5 disagreed, but ES6 conforms to Web reality:
     //   https://bugs.ecmascript.org/show_bug.cgi?id=157
-    bool ignored;
-    if (!tokenStream.matchToken(&ignored, TOK_SEMI))
-        return null();
+    bool matched;
+    if (!tokenStream.matchToken(&matched, TOK_SEMI))
+        return null();
+    if (!matched)
+        tokenStream.addModifierException(TokenStream::OperandIsNone);
     return handler.newDoWhileStatement(body, cond, TokenPos(begin, pos().end));
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::whileStatement(YieldHandling yieldHandling)
 {
     uint32_t begin = pos().begin;
@@ -4916,16 +4929,17 @@ Parser<FullParseHandler>::forStatement(Y
     bool isForDecl = false;
 
     /* Non-null when isForDecl is true for a 'for (let ...)' statement. */
     RootedStaticBlockObject blockObj(context);
 
     /* Set to 'x' in 'for (x ;... ;...)' or 'for (x in ...)'. */
     ParseNode* pn1;
 
+    TokenStream::Modifier modifier = TokenStream::Operand;
     {
         TokenKind tt;
         if (!tokenStream.peekToken(&tt, TokenStream::Operand))
             return null();
         if (tt == TOK_SEMI) {
             pn1 = nullptr;
         } else {
             // Set pn1 to a variable list or an initializing expression.
@@ -4934,37 +4948,38 @@ Parser<FullParseHandler>::forStatement(Y
             // to trigger |for|-specific parsing for that one position.  In a
             // normal variable declaration, any initializer may be an |in|
             // expression.  But for declarations at the start of a for-loop
             // head, initializers can't contain |in|.  (Such syntax conflicts
             // with ES5's |for (var i = 0 in foo)| syntax, removed in ES6, that
             // we "support" by ignoring the |= 0|.)
             if (tt == TOK_VAR) {
                 isForDecl = true;
-                tokenStream.consumeKnownToken(tt);
+                tokenStream.consumeKnownToken(tt, TokenStream::Operand);
                 pn1 = variables(yieldHandling, PNK_VAR, InForInit);
             } else if (tt == TOK_LET || tt == TOK_CONST) {
                 handler.disableSyntaxParser();
                 bool constDecl = tt == TOK_CONST;
-                tokenStream.consumeKnownToken(tt);
+                tokenStream.consumeKnownToken(tt, TokenStream::Operand);
                 isForDecl = true;
                 blockObj = StaticBlockObject::create(context);
                 if (!blockObj)
                     return null();
                 pn1 = variables(yieldHandling, constDecl ? PNK_CONST : PNK_LET, InForInit,
                                 nullptr, blockObj, DontHoistVars);
             } else {
                 // Pass |InProhibited| when parsing an expression so that |in|
                 // isn't parsed in a RelationalExpression as a binary operator.
                 // In this context, |in| is part of a for-in loop -- *not* part
                 // of a binary expression.
                 pn1 = expr(InProhibited, yieldHandling);
             }
             if (!pn1)
                 return null();
+            modifier = TokenStream::None;
         }
     }
 
     MOZ_ASSERT_IF(isForDecl, pn1->isArity(PN_LIST));
     MOZ_ASSERT(!!blockObj == (isForDecl && pn1->isOp(JSOP_NOP)));
 
     // All forms of for-loop (for(;;), for-in, for-of) generate an implicit
     // block to store any lexical variables declared by the loop-head.  We
@@ -5087,16 +5102,17 @@ Parser<FullParseHandler>::forStatement(Y
                 return null();
         }
 
         pn3 = (headKind == PNK_FOROF)
               ? assignExpr(InAllowed, yieldHandling)
               : expr(InAllowed, yieldHandling);
         if (!pn3)
             return null();
+        modifier = TokenStream::None;
 
         if (blockObj) {
             /*
              * Now that the pn3 has been parsed, push the let scope. To hold
              * the blockObj for the emitter, wrap the PNK_LEXICALSCOPE node
              * created by pushLetScope around the for's initializer. This also
              * serves to indicate the let-decl to the emitter.
              */
@@ -5174,42 +5190,46 @@ Parser<FullParseHandler>::forStatement(Y
             } else {
                 pn1 = handler.newFreshenBlock(pn1->pn_pos);
                 if (!pn1)
                     return null();
             }
         }
 
         /* Parse the loop condition or null into pn2. */
-        MUST_MATCH_TOKEN(TOK_SEMI, JSMSG_SEMI_AFTER_FOR_INIT);
+        MUST_MATCH_TOKEN_MOD(TOK_SEMI, modifier, JSMSG_SEMI_AFTER_FOR_INIT);
         TokenKind tt;
         if (!tokenStream.peekToken(&tt, TokenStream::Operand))
             return null();
         if (tt == TOK_SEMI) {
             pn2 = nullptr;
+            modifier = TokenStream::Operand;
         } else {
             pn2 = expr(InAllowed, yieldHandling);
             if (!pn2)
                 return null();
+            modifier = TokenStream::None;
         }
 
         /* Parse the update expression or null into pn3. */
-        MUST_MATCH_TOKEN(TOK_SEMI, JSMSG_SEMI_AFTER_FOR_COND);
+        MUST_MATCH_TOKEN_MOD(TOK_SEMI, modifier, JSMSG_SEMI_AFTER_FOR_COND);
         if (!tokenStream.peekToken(&tt, TokenStream::Operand))
             return null();
         if (tt == TOK_RP) {
             pn3 = nullptr;
+            modifier = TokenStream::Operand;
         } else {
             pn3 = expr(InAllowed, yieldHandling);
             if (!pn3)
                 return null();
-        }
-    }
-
-    MUST_MATCH_TOKEN(TOK_RP, JSMSG_PAREN_AFTER_FOR_CTRL);
+            modifier = TokenStream::None;
+        }
+    }
+
+    MUST_MATCH_TOKEN_MOD(TOK_RP, modifier, JSMSG_PAREN_AFTER_FOR_CTRL);
 
     TokenPos headPos(begin, pos().end);
     ParseNode* forHead = handler.newForHead(headKind, pn1, pn2, pn3, headPos);
     if (!forHead)
         return null();
 
     /* Parse the loop body. */
     ParseNode* body = statement(yieldHandling);
@@ -5259,38 +5279,40 @@ Parser<SyntaxParseHandler>::forStatement
 
     /* True if we have 'for (var ...)'. */
     bool isForDecl = false;
     bool simpleForDecl = true;
 
     /* Set to 'x' in 'for (x ;... ;...)' or 'for (x in ...)'. */
     Node lhsNode;
 
+    TokenStream::Modifier modifier = TokenStream::Operand;
     {
         TokenKind tt;
         if (!tokenStream.peekToken(&tt, TokenStream::Operand))
             return null();
         if (tt == TOK_SEMI) {
             lhsNode = null();
         } else {
             /* Set lhsNode to a var list or an initializing expression. */
             if (tt == TOK_VAR) {
                 isForDecl = true;
-                tokenStream.consumeKnownToken(tt);
+                tokenStream.consumeKnownToken(tt, TokenStream::Operand);
                 lhsNode = variables(yieldHandling, PNK_VAR, InForInit, &simpleForDecl);
             }
             else if (tt == TOK_CONST || tt == TOK_LET) {
                 JS_ALWAYS_FALSE(abortIfSyntaxParser());
                 return null();
             }
             else {
                 lhsNode = expr(InProhibited, yieldHandling);
             }
             if (!lhsNode)
                 return null();
+            modifier = TokenStream::None;
         }
     }
 
     // If there's an |in| keyword here, it's a for-in loop, by dint of careful
     // parsing of |pn1|.
     bool isForIn = false, isForOf = false;
     if (lhsNode) {
         if (!matchInOrOf(&isForIn, &isForOf))
@@ -5314,38 +5336,43 @@ Parser<SyntaxParseHandler>::forStatement
             return null();
         }
 
         if (!isForDecl && !checkAndMarkAsAssignmentLhs(lhsNode, PlainAssignment))
             return null();
 
         if (!(isForIn ? expr(InAllowed, yieldHandling) : assignExpr(InAllowed, yieldHandling)))
             return null();
+        modifier = TokenStream::None;
     } else {
         /* Parse the loop condition or null. */
-        MUST_MATCH_TOKEN(TOK_SEMI, JSMSG_SEMI_AFTER_FOR_INIT);
+        MUST_MATCH_TOKEN_MOD(TOK_SEMI, modifier, JSMSG_SEMI_AFTER_FOR_INIT);
         TokenKind tt;
         if (!tokenStream.peekToken(&tt, TokenStream::Operand))
             return null();
+        modifier = TokenStream::Operand;
         if (tt != TOK_SEMI) {
             if (!expr(InAllowed, yieldHandling))
                 return null();
+            modifier = TokenStream::None;
         }
 
         /* Parse the update expression or null. */
-        MUST_MATCH_TOKEN(TOK_SEMI, JSMSG_SEMI_AFTER_FOR_COND);
+        MUST_MATCH_TOKEN_MOD(TOK_SEMI, modifier, JSMSG_SEMI_AFTER_FOR_COND);
         if (!tokenStream.peekToken(&tt, TokenStream::Operand))
             return null();
+        modifier = TokenStream::Operand;
         if (tt != TOK_RP) {
             if (!expr(InAllowed, yieldHandling))
                 return null();
-        }
-    }
-
-    MUST_MATCH_TOKEN(TOK_RP, JSMSG_PAREN_AFTER_FOR_CTRL);
+            modifier = TokenStream::None;
+        }
+    }
+
+    MUST_MATCH_TOKEN_MOD(TOK_RP, modifier, JSMSG_PAREN_AFTER_FOR_CTRL);
 
     /* Parse the loop body. */
     if (!statement(yieldHandling))
         return null();
 
     return SyntaxParseHandler::NodeGeneric;
 }
 
@@ -5374,17 +5401,17 @@ Parser<ParseHandler>::switchStatement(Yi
         return null();
 
     Node saveBlock = pc->blockNode;
     pc->blockNode = caseList;
 
     bool seenDefault = false;
     TokenKind tt;
     while (true) {
-        if (!tokenStream.getToken(&tt))
+        if (!tokenStream.getToken(&tt, TokenStream::Operand))
             return null();
         if (tt == TOK_RC)
             break;
         uint32_t caseBegin = pos().begin;
 
         Node caseExpr;
         switch (tt) {
           case TOK_DEFAULT:
@@ -5517,17 +5544,17 @@ Parser<ParseHandler>::continueStatement(
                 report(ParseError, false, null(), JSMSG_BAD_CONTINUE);
                 return null();
             }
             if (stmt->isLoop())
                 break;
         }
     }
 
-    if (!MatchOrInsertSemicolon(tokenStream))
+    if (!MatchOrInsertSemicolon(tokenStream, TokenStream::Operand))
         return null();
 
     return handler.newContinueStatement(label, TokenPos(begin, pos().end));
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::breakStatement(YieldHandling yieldHandling)
@@ -5554,17 +5581,17 @@ Parser<ParseHandler>::breakStatement(Yie
                 report(ParseError, false, null(), JSMSG_TOUGH_BREAK);
                 return null();
             }
             if (stmt->isLoop() || stmt->type == StmtType::SWITCH)
                 break;
         }
     }
 
-    if (!MatchOrInsertSemicolon(tokenStream))
+    if (!MatchOrInsertSemicolon(tokenStream, TokenStream::Operand))
         return null();
 
     return handler.newBreakStatement(label, TokenPos(begin, pos().end));
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::returnStatement(YieldHandling yieldHandling)
@@ -5576,33 +5603,35 @@ Parser<ParseHandler>::returnStatement(Yi
 
     // Parse an optional operand.
     //
     // This is ugly, but we don't want to require a semicolon.
     Node exprNode;
     TokenKind tt;
     if (!tokenStream.peekTokenSameLine(&tt, TokenStream::Operand))
         return null();
+    TokenStream::Modifier modifier = TokenStream::Operand;
     switch (tt) {
       case TOK_EOL:
       case TOK_EOF:
       case TOK_SEMI:
       case TOK_RC:
         exprNode = null();
         pc->funHasReturnVoid = true;
         break;
       default: {
         exprNode = expr(InAllowed, yieldHandling);
         if (!exprNode)
             return null();
+        modifier = TokenStream::None;
         pc->funHasReturnExpr = true;
       }
     }
 
-    if (!MatchOrInsertSemicolon(tokenStream))
+    if (!MatchOrInsertSemicolon(tokenStream, modifier))
         return null();
 
     Node genrval = null();
     if (pc->isStarGenerator()) {
         genrval = newName(context->names().dotGenRVal);
         if (!genrval)
             return null();
         if (!noteNameUse(context->names().dotGenRVal, genrval))
@@ -5671,20 +5700,21 @@ Parser<ParseHandler>::yieldExpression(In
           case TOK_SEMI:
           case TOK_RC:
           case TOK_RB:
           case TOK_RP:
           case TOK_COLON:
           case TOK_COMMA:
             // No value.
             exprNode = null();
+            tokenStream.addModifierException(TokenStream::NoneIsOperand);
             break;
           case TOK_MUL:
             kind = PNK_YIELD_STAR;
-            tokenStream.consumeKnownToken(TOK_MUL);
+            tokenStream.consumeKnownToken(TOK_MUL, TokenStream::Operand);
             // Fall through.
           default:
             exprNode = assignExpr(inHandling, YieldIsKeyword);
             if (!exprNode)
                 return null();
         }
         return newYieldExpression(begin, exprNode, kind == PNK_YIELD_STAR);
       }
@@ -5733,16 +5763,17 @@ Parser<ParseHandler>::yieldExpression(In
           case TOK_SEMI:
           case TOK_RC:
           case TOK_RB:
           case TOK_RP:
           case TOK_COLON:
           case TOK_COMMA:
             // No value.
             exprNode = null();
+            tokenStream.addModifierException(TokenStream::NoneIsOperand);
             break;
           default:
             exprNode = assignExpr(inHandling, YieldIsKeyword);
             if (!exprNode)
                 return null();
         }
 
         return newYieldExpression(begin, exprNode);
@@ -5905,17 +5936,17 @@ Parser<ParseHandler>::tryStatement(Yield
     {
         MUST_MATCH_TOKEN(TOK_LC, JSMSG_CURLY_BEFORE_TRY);
         AutoPushStmtInfoPC stmtInfo(*this, StmtType::TRY);
         if (!stmtInfo.generateBlockId())
             return null();
         innerBlock = statements(yieldHandling);
         if (!innerBlock)
             return null();
-        MUST_MATCH_TOKEN(TOK_RC, JSMSG_CURLY_AFTER_TRY);
+        MUST_MATCH_TOKEN_MOD(TOK_RC, TokenStream::Operand, JSMSG_CURLY_AFTER_TRY);
     }
 
     bool hasUnconditionalCatch = false;
     Node catchList = null();
     TokenKind tt;
     if (!tokenStream.getToken(&tt))
         return null();
     if (tt == TOK_CATCH) {
@@ -6017,17 +6048,17 @@ Parser<ParseHandler>::tryStatement(Yield
             }
 #endif
             MUST_MATCH_TOKEN(TOK_RP, JSMSG_PAREN_AFTER_CATCH);
 
             MUST_MATCH_TOKEN(TOK_LC, JSMSG_CURLY_BEFORE_CATCH);
             Node catchBody = statements(yieldHandling);
             if (!catchBody)
                 return null();
-            MUST_MATCH_TOKEN(TOK_RC, JSMSG_CURLY_AFTER_CATCH);
+            MUST_MATCH_TOKEN_MOD(TOK_RC, TokenStream::Operand, JSMSG_CURLY_AFTER_CATCH);
 
             if (!catchGuard)
                 hasUnconditionalCatch = true;
 
             if (!handler.addCatchBlock(catchList, pnblock, catchName, catchGuard, catchBody))
                 return null();
             handler.setEndPosition(catchList, pos().end);
             handler.setEndPosition(pnblock, pos().end);
@@ -6042,17 +6073,17 @@ Parser<ParseHandler>::tryStatement(Yield
     if (tt == TOK_FINALLY) {
         MUST_MATCH_TOKEN(TOK_LC, JSMSG_CURLY_BEFORE_FINALLY);
         AutoPushStmtInfoPC stmtInfo(*this, StmtType::TRY);
         if (!stmtInfo.generateBlockId())
             return null();
         finallyBlock = statements(yieldHandling);
         if (!finallyBlock)
             return null();
-        MUST_MATCH_TOKEN(TOK_RC, JSMSG_CURLY_AFTER_FINALLY);
+        MUST_MATCH_TOKEN_MOD(TOK_RC, TokenStream::Operand, JSMSG_CURLY_AFTER_FINALLY);
     } else {
         tokenStream.ungetToken();
     }
     if (!catchList && !finallyBlock) {
         report(ParseError, false, null(), JSMSG_CATCH_OR_FINALLY);
         return null();
     }
 
@@ -6060,17 +6091,17 @@ Parser<ParseHandler>::tryStatement(Yield
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::debuggerStatement()
 {
     TokenPos p;
     p.begin = pos().begin;
-    if (!MatchOrInsertSemicolon(tokenStream))
+    if (!MatchOrInsertSemicolon(tokenStream, TokenStream::Operand))
         return null();
     p.end = pos().end;
 
     pc->sc->setBindingsAccessedDynamically();
     pc->sc->setHasDebuggerStatement();
 
     return handler.newDebuggerStatement(p);
 }
@@ -6705,17 +6736,17 @@ Parser<ParseHandler>::assignExpr(InHandl
             return null();
         }
 
         tokenStream.seek(start);
         if (!abortIfSyntaxParser())
             return null();
 
         TokenKind ignored;
-        if (!tokenStream.peekToken(&ignored))
+        if (!tokenStream.peekToken(&ignored, TokenStream::Operand))
             return null();
 
         if (pc->sc->isFunctionBox() && pc->sc->asFunctionBox()->isDerivedClassConstructor()) {
             report(ParseError, false, null(), JSMSG_DISABLED_DERIVED_CLASS, "arrow functions");
             return null();
         }
 
         return functionDef(inHandling, yieldHandling, nullptr, Arrow, NotGenerator);
@@ -6948,17 +6979,17 @@ Parser<ParseHandler>::unaryExpr(YieldHan
       }
 
       default: {
         Node pn = memberExpr(yieldHandling, tt, /* allowCallSyntax = */ true, invoked);
         if (!pn)
             return null();
 
         /* Don't look across a newline boundary for a postfix incop. */
-        if (!tokenStream.peekTokenSameLine(&tt, TokenStream::Operand))
+        if (!tokenStream.peekTokenSameLine(&tt))
             return null();
         if (tt == TOK_INC || tt == TOK_DEC) {
             tokenStream.consumeKnownToken(tt);
             AssignmentFlavor flavor = (tt == TOK_INC) ? IncrementAssignment : DecrementAssignment;
             if (!checkAndMarkAsIncOperand(pn, flavor))
                 return null();
             return handler.newUnary((tt == TOK_INC) ? PNK_POSTINCREMENT : PNK_POSTDECREMENT,
                                     JSOP_NOP,
@@ -8267,35 +8298,36 @@ Parser<ParseHandler>::arrayInitializer(Y
          * determine their type.
          */
         handler.setListFlag(literal, PNX_NONCONST);
     } else {
         tokenStream.ungetToken();
 
         bool spread = false, missingTrailingComma = false;
         uint32_t index = 0;
+        TokenStream::Modifier modifier = TokenStream::Operand;
         for (; ; index++) {
             if (index == NativeObject::NELEMENTS_LIMIT) {
                 report(ParseError, false, null(), JSMSG_ARRAY_INIT_TOO_BIG);
                 return null();
             }
 
             TokenKind tt;
             if (!tokenStream.peekToken(&tt, TokenStream::Operand))
                 return null();
             if (tt == TOK_RB)
                 break;
 
             if (tt == TOK_COMMA) {
-                tokenStream.consumeKnownToken(TOK_COMMA);
+                tokenStream.consumeKnownToken(TOK_COMMA, TokenStream::Operand);
                 if (!handler.addElision(literal, pos()))
                     return null();
             } else if (tt == TOK_TRIPLEDOT) {
                 spread = true;
-                tokenStream.consumeKnownToken(TOK_TRIPLEDOT);
+                tokenStream.consumeKnownToken(TOK_TRIPLEDOT, TokenStream::Operand);
                 uint32_t begin = pos().begin;
                 Node inner = assignExpr(InAllowed, yieldHandling);
                 if (!inner)
                     return null();
                 if (!handler.addSpreadElement(literal, begin, inner))
                     return null();
             } else {
                 Node element = assignExpr(InAllowed, yieldHandling);
@@ -8308,16 +8340,17 @@ Parser<ParseHandler>::arrayInitializer(Y
 
             if (tt != TOK_COMMA) {
                 /* If we didn't already match TOK_COMMA in above case. */
                 bool matched;
                 if (!tokenStream.matchToken(&matched, TOK_COMMA))
                     return null();
                 if (!matched) {
                     missingTrailingComma = true;
+                    modifier = TokenStream::None;
                     break;
                 }
             }
         }
 
         /*
          * At this point, (index == 0 && missingTrailingComma) implies one
          * element initialiser was parsed.
@@ -8362,23 +8395,24 @@ Parser<ParseHandler>::arrayInitializer(Y
          * slot's stack index from fp->spbase.
          *
          * The legacy array comprehension iteration step, array.push(i * j) in
          * the example above, is done by <i * j>; JSOP_ARRAYPUSH <array>, where
          * <array> is the index of array's stack slot.
          */
         if (index == 0 && !spread) {
             bool matched;
-            if (!tokenStream.matchToken(&matched, TOK_FOR))
+            if (!tokenStream.matchToken(&matched, TOK_FOR, modifier))
                 return null();
             if (matched && missingTrailingComma)
                 return legacyArrayComprehension(literal);
-        }
-
-        MUST_MATCH_TOKEN(TOK_RB, JSMSG_BRACKET_AFTER_LIST);
+            if (matched)
+                modifier = TokenStream::None;
+        }
+        MUST_MATCH_TOKEN_MOD(TOK_RB, modifier, JSMSG_BRACKET_AFTER_LIST);
     }
     handler.setEndPosition(literal, pos().end);
     return literal;
 }
 
 static JSAtom*
 DoubleToAtom(ExclusiveContext* cx, double value)
 {
@@ -8540,16 +8574,17 @@ Parser<ParseHandler>::propertyList(Yield
                 if (!propname)
                     return null();
             } else {
                 // Not an accessor property after all.
                 tokenStream.ungetToken();
                 propname = handler.newObjectLiteralPropertyName(atom, pos());
                 if (!propname)
                     return null();
+                tokenStream.addModifierException(TokenStream::NoneIsKeywordIsName);
                 op = JSOP_INITPROP;
                 break;
             }
 
             MOZ_ASSERT(op == JSOP_INITPROP_GETTER || op == JSOP_INITPROP_SETTER);
             break;
           }
 
@@ -8567,18 +8602,20 @@ Parser<ParseHandler>::propertyList(Yield
             }
             break;
           }
 
           default:
             // There is never a case in which |static *(| can make a meaningful method definition.
             if (isStatic && !isGenerator) {
                 // Turns out it wasn't static. Put it back and pretend it was a name all along.
+                tokenStream.ungetToken();
+                if (isStatic)
+                    tokenStream.addModifierException(TokenStream::NoneIsKeywordIsName);
                 isStatic = false;
-                tokenStream.ungetToken();
                 atom = tokenStream.currentName();
                 propname = handler.newObjectLiteralPropertyName(atom->asPropertyName(), pos());
                 if (!propname)
                     return null();
             } else {
                 report(ParseError, false, null(), JSMSG_BAD_PROP_ID);
                 return null();
             }
@@ -8803,17 +8840,17 @@ Parser<ParseHandler>::primaryExpr(YieldH
         TokenKind next;
         if (!tokenStream.peekToken(&next, TokenStream::Operand))
             return null();
         if (next != TOK_RP)
             return parenExprOrGeneratorComprehension(yieldHandling);
 
         // Not valid expression syntax, but this is valid in an arrow function
         // with no params: `() => body`.
-        tokenStream.consumeKnownToken(next);
+        tokenStream.consumeKnownToken(next, TokenStream::Operand);
 
         if (!tokenStream.peekToken(&next))
             return null();
         if (next != TOK_ARROW) {
             report(ParseError, false, null(), JSMSG_UNEXPECTED_TOKEN,
                    "expression", TokenKindToDesc(TOK_RP));
             return null();
         }
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@@ -1640,16 +1640,23 @@ TokenStream::getTokenInternal(TokenKind*
     MOZ_CRASH("should have jumped to |out| or |error|");
 
   out:
     if (flags.hitOOM)
         return reportError(JSMSG_OUT_OF_MEMORY);
 
     flags.isDirtyLine = true;
     tp->pos.end = userbuf.offset();
+#ifdef DEBUG
+    // Save the modifier used to get this token, so that if an ungetToken()
+    // occurs and then the token is re-gotten (or peeked, etc.), we can assert
+    // that both gets have used the same modifiers.
+    tp->modifier = modifier;
+    tp->modifierExceptions = NoException;
+#endif
     MOZ_ASSERT(IsTokenSane(tp));
     *ttp = tp->type;
     return true;
 
   error:
     if (flags.hitOOM)
         return reportError(JSMSG_OUT_OF_MEMORY);
 
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -5,16 +5,18 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef frontend_TokenStream_h
 #define frontend_TokenStream_h
 
 // JS lexical scanner interface.
 
 #include "mozilla/ArrayUtils.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
 #include "mozilla/DebugOnly.h"
 #include "mozilla/PodOperations.h"
 #include "mozilla/UniquePtr.h"
 
 #include <stdarg.h>
 #include <stddef.h>
 #include <stdio.h>
 
@@ -71,32 +73,76 @@ struct TokenPos {
 
     bool encloses(const TokenPos& pos) const {
         return begin <= pos.begin && pos.end <= end;
     }
 };
 
 enum DecimalPoint { NoDecimal = false, HasDecimal = true };
 
+class TokenStream;
+
 struct Token
 {
+  private:
+    // Sometimes the parser needs to inform the tokenizer to interpret
+    // subsequent text in a particular manner: for example, to tokenize a
+    // keyword as an identifier, not as the actual keyword, on the right-hand
+    // side of a dotted property access.  Such information is communicated to
+    // the tokenizer as a Modifier when getting the next token.
+    //
+    // Ideally this definition would reside in TokenStream as that's the real
+    // user, but the debugging-use of it here causes a cyclic dependency (and
+    // C++ provides no way to forward-declare an enum inside a class).  So
+    // define it here, then typedef it into TokenStream with static consts to
+    // bring the initializers into scope.
+    enum Modifier
+    {
+        // Normal operation.
+        None,
+
+        // Looking for an operand, not an operator.  In practice, this means
+        // that when '/' is seen, we look for a regexp instead of just returning
+        // TOK_DIV.
+        Operand,
+
+        // Treat keywords as names by returning TOK_NAME.
+        KeywordIsName,
+
+        // Treat subsequent characters as the tail of a template literal, after
+        // a template substitution, beginning with a "}", continuing with zero
+        // or more template literal characters, and ending with either "${" or
+        // the end of the template literal.  For example:
+        //
+        //   var entity = "world";
+        //   var s = `Hello ${entity}!`;
+        //                          ^ TemplateTail context
+        TemplateTail,
+    };
+    friend class TokenStream;
+
+  public:
     TokenKind           type;           // char value or above enumerator
     TokenPos            pos;            // token position in file
     union {
       private:
         friend struct Token;
         PropertyName*   name;          // non-numeric atom
         JSAtom*         atom;          // potentially-numeric atom
         struct {
             double      value;          // floating point number
             DecimalPoint decimalPoint;  // literal contains '.'
         } number;
         RegExpFlag      reflags;        // regexp flags; use tokenbuf to access
                                         //   regexp chars
     } u;
+#ifdef DEBUG
+    Modifier modifier;                  // Modifier used to get this token
+    uint8_t modifierExceptions;         // Bitwise OR of modifier exceptions
+#endif
 
     // This constructor is necessary only for MSVC 2013 and how it compiles the
     // initialization of TokenStream::tokens.  That field is initialized as
     // tokens() in the constructor init-list.  This *should* zero the entire
     // array, then (because Token has a non-trivial constructor, because
     // TokenPos has a user-provided constructor) call the implicit Token
     // constructor on each element, which would call the TokenPos constructor
     // for Token::pos and do nothing.  (All of which is equivalent to just
@@ -354,38 +400,119 @@ class MOZ_STACK_CLASS TokenStream
         bool hitOOM:1;          // Hit OOM.
 
         Flags()
           : isEOF(), isDirtyLine(), sawOctalEscape(), hadError(), hitOOM()
         {}
     };
 
   public:
-    // Sometimes the parser needs to modify how tokens are created.
-    enum Modifier
+    typedef Token::Modifier Modifier;
+    static MOZ_CONSTEXPR_VAR Modifier None = Token::None;
+    static MOZ_CONSTEXPR_VAR Modifier Operand = Token::Operand;
+    static MOZ_CONSTEXPR_VAR Modifier KeywordIsName = Token::KeywordIsName;
+    static MOZ_CONSTEXPR_VAR Modifier TemplateTail = Token::TemplateTail;
+
+    enum ModifierException
     {
-        None,           // Normal operation.
-        Operand,        // Looking for an operand, not an operator.  In
-                        //   practice, this means that when '/' is seen,
-                        //   we look for a regexp instead of just returning
-                        //   TOK_DIV.
-        KeywordIsName,  // Treat keywords as names by returning TOK_NAME.
-        TemplateTail,   // Treat next characters as part of a template string
+        NoException = 0x00,
+
+        // If a semicolon is inserted automatically, the next token is already
+        // gotten with None, but we expect Operand.
+        NoneIsOperand = 0x01,
+
+        // If an yield expression operand is omitted, the next token is already
+        // gotten with Operand, but we expect operator (None).
+        OperandIsNone = 0x02,
+
+        // If name of method definition is `get` or `set`, the next token is
+        // already gotten with KeywordIsName, but we expect None.
+        NoneIsKeywordIsName = 0x04,
     };
 
+    void addModifierException(ModifierException modifierException) {
+#ifdef DEBUG
+        const Token& next = nextToken();
+        switch (modifierException) {
+          case NoneIsOperand:
+            MOZ_ASSERT(next.modifier == Operand);
+            MOZ_ASSERT(next.type != TOK_DIV && next.type != TOK_REGEXP,
+                       "next token requires contextual specifier to be parsed unambiguously");
+            break;
+          case OperandIsNone:
+            // Non-Operand token after yield/continue/break already has
+            // NoneIsOperand exception.
+            MOZ_ASSERT(next.modifier == None ||
+                       ((next.modifierExceptions & NoneIsOperand) && next.modifier == Operand));
+            MOZ_ASSERT(next.type != TOK_DIV && next.type != TOK_REGEXP,
+                       "next token requires contextual specifier to be parsed unambiguously");
+            break;
+          case NoneIsKeywordIsName:
+            MOZ_ASSERT(next.modifier == KeywordIsName);
+            MOZ_ASSERT(next.type != TOK_NAME);
+            break;
+          default:
+            MOZ_CRASH("unexpected modifier exception");
+        }
+        tokens[(cursor + 1) & ntokensMask].modifierExceptions |= modifierException;
+#endif
+    }
+
+    bool hasLookahead() { return lookahead > 0; }
+
+    Modifier getLookaheadModifier() {
+#ifdef DEBUG
+        return nextToken().modifier;
+#else
+        return None;
+#endif
+    }
+
+    void
+    verifyConsistentModifier(Modifier modifier, Token lookaheadToken) {
+#ifdef DEBUG
+        // Easy case: modifiers match.
+        if (modifier == lookaheadToken.modifier)
+            return;
+
+        if (lookaheadToken.modifierExceptions & OperandIsNone) {
+            // getToken(Operand) permissibly following getToken().
+            if (modifier == Operand && lookaheadToken.modifier == None)
+                return;
+        }
+
+        if (lookaheadToken.modifierExceptions & NoneIsOperand) {
+            // getToken() permissibly following getToken(Operand).
+            if (modifier == None && lookaheadToken.modifier == Operand)
+                return;
+        }
+
+        if (lookaheadToken.modifierExceptions & NoneIsKeywordIsName) {
+            // getToken() permissibly following getToken(KeywordIsName).
+            if (modifier == None && lookaheadToken.modifier == KeywordIsName)
+                return;
+        }
+
+        MOZ_ASSERT_UNREACHABLE("this token was previously looked up with a "
+                               "different modifier, potentially making "
+                               "tokenization non-deterministic");
+#endif
+    }
+
     // Advance to the next token.  If the token stream encountered an error,
     // return false.  Otherwise return true and store the token kind in |*ttp|.
     bool getToken(TokenKind* ttp, Modifier modifier = None) {
         // Check for a pushed-back token resulting from mismatching lookahead.
         if (lookahead != 0) {
             MOZ_ASSERT(!flags.hadError);
             lookahead--;
             cursor = (cursor + 1) & ntokensMask;
             TokenKind tt = currentToken().type;
             MOZ_ASSERT(tt != TOK_EOL);
+            verifyConsistentModifier(modifier, currentToken());
             *ttp = tt;
             return true;
         }
 
         return getTokenInternal(ttp, modifier);
     }
 
     // Push the last scanned token back into the stream.
@@ -393,36 +520,38 @@ class MOZ_STACK_CLASS TokenStream
         MOZ_ASSERT(lookahead < maxLookahead);
         lookahead++;
         cursor = (cursor - 1) & ntokensMask;
     }
 
     bool peekToken(TokenKind* ttp, Modifier modifier = None) {
         if (lookahead > 0) {
             MOZ_ASSERT(!flags.hadError);
-            *ttp = tokens[(cursor + 1) & ntokensMask].type;
+            verifyConsistentModifier(modifier, nextToken());
+            *ttp = nextToken().type;
             return true;
         }
         if (!getTokenInternal(ttp, modifier))
             return false;
         ungetToken();
         return true;
     }
 
     bool peekTokenPos(TokenPos* posp, Modifier modifier = None) {
         if (lookahead == 0) {
             TokenKind tt;
             if (!getTokenInternal(&tt, modifier))
                 return false;
             ungetToken();
-            MOZ_ASSERT(lookahead != 0);
+            MOZ_ASSERT(hasLookahead());
         } else {
             MOZ_ASSERT(!flags.hadError);
+            verifyConsistentModifier(modifier, nextToken());
         }
-        *posp = tokens[(cursor + 1) & ntokensMask].pos;
+        *posp = nextToken().pos;
         return true;
     }
 
     // This is like peekToken(), with one exception:  if there is an EOL
     // between the end of the current token and the start of the next token, it
     // return true and store TOK_EOL in |*ttp|.  In that case, no token with
     // TOK_EOL is actually created, just a TOK_EOL TokenKind is returned, and
     // currentToken() shouldn't be consulted.  (This is the only place TOK_EOL
@@ -437,17 +566,18 @@ class MOZ_STACK_CLASS TokenStream
         // stronger condition than what we are looking for, and we don't need
         // to return TOK_EOL.
         if (lookahead != 0) {
             bool onThisLine;
             if (!srcCoords.isOnThisLine(curr.pos.end, lineno, &onThisLine))
                 return reportError(JSMSG_OUT_OF_MEMORY);
             if (onThisLine) {
                 MOZ_ASSERT(!flags.hadError);
-                *ttp = tokens[(cursor + 1) & ntokensMask].type;
+                verifyConsistentModifier(modifier, nextToken());
+                *ttp = nextToken().type;
                 return true;
             }
         }
 
         // The above check misses two cases where we don't have to return
         // TOK_EOL.
         // - The next token starts on the same line, but is a multi-line token.
         // - The next token starts on the same line, but lookahead==2 and there
@@ -475,20 +605,20 @@ class MOZ_STACK_CLASS TokenStream
             *matchedp = true;
         } else {
             ungetToken();
             *matchedp = false;
         }
         return true;
     }
 
-    void consumeKnownToken(TokenKind tt) {
+    void consumeKnownToken(TokenKind tt, Modifier modifier = None) {
         bool matched;
-        MOZ_ASSERT(lookahead != 0);
-        MOZ_ALWAYS_TRUE(matchToken(&matched, tt));
+        MOZ_ASSERT(hasLookahead());
+        MOZ_ALWAYS_TRUE(matchToken(&matched, tt, modifier));
         MOZ_ALWAYS_TRUE(matched);
     }
 
     bool matchContextualKeyword(bool* matchedp, Handle<PropertyName*> keyword) {
         TokenKind token;
         if (!getToken(&token))
             return false;
         if (token == TOK_NAME && currentToken().name() == keyword) {
@@ -816,16 +946,21 @@ class MOZ_STACK_CLASS TokenStream
     void skipChars(int n) {
         while (--n >= 0)
             getChar();
     }
 
     void updateLineInfoForEOL();
     void updateFlagsForEOL();
 
+    const Token& nextToken() {
+        MOZ_ASSERT(hasLookahead());
+        return tokens[(cursor + 1) & ntokensMask];
+    }
+
     // Options used for parsing/tokenizing.
     const ReadOnlyCompileOptions& options_;
 
     Token               tokens[ntokens];    // circular token buffer
     unsigned            cursor;             // index of last parsed token
     unsigned            lookahead;          // count of lookahead tokens
     unsigned            lineno;             // current line number
     Flags               flags;              // flags -- see above