Bug 1066827 - Part4: Check error in TokenStream.matchToken. r=Waldo
☠☠ backed out by da1125a02791 ☠ ☠
authorTooru Fujisawa <arai_a@mac.com>
Fri, 24 Oct 2014 21:01:00 +0200
changeset 212609 f023c533e279fd4bd850c5cb89be9743bf2610eb
parent 212608 09e37ba34968d6b850c9057152ae1f50a95f6461
child 212610 7d9e3126b05b2415b89ffba7010de560d9f3b01b
push id27721
push usercbook@mozilla.com
push dateTue, 28 Oct 2014 14:55:05 +0000
treeherdermozilla-central@c0ddb1b098ec [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersWaldo
bugs1066827
milestone36.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1066827 - Part4: Check error in TokenStream.matchToken. r=Waldo
js/src/frontend/Parser.cpp
js/src/frontend/Parser.h
js/src/frontend/TokenStream.h
--- a/js/src/frontend/Parser.cpp
+++ b/js/src/frontend/Parser.cpp
@@ -693,17 +693,20 @@ Parser<ParseHandler>::parse(JSObject *ch
                                         &globalsc, /* newDirectives = */ nullptr,
                                         /* staticLevel = */ 0, /* bodyid = */ 0,
                                         /* blockScopeDepth = */ 0);
     if (!globalpc.init(tokenStream))
         return null();
 
     Node pn = statements();
     if (pn) {
-        if (!tokenStream.matchToken(TOK_EOF)) {
+        bool matched;
+        if (!tokenStream.matchToken(&matched, TOK_EOF))
+            return null();
+        if (!matched) {
             TokenKind tt;
             if (!tokenStream.peekToken(&tt))
                 return null();
             report(ParseError, false, null(), JSMSG_GARBAGE_AFTER_INPUT,
                    "script", TokenKindToDesc(tt));
             return null();
         }
         if (foldConstants) {
@@ -815,17 +818,20 @@ Parser<FullParseHandler>::standaloneFunc
         if (!defineArg(fn, formals[i]))
             return null();
     }
 
     ParseNode *pn = functionBody(Statement, StatementListBody);
     if (!pn)
         return null();
 
-    if (!tokenStream.matchToken(TOK_EOF)) {
+    bool matched;
+    if (!tokenStream.matchToken(&matched, TOK_EOF))
+        return null();
+    if (!matched) {
         TokenKind tt;
         if (!tokenStream.peekToken(&tt))
             return null();
         report(ParseError, false, null(), JSMSG_GARBAGE_AFTER_INPUT,
                "function body", TokenKindToDesc(tt));
         return null();
     }
 
@@ -1226,18 +1232,18 @@ MatchOrInsertSemicolon(TokenStream &ts)
     if (!ts.peekTokenSameLine(&tt, TokenStream::Operand))
         return false;
     if (tt != TOK_EOF && tt != TOK_EOL && tt != TOK_SEMI && tt != TOK_RC) {
         /* Advance the scanner for proper error location reporting. */
         ts.consumeKnownToken(tt);
         ts.reportError(JSMSG_SEMI_BEFORE_STMNT);
         return false;
     }
-    (void) ts.matchToken(TOK_SEMI);
-    return true;
+    bool ignored;
+    return ts.matchToken(&ignored, TOK_SEMI);
 }
 
 template <typename ParseHandler>
 typename ParseHandler::DefinitionNode
 Parser<ParseHandler>::getOrCreateLexicalDependency(ParseContext<ParseHandler> *pc, JSAtom *atom)
 {
     AtomDefnAddPtr p = pc->lexdeps->lookupForAdd(atom);
     if (p)
@@ -1552,22 +1558,32 @@ Parser<ParseHandler>::functionArguments(
         funbox->setStart(tokenStream);
     }
 
     Node argsbody = handler.newList(PNK_ARGSBODY);
     if (!argsbody)
         return false;
     handler.setFunctionBody(funcpn, argsbody);
 
-    if (parenFreeArrow || !tokenStream.matchToken(TOK_RP)) {
+    bool hasArguments = false;
+    if (parenFreeArrow) {
+        hasArguments = true;
+    } else {
+        bool matched;
+        if (!tokenStream.matchToken(&matched, TOK_RP))
+            return false;
+        if (!matched)
+            hasArguments = true;
+    }
+    if (hasArguments) {
         bool hasDefaults = false;
         Node duplicatedArg = null();
         Node list = null();
 
-        do {
+        while (true) {
             if (*hasRest) {
                 report(ParseError, false, null(), JSMSG_PARAMETER_AFTER_REST);
                 return false;
             }
 
             TokenKind tt;
             if (!tokenStream.getToken(&tt))
                 return false;
@@ -1653,17 +1669,20 @@ Parser<ParseHandler>::functionArguments(
                 if (parenFreeArrow)
                     funbox->setStart(tokenStream);
 
                 RootedPropertyName name(context, tokenStream.currentName());
                 bool disallowDuplicateArgs = funbox->hasDestructuringArgs || hasDefaults;
                 if (!defineArg(funcpn, name, disallowDuplicateArgs, &duplicatedArg))
                     return false;
 
-                if (tokenStream.matchToken(TOK_ASSIGN)) {
+                bool matched;
+                if (!tokenStream.matchToken(&matched, TOK_ASSIGN))
+                    return false;
+                if (matched) {
                     // A default argument without parentheses would look like:
                     // a = expr => body, but both operators are right-associative, so
                     // that would have been parsed as a = (expr => body) instead.
                     // Therefore it's impossible to get here with parenFreeArrow.
                     MOZ_ASSERT(!parenFreeArrow);
 
                     if (*hasRest) {
                         report(ParseError, false, null(), JSMSG_REST_WITH_DEFAULT);
@@ -1688,17 +1707,26 @@ Parser<ParseHandler>::functionArguments(
 
                 break;
               }
 
               default:
                 report(ParseError, false, null(), JSMSG_MISSING_FORMAL);
                 return false;
             }
-        } while (!parenFreeArrow && tokenStream.matchToken(TOK_COMMA));
+
+            if (parenFreeArrow)
+                break;
+
+            bool matched;
+            if (!tokenStream.matchToken(&matched, TOK_COMMA))
+                return false;
+            if (!matched)
+                break;
+        }
 
         if (!parenFreeArrow) {
             TokenKind tt;
             if (!tokenStream.getToken(&tt))
                 return false;
             if (tt != TOK_RP) {
                 report(ParseError, false, null(), JSMSG_PAREN_AFTER_FORMAL);
                 return false;
@@ -2433,19 +2461,24 @@ Parser<ParseHandler>::functionArgsAndBod
         report(ParseError, false, null(), JSMSG_ACCESSOR_WRONG_ARGS, "getter", "no", "s");
         return false;
     }
     if (type == Setter && fun->nargs() != 1) {
         report(ParseError, false, null(), JSMSG_ACCESSOR_WRONG_ARGS, "setter", "one", "");
         return false;
     }
 
-    if (kind == Arrow && !tokenStream.matchToken(TOK_ARROW)) {
-        report(ParseError, false, null(), JSMSG_BAD_ARROW_ARGS);
-        return false;
+    if (kind == Arrow) {
+        bool matched;
+        if (!tokenStream.matchToken(&matched, TOK_ARROW))
+            return false;
+        if (!matched) {
+            report(ParseError, false, null(), JSMSG_BAD_ARROW_ARGS);
+            return false;
+        }
     }
 
     // Parse the function body.
     FunctionBodyType bodyType = StatementListBody;
     TokenKind tt;
     if (!tokenStream.getToken(&tt, TokenStream::Operand))
         return false;
     if (tt != TOK_LC) {
@@ -2467,17 +2500,20 @@ Parser<ParseHandler>::functionArgsAndBod
         return false;
 
     if (fun->name() && !checkStrictBinding(fun->name(), pn))
         return false;
 
 #if JS_HAS_EXPR_CLOSURES
     if (bodyType == StatementListBody) {
 #endif
-        if (!tokenStream.matchToken(TOK_RC)) {
+        bool matched;
+        if (!tokenStream.matchToken(&matched, TOK_RC))
+            return false;
+        if (!matched) {
             report(ParseError, false, null(), JSMSG_CURLY_AFTER_BODY);
             return false;
         }
         funbox->bufEnd = pos().begin + 1;
 #if JS_HAS_EXPR_CLOSURES
     } else {
         if (tokenStream.hadError())
             return false;
@@ -3515,38 +3551,51 @@ Parser<ParseHandler>::letBlock(LetContex
         return null();
 
     Node pnlet = handler.newBinary(PNK_LET, vars, block);
     if (!pnlet)
         return null();
     handler.setBeginPosition(pnlet, begin);
 
     bool needExprStmt = false;
-    if (letContext == LetStatement && !tokenStream.matchToken(TOK_LC, TokenStream::Operand)) {
-        /*
-         * Strict mode eliminates a grammar ambiguity with unparenthesized
-         * LetExpressions in an ExpressionStatement. If followed immediately
-         * by an arguments list, it's ambiguous whether the let expression
-         * is the callee or the call is inside the let expression body.
-         *
-         * See bug 569464.
-         */
-        if (!report(ParseStrictError, pc->sc->strict, pnlet,
-                    JSMSG_STRICT_CODE_LET_EXPR_STMT))
-        {
-            return null();
-        }
-
-        /*
-         * If this is really an expression in let statement guise, then we
-         * need to wrap the PNK_LET node in a PNK_SEMI node so that we pop
-         * the return value of the expression.
-         */
-        needExprStmt = true;
-        letContext = LetExpresion;
+    if (letContext == LetStatement) {
+        bool matched;
+        if (!tokenStream.matchToken(&matched, TOK_LC, TokenStream::Operand))
+            return null();
+        if (!matched) {
+            /*
+             * Strict mode eliminates a grammar ambiguity with unparenthesized
+             * LetExpressions in an ExpressionStatement. If followed immediately
+             * by an arguments list, it's ambiguous whether the let expression
+             * is the callee or the call is inside the let expression body.
+             *
+             *   function id(x) { return x; }
+             *   var x = "outer";
+             *   // Does this parse as
+             *   //   (let (loc = "inner") id)(loc) // "outer"
+             *   // or as
+             *   //   let (loc = "inner") (id(loc)) // "inner"
+             *   let (loc = "inner") id(loc);
+             *
+             * See bug 569464.
+             */
+            if (!report(ParseStrictError, pc->sc->strict, pnlet,
+                        JSMSG_STRICT_CODE_LET_EXPR_STMT))
+            {
+                return null();
+            }
+
+            /*
+             * If this is really an expression in let statement guise, then we
+             * need to wrap the PNK_LET node in a PNK_SEMI node so that we pop
+             * the return value of the expression.
+             */
+            needExprStmt = true;
+            letContext = LetExpresion;
+        }
     }
 
     Node expr;
     if (letContext == LetStatement) {
         expr = statements();
         if (!expr)
             return null();
         MUST_MATCH_TOKEN(TOK_RC, JSMSG_CURLY_AFTER_LET);
@@ -3666,117 +3715,131 @@ Parser<ParseHandler>::variables(ParseNod
     BindData<ParseHandler> data(context);
     if (kind == PNK_LET)
         data.initLet(varContext, blockObj, JSMSG_TOO_MANY_LOCALS);
     else
         data.initVarOrConst(op);
 
     bool first = true;
     Node pn2;
-    do {
-        if (psimple && !first)
-            *psimple = false;
-        first = false;
-
-        TokenKind tt;
-        if (!tokenStream.getToken(&tt))
-            return null();
-        if (tt == TOK_LB || tt == TOK_LC) {
-            if (psimple)
+    while (true) {
+        do {
+            if (psimple && !first)
                 *psimple = false;
-
-            pc->inDeclDestructuring = true;
-            pn2 = primaryExpr(tt);
-            pc->inDeclDestructuring = false;
-            if (!pn2)
+            first = false;
+
+            TokenKind tt;
+            if (!tokenStream.getToken(&tt))
                 return null();
-
-            bool ignored;
-            bool parsingForInOrOfInit = pc->parsingForInit && matchInOrOf(&ignored);
-
-            // See comment below for bindBeforeInitializer in the code that
-            // handles the non-destructuring case.
-            bool bindBeforeInitializer = kind != PNK_LET || parsingForInOrOfInit;
-            if (bindBeforeInitializer && !checkDestructuring(&data, pn2))
-                return null();
-
-            if (parsingForInOrOfInit) {
-                tokenStream.ungetToken();
+            if (tt == TOK_LB || tt == TOK_LC) {
+                if (psimple)
+                    *psimple = false;
+
+                pc->inDeclDestructuring = true;
+                pn2 = primaryExpr(tt);
+                pc->inDeclDestructuring = false;
+                if (!pn2)
+                    return null();
+
+                bool parsingForInOrOfInit = false;
+                if (pc->parsingForInit) {
+                    bool isForIn, isForOf;
+                    if (!matchInOrOf(&isForIn, &isForOf))
+                        return null();
+                    parsingForInOrOfInit = isForIn || isForOf;
+                }
+
+                // See comment below for bindBeforeInitializer in the code that
+                // handles the non-destructuring case.
+                bool bindBeforeInitializer = kind != PNK_LET || parsingForInOrOfInit;
+                if (bindBeforeInitializer && !checkDestructuring(&data, pn2))
+                    return null();
+
+                if (parsingForInOrOfInit) {
+                    tokenStream.ungetToken();
+                    handler.addList(pn, pn2);
+                    break;
+                }
+
+                MUST_MATCH_TOKEN(TOK_ASSIGN, JSMSG_BAD_DESTRUCT_DECL);
+
+                Node init = assignExpr();
+                if (!init)
+                    return null();
+
+                if (!bindBeforeInitializer && !checkDestructuring(&data, pn2))
+                    return null();
+
+                pn2 = handler.newBinaryOrAppend(PNK_ASSIGN, pn2, init, pc);
+                if (!pn2)
+                    return null();
                 handler.addList(pn, pn2);
-                continue;
+                break;
             }
 
-            MUST_MATCH_TOKEN(TOK_ASSIGN, JSMSG_BAD_DESTRUCT_DECL);
-
-            Node init = assignExpr();
-            if (!init)
-                return null();
-
-            if (!bindBeforeInitializer && !checkDestructuring(&data, pn2))
-                return null();
-
-            pn2 = handler.newBinaryOrAppend(PNK_ASSIGN, pn2, init, pc);
+            if (tt != TOK_NAME) {
+                if (tt == TOK_YIELD) {
+                    if (!checkYieldNameValidity())
+                        return null();
+                } else {
+                    report(ParseError, false, null(), JSMSG_NO_VARIABLE_NAME);
+                    return null();
+                }
+            }
+
+            RootedPropertyName name(context, tokenStream.currentName());
+            pn2 = newBindingNode(name, kind == PNK_VAR || kind == PNK_CONST, varContext);
             if (!pn2)
                 return null();
+            if (data.op == JSOP_DEFCONST)
+                handler.setFlag(pn2, PND_CONST);
+            data.pn = pn2;
+
             handler.addList(pn, pn2);
-            continue;
-        }
-
-        if (tt != TOK_NAME) {
-            if (tt == TOK_YIELD) {
-                if (!checkYieldNameValidity())
+
+            bool matched;
+            if (!tokenStream.matchToken(&matched, TOK_ASSIGN))
+                return null();
+            if (matched) {
+                if (psimple)
+                    *psimple = false;
+
+                // In ES6, lexical bindings may not be accessed until
+                // initialized. So a declaration of the form |let x = x| results
+                // in a ReferenceError, as the 'x' on the RHS is accessing the let
+                // binding before it is initialized.
+                //
+                // If we are not parsing a let declaration, bind the name
+                // now. Otherwise we must wait until after parsing the initializing
+                // assignment.
+                bool bindBeforeInitializer = kind != PNK_LET;
+                if (bindBeforeInitializer && !data.binder(&data, name, this))
+                    return null();
+
+                Node init = assignExpr();
+                if (!init)
+                    return null();
+
+                if (!bindBeforeInitializer && !data.binder(&data, name, this))
+                    return null();
+
+                if (!handler.finishInitializerAssignment(pn2, init, data.op))
                     return null();
             } else {
-                report(ParseError, false, null(), JSMSG_NO_VARIABLE_NAME);
-                return null();
+                if (!data.binder(&data, name, this))
+                    return null();
             }
-        }
-
-        RootedPropertyName name(context, tokenStream.currentName());
-        pn2 = newBindingNode(name, kind == PNK_VAR || kind == PNK_CONST, varContext);
-        if (!pn2)
-            return null();
-        if (data.op == JSOP_DEFCONST)
-            handler.setFlag(pn2, PND_CONST);
-        data.pn = pn2;
-
-        handler.addList(pn, pn2);
-
-        if (tokenStream.matchToken(TOK_ASSIGN)) {
-            if (psimple)
-                *psimple = false;
-
-            // In ES6, lexical bindings may not be accessed until
-            // initialized. So a declaration of the form |let x = x| results
-            // in a ReferenceError, as the 'x' on the RHS is accessing the let
-            // binding before it is initialized.
-            //
-            // If we are not parsing a let declaration, bind the name
-            // now. Otherwise we must wait until after parsing the initializing
-            // assignment.
-            bool bindBeforeInitializer = kind != PNK_LET;
-            if (bindBeforeInitializer && !data.binder(&data, name, this))
-                return null();
-
-            Node init = assignExpr();
-            if (!init)
-                return null();
-
-            if (!bindBeforeInitializer && !data.binder(&data, name, this))
-                return null();
-
-            if (!handler.finishInitializerAssignment(pn2, init, data.op))
-                return null();
-        } else {
-            if (!data.binder(&data, name, this))
-                return null();
-        }
-
-
-    } while (tokenStream.matchToken(TOK_COMMA));
+        } while (false);
+
+        bool matched;
+        if (!tokenStream.matchToken(&matched, TOK_COMMA))
+            return null();
+        if (!matched)
+            break;
+    }
 
     return pn;
 }
 
 template <>
 ParseNode *
 Parser<FullParseHandler>::letDeclaration()
 {
@@ -3977,17 +4040,17 @@ Parser<ParseHandler>::importDeclaration(
                 return null();
 
             Node importSpec = handler.newBinary(PNK_IMPORT_SPEC, importName, bindingName);
             if (!importSpec)
                 return null();
 
             handler.addList(importSpecSet, importSpec);
         } else {
-            do {
+            while (true) {
                 // Handle the forms |import {} from 'a'| and
                 // |import { ..., } from 'a'| (where ... is non empty), by
                 // escaping the loop early if the next token is }.
                 if (!tokenStream.peekToken(&tt, TokenStream::KeywordIsName))
                     return null();
                 if (tt == TOK_RC)
                     break;
 
@@ -4025,17 +4088,23 @@ Parser<ParseHandler>::importDeclaration(
                 if (!bindingName)
                     return null();
 
                 Node importSpec = handler.newBinary(PNK_IMPORT_SPEC, importName, bindingName);
                 if (!importSpec)
                     return null();
 
                 handler.addList(importSpecSet, importSpec);
-            } while (tokenStream.matchToken(TOK_COMMA));
+
+                bool matched;
+                if (!tokenStream.matchToken(&matched, TOK_COMMA))
+                    return null();
+                if (!matched)
+                    break;
+            }
 
             MUST_MATCH_TOKEN(TOK_RC, JSMSG_RC_AFTER_IMPORT_SPEC_LIST);
         }
 
         if (!tokenStream.getToken(&tt))
             return null();
         if (tt != TOK_NAME || tokenStream.currentName() != context->names().from) {
             report(ParseError, false, null(), JSMSG_FROM_AFTER_IMPORT_SPEC_SET);
@@ -4093,17 +4162,17 @@ Parser<ParseHandler>::exportDeclaration(
     switch (tt) {
       case TOK_LC:
       case TOK_MUL:
         kid = handler.newList(PNK_EXPORT_SPEC_LIST);
         if (!kid)
             return null();
 
         if (tt == TOK_LC) {
-            do {
+            while (true) {
                 // Handle the forms |export {}| and |export { ..., }| (where ...
                 // is non empty), by escaping the loop early if the next token
                 // is }.
                 if (!tokenStream.peekToken(&tt))
                     return null();
                 if (tt == TOK_RC)
                     break;
 
@@ -4128,17 +4197,23 @@ Parser<ParseHandler>::exportDeclaration(
                 if (!exportName)
                     return null();
 
                 Node exportSpec = handler.newBinary(PNK_EXPORT_SPEC, bindingName, exportName);
                 if (!exportSpec)
                     return null();
 
                 handler.addList(kid, exportSpec);
-            } while (tokenStream.matchToken(TOK_COMMA));
+
+                bool matched;
+                if (!tokenStream.matchToken(&matched, TOK_COMMA))
+                    return null();
+                if (!matched)
+                    break;
+            }
 
             MUST_MATCH_TOKEN(TOK_RC, JSMSG_RC_AFTER_EXPORT_SPEC_LIST);
         } else {
             // Handle the form |export *| by adding a special export batch
             // specifier to the list.
             Node exportSpec = handler.newNullary(PNK_EXPORT_BATCH_SPEC, JSOP_NOP, pos());
             if (!kid)
                 return null();
@@ -4247,17 +4322,20 @@ Parser<ParseHandler>::ifStatement()
 
     StmtInfoPC stmtInfo(context);
     PushStatementPC(pc, &stmtInfo, STMT_IF);
     Node thenBranch = statement();
     if (!thenBranch)
         return null();
 
     Node elseBranch;
-    if (tokenStream.matchToken(TOK_ELSE, TokenStream::Operand)) {
+    bool matched;
+    if (!tokenStream.matchToken(&matched, TOK_ELSE, TokenStream::Operand))
+        return null();
+    if (matched) {
         stmtInfo.type = STMT_ELSE;
         elseBranch = statement();
         if (!elseBranch)
             return null();
     } else {
         elseBranch = null();
     }
 
@@ -4281,17 +4359,19 @@ Parser<ParseHandler>::doWhileStatement()
         return null();
     PopStatementPC(tokenStream, pc);
 
     // The semicolon after do-while is even more optional than most
     // semicolons in JS.  Web compat required this by 2004:
     //   http://bugzilla.mozilla.org/show_bug.cgi?id=238945
     // ES3 and ES5 disagreed, but ES6 conforms to Web reality:
     //   https://bugs.ecmascript.org/show_bug.cgi?id=157
-    tokenStream.matchToken(TOK_SEMI);
+    bool ignored;
+    if (!tokenStream.matchToken(&ignored, TOK_SEMI))
+        return null();
     return handler.newDoWhileStatement(body, cond, TokenPos(begin, pos().end));
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::whileStatement()
 {
     uint32_t begin = pos().begin;
@@ -4304,27 +4384,26 @@ Parser<ParseHandler>::whileStatement()
     if (!body)
         return null();
     PopStatementPC(tokenStream, pc);
     return handler.newWhileStatement(begin, cond, body);
 }
 
 template <typename ParseHandler>
 bool
-Parser<ParseHandler>::matchInOrOf(bool *isForOfp)
-{
-    if (tokenStream.matchToken(TOK_IN)) {
-        *isForOfp = false;
-        return true;
-    }
-    if (tokenStream.matchContextualKeyword(context->names().of)) {
-        *isForOfp = true;
-        return true;
-    }
-    return false;
+Parser<ParseHandler>::matchInOrOf(bool *isForInp, bool *isForOfp)
+{
+    TokenKind tt;
+    if (!tokenStream.getToken(&tt))
+        return false;
+    *isForInp = tt == TOK_IN;
+    *isForOfp = tt == TOK_NAME && tokenStream.currentToken().name() == context->names().of;
+    if (!*isForInp && !*isForOfp)
+        tokenStream.ungetToken();
+    return true;
 }
 
 template <>
 bool
 Parser<FullParseHandler>::isValidForStatementLHS(ParseNode *pn1, JSVersion version,
                                                  bool isForDecl, bool isForEach,
                                                  ParseNodeKind headKind)
 {
@@ -4471,19 +4550,23 @@ Parser<FullParseHandler>::forStatement()
      * keyword here, even if JavaScript recognizes 'in' as an operator,
      * as we've excluded 'in' from being parsed in RelExpr by setting
      * pc->parsingForInit.
      */
     StmtInfoPC letStmt(context); /* used if blockObj != nullptr. */
     ParseNode *pn2, *pn3;      /* forHead->pn_kid2 and pn_kid3. */
     ParseNodeKind headKind = PNK_FORHEAD;
     if (pn1) {
-        bool isForOf;
-        if (matchInOrOf(&isForOf))
-            headKind = isForOf ? PNK_FOROF : PNK_FORIN;
+        bool isForIn, isForOf;
+        if (!matchInOrOf(&isForIn, &isForOf))
+            return null();
+        if (isForIn)
+            headKind = PNK_FORIN;
+        else if (isForOf)
+            headKind = PNK_FOROF;
     }
 
     if (headKind == PNK_FOROF || headKind == PNK_FORIN) {
         /*
          * Parse the rest of the for/in or for/of head.
          *
          * Here pn1 is everything to the left of 'in' or 'of'. At the end of
          * this block, pn1 is a decl or nullptr, pn2 is the assignment target
@@ -4769,18 +4852,22 @@ Parser<SyntaxParseHandler>::forStatement
     }
 
     /*
      * We can be sure that it's a for/in loop if there's still an 'in'
      * keyword here, even if JavaScript recognizes 'in' as an operator,
      * as we've excluded 'in' from being parsed in RelExpr by setting
      * pc->parsingForInit.
      */
-    bool isForOf;
-    if (lhsNode && matchInOrOf(&isForOf)) {
+    bool isForIn = false, isForOf = false;
+    if (lhsNode) {
+        if (!matchInOrOf(&isForIn, &isForOf))
+            return null();
+    }
+    if (isForIn || isForOf) {
         /* Parse the rest of the for/in or for/of head. */
         forStmt.type = isForOf ? STMT_FOR_OF_LOOP : STMT_FOR_IN_LOOP;
 
         /* Check that the left side of the 'in' or 'of' is valid. */
         if (!isForDecl &&
             lhsNode != SyntaxParseHandler::NodeName &&
             lhsNode != SyntaxParseHandler::NodeGetProp &&
             lhsNode != SyntaxParseHandler::NodeLValue)
@@ -5456,17 +5543,20 @@ Parser<ParseHandler>::tryStatement()
 
             Node catchGuard = null();
 #if JS_HAS_CATCH_GUARD
             /*
              * We use 'catch (x if x === 5)' (not 'catch (x : x === 5)')
              * to avoid conflicting with the JS2/ECMAv4 type annotation
              * catchguard syntax.
              */
-            if (tokenStream.matchToken(TOK_IF)) {
+            bool matched;
+            if (!tokenStream.matchToken(&matched, TOK_IF))
+                return null();
+            if (matched) {
                 catchGuard = expr();
                 if (!catchGuard)
                     return null();
             }
 #endif
             MUST_MATCH_TOKEN(TOK_RP, JSMSG_PAREN_AFTER_CATCH);
 
             MUST_MATCH_TOKEN(TOK_LC, JSMSG_CURLY_BEFORE_CATCH);
@@ -5636,31 +5726,42 @@ Parser<ParseHandler>::statement(bool can
     }
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::expr()
 {
     Node pn = assignExpr();
-    if (pn && tokenStream.matchToken(TOK_COMMA)) {
+    if (!pn)
+        return null();
+
+    bool matched;
+    if (!tokenStream.matchToken(&matched, TOK_COMMA))
+        return null();
+    if (matched) {
         Node seq = handler.newList(PNK_COMMA, pn);
         if (!seq)
             return null();
-        do {
+        while (true) {
             if (handler.isUnparenthesizedYield(pn)) {
                 report(ParseError, false, pn, JSMSG_BAD_GENERATOR_SYNTAX, js_yield_str);
                 return null();
             }
 
             pn = assignExpr();
             if (!pn)
                 return null();
             handler.addList(seq, pn);
-        } while (tokenStream.matchToken(TOK_COMMA));
+
+            if (!tokenStream.matchToken(&matched, TOK_COMMA))
+                return null();
+            if (!matched)
+                break;
+        }
         return seq;
     }
     return pn;
 }
 
 static const JSOp ParseNodeKindToJSOp[] = {
     JSOP_OR,
     JSOP_AND,
@@ -6494,17 +6595,17 @@ Parser<FullParseHandler>::legacyComprehe
         return null();
 
     if (!transplanter.transplant(bodyExpr))
         return null();
 
     MOZ_ASSERT(pc->staticScope && pc->staticScope == pn->pn_objbox->object);
     data.initLet(HoistVars, &pc->staticScope->as<StaticBlockObject>(), JSMSG_ARRAY_INIT_TOO_BIG);
 
-    do {
+    while (true) {
         /*
          * FOR node is binary, left is loop control and right is body.  Use
          * index to count each block-local let-variable on the left-hand side
          * of the in/of.
          */
         pn2 = BinaryNode::create(PNK_FOR, &handler);
         if (!pn2)
             return null();
@@ -6545,18 +6646,20 @@ Parser<FullParseHandler>::legacyComprehe
                 return null();
             break;
 
           default:
             report(ParseError, false, null(), JSMSG_NO_VARIABLE_NAME);
             return null();
         }
 
-        bool isForOf;
-        if (!matchInOrOf(&isForOf)) {
+        bool isForIn, isForOf;
+        if (!matchInOrOf(&isForIn, &isForOf))
+            return null();
+        if (!isForIn && !isForOf) {
             report(ParseError, false, null(), JSMSG_IN_AFTER_FOR_NAME);
             return null();
         }
         ParseNodeKind headKind = PNK_FORIN;
         if (isForOf) {
             if (pn2->pn_iflags != JSITER_ENUMERATE) {
                 MOZ_ASSERT(pn2->pn_iflags == (JSITER_FOREACH | JSITER_ENUMERATE));
                 report(ParseError, false, null(), JSMSG_BAD_FOR_EACH_LOOP);
@@ -6630,19 +6733,28 @@ Parser<FullParseHandler>::legacyComprehe
         if (!pn3)
             return null();
 
         pn2->pn_left = handler.newTernary(headKind, lets, pn3, pn4);
         if (!pn2->pn_left)
             return null();
         *pnp = pn2;
         pnp = &pn2->pn_right;
-    } while (tokenStream.matchToken(TOK_FOR));
-
-    if (tokenStream.matchToken(TOK_IF)) {
+
+        bool matched;
+        if (!tokenStream.matchToken(&matched, TOK_FOR))
+            return null();
+        if (!matched)
+            break;
+    }
+
+    bool matched;
+    if (!tokenStream.matchToken(&matched, TOK_IF))
+        return null();
+    if (matched) {
         pn2 = TernaryNode::create(PNK_IF, &handler);
         if (!pn2)
             return null();
         pn2->pn_kid1 = condition();
         if (!pn2->pn_kid1)
             return null();
         *pnp = pn2;
         pnp = &pn2->pn_kid2;
@@ -6985,20 +7097,25 @@ Parser<ParseHandler>::comprehensionIf(Ge
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::comprehensionTail(GeneratorKind comprehensionKind)
 {
     JS_CHECK_RECURSION(context, return null());
 
-    if (tokenStream.matchToken(TOK_FOR, TokenStream::Operand))
+    bool matched;
+    if (!tokenStream.matchToken(&matched, TOK_FOR, TokenStream::Operand))
+        return null();
+    if (matched)
         return comprehensionFor(comprehensionKind);
 
-    if (tokenStream.matchToken(TOK_IF, TokenStream::Operand))
+    if (!tokenStream.matchToken(&matched, TOK_IF, TokenStream::Operand))
+        return null();
+    if (matched)
         return comprehensionIf(comprehensionKind);
 
     uint32_t begin = pos().begin;
 
     Node bodyExpr = assignExpr();
     if (!bodyExpr)
         return null();
 
@@ -7096,28 +7213,33 @@ Parser<ParseHandler>::assignExprWithoutY
     }
     return res;
 }
 
 template <typename ParseHandler>
 bool
 Parser<ParseHandler>::argumentList(Node listNode, bool *isSpread)
 {
-    if (tokenStream.matchToken(TOK_RP, TokenStream::Operand)) {
+    bool matched;
+    if (!tokenStream.matchToken(&matched, TOK_RP, TokenStream::Operand))
+        return false;
+    if (matched) {
         handler.setEndPosition(listNode, pos().end);
         return true;
     }
 
     uint32_t startYieldOffset = pc->lastYieldOffset;
     bool arg0 = true;
 
-    do {
+    while (true) {
         bool spread = false;
         uint32_t begin = 0;
-        if (tokenStream.matchToken(TOK_TRIPLEDOT, TokenStream::Operand)) {
+        if (!tokenStream.matchToken(&matched, TOK_TRIPLEDOT, TokenStream::Operand))
+            return false;
+        if (matched) {
             spread = true;
             begin = pos().begin;
             *isSpread = true;
         }
 
         Node argNode = assignExpr();
         if (!argNode)
             return false;
@@ -7132,42 +7254,52 @@ Parser<ParseHandler>::argumentList(Node 
             if (!tokenStream.peekToken(&tt))
                 return false;
             if (tt == TOK_COMMA) {
                 report(ParseError, false, argNode, JSMSG_BAD_GENERATOR_SYNTAX, js_yield_str);
                 return false;
             }
         }
 #if JS_HAS_GENERATOR_EXPRS
-        if (!spread && tokenStream.matchToken(TOK_FOR)) {
-            if (pc->lastYieldOffset != startYieldOffset) {
-                reportWithOffset(ParseError, false, pc->lastYieldOffset,
-                                 JSMSG_BAD_GENEXP_BODY, js_yield_str);
-                return false;
-            }
-            argNode = legacyGeneratorExpr(argNode);
-            if (!argNode)
+        if (!spread) {
+            if (!tokenStream.matchToken(&matched, TOK_FOR))
                 return false;
-            if (!arg0) {
-                report(ParseError, false, argNode, JSMSG_BAD_GENERATOR_SYNTAX, js_generator_str);
-                return false;
-            }
-            TokenKind tt;
-            if (!tokenStream.peekToken(&tt))
-                return false;
-            if (tt == TOK_COMMA) {
-                report(ParseError, false, argNode, JSMSG_BAD_GENERATOR_SYNTAX, js_generator_str);
-                return false;
+            if (matched) {
+                if (pc->lastYieldOffset != startYieldOffset) {
+                    reportWithOffset(ParseError, false, pc->lastYieldOffset,
+                                     JSMSG_BAD_GENEXP_BODY, js_yield_str);
+                    return false;
+                }
+                argNode = legacyGeneratorExpr(argNode);
+                if (!argNode)
+                    return false;
+                if (!arg0) {
+                    report(ParseError, false, argNode, JSMSG_BAD_GENERATOR_SYNTAX, js_generator_str);
+                    return false;
+                }
+                TokenKind tt;
+                if (!tokenStream.peekToken(&tt))
+                    return false;
+                if (tt == TOK_COMMA) {
+                    report(ParseError, false, argNode, JSMSG_BAD_GENERATOR_SYNTAX, js_generator_str);
+                    return false;
+                }
             }
         }
 #endif
         arg0 = false;
 
         handler.addList(listNode, argNode);
-    } while (tokenStream.matchToken(TOK_COMMA));
+
+        bool matched;
+        if (!tokenStream.matchToken(&matched, TOK_COMMA))
+            return false;
+        if (!matched)
+            break;
+    }
 
     TokenKind tt;
     if (!tokenStream.getToken(&tt))
         return false;
     if (tt != TOK_RP) {
         report(ParseError, false, null(), JSMSG_PAREN_AFTER_ARGS);
         return false;
     }
@@ -7194,17 +7326,20 @@ Parser<ParseHandler>::memberExpr(TokenKi
         if (!tokenStream.getToken(&tt, TokenStream::Operand))
             return null();
         Node ctorExpr = memberExpr(tt, false);
         if (!ctorExpr)
             return null();
 
         handler.addList(lhs, ctorExpr);
 
-        if (tokenStream.matchToken(TOK_LP)) {
+        bool matched;
+        if (!tokenStream.matchToken(&matched, TOK_LP))
+            return null();
+        if (matched) {
             bool isSpread = false;
             if (!argumentList(lhs, &isSpread))
                 return null();
             if (isSpread)
                 handler.setOp(lhs, JSOP_SPREADNEW);
         }
     } else {
         lhs = primaryExpr(tt);
@@ -7378,26 +7513,31 @@ Parser<ParseHandler>::arrayInitializer()
 {
     MOZ_ASSERT(tokenStream.isCurrentTokenType(TOK_LB));
 
     uint32_t begin = pos().begin;
     Node literal = handler.newArrayLiteral(begin, pc->blockidGen);
     if (!literal)
         return null();
 
-    if (tokenStream.matchToken(TOK_RB, TokenStream::Operand)) {
+    TokenKind tt;
+    if (!tokenStream.getToken(&tt, TokenStream::Operand))
+        return null();
+    if (tt == TOK_RB) {
         /*
          * Mark empty arrays as non-constant, since we cannot easily
          * determine their type.
          */
         handler.setListFlag(literal, PNX_NONCONST);
-    } else if (tokenStream.matchToken(TOK_FOR, TokenStream::Operand)) {
+    } else if (tt == TOK_FOR) {
         // ES6 array comprehension.
         return arrayComprehension(begin);
     } else {
+        tokenStream.ungetToken();
+
         bool spread = false, missingTrailingComma = false;
         uint32_t index = 0;
         for (; ; index++) {
             if (index == NativeObject::NELEMENTS_LIMIT) {
                 report(ParseError, false, null(), JSMSG_ARRAY_INIT_TOO_BIG);
                 return null();
             }
 
@@ -7427,17 +7567,20 @@ Parser<ParseHandler>::arrayInitializer()
                 if (foldConstants && !FoldConstants(context, &element, this))
                     return null();
                 if (!handler.addArrayElement(literal, element))
                     return null();
             }
 
             if (tt != TOK_COMMA) {
                 /* If we didn't already match TOK_COMMA in above case. */
-                if (!tokenStream.matchToken(TOK_COMMA)) {
+                bool matched;
+                if (!tokenStream.matchToken(&matched, TOK_COMMA))
+                    return null();
+                if (!matched) {
                     missingTrailingComma = true;
                     break;
                 }
             }
         }
 
         /*
          * At this point, (index == 0 && missingTrailingComma) implies one
@@ -7480,18 +7623,23 @@ Parser<ParseHandler>::arrayInitializer()
          * time. A block-local var is accessed by the JSOP_GETLOCAL and
          * JSOP_SETLOCAL ops. These ops have an immediate operand, the local
          * slot's stack index from fp->spbase.
          *
          * The legacy array comprehension iteration step, array.push(i * j) in
          * the example above, is done by <i * j>; JSOP_ARRAYPUSH <array>, where
          * <array> is the index of array's stack slot.
          */
-        if (index == 0 && !spread && tokenStream.matchToken(TOK_FOR) && missingTrailingComma)
-            return legacyArrayComprehension(literal);
+        if (index == 0 && !spread) {
+            bool matched;
+            if (!tokenStream.matchToken(&matched, TOK_FOR))
+                return null();
+            if (matched && missingTrailingComma)
+                return legacyArrayComprehension(literal);
+        }
 
         MUST_MATCH_TOKEN(TOK_RB, JSMSG_BRACKET_AFTER_LIST);
     }
     handler.setEndPosition(literal, pos().end);
     return literal;
 }
 
 static JSAtom*
@@ -7898,34 +8046,39 @@ Parser<ParseHandler>::primaryExpr(TokenK
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::parenExprOrGeneratorComprehension()
 {
     MOZ_ASSERT(tokenStream.isCurrentTokenType(TOK_LP));
     uint32_t begin = pos().begin;
     uint32_t startYieldOffset = pc->lastYieldOffset;
 
-    if (tokenStream.matchToken(TOK_FOR, TokenStream::Operand))
+    bool matched;
+    if (!tokenStream.matchToken(&matched, TOK_FOR, TokenStream::Operand))
+        return null();
+    if (matched)
         return generatorComprehension(begin);
 
     /*
      * Always accept the 'in' operator in a parenthesized expression,
      * where it's unambiguous, even if we might be parsing the init of a
      * for statement.
      */
     bool oldParsingForInit = pc->parsingForInit;
     pc->parsingForInit = false;
     Node pn = expr();
     pc->parsingForInit = oldParsingForInit;
 
     if (!pn)
         return null();
 
 #if JS_HAS_GENERATOR_EXPRS
-    if (tokenStream.matchToken(TOK_FOR)) {
+    if (!tokenStream.matchToken(&matched, TOK_FOR))
+        return null();
+    if (matched) {
         if (pc->lastYieldOffset != startYieldOffset) {
             reportWithOffset(ParseError, false, pc->lastYieldOffset,
                              JSMSG_BAD_GENEXP_BODY, js_yield_str);
             return null();
         }
         if (handler.isOperationWithoutParens(pn, PNK_COMMA)) {
             report(ParseError, false, null(),
                    JSMSG_BAD_GENERATOR_SYNTAX, js_generator_str);
@@ -7991,17 +8144,20 @@ Parser<ParseHandler>::exprInParens()
     pc->parsingForInit = false;
     Node pn = expr();
     pc->parsingForInit = oldParsingForInit;
 
     if (!pn)
         return null();
 
 #if JS_HAS_GENERATOR_EXPRS
-    if (tokenStream.matchToken(TOK_FOR)) {
+    bool matched;
+    if (!tokenStream.matchToken(&matched, TOK_FOR))
+        return null();
+    if (matched) {
         if (pc->lastYieldOffset != startYieldOffset) {
             reportWithOffset(ParseError, false, pc->lastYieldOffset,
                              JSMSG_BAD_GENEXP_BODY, js_yield_str);
             return null();
         }
         if (handler.isOperationWithoutParens(pn, PNK_COMMA)) {
             report(ParseError, false, null(),
                    JSMSG_BAD_GENERATOR_SYNTAX, js_generator_str);
--- a/js/src/frontend/Parser.h
+++ b/js/src/frontend/Parser.h
@@ -617,17 +617,17 @@ class Parser : private JS::AutoGCRooter,
     enum AssignmentFlavor {
         PlainAssignment,
         CompoundAssignment,
         KeyedDestructuringAssignment,
         IncDecAssignment
     };
 
     bool checkAndMarkAsAssignmentLhs(Node pn, AssignmentFlavor flavor);
-    bool matchInOrOf(bool *isForOfp);
+    bool matchInOrOf(bool *isForInp, bool *isForOfp);
 
     bool checkFunctionArguments();
     bool makeDefIntoUse(Definition *dn, Node pn, JSAtom *atom, bool *pbodyLevelHoistedUse);
     bool checkFunctionDefinition(HandlePropertyName funName, Node *pn, FunctionSyntaxKind kind,
                                  bool *pbodyProcessed, bool *pbodyLevelHoistedUse);
     bool finishFunctionDefinition(Node pn, FunctionBox *funbox, Node prelude, Node body);
     bool addFreeVariablesFromLazyFunction(JSFunction *fun, ParseContext<ParseHandler> *pc,
                                           bool bodyLevelHoistedUse);
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -448,31 +448,34 @@ class MOZ_STACK_CLASS TokenStream
 
         *ttp = srcCoords.lineNum(curr.pos.end) == srcCoords.lineNum(next.pos.begin)
              ? next.type
              : TOK_EOL;
         return true;
     }
 
     // Get the next token from the stream if its kind is |tt|.
-    bool matchToken(TokenKind tt, Modifier modifier = None) {
+    bool matchToken(bool *matchedp, TokenKind tt, Modifier modifier = None) {
         TokenKind token;
-        if (!getToken(&token, modifier)) {
-            ungetToken();
+        if (!getToken(&token, modifier))
             return false;
+        if (token == tt) {
+            *matchedp = true;
+        } else {
+            ungetToken();
+            *matchedp = false;
         }
-        if (token == tt)
-            return true;
-        ungetToken();
-        return false;
+        return true;
     }
 
     void consumeKnownToken(TokenKind tt) {
+        bool matched;
         MOZ_ASSERT(lookahead != 0);
-        JS_ALWAYS_TRUE(matchToken(tt));
+        MOZ_ALWAYS_TRUE(matchToken(&matched, tt));
+        MOZ_ALWAYS_TRUE(matched);
     }
 
     bool matchContextualKeyword(Handle<PropertyName*> keyword) {
         TokenKind token;
         if (!getToken(&token)) {
             ungetToken();
             return false;
         }