Bug 747831 - Record buffer offset in ParseNodes instead of line number and column index. r=jorendorff.
☠☠ backed out by cad5306d569e ☠ ☠
authorNicholas Nethercote <nnethercote@mozilla.com>
Mon, 11 Mar 2013 15:56:58 -0700
changeset 125486 d1b71de5bbc128ecc1ae32fea556df104502097a
parent 125485 6b8e128a989c40614621dd72a437d9e66435afd2
child 125487 e9e5e2a8a52bdaecbb6052ca2ed583e39a8b8673
push id24459
push useremorley@mozilla.com
push dateWed, 20 Mar 2013 11:46:36 +0000
treeherdermozilla-central@1d6fe70c79c5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjorendorff
bugs747831
milestone22.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 747831 - Record buffer offset in ParseNodes instead of line number and column index. r=jorendorff.
js/src/frontend/BytecodeEmitter.cpp
js/src/frontend/BytecodeEmitter.h
js/src/frontend/FullParseHandler.h
js/src/frontend/ParseNode.h
js/src/frontend/Parser.cpp
js/src/frontend/SyntaxParseHandler.h
js/src/frontend/TokenStream.cpp
js/src/frontend/TokenStream.h
js/src/ion/AsmJS.cpp
js/src/jsfun.cpp
js/src/jsprvtd.h
js/src/jsreflect.cpp
--- a/js/src/frontend/BytecodeEmitter.cpp
+++ b/js/src/frontend/BytecodeEmitter.cpp
@@ -85,30 +85,30 @@ struct frontend::StmtInfoBCE : public St
         JS_ASSERT(type == STMT_TRY || type == STMT_FINALLY);
         return continues;
     }
 };
 
 BytecodeEmitter::BytecodeEmitter(BytecodeEmitter *parent,
                                  Parser<FullParseHandler> *parser, SharedContext *sc,
                                  HandleScript script, HandleScript evalCaller, bool hasGlobalScope,
-                                 unsigned lineno, bool selfHostingMode)
+                                 uint32_t lineNum, bool selfHostingMode)
   : sc(sc),
     parent(parent),
     script(sc->context, script),
-    prolog(sc->context, lineno),
-    main(sc->context, lineno),
+    prolog(sc->context, lineNum),
+    main(sc->context, lineNum),
     current(&main),
     parser(parser),
     evalCaller(evalCaller),
     topStmt(NULL),
     topScopeStmt(NULL),
     blockChain(sc->context),
     atomIndices(sc->context),
-    firstLine(lineno),
+    firstLine(lineNum),
     stackDepth(0), maxStackDepth(0),
     tryNoteList(sc->context),
     arrayCompDepth(0),
     emitLevel(0),
     constList(sc->context),
     typesetCount(0),
     hasSingletons(false),
     emittingForInit(false),
@@ -336,20 +336,23 @@ EmitBackPatchOp(JSContext *cx, BytecodeE
     delta = offset - *lastp;
     *lastp = offset;
     JS_ASSERT(delta > 0);
     return EmitJump(cx, bce, JSOP_BACKPATCH, delta);
 }
 
 /* Updates line number notes, not column notes. */
 static inline bool
-UpdateLineNumberNotes(JSContext *cx, BytecodeEmitter *bce, unsigned line)
+UpdateLineNumberNotes(JSContext *cx, BytecodeEmitter *bce, uint32_t offset)
 {
-    unsigned delta = line - bce->currentLine();
-    if (delta != 0) {
+    TokenStream *ts = &bce->parser->tokenStream;
+    if (!ts->srcCoords.isOnThisLine(offset, bce->currentLine())) {
+        unsigned line = ts->srcCoords.lineNum(offset);
+        unsigned delta = line - bce->currentLine();
+
         /*
          * Encode any change in the current source line number by using
          * either several SRC_NEWLINE notes or just one SRC_SETLINE note,
          * whichever consumes less space.
          *
          * NB: We handle backward line number deltas (possible with for
          * loops where the update part is emitted after the body, but its
          * line number is <= any line number in the body) here by letting
@@ -368,37 +371,37 @@ UpdateLineNumberNotes(JSContext *cx, Byt
             } while (--delta != 0);
         }
     }
     return true;
 }
 
 /* A function, so that we avoid macro-bloating all the other callsites. */
 static bool
-UpdateSourceCoordNotes(JSContext *cx, BytecodeEmitter *bce, TokenPtr pos)
+UpdateSourceCoordNotes(JSContext *cx, BytecodeEmitter *bce, uint32_t offset)
 {
-    if (!UpdateLineNumberNotes(cx, bce, pos.lineno))
-        return false;
-
-    ptrdiff_t colspan = ptrdiff_t(pos.index) -
-                        ptrdiff_t(bce->current->lastColumn);
+    if (!UpdateLineNumberNotes(cx, bce, offset))
+        return false;
+
+    uint32_t columnIndex = bce->parser->tokenStream.srcCoords.columnIndex(offset);
+    ptrdiff_t colspan = ptrdiff_t(columnIndex) - ptrdiff_t(bce->current->lastColumn);
     if (colspan != 0) {
         if (colspan < 0) {
             colspan += SN_COLSPAN_DOMAIN;
         } else if (colspan >= SN_COLSPAN_DOMAIN / 2) {
             // If the column span is so large that we can't store it, then just
             // discard this information because column information would most
             // likely be useless anyway once the column numbers are ~4000000.
             // This has been known to happen with scripts that have been
             // minimized and put into all one line.
             return true;
         }
         if (NewSrcNote2(cx, bce, SRC_COLSPAN, colspan) < 0)
             return false;
-        bce->current->lastColumn = pos.index;
+        bce->current->lastColumn = columnIndex;
     }
     return true;
 }
 
 static ptrdiff_t
 EmitLoopHead(JSContext *cx, BytecodeEmitter *bce, ParseNode *nextpn)
 {
     if (nextpn) {
@@ -1664,41 +1667,43 @@ BytecodeEmitter::tellDebuggerAboutCompil
 
 bool
 BytecodeEmitter::reportError(ParseNode *pn, unsigned errorNumber, ...)
 {
     TokenPos pos = pn ? pn->pn_pos : tokenStream()->currentToken().pos;
 
     va_list args;
     va_start(args, errorNumber);
-    bool result = tokenStream()->reportCompileErrorNumberVA(pos, JSREPORT_ERROR, errorNumber, args);
+    bool result = tokenStream()->reportCompileErrorNumberVA(pos.begin, JSREPORT_ERROR,
+                                                            errorNumber, args);
     va_end(args);
     return result;
 }
 
 bool
 BytecodeEmitter::reportStrictWarning(ParseNode *pn, unsigned errorNumber, ...)
 {
     TokenPos pos = pn ? pn->pn_pos : tokenStream()->currentToken().pos;
 
     va_list args;
     va_start(args, errorNumber);
-    bool result = tokenStream()->reportStrictWarningErrorNumberVA(pos, errorNumber, args);
+    bool result = tokenStream()->reportStrictWarningErrorNumberVA(pos.begin, errorNumber, args);
     va_end(args);
     return result;
 }
 
 bool
 BytecodeEmitter::reportStrictModeError(ParseNode *pn, unsigned errorNumber, ...)
 {
     TokenPos pos = pn ? pn->pn_pos : tokenStream()->currentToken().pos;
 
     va_list args;
     va_start(args, errorNumber);
-    bool result = tokenStream()->reportStrictModeErrorNumberVA(pos, sc->strict, errorNumber, args);
+    bool result = tokenStream()->reportStrictModeErrorNumberVA(pos.begin, sc->strict,
+                                                               errorNumber, args);
     va_end(args);
     return result;
 }
 
 static bool
 EmitNameOp(JSContext *cx, BytecodeEmitter *bce, ParseNode *pn, bool callContext)
 {
     JSOp op;
@@ -4305,21 +4310,21 @@ EmitNormalFor(JSContext *cx, BytecodeEmi
         if (op == JSOP_POP && !EmitTree(cx, bce, pn3))
             return false;
 
         /* Always emit the POP or NOP, to help the decompiler. */
         if (Emit1(cx, bce, op) < 0)
             return false;
 
         /* Restore the absolute line number for source note readers. */
-        ptrdiff_t lineno = pn->pn_pos.end.lineno;
-        if (bce->currentLine() != (unsigned) lineno) {
-            if (NewSrcNote2(cx, bce, SRC_SETLINE, lineno) < 0)
+        uint32_t lineNum = bce->parser->tokenStream.srcCoords.lineNum(pn->pn_pos.end);
+        if (bce->currentLine() != lineNum) {
+            if (NewSrcNote2(cx, bce, SRC_SETLINE, ptrdiff_t(lineNum)) < 0)
                 return false;
-            bce->current->currentLine = (unsigned) lineno;
+            bce->current->currentLine = lineNum;
             bce->current->lastColumn = 0;
         }
     }
 
     ptrdiff_t tmp3 = bce->offset();
 
     if (forHead->pn_kid2) {
         /* Fix up the goto from top to target the loop condition. */
@@ -4402,18 +4407,19 @@ EmitFunc(JSContext *cx, BytecodeEmitter 
         // Do asm.js compilation at the beginning of emitting to avoid
         // compiling twice when JS_BufferIsCompilableUnit and to know whether
         // to emit JSOP_LINKASMJS. Don't fold constants as this will
         // misrepresent the source JS as written to the type checker.
         if (funbox->useAsm && !CompileAsmJS(cx, *bce->tokenStream(), pn, script))
             return false;
 #endif
 
+        uint32_t lineNum = bce->parser->tokenStream.srcCoords.lineNum(pn->pn_pos.begin);
         BytecodeEmitter bce2(bce, bce->parser, funbox, script, bce->evalCaller,
-                             bce->hasGlobalScope, pn->pn_pos.begin.lineno, bce->selfHostingMode);
+                             bce->hasGlobalScope, lineNum, bce->selfHostingMode);
         if (!bce2.init())
             return false;
 
         /* We measured the max scope depth when we parsed the function. */
         if (!EmitFunctionScript(cx, &bce2, pn->pn_body))
             return false;
     }
 
@@ -4735,17 +4741,17 @@ EmitStatement(JSContext *cx, BytecodeEmi
         if (op != JSOP_NOP) {
             if (!EmitTree(cx, bce, pn2))
                 return false;
             if (Emit1(cx, bce, op) < 0)
                 return false;
         }
     } else if (!pn->isDirectivePrologueMember()) {
         /* Don't complain about directive prologue members; just don't emit their code. */
-        bce->current->currentLine = pn2->pn_pos.begin.lineno;
+        bce->current->currentLine = bce->parser->tokenStream.srcCoords.lineNum(pn2->pn_pos.begin);
         bce->current->lastColumn = 0;
         if (!bce->reportStrictWarning(pn2, JSMSG_USELESS_EXPR))
             return false;
     }
 
     return true;
 }
 
@@ -4950,18 +4956,20 @@ EmitCallOrNew(JSContext *cx, BytecodeEmi
                 return false;
         }
         bce->emittingForInit = oldEmittingForInit;
     }
 
     if (Emit3(cx, bce, pn->getOp(), ARGC_HI(argc), ARGC_LO(argc)) < 0)
         return false;
     CheckTypeSet(cx, bce, pn->getOp());
-    if (pn->isOp(JSOP_EVAL))
-        EMIT_UINT16_IMM_OP(JSOP_LINENO, pn->pn_pos.begin.lineno);
+    if (pn->isOp(JSOP_EVAL)) {
+        uint32_t lineNum = bce->parser->tokenStream.srcCoords.lineNum(pn->pn_pos.begin);
+        EMIT_UINT16_IMM_OP(JSOP_LINENO, lineNum);
+    }
     if (pn->pn_xflags & PNX_SETCALL) {
         if (Emit1(cx, bce, JSOP_SETCALL) < 0)
             return false;
     }
     return true;
 }
 
 static bool
@@ -5469,17 +5477,17 @@ frontend::EmitTree(JSContext *cx, Byteco
 
     EmitLevelManager elm(bce);
 
     bool ok = true;
     ptrdiff_t top = bce->offset();
     pn->pn_offset = top;
 
     /* Emit notes to tell the current bytecode's source line number. */
-    if (!UpdateLineNumberNotes(cx, bce, pn->pn_pos.begin.lineno))
+    if (!UpdateLineNumberNotes(cx, bce, pn->pn_pos.begin))
         return false;
 
     switch (pn->getKind()) {
       case PNK_FUNCTION:
         ok = EmitFunc(cx, bce, pn);
         break;
 
       case PNK_ARGSBODY:
--- a/js/src/frontend/BytecodeEmitter.h
+++ b/js/src/frontend/BytecodeEmitter.h
@@ -73,22 +73,22 @@ struct BytecodeEmitter
     BytecodeEmitter *const parent;  /* enclosing function or global context */
 
     Rooted<JSScript*> script;       /* the JSScript we're ultimately producing */
 
     struct EmitSection {
         BytecodeVector code;        /* bytecode */
         SrcNotesVector notes;       /* source notes, see below */
         ptrdiff_t   lastNoteOffset; /* code offset for last source note */
-        unsigned    currentLine;    /* line number for tree-based srcnote gen */
-        unsigned    lastColumn;     /* zero-based column index on currentLine of
+        uint32_t    currentLine;    /* line number for tree-based srcnote gen */
+        uint32_t    lastColumn;     /* zero-based column index on currentLine of
                                        last SRC_COLSPAN-annotated opcode */
 
-        EmitSection(JSContext *cx, unsigned lineno)
-          : code(cx), notes(cx), lastNoteOffset(0), currentLine(lineno), lastColumn(0)
+        EmitSection(JSContext *cx, uint32_t lineNum)
+          : code(cx), notes(cx), lastNoteOffset(0), currentLine(lineNum), lastColumn(0)
         {}
     };
     EmitSection prolog, main, *current;
 
     /* the parser */
     Parser<FullParseHandler> *const parser;
 
     HandleScript    evalCaller;     /* scripted caller info for eval and dbgapi */
@@ -136,17 +136,17 @@ struct BytecodeEmitter
     /*
      * Note that BytecodeEmitters are magic: they own the arena "top-of-stack"
      * space above their tempMark points. This means that you cannot alloc from
      * tempLifoAlloc and save the pointer beyond the next BytecodeEmitter
      * destruction.
      */
     BytecodeEmitter(BytecodeEmitter *parent, Parser<FullParseHandler> *parser, SharedContext *sc,
                     HandleScript script, HandleScript evalCaller, bool hasGlobalScope,
-                    unsigned lineno, bool selfHostingMode = false);
+                    uint32_t lineNum, bool selfHostingMode = false);
     bool init();
 
     bool isAliasedName(ParseNode *pn);
 
     JS_ALWAYS_INLINE
     bool makeAtomIndex(JSAtom *atom, jsatomid *indexp) {
         AtomIndexAddPtr p = atomIndices->lookupForAdd(atom);
         if (p) {
--- a/js/src/frontend/FullParseHandler.h
+++ b/js/src/frontend/FullParseHandler.h
@@ -130,29 +130,29 @@ class FullParseHandler
     }
 
     ParseNode *newTernary(ParseNodeKind kind,
                           ParseNode *first, ParseNode *second, ParseNode *third,
                           JSOp op = JSOP_NOP) {
         return new_<TernaryNode>(kind, op, first, second, third);
     }
 
-    ParseNode *newBreak(PropertyName *label, const TokenPtr &begin, const TokenPtr &end) {
+    ParseNode *newBreak(PropertyName *label, uint32_t begin, uint32_t end) {
         return new_<BreakStatement>(label, begin, end);
     }
-    ParseNode *newContinue(PropertyName *label, const TokenPtr &begin, const TokenPtr &end) {
+    ParseNode *newContinue(PropertyName *label, uint32_t begin, uint32_t end) {
         return new_<ContinueStatement>(label, begin, end);
     }
     ParseNode *newDebuggerStatement(const TokenPos &pos) {
         return new_<DebuggerStatement>(pos);
     }
-    ParseNode *newPropertyAccess(ParseNode *pn, PropertyName *name, const TokenPtr &end) {
+    ParseNode *newPropertyAccess(ParseNode *pn, PropertyName *name, uint32_t end) {
         return new_<PropertyAccess>(pn, name, pn->pn_pos.begin, end);
     }
-    ParseNode *newPropertyByValue(ParseNode *pn, ParseNode *kid, const TokenPtr &end) {
+    ParseNode *newPropertyByValue(ParseNode *pn, ParseNode *kid, uint32_t end) {
         return new_<PropertyByValue>(pn, kid, pn->pn_pos.begin, end);
     }
 
     inline bool addCatchBlock(ParseNode *catchList, ParseNode *letBlock,
                               ParseNode *catchName, ParseNode *catchGuard, ParseNode *catchBody);
 
     inline void morphNameIntoLabel(ParseNode *name, ParseNode *statement);
 
@@ -172,25 +172,25 @@ class FullParseHandler
     }
 
     inline void noteLValue(ParseNode *pn);
     inline bool finishInitializerAssignment(ParseNode *pn, ParseNode *init, JSOp op);
 
     void setBeginPosition(ParseNode *pn, ParseNode *oth) {
         setBeginPosition(pn, oth->pn_pos.begin);
     }
-    void setBeginPosition(ParseNode *pn, const TokenPtr &begin) {
+    void setBeginPosition(ParseNode *pn, uint32_t begin) {
         pn->pn_pos.begin = begin;
         JS_ASSERT(pn->pn_pos.begin <= pn->pn_pos.end);
     }
 
     void setEndPosition(ParseNode *pn, ParseNode *oth) {
         setEndPosition(pn, oth->pn_pos.end);
     }
-    void setEndPosition(ParseNode *pn, const TokenPtr &end) {
+    void setEndPosition(ParseNode *pn, uint32_t end) {
         pn->pn_pos.end = end;
         JS_ASSERT(pn->pn_pos.begin <= pn->pn_pos.end);
     }
 
     TokenPos getPosition(ParseNode *pn) {
         return pn->pn_pos;
     }
 
--- a/js/src/frontend/ParseNode.h
+++ b/js/src/frontend/ParseNode.h
@@ -425,20 +425,18 @@ struct ParseNode {
     void operator=(const ParseNode &other) MOZ_DELETE;
 
   public:
     ParseNode(ParseNodeKind kind, JSOp op, ParseNodeArity arity)
       : pn_type(kind), pn_op(op), pn_arity(arity), pn_parens(0), pn_used(0), pn_defn(0),
         pn_offset(0), pn_next(NULL), pn_link(NULL)
     {
         JS_ASSERT(kind < PNK_LIMIT);
-        pn_pos.begin.index = 0;
-        pn_pos.begin.lineno = 0;
-        pn_pos.end.index = 0;
-        pn_pos.end.lineno = 0;
+        pn_pos.begin = 0;
+        pn_pos.end = 0;
         memset(&pn_u, 0, sizeof pn_u);
     }
 
     ParseNode(ParseNodeKind kind, JSOp op, ParseNodeArity arity, const TokenPos &pos)
       : pn_type(kind), pn_op(op), pn_arity(arity), pn_parens(0), pn_used(0), pn_defn(0),
         pn_pos(pos), pn_offset(0), pn_next(NULL), pn_link(NULL)
     {
         JS_ASSERT(kind < PNK_LIMIT);
@@ -957,18 +955,17 @@ struct NameNode : public ParseNode {
 struct LexicalScopeNode : public ParseNode {
     static inline LexicalScopeNode *create(ParseNodeKind kind, FullParseHandler *handler) {
         return (LexicalScopeNode *) ParseNode::create(kind, PN_NAME, handler);
     }
 };
 
 class LoopControlStatement : public ParseNode {
   protected:
-    LoopControlStatement(ParseNodeKind kind, PropertyName *label,
-                         const TokenPtr &begin, const TokenPtr &end)
+    LoopControlStatement(ParseNodeKind kind, PropertyName *label, uint32_t begin, uint32_t end)
       : ParseNode(kind, JSOP_NOP, PN_NULLARY, TokenPos::make(begin, end))
     {
         JS_ASSERT(kind == PNK_BREAK || kind == PNK_CONTINUE);
         pn_u.loopControl.label = label;
     }
 
   public:
     /* Label associated with this break/continue statement, if any. */
@@ -981,31 +978,31 @@ class LoopControlStatement : public Pars
         JS_ASSERT_IF(match, node.isArity(PN_NULLARY));
         JS_ASSERT_IF(match, node.isOp(JSOP_NOP));
         return match;
     }
 };
 
 class BreakStatement : public LoopControlStatement {
   public:
-    BreakStatement(PropertyName *label, const TokenPtr &begin, const TokenPtr &end)
+    BreakStatement(PropertyName *label, uint32_t begin, uint32_t end)
       : LoopControlStatement(PNK_BREAK, label, begin, end)
     { }
 
     static bool test(const ParseNode &node) {
         bool match = node.isKind(PNK_BREAK);
         JS_ASSERT_IF(match, node.isArity(PN_NULLARY));
         JS_ASSERT_IF(match, node.isOp(JSOP_NOP));
         return match;
     }
 };
 
 class ContinueStatement : public LoopControlStatement {
   public:
-    ContinueStatement(PropertyName *label, TokenPtr &begin, TokenPtr &end)
+    ContinueStatement(PropertyName *label, uint32_t begin, uint32_t end)
       : LoopControlStatement(PNK_CONTINUE, label, begin, end)
     { }
 
     static bool test(const ParseNode &node) {
         bool match = node.isKind(PNK_CONTINUE);
         JS_ASSERT_IF(match, node.isArity(PN_NULLARY));
         JS_ASSERT_IF(match, node.isOp(JSOP_NOP));
         return match;
@@ -1067,18 +1064,17 @@ class BooleanLiteral : public ParseNode 
   public:
     BooleanLiteral(bool b, const TokenPos &pos)
       : ParseNode(b ? PNK_TRUE : PNK_FALSE, b ? JSOP_TRUE : JSOP_FALSE, PN_NULLARY, pos)
     { }
 };
 
 class PropertyAccess : public ParseNode {
   public:
-    PropertyAccess(ParseNode *lhs, PropertyName *name,
-                   const TokenPtr &begin, const TokenPtr &end)
+    PropertyAccess(ParseNode *lhs, PropertyName *name, uint32_t begin, uint32_t end)
       : ParseNode(PNK_DOT, JSOP_GETPROP, PN_NAME, TokenPos::make(begin, end))
     {
         JS_ASSERT(lhs != NULL);
         JS_ASSERT(name != NULL);
         pn_u.name.expr = lhs;
         pn_u.name.atom = name;
     }
 
@@ -1094,18 +1090,17 @@ class PropertyAccess : public ParseNode 
 
     PropertyName &name() const {
         return *pn_u.name.atom->asPropertyName();
     }
 };
 
 class PropertyByValue : public ParseNode {
   public:
-    PropertyByValue(ParseNode *lhs, ParseNode *propExpr,
-                    const TokenPtr &begin, const TokenPtr &end)
+    PropertyByValue(ParseNode *lhs, ParseNode *propExpr, uint32_t begin, uint32_t end)
       : ParseNode(PNK_ELEM, JSOP_GETELEM, PN_BINARY, TokenPos::make(begin, end))
     {
         pn_u.binary.left = lhs;
         pn_u.binary.right = propExpr;
     }
 };
 
 #ifdef DEBUG
--- a/js/src/frontend/Parser.cpp
+++ b/js/src/frontend/Parser.cpp
@@ -329,33 +329,34 @@ ParseContext<ParseHandler>::generateFunc
 
     return true;
 }
 
 template <typename ParseHandler>
 bool
 Parser<ParseHandler>::report(ParseReportKind kind, bool strict, Node pn, unsigned errorNumber, ...)
 {
-    TokenPos pos = pn ? handler.getPosition(pn) : tokenStream.currentToken().pos;
+    uint32_t offset = (pn ? handler.getPosition(pn) : tokenStream.currentToken().pos).begin;
 
     va_list args;
     va_start(args, errorNumber);
     bool result = false;
     switch (kind) {
       case ParseError:
-        result = tokenStream.reportCompileErrorNumberVA(pos, JSREPORT_ERROR, errorNumber, args);
+        result = tokenStream.reportCompileErrorNumberVA(offset, JSREPORT_ERROR, errorNumber, args);
         break;
       case ParseWarning:
-        result = tokenStream.reportCompileErrorNumberVA(pos, JSREPORT_WARNING, errorNumber, args);
+        result =
+            tokenStream.reportCompileErrorNumberVA(offset, JSREPORT_WARNING, errorNumber, args);
         break;
       case ParseStrictWarning:
-        result = tokenStream.reportStrictWarningErrorNumberVA(pos, errorNumber, args);
+        result = tokenStream.reportStrictWarningErrorNumberVA(offset, errorNumber, args);
         break;
       case ParseStrictError:
-        result = tokenStream.reportStrictModeErrorNumberVA(pos, strict, errorNumber, args);
+        result = tokenStream.reportStrictModeErrorNumberVA(offset, strict, errorNumber, args);
         break;
     }
     va_end(args);
     return result;
 }
 
 template <typename ParseHandler>
 Parser<ParseHandler>::Parser(JSContext *cx, const CompileOptions &options,
@@ -1507,17 +1508,17 @@ Parser<ParseHandler>::functionArguments(
         if (tokenStream.getToken() != TOK_LP) {
             report(ParseError, false, null(),
                    kind == Arrow ? JSMSG_BAD_ARROW_ARGS : JSMSG_PAREN_BEFORE_FORMAL);
             return false;
         }
 
         // Record the start of function source (for FunctionToString). If we
         // are parenFreeArrow, we will set this below, after consuming the NAME.
-        funbox->bufStart = tokenStream.offsetOfToken(tokenStream.currentToken());
+        funbox->bufStart = tokenStream.currentToken().pos.begin;
     }
 
     hasRest = false;
 
     Node argsbody = handler.newList(PNK_ARGSBODY);
     if (!argsbody)
         return false;
     handler.setFunctionBody(funcpn, argsbody);
@@ -1608,17 +1609,17 @@ Parser<ParseHandler>::functionArguments(
                     return false;
                 }
                 /* Fall through */
               }
 
               case TOK_NAME:
               {
                 if (parenFreeArrow)
-                    funbox->bufStart = tokenStream.offsetOfToken(tokenStream.currentToken());
+                    funbox->bufStart = tokenStream.currentToken().pos.begin;
 
                 RootedPropertyName name(context, tokenStream.currentToken().name());
                 bool disallowDuplicateArgs = destructuringArg || hasDefaults;
                 if (!defineArg(funcpn, name, disallowDuplicateArgs, &duplicatedArg))
                     return false;
 
                 if (tokenStream.matchToken(TOK_ASSIGN)) {
                     // A default argument without parentheses would look like:
@@ -1975,23 +1976,22 @@ Parser<ParseHandler>::functionArgsAndBod
 
 #if JS_HAS_EXPR_CLOSURES
     if (bodyType == StatementListBody) {
 #endif
         if (!tokenStream.matchToken(TOK_RC)) {
             report(ParseError, false, null(), JSMSG_CURLY_AFTER_BODY);
             return false;
         }
-        funbox->bufEnd = tokenStream.offsetOfToken(tokenStream.currentToken()) + 1;
+        funbox->bufEnd = tokenStream.currentToken().pos.begin + 1;
 #if JS_HAS_EXPR_CLOSURES
     } else {
-        // We shouldn't call endOffset if the tokenizer got an error.
         if (tokenStream.hadError())
             return false;
-        funbox->bufEnd = tokenStream.endOffset(tokenStream.currentToken());
+        funbox->bufEnd = tokenStream.currentToken().pos.end;
         if (kind == Statement && !MatchOrInsertSemicolon(context, &tokenStream))
             return false;
     }
 #endif
 
     if (!finishFunctionDefinition(pn, funbox, prelude, body, outerpc))
         return false;
 
@@ -2092,18 +2092,17 @@ Parser<ParseHandler>::functionExpr()
 static inline bool
 IsEscapeFreeStringLiteral(const TokenPos &pos, JSAtom *str)
 {
     /*
      * If the string's length in the source code is its length as a value,
      * accounting for the quotes, then it must not contain any escape
      * sequences or line continuations.
      */
-    return (pos.begin.lineno == pos.end.lineno &&
-            pos.begin.index + str->length() + 2 == pos.end.index);
+    return pos.begin + str->length() + 2 == pos.end;
 }
 
 /*
  * Recognize Directive Prologue members and directives. Assuming |pn| is a
  * candidate for membership in a directive prologue, recognize directives and
  * set |pc|'s flags accordingly. If |pn| is indeed part of a prologue, set its
  * |pn_prologue| flag.
  *
@@ -3021,17 +3020,17 @@ typename ParseHandler::Node
 Parser<ParseHandler>::letBlock(LetContext letContext)
 {
     JS_ASSERT(tokenStream.currentToken().type == TOK_LET);
 
     RootedStaticBlockObject blockObj(context, StaticBlockObject::create(context));
     if (!blockObj)
         return null();
 
-    TokenPtr begin = tokenStream.currentToken().pos.begin;
+    uint32_t begin = tokenStream.currentToken().pos.begin;
 
     MUST_MATCH_TOKEN(TOK_LP, JSMSG_PAREN_BEFORE_LET);
 
     Node vars = variables(PNK_LET, NULL, blockObj, DontHoistVars);
     if (!vars)
         return null();
 
     MUST_MATCH_TOKEN(TOK_RP, JSMSG_PAREN_AFTER_LET);
@@ -3776,17 +3775,17 @@ Parser<SyntaxParseHandler>::forStatement
     return SyntaxParseHandler::NodeGeneric;
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::tryStatement()
 {
     JS_ASSERT(tokenStream.isCurrentTokenType(TOK_TRY));
-    TokenPtr begin = tokenStream.currentToken().pos.begin;
+    uint32_t begin = tokenStream.currentToken().pos.begin;
 
     /*
      * try nodes are ternary.
      * kid1 is the try statement
      * kid2 is the catch node list or null
      * kid3 is the finally statement
      *
      * catch nodes are ternary.
@@ -3946,17 +3945,17 @@ Parser<ParseHandler>::tryStatement()
     return pn;
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::withStatement()
 {
     JS_ASSERT(tokenStream.isCurrentTokenType(TOK_WITH));
-    TokenPtr begin = tokenStream.currentToken().pos.begin;
+    uint32_t begin = tokenStream.currentToken().pos.begin;
 
     // In most cases, we want the constructs forbidden in strict mode code to be
     // a subset of those that JSOPTION_STRICT warns about, and we should use
     // reportStrictModeError.  However, 'with' is the sole instance of a
     // construct that is forbidden in strict mode code, but doesn't even merit a
     // warning under JSOPTION_STRICT.  See
     // https://bugzilla.mozilla.org/show_bug.cgi?id=514576#c1.
     if (pc->sc->strict && !report(ParseStrictError, true, null(), JSMSG_STRICT_CODE_WITH))
@@ -4183,17 +4182,17 @@ Parser<ParseHandler>::statement()
     JS_CHECK_RECURSION(context, return null());
 
     switch (tokenStream.getToken(TSF_OPERAND)) {
       case TOK_FUNCTION:
         return functionStmt();
 
       case TOK_IF:
       {
-        TokenPtr begin = tokenStream.currentToken().pos.begin;
+        uint32_t begin = tokenStream.currentToken().pos.begin;
 
         /* An IF node has three kids: condition, then, and optional else. */
         Node cond = condition();
         if (!cond)
             return null();
 
         StmtInfoPC stmtInfo(context);
         PushStatementPC(pc, &stmtInfo, STMT_IF);
@@ -4225,17 +4224,17 @@ Parser<ParseHandler>::statement()
         return pn;
       }
 
       case TOK_SWITCH:
         return switchStatement();
 
       case TOK_WHILE:
       {
-        TokenPtr begin = tokenStream.currentToken().pos.begin;
+        uint32_t begin = tokenStream.currentToken().pos.begin;
         StmtInfoPC stmtInfo(context);
         PushStatementPC(pc, &stmtInfo, STMT_WHILE_LOOP);
         Node cond = condition();
         if (!cond)
             return null();
         Node body = statement();
         if (!body)
             return null();
@@ -4244,17 +4243,17 @@ Parser<ParseHandler>::statement()
         if (!pn)
             return null();
         handler.setBeginPosition(pn, begin);
         return pn;
       }
 
       case TOK_DO:
       {
-        TokenPtr begin = tokenStream.currentToken().pos.begin;
+        uint32_t begin = tokenStream.currentToken().pos.begin;
         StmtInfoPC stmtInfo(context);
         PushStatementPC(pc, &stmtInfo, STMT_DO_LOOP);
         Node body = statement();
         if (!body)
             return null();
         MUST_MATCH_TOKEN(TOK_WHILE, JSMSG_WHILE_AFTER_DO);
         Node cond = condition();
         if (!cond)
@@ -4281,17 +4280,17 @@ Parser<ParseHandler>::statement()
       case TOK_FOR:
         return forStatement();
 
       case TOK_TRY:
         return tryStatement();
 
       case TOK_THROW:
       {
-        TokenPtr begin = tokenStream.currentToken().pos.begin;
+        uint32_t begin = tokenStream.currentToken().pos.begin;
 
         /* ECMA-262 Edition 3 says 'throw [no LineTerminator here] Expr'. */
         TokenKind tt = tokenStream.peekTokenSameLine(TSF_OPERAND);
         if (tt == TOK_ERROR)
             return null();
         if (tt == TOK_EOF || tt == TOK_EOL || tt == TOK_SEMI || tt == TOK_RC) {
             report(ParseError, false, null(), JSMSG_SYNTAX_ERROR);
             return null();
@@ -4314,21 +4313,21 @@ Parser<ParseHandler>::statement()
         return null();
 
       case TOK_FINALLY:
         report(ParseError, false, null(), JSMSG_FINALLY_WITHOUT_TRY);
         return null();
 
       case TOK_BREAK:
       {
-        TokenPtr begin = tokenStream.currentToken().pos.begin;
+        uint32_t begin = tokenStream.currentToken().pos.begin;
         RootedPropertyName label(context);
         if (!MatchLabel(context, &tokenStream, &label))
             return null();
-        TokenPtr end = tokenStream.currentToken().pos.end;
+        uint32_t end = tokenStream.currentToken().pos.end;
         pn = handler.newBreak(label, begin, end);
         if (!pn)
             return null();
         StmtInfoPC *stmt = pc->topStmt;
         if (label) {
             for (; ; stmt = stmt->down) {
                 if (!stmt) {
                     report(ParseError, false, null(), JSMSG_LABEL_NOT_FOUND);
@@ -4347,21 +4346,21 @@ Parser<ParseHandler>::statement()
                     break;
             }
         }
         break;
       }
 
       case TOK_CONTINUE:
       {
-        TokenPtr begin = tokenStream.currentToken().pos.begin;
+        uint32_t begin = tokenStream.currentToken().pos.begin;
         RootedPropertyName label(context);
         if (!MatchLabel(context, &tokenStream, &label))
             return null();
-        TokenPtr end = tokenStream.currentToken().pos.begin;
+        uint32_t end = tokenStream.currentToken().pos.end;
         pn = handler.newContinue(label, begin, end);
         if (!pn)
             return null();
         StmtInfoPC *stmt = pc->topStmt;
         if (label) {
             for (StmtInfoPC *stmt2 = NULL; ; stmt = stmt->down) {
                 if (!stmt) {
                     report(ParseError, false, null(), JSMSG_LABEL_NOT_FOUND);
@@ -4960,17 +4959,17 @@ Parser<ParseHandler>::assignExpr()
       case TOK_DIVASSIGN:    kind = PNK_DIVASSIGN;    break;
       case TOK_MODASSIGN:    kind = PNK_MODASSIGN;    break;
 
       case TOK_ARROW: {
         tokenStream.seek(start);
 
         if (tokenStream.getToken() == TOK_ERROR)
             return null();
-        size_t offset = tokenStream.offsetOfToken(tokenStream.currentToken());
+        size_t offset = tokenStream.currentToken().pos.begin;
         tokenStream.ungetToken();
 
         return functionDef(NullPtr(), start, offset, Normal, Arrow);
       }
 
       default:
         JS_ASSERT(!tokenStream.isCurrentTokenAssignment());
         tokenStream.ungetToken();
@@ -5134,32 +5133,32 @@ Parser<ParseHandler>::unaryExpr()
       case TOK_PLUS:
         return unaryOpExpr(PNK_POS, JSOP_POS);
       case TOK_MINUS:
         return unaryOpExpr(PNK_NEG, JSOP_NEG);
 
       case TOK_INC:
       case TOK_DEC:
       {
-        TokenPtr begin = tokenStream.currentToken().pos.begin;
+        uint32_t begin = tokenStream.currentToken().pos.begin;
         pn2 = memberExpr(true);
         if (!pn2)
             return null();
         pn = handler.newUnary((tt == TOK_INC) ? PNK_PREINCREMENT : PNK_PREDECREMENT, pn2);
         if (!pn)
             return null();
         handler.setBeginPosition(pn, begin);
         if (!setIncOpKid(pn, pn2, tt, true))
             return null();
         break;
       }
 
       case TOK_DELETE:
       {
-        TokenPtr begin = tokenStream.currentToken().pos.begin;
+        uint32_t begin = tokenStream.currentToken().pos.begin;
         pn2 = unaryExpr();
         if (!pn2)
             return null();
 
         if (!checkDeleteExpression(&pn2))
             return null();
 
         pn = handler.newUnary(PNK_DELETE, pn2);
@@ -6063,17 +6062,17 @@ Parser<ParseHandler>::memberExpr(bool al
     while ((tt = tokenStream.getToken()) > TOK_EOF) {
         Node nextMember;
         if (tt == TOK_DOT) {
             tt = tokenStream.getToken(TSF_KEYWORD_IS_NAME);
             if (tt == TOK_ERROR)
                 return null();
             if (tt == TOK_NAME) {
                 PropertyName *field = tokenStream.currentToken().name();
-                TokenPtr end = tokenStream.currentToken().pos.end;
+                uint32_t end = tokenStream.currentToken().pos.end;
                 nextMember = handler.newPropertyAccess(lhs, field, end);
                 if (!nextMember)
                     return null();
             } else {
                 report(ParseError, false, null(), JSMSG_NAME_AFTER_DOT);
                 return null();
             }
         } else if (tt == TOK_LB) {
@@ -6087,17 +6086,17 @@ Parser<ParseHandler>::memberExpr(bool al
              * Do folding so we don't have roundtrip changes for cases like:
              * function (obj) { return obj["a" + "b"] }
              */
             if (foldConstants && !FoldConstants(context, &propExpr, this))
                 return null();
 
             PropertyName *name = foldPropertyByValue(propExpr);
 
-            TokenPtr end = tokenStream.currentToken().pos.end;
+            uint32_t end = tokenStream.currentToken().pos.end;
             if (name)
                 nextMember = handler.newPropertyAccess(lhs, name, end);
             else
                 nextMember = handler.newPropertyByValue(lhs, propExpr, end);
             if (!nextMember)
                 return null();
         } else if (allowCallSyntax && tt == TOK_LP) {
             nextMember = handler.newList(PNK_CALL, null(), JSOP_CALL);
@@ -6690,17 +6689,17 @@ Parser<ParseHandler>::primaryExpr(TokenK
     return pn;
 }
 
 template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::parenExpr(bool *genexp)
 {
     JS_ASSERT(tokenStream.currentToken().type == TOK_LP);
-    TokenPtr begin = tokenStream.currentToken().pos.begin;
+    uint32_t begin = tokenStream.currentToken().pos.begin;
 
     if (genexp)
         *genexp = false;
 
     GenexpGuard<ParseHandler> guard(this);
 
     Node pn = bracketedExpr();
     if (!pn)
--- a/js/src/frontend/SyntaxParseHandler.h
+++ b/js/src/frontend/SyntaxParseHandler.h
@@ -46,17 +46,17 @@ class SyntaxParseHandler
     Node newAtom(ParseNodeKind kind, JSAtom *atom, JSOp op = JSOP_NOP) {
         if (kind == PNK_STRING) {
             lastAtom = atom;
             lastStringPos = tokenStream.currentToken().pos;
         }
         return NodeString;
     }
     Node newNumber(double value, DecimalPoint decimalPoint = NoDecimal) { return NodeGeneric; }
-    Node newNumber(const Token &tok) { return NodeGeneric; }
+    Node newNumber(Token tok) { return NodeGeneric; }
     Node newBooleanLiteral(bool cond, const TokenPos &pos) { return NodeGeneric; }
     Node newThisLiteral(const TokenPos &pos) { return NodeGeneric; }
     Node newNullLiteral(const TokenPos &pos) { return NodeGeneric; }
     Node newConditional(Node cond, Node thenExpr, Node elseExpr) { return NodeGeneric; }
 
     Node newNullary(ParseNodeKind kind) { return NodeGeneric; }
 
     Node newUnary(ParseNodeKind kind, Node kid, JSOp op = JSOP_NOP) {
@@ -77,25 +77,25 @@ class SyntaxParseHandler
         return NodeGeneric;
     }
     void setBinaryRHS(Node pn, Node rhs) {}
 
     Node newTernary(ParseNodeKind kind, Node first, Node second, Node third, JSOp op = JSOP_NOP) {
         return NodeGeneric;
     }
 
-    Node newBreak(PropertyName *label, const TokenPtr &begin, const TokenPtr &end) {
+    Node newBreak(PropertyName *label, uint32_t begin, uint32_t end) {
         return NodeGeneric;
     }
-    Node newContinue(PropertyName *label, const TokenPtr &begin, const TokenPtr &end) {
+    Node newContinue(PropertyName *label, uint32_t begin, uint32_t end) {
         return NodeGeneric;
     }
     Node newDebuggerStatement(const TokenPos &pos) { return NodeGeneric; }
-    Node newPropertyAccess(Node pn, PropertyName *name, const TokenPtr &end) { return NodeLValue; }
-    Node newPropertyByValue(Node pn, Node kid, const TokenPtr &end) { return NodeLValue; }
+    Node newPropertyAccess(Node pn, PropertyName *name, uint32_t end) { return NodeLValue; }
+    Node newPropertyByValue(Node pn, Node kid, uint32_t end) { return NodeLValue; }
 
     bool addCatchBlock(Node catchList, Node letBlock,
                        Node catchName, Node catchGuard, Node catchBody) { return true; }
 
     void morphNameIntoLabel(Node name, Node statement) {}
     void setLeaveBlockResult(Node block, Node kid, bool leaveBlockExpr) {}
 
     void setLastFunctionArgumentDefault(Node funcpn, Node pn) {}
@@ -109,20 +109,20 @@ class SyntaxParseHandler
         // syntax parser does not handle.
         return false;
     }
 
     void noteLValue(Node pn) {}
     bool finishInitializerAssignment(Node pn, Node init, JSOp op) { return true; }
 
     void setBeginPosition(Node pn, Node oth) {}
-    void setBeginPosition(Node pn, const TokenPtr &begin) {}
+    void setBeginPosition(Node pn, uint32_t begin) {}
 
     void setEndPosition(Node pn, Node oth) {}
-    void setEndPosition(Node pn, const TokenPtr &end) {}
+    void setEndPosition(Node pn, uint32_t end) {}
 
     TokenPos getPosition(Node pn) {
         return tokenStream.currentToken().pos;
     }
 
     Node newList(ParseNodeKind kind, Node kid = NodeGeneric, JSOp op = JSOP_NOP) {
         return NodeGeneric;
     }
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@@ -102,25 +102,142 @@ frontend::IsIdentifier(JSLinearString *s
     while (++chars != end) {
         c = *chars;
         if (!IsIdentifierPart(c))
             return false;
     }
     return true;
 }
 
+const uint32_t TokenStream::SourceCoords::MAX_PTR;
+
+TokenStream::SourceCoords::SourceCoords(JSContext *cx, uint32_t ln)
+  : lineStartOffsets_(cx), initialLineNum_(ln), lastLineIndex_(0)
+{
+    // The first line begins at buffer offset 0.  MAX_PTR is the sentinel.  The
+    // appends cannot fail because |lineStartOffsets_| has statically-allocated
+    // elements.
+    JS_ASSERT(lineStartOffsets_.capacity() >= 2);
+    (void)lineStartOffsets_.reserve(2);
+    lineStartOffsets_.infallibleAppend(0);
+    lineStartOffsets_.infallibleAppend(MAX_PTR);
+}
+
+JS_ALWAYS_INLINE void
+TokenStream::SourceCoords::add(uint32_t lineNum, uint32_t lineStartOffset)
+{
+    uint32_t lineIndex = lineNumToIndex(lineNum);
+    uint32_t sentinelIndex = lineStartOffsets_.length() - 1;
+
+    JS_ASSERT(lineStartOffsets_[0] == 0 && lineStartOffsets_[sentinelIndex] == MAX_PTR);
+
+    if (lineIndex == sentinelIndex) {
+        // We haven't seen this newline before.  Update lineStartOffsets_.
+        // We ignore any failures due to OOM -- because we always have a
+        // sentinel node, it'll just be like the newline wasn't present.  I.e.
+        // the line numbers will be wrong, but the code won't crash or anything
+        // like that.
+        lineStartOffsets_[lineIndex] = lineStartOffset;
+        (void)lineStartOffsets_.append(MAX_PTR);
+
+    } else {
+        // We have seen this newline before (and ungot it).  Do nothing (other
+        // than checking it hasn't mysteriously changed).
+        JS_ASSERT(lineStartOffsets_[lineIndex] == lineStartOffset);
+    }
+}
+
+JS_ALWAYS_INLINE uint32_t
+TokenStream::SourceCoords::lineIndexOf(uint32_t offset) const
+{
+    uint32_t iMin, iMax, iMid;
+
+    if (lineStartOffsets_[lastLineIndex_] <= offset) {
+        // If we reach here, offset is on a line the same as or higher than
+        // last time.  Check first for the +0, +1, +2 cases, because they
+        // typically cover 85--98% of cases.
+        if (offset < lineStartOffsets_[lastLineIndex_ + 1])
+            return lastLineIndex_;      // lineIndex is same as last time
+
+        // If we reach here, there must be at least one more entry (plus the
+        // sentinel).  Try it.
+        lastLineIndex_++;
+        if (offset < lineStartOffsets_[lastLineIndex_ + 1])
+            return lastLineIndex_;      // lineIndex is one higher than last time
+
+        // The same logic applies here.
+        lastLineIndex_++;
+        if (offset < lineStartOffsets_[lastLineIndex_ + 1]) {
+            return lastLineIndex_;      // lineIndex is two higher than last time
+        }
+
+        // No luck.  Oh well, we have a better-than-default starting point for
+        // the binary search.
+        iMin = lastLineIndex_ + 1;
+        JS_ASSERT(iMin < lineStartOffsets_.length() - 1);   // -1 due to the sentinel
+
+    } else {
+        iMin = 0;
+    }
+
+    // This is a binary search with deferred detection of equality, which was
+    // marginally faster in this case than a standard binary search.
+    // The -2 is because |lineStartOffsets_.length() - 1| is the sentinel, and we
+    // want one before that.
+    iMax = lineStartOffsets_.length() - 2;
+    while (iMax > iMin) {
+        iMid = (iMin + iMax) / 2;
+        if (offset >= lineStartOffsets_[iMid + 1])
+            iMin = iMid + 1;    // offset is above lineStartOffsets_[iMid]
+        else
+            iMax = iMid;        // offset is below or within lineStartOffsets_[iMid]
+    }
+    JS_ASSERT(iMax == iMin);
+    JS_ASSERT(lineStartOffsets_[iMin] <= offset && offset < lineStartOffsets_[iMin + 1]);
+    lastLineIndex_ = iMin;
+    return iMin;
+}
+
+uint32_t
+TokenStream::SourceCoords::lineNum(uint32_t offset) const
+{
+    uint32_t lineIndex = lineIndexOf(offset);
+    return lineIndexToNum(lineIndex);
+}
+
+uint32_t
+TokenStream::SourceCoords::columnIndex(uint32_t offset) const
+{
+    uint32_t lineIndex = lineIndexOf(offset);
+    uint32_t lineStartOffset = lineStartOffsets_[lineIndex];
+    JS_ASSERT(offset >= lineStartOffset);
+    return offset - lineStartOffset;
+}
+
+void
+TokenStream::SourceCoords::lineNumAndColumnIndex(uint32_t offset, uint32_t *lineNum,
+                                                 uint32_t *columnIndex) const
+{
+    uint32_t lineIndex = lineIndexOf(offset);
+    *lineNum = lineIndexToNum(lineIndex);
+    uint32_t lineStartOffset = lineStartOffsets_[lineIndex];
+    JS_ASSERT(offset >= lineStartOffset);
+    *columnIndex = offset - lineStartOffset;
+}
+
 #ifdef _MSC_VER
 #pragma warning(push)
 #pragma warning(disable:4351)
 #endif
 
 /* Initialize members that aren't initialized in |init|. */
 TokenStream::TokenStream(JSContext *cx, const CompileOptions &options,
                          const jschar *base, size_t length, StrictModeGetter *smg)
-  : tokens(),
+  : srcCoords(cx, options.lineno),
+    tokens(),
     cursor(),
     lookahead(),
     lineno(options.lineno),
     flags(),
     linebase(base),
     prevLinebase(NULL),
     userbuf(cx, base, length),
     filename(options.filename),
@@ -143,19 +260,19 @@ TokenStream::TokenStream(JSContext *cx, 
     void *listenerData = cx->runtime->debugHooks.sourceHandlerData;
 
     if (listener)
         listener(options.filename, options.lineno, base, length, &listenerTSData, listenerData);
 
     /*
      * This table holds all the token kinds that satisfy these properties:
      * - A single char long.
-     * - Cannot be a prefix of any longer token (eg. '+' is excluded because
+     * - Cannot be a prefix of any longer token (e.g. '+' is excluded because
      *   '+=' is a valid token).
-     * - Doesn't need tp->t_op set (eg. this excludes '~').
+     * - Doesn't need tp->t_op set (e.g. this excludes '~').
      *
      * The few token kinds satisfying these properties cover roughly 35--45%
      * of the tokens seen in practice.
      *
      * Nb: oneCharTokens, maybeEOL and maybeStrSpecial could be static, but
      * initializing them this way is a bit easier.  Don't worry, the time to
      * initialize them for each TokenStream is trivial.  See bug 639420.
      */
@@ -182,28 +299,16 @@ TokenStream::TokenStream(JSContext *cx, 
     maybeStrSpecial[unsigned('"')] = true;
     maybeStrSpecial[unsigned('\'')] = true;
     maybeStrSpecial[unsigned('\\')] = true;
     maybeStrSpecial[unsigned('\n')] = true;
     maybeStrSpecial[unsigned('\r')] = true;
     maybeStrSpecial[unsigned(LINE_SEPARATOR & 0xff)] = true;
     maybeStrSpecial[unsigned(PARA_SEPARATOR & 0xff)] = true;
     maybeStrSpecial[unsigned(EOF & 0xff)] = true;
-
-    /*
-     * Set |ln| as the beginning line number of the ungot "current token", so
-     * that js::Parser::statements (and potentially other such methods, in the
-     * future) can create parse nodes with good source coordinates before they
-     * explicitly get any tokens.
-     *
-     * Switching the parser/lexer so we always get the next token ahead of the
-     * parser needing it (the so-called "pump-priming" model) might be a better
-     * way to address the dependency from statements on the current token.
-     */
-    tokens[0].pos.begin.lineno = tokens[0].pos.end.lineno = options.lineno;
 }
 
 #ifdef _MSC_VER
 #pragma warning(pop)
 #endif
 
 TokenStream::~TokenStream()
 {
@@ -223,16 +328,17 @@ TokenStream::~TokenStream()
 #endif
 
 JS_ALWAYS_INLINE void
 TokenStream::updateLineInfoForEOL()
 {
     prevLinebase = linebase;
     linebase = userbuf.addressOfNextRawChar();
     lineno++;
+    srcCoords.add(lineno, linebase - userbuf.base());
 }
 
 JS_ALWAYS_INLINE void
 TokenStream::updateFlagsForEOL()
 {
     flags &= ~TSF_DIRTYLINE;
     flags |= TSF_EOL;
 }
@@ -412,29 +518,29 @@ TokenStream::seek(const Position &pos)
 void
 TokenStream::positionAfterLastFunctionKeyword(Position &pos)
 {
     JS_ASSERT(lastFunctionKeyword.buf > userbuf.base());
     PodAssign(&pos, &lastFunctionKeyword);
 }
 
 bool
-TokenStream::reportStrictModeErrorNumberVA(const TokenPos &pos, bool strictMode, unsigned errorNumber,
+TokenStream::reportStrictModeErrorNumberVA(uint32_t offset, bool strictMode, unsigned errorNumber,
                                            va_list args)
 {
     /* In strict mode code, this is an error, not merely a warning. */
     unsigned flags = JSREPORT_STRICT;
     if (strictMode)
         flags |= JSREPORT_ERROR;
     else if (cx->hasStrictOption())
         flags |= JSREPORT_WARNING;
     else
         return true;
- 
-    return reportCompileErrorNumberVA(pos, flags, errorNumber, args);
+
+    return reportCompileErrorNumberVA(offset, flags, errorNumber, args);
 }
 
 void
 CompileError::throwError()
 {
     /*
      * If there's a runtime exception type associated with this error
      * number, set that as the pending exception.  For errors occuring at
@@ -480,71 +586,70 @@ CompileError::~CompileError()
         }
         js_free(report.messageArgs);
     }
 
     PodZero(&report);
 }
 
 bool
-TokenStream::reportCompileErrorNumberVA(const TokenPos &pos, unsigned flags, unsigned errorNumber,
+TokenStream::reportCompileErrorNumberVA(uint32_t offset, unsigned flags, unsigned errorNumber,
                                         va_list args)
 {
     bool warning = JSREPORT_IS_WARNING(flags);
 
     if (warning && cx->hasWErrorOption()) {
         flags &= ~JSREPORT_WARNING;
         warning = false;
     }
 
     CompileError err(cx);
 
     err.report.flags = flags;
     err.report.errorNumber = errorNumber;
     err.report.filename = filename;
     err.report.originPrincipals = originPrincipals;
-    err.report.lineno = pos.begin.lineno;
+    err.report.lineno = srcCoords.lineNum(offset);
 
     err.argumentsType = (flags & JSREPORT_UC) ? ArgumentsAreUnicode : ArgumentsAreASCII;
 
     if (!js_ExpandErrorArguments(cx, js_GetErrorMessage, NULL, errorNumber, &err.message,
                                  &err.report, err.argumentsType, args))
     {
         return false;
     }
 
     /*
      * Given a token, T, that we want to complain about: if T's (starting)
      * lineno doesn't match TokenStream's lineno, that means we've scanned past
      * the line that T starts on, which makes it hard to print some or all of
      * T's (starting) line for context.
      *
      * So we don't even try, leaving report.linebuf and friends zeroed.  This
-     * means that any error involving a multi-line token (eg. an unterminated
+     * means that any error involving a multi-line token (e.g. an unterminated
      * multi-line string literal) won't have a context printed.
      */
     if (err.report.lineno == lineno) {
-        const jschar *tokptr = linebase + pos.begin.index;
+        const jschar *tokenStart = userbuf.base() + offset;
 
         // We show only a portion (a "window") of the line around the erroneous
         // token -- the first char in the token, plus |windowRadius| chars
         // before it and |windowRadius - 1| chars after it.  This is because
         // lines can be very long and printing the whole line is (a) not that
         // helpful, and (b) can waste a lot of memory.  See bug 634444.
         static const size_t windowRadius = 60;
 
         // Truncate at the front if necessary.
-        const jschar *windowBase = (linebase + windowRadius < tokptr)
-                                 ? tokptr - windowRadius
+        const jschar *windowBase = (linebase + windowRadius < tokenStart)
+                                 ? tokenStart - windowRadius
                                  : linebase;
-        size_t nTrunc = windowBase - linebase;
-        uint32_t windowIndex = pos.begin.index - nTrunc;
+        uint32_t windowOffset = tokenStart - windowBase;
 
         // Find EOL, or truncate at the back if necessary.
-        const jschar *windowLimit = userbuf.findEOLMax(tokptr, windowRadius);
+        const jschar *windowLimit = userbuf.findEOLMax(tokenStart, windowRadius);
         size_t windowLength = windowLimit - windowBase;
         JS_ASSERT(windowLength <= windowRadius * 2);
 
         // Create the windowed strings.
         StringBuffer windowBuf(cx);
         if (!windowBuf.append(windowBase, windowLength) || !windowBuf.append((jschar)0))
             return false;
 
@@ -553,72 +658,73 @@ TokenStream::reportCompileErrorNumberVA(
         err.report.uclinebuf = windowBuf.extractWellSized();
         if (!err.report.uclinebuf)
             return false;
         TwoByteChars tbchars(err.report.uclinebuf, windowLength);
         err.report.linebuf = LossyTwoByteCharsToNewLatin1CharsZ(cx, tbchars).c_str();
         if (!err.report.linebuf)
             return false;
 
-        // The lineno check above means we should only see single-line tokens here.
-        JS_ASSERT(pos.begin.lineno == pos.end.lineno);
-        err.report.tokenptr = err.report.linebuf + windowIndex;
-        err.report.uctokenptr = err.report.uclinebuf + windowIndex;
+        err.report.tokenptr = err.report.linebuf + windowOffset;
+        err.report.uctokenptr = err.report.uclinebuf + windowOffset;
     }
 
     err.throwError();
 
     return warning;
 }
 
 bool
 TokenStream::reportStrictModeError(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
-    bool result = reportStrictModeErrorNumberVA(currentToken().pos, strictMode(), errorNumber, args);
+    bool result = reportStrictModeErrorNumberVA(currentToken().pos.begin, strictMode(),
+                                                errorNumber, args);
     va_end(args);
     return result;
 }
 
 bool
 TokenStream::reportError(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
-    bool result = reportCompileErrorNumberVA(currentToken().pos, JSREPORT_ERROR, errorNumber, args);
+    bool result = reportCompileErrorNumberVA(currentToken().pos.begin, JSREPORT_ERROR, errorNumber,
+                                             args);
     va_end(args);
     return result;
 }
 
 bool
 TokenStream::reportWarning(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
-    bool result = reportCompileErrorNumberVA(currentToken().pos, JSREPORT_WARNING, errorNumber, args);
+    bool result = reportCompileErrorNumberVA(currentToken().pos.begin, JSREPORT_WARNING,
+                                             errorNumber, args);
     va_end(args);
     return result;
 }
 
 bool
-TokenStream::reportStrictWarningErrorNumberVA(const TokenPos &pos, unsigned errorNumber, va_list args)
+TokenStream::reportStrictWarningErrorNumberVA(uint32_t offset, unsigned errorNumber, va_list args)
 {
     if (!cx->hasStrictOption())
         return true;
 
-    return reportCompileErrorNumberVA(pos, JSREPORT_STRICT | JSREPORT_WARNING, errorNumber, args);
+    return reportCompileErrorNumberVA(offset, JSREPORT_STRICT|JSREPORT_WARNING, errorNumber, args);
 }
 
 void
-TokenStream::reportAsmJSError(ParseNode *pn, unsigned errorNumber, ...)
+TokenStream::reportAsmJSError(uint32_t offset, unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
-    reportCompileErrorNumberVA(pn->pn_pos, JSREPORT_WARNING, errorNumber, args);
+    reportCompileErrorNumberVA(offset, JSREPORT_WARNING, errorNumber, args);
     va_end(args);
 }
 
 /*
  * We have encountered a '\': check for a Unicode escape sequence after it.
  * Return 'true' and the character code value (by value) if we found a
  * Unicode escape sequence.  Otherwise, return 'false'.  In both cases, do not
  * advance along the buffer.
@@ -656,41 +762,16 @@ TokenStream::matchUnicodeEscapeIdent(int
 {
     if (peekUnicodeEscape(cp) && IsIdentifierPart(*cp)) {
         skipChars(5);
         return true;
     }
     return false;
 }
 
-size_t
-TokenStream::endOffset(const Token &tok)
-{
-    uint32_t lineno = tok.pos.begin.lineno;
-    JS_ASSERT(lineno <= tok.pos.end.lineno);
-    const jschar *end;
-    if (lineno < tok.pos.end.lineno) {
-        TokenBuf buf(cx, tok.ptr, userbuf.addressOfNextRawChar() - userbuf.base());
-        for (; lineno < tok.pos.end.lineno; lineno++) {
-            jschar c;
-            do {
-                JS_ASSERT(buf.hasRawChars());
-                c = buf.getRawChar();
-            } while (!TokenBuf::isRawEOLChar(c));
-            if (c == '\r' && buf.hasRawChars())
-                buf.matchRawChar('\n');
-        }
-        end = buf.addressOfNextRawChar() + tok.pos.end.index;
-    } else {
-        end = tok.ptr + (tok.pos.end.index - tok.pos.begin.index);
-    }
-    JS_ASSERT(end <= userbuf.addressOfNextRawChar());
-    return end - userbuf.base();
-}
-
 /*
  * Helper function which returns true if the first length(q) characters in p are
  * the same as the characters in q.
  */
 static bool
 CharsMatch(const jschar *p, const char *q) {
     while (*q) {
         if (*p++ != *q++)
@@ -751,19 +832,21 @@ TokenStream::getAtSourceMappingURL(bool 
     return true;
 }
 
 Token *
 TokenStream::newToken(ptrdiff_t adjust)
 {
     cursor = (cursor + 1) & ntokensMask;
     Token *tp = &tokens[cursor];
-    tp->ptr = userbuf.addressOfNextRawChar() + adjust;
-    tp->pos.begin.index = tp->ptr - linebase;
-    tp->pos.begin.lineno = tp->pos.end.lineno = lineno;
+    tp->pos.begin = userbuf.addressOfNextRawChar() + adjust - userbuf.base();
+
+    // NOTE: tp->pos.end is not set until the very end of getTokenInternal().
+    MOZ_MAKE_MEM_UNDEFINED(&tp->pos.end, sizeof(tp->pos.end));
+
     return tp;
 }
 
 JS_ALWAYS_INLINE JSAtom *
 TokenStream::atomize(JSContext *cx, CharBuffer &cb)
 {
     return AtomizeChars<CanGC>(cx, cb.begin(), cb.length());
 }
@@ -774,24 +857,19 @@ IsTokenSane(Token *tp)
 {
     /*
      * Nb: TOK_EOL should never be used in an actual Token;  it should only be
      * returned as a TokenKind from peekTokenSameLine().
      */
     if (tp->type < TOK_ERROR || tp->type >= TOK_LIMIT || tp->type == TOK_EOL)
         return false;
 
-    if (tp->pos.begin.lineno == tp->pos.end.lineno) {
-        if (tp->pos.begin.index > tp->pos.end.index)
-            return false;
-    } else {
-        /* Only string tokens can be multi-line. */
-        if (tp->type != TOK_STRING)
-            return false;
-    }
+    if (tp->pos.end < tp->pos.begin)
+        return false;
+
     return true;
 }
 #endif
 
 bool
 TokenStream::putIdentInTokenbuf(const jschar *identStart)
 {
     int32_t c, qc;
@@ -1183,17 +1261,16 @@ TokenStream::getTokenInternal()
                 }
             }
             if (!tokenbuf.append(c))
                 goto error;
         }
         JSAtom *atom = atomize(cx, tokenbuf);
         if (!atom)
             goto error;
-        tp->pos.end.lineno = lineno;
         tp->setAtom(JSOP_STRING, atom);
         tt = TOK_STRING;
         goto out;
     }
 
     /*
      * Look for a decimal number.
      */
@@ -1532,17 +1609,17 @@ TokenStream::getTokenInternal()
                     break;
                 getChar();
                 length++;
             }
 
             c = peekChar();
             if (JS7_ISLET(c)) {
                 char buf[2] = { '\0', '\0' };
-                tp->pos.begin.index += length + 1;
+                tp->pos.begin += length + 1;
                 buf[0] = char(c);
                 reportError(JSMSG_BAD_REGEXP_FLAG, buf);
                 (void) getChar();
                 goto error;
             }
             tp->setRegExpFlags(reflags);
             tt = TOK_REGEXP;
             break;
@@ -1581,30 +1658,24 @@ TokenStream::getTokenInternal()
       badchar:
       default:
         reportError(JSMSG_ILLEGAL_CHARACTER);
         goto error;
     }
 
   out:
     flags |= TSF_DIRTYLINE;
-    tp->pos.end.index = userbuf.addressOfNextRawChar() - linebase;
+    tp->pos.end = userbuf.addressOfNextRawChar() - userbuf.base();
     tp->type = tt;
     JS_ASSERT(IsTokenSane(tp));
     return tt;
 
   error:
-    /*
-     * For erroneous multi-line tokens we won't have changed end.lineno (it'll
-     * still be equal to begin.lineno) so we revert end.index to be equal to
-     * begin.index + 1 (as if it's a 1-char token) to avoid having inconsistent
-     * begin/end positions.  end.index isn't used in error messages anyway.
-     */
     flags |= TSF_DIRTYLINE;
-    tp->pos.end.index = tp->pos.begin.index + 1;
+    tp->pos.end = userbuf.addressOfNextRawChar() - userbuf.base();
     tp->type = TOK_ERROR;
     JS_ASSERT(IsTokenSane(tp));
     onError();
     return TOK_ERROR;
 }
 
 void
 TokenStream::onError()
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -173,52 +173,21 @@ TokenKindIsDecl(TokenKind tt)
 {
 #if JS_HAS_BLOCK_SCOPE
     return tt == TOK_VAR || tt == TOK_LET;
 #else
     return tt == TOK_VAR;
 #endif
 }
 
-struct TokenPtr {
-    uint32_t            index;          /* index of char in physical line */
-    uint32_t            lineno;         /* physical line number */
-
-    bool operator==(const TokenPtr& bptr) const {
-        return index == bptr.index && lineno == bptr.lineno;
-    }
-
-    bool operator!=(const TokenPtr& bptr) const {
-        return index != bptr.index || lineno != bptr.lineno;
-    }
-
-    bool operator <(const TokenPtr& bptr) const {
-        return lineno < bptr.lineno ||
-               (lineno == bptr.lineno && index < bptr.index);
-    }
+struct TokenPos {
+    uint32_t          begin;          /* offset of the token's first char */
+    uint32_t          end;            /* offset of 1 past the token's last char */
 
-    bool operator <=(const TokenPtr& bptr) const {
-        return lineno < bptr.lineno ||
-               (lineno == bptr.lineno && index <= bptr.index);
-    }
-
-    bool operator >(const TokenPtr& bptr) const {
-        return !(*this <= bptr);
-    }
-
-    bool operator >=(const TokenPtr& bptr) const {
-        return !(*this < bptr);
-    }
-};
-
-struct TokenPos {
-    TokenPtr          begin;          /* first character and line of token */
-    TokenPtr          end;            /* index 1 past last char, last line */
-
-    static TokenPos make(const TokenPtr &begin, const TokenPtr &end) {
+    static TokenPos make(uint32_t begin, uint32_t end) {
         JS_ASSERT(begin <= end);
         TokenPos pos = {begin, end};
         return pos;
     }
 
     /* Return a TokenPos that covers left, right, and anything in between. */
     static TokenPos box(const TokenPos &left, const TokenPos &right) {
         JS_ASSERT(left.begin <= left.end);
@@ -257,17 +226,16 @@ struct TokenPos {
     }
 };
 
 enum DecimalPoint { NoDecimal = false, HasDecimal = true };
 
 struct Token {
     TokenKind           type;           /* char value or above enumerator */
     TokenPos            pos;            /* token position in file */
-    const jschar        *ptr;           /* beginning of token in line buffer */
     union {
         struct {                        /* name or string literal */
             JSOp        op;             /* operator, for minimal parser */
             union {
               private:
                 friend struct Token;
                 PropertyName *name;     /* non-numeric atom */
                 JSAtom       *atom;     /* potentially-numeric atom */
@@ -429,28 +397,25 @@ class TokenStream
 
     TokenStream(JSContext *cx, const CompileOptions &options,
                 const jschar *base, size_t length, StrictModeGetter *smg);
 
     ~TokenStream();
 
     /* Accessors. */
     JSContext *getContext() const { return cx; }
-    bool onCurrentLine(const TokenPos &pos) const { return lineno == pos.end.lineno; }
+    bool onCurrentLine(const TokenPos &pos) const { return srcCoords.isOnThisLine(pos.end, lineno); }
     const Token &currentToken() const { return tokens[cursor]; }
     bool isCurrentTokenType(TokenKind type) const {
         return currentToken().type == type;
     }
     bool isCurrentTokenType(TokenKind type1, TokenKind type2) const {
         TokenKind type = currentToken().type;
         return type == type1 || type == type2;
     }
-    size_t offsetOfToken(const Token &tok) const {
-        return tok.ptr - userbuf.base();
-    }
     const CharBuffer &getTokenbuf() const { return tokenbuf; }
     const char *getFilename() const { return filename; }
     unsigned getLineno() const { return lineno; }
     JSVersion versionNumber() const { return VersionNumber(version); }
     JSVersion versionWithFlags() const { return version; }
     bool hadError() const { return !!(flags & TSF_HAD_ERROR); }
 
     bool isCurrentTokenEquality() const {
@@ -478,25 +443,25 @@ class TokenStream
 
     // TokenStream-specific error reporters.
     bool reportError(unsigned errorNumber, ...);
     bool reportWarning(unsigned errorNumber, ...);
 
     // General-purpose error reporters.  You should avoid calling these
     // directly, and instead use the more succinct alternatives (e.g.
     // reportError()) in TokenStream, Parser, and BytecodeEmitter.
-    bool reportCompileErrorNumberVA(const TokenPos &pos, unsigned flags, unsigned errorNumber,
+    bool reportCompileErrorNumberVA(uint32_t offset, unsigned flags, unsigned errorNumber,
                                     va_list args);
-    bool reportStrictModeErrorNumberVA(const TokenPos &pos, bool strictMode, unsigned errorNumber,
+    bool reportStrictModeErrorNumberVA(uint32_t offset, bool strictMode, unsigned errorNumber,
                                        va_list args);
-    bool reportStrictWarningErrorNumberVA(const TokenPos &pos, unsigned errorNumber,
+    bool reportStrictWarningErrorNumberVA(uint32_t offset, unsigned errorNumber,
                                           va_list args);
 
     // asm.js reporter
-    void reportAsmJSError(ParseNode *pn, unsigned errorNumber, ...);
+    void reportAsmJSError(uint32_t offset, unsigned errorNumber, ...);
 
   private:
     // These are private because they should only be called by the tokenizer
     // while tokenizing not by, for example, BytecodeEmitter.
     bool reportStrictModeError(unsigned errorNumber, ...);
     bool strictMode() const { return strictModeGetter && strictModeGetter->strictMode(); }
 
     void onError();
@@ -555,42 +520,35 @@ class TokenStream
      */
     void ungetToken() {
         JS_ASSERT(lookahead < ntokensMask);
         lookahead++;
         cursor = (cursor - 1) & ntokensMask;
     }
 
     TokenKind peekToken() {
-        if (lookahead != 0) {
-            JS_ASSERT(lookahead <= maxLookahead);
+        if (lookahead != 0)
             return tokens[(cursor + 1) & ntokensMask].type;
-        }
         TokenKind tt = getTokenInternal();
         ungetToken();
         return tt;
     }
 
     TokenKind peekToken(unsigned withFlags) {
         Flagger flagger(this, withFlags);
         return peekToken();
     }
 
     TokenKind peekTokenSameLine(unsigned withFlags = 0) {
-        if (lookahead != 0) {
-            JS_ASSERT(lookahead <= maxLookahead);
-            Token &nextToken = tokens[(cursor + 1) & ntokensMask];
-            return currentToken().pos.end.lineno == nextToken.pos.begin.lineno
-                   ? nextToken.type
-                   : TOK_EOL;
-        }
-
         if (!onCurrentLine(currentToken().pos))
             return TOK_EOL;
 
+        if (lookahead != 0)
+            return tokens[(cursor + 1) & ntokensMask].type;
+
         /*
          * This is the only place TOK_EOL is produced.  No token with TOK_EOL
          * is created, just a TOK_EOL TokenKind is returned.
          */
         flags &= ~TSF_EOL;
         TokenKind tt = getToken(withFlags);
         if (flags & TSF_EOL) {
             tt = TOK_EOL;
@@ -630,21 +588,16 @@ class TokenStream
         unsigned lookahead;
         Token lookaheadTokens[maxLookahead];
     };
 
     void tell(Position *);
     void seek(const Position &pos);
     void positionAfterLastFunctionKeyword(Position &pos);
 
-    /*
-     * Return the offset into the source buffer of the end of the token.
-     */
-    size_t endOffset(const Token &tok);
-
     size_t positionToOffset(const Position &pos) const {
         return pos.buf - userbuf.base();
     }
 
     bool hasSourceMap() const {
         return sourceMap != NULL;
     }
 
@@ -669,16 +622,85 @@ class TokenStream
      * and topp are null, report a SyntaxError ("if is a reserved identifier")
      * and return false. If ttp and topp are non-null, return true with the
      * keyword's TokenKind in *ttp and its JSOp in *topp.
      *
      * ttp and topp must be either both null or both non-null.
      */
     bool checkForKeyword(const jschar *s, size_t length, TokenKind *ttp, JSOp *topp);
 
+    // This class maps a userbuf offset (which is 0-indexed) to a line number
+    // (which is 1-indexed) and a column index (which is 0-indexed).
+    class SourceCoords
+    {
+        // For a given buffer holding source code, |lineStartOffsets_| has one
+        // element per line of source code, plus one sentinel element.  Each
+        // non-sentinel element holds the buffer offset for the start of the
+        // corresponding line of source code.  For this example script:
+        //
+        // 1  // xyz            [line starts at offset 0]
+        // 2  var x;            [line starts at offset 7]
+        // 3                    [line starts at offset 14]
+        // 4  var y;            [line starts at offset 15]
+        //
+        // |lineStartOffsets_| is:
+        //
+        //   [0, 7, 14, 15, MAX_PTR]
+        //
+        // To convert a "line number" to a "line index" (i.e. an index into
+        // |lineStartOffsets_|), subtract |initialLineNum_|.  E.g. line 3's
+        // line index is (3 - initialLineNum_), which is 2.  Therefore
+        // lineStartOffsets_[2] holds the buffer offset for the start of line 3,
+        // which is 14.  (Note that |initialLineNum_| is often 1, but not
+        // always.)
+        //
+        // The first element is always 0, and the last element is always the
+        // MAX_PTR sentinel.
+        //
+        // offset-to-line/column lookups are O(log n) in the worst case (binary
+        // search), but in practice they're heavily clustered and we do better
+        // than that by using the previous lookup's result (lastLineIndex_) as
+        // a starting point.
+        //
+        // Checking if an offset lies within a particular line number
+        // (isOnThisLine()) is O(1).
+        //
+        Vector<uint32_t, 128> lineStartOffsets_;
+        uint32_t            initialLineNum_;
+
+        // This is mutable because it's modified on every search, but that fact
+        // isn't visible outside this class.
+        mutable uint32_t    lastLineIndex_;
+
+        uint32_t lineIndexOf(uint32_t offset) const;
+
+        static const uint32_t MAX_PTR = UINT32_MAX;
+
+        uint32_t lineIndexToNum(uint32_t lineIndex) const { return lineIndex + initialLineNum_; }
+        uint32_t lineNumToIndex(uint32_t lineNum)   const { return lineNum   - initialLineNum_; }
+
+      public:
+        SourceCoords(JSContext *cx, uint32_t ln);
+
+        void add(uint32_t lineNum, uint32_t lineStartOffset);
+
+        bool isOnThisLine(uint32_t offset, uint32_t lineNum) const {
+            uint32_t lineIndex = lineNumToIndex(lineNum);
+            JS_ASSERT(lineIndex + 1 < lineStartOffsets_.length());  // +1 due to sentinel
+            return lineStartOffsets_[lineIndex] <= offset &&
+                   offset < lineStartOffsets_[lineIndex + 1];
+        }
+
+        uint32_t lineNum(uint32_t offset) const;
+        uint32_t columnIndex(uint32_t offset) const;
+        void lineNumAndColumnIndex(uint32_t offset, uint32_t *lineNum, uint32_t *columnIndex) const;
+    };
+
+    SourceCoords srcCoords;
+
   private:
     /*
      * This is the low-level interface to the JS source code buffer.  It just
      * gets raw chars, basically.  TokenStreams functions are layered on top
      * and do some extra stuff like converting all EOL sequences to '\n',
      * tracking the line number, and setting the TSF_EOF flag.  (The "raw" in
      * "raw chars" refers to the lack of EOL sequence normalization.)
      */
--- a/js/src/ion/AsmJS.cpp
+++ b/js/src/ion/AsmJS.cpp
@@ -1087,17 +1087,18 @@ class ModuleCompiler
         errorString_(NULL),
         errorNode_(NULL),
         tokenStream_(ts),
         currentPass_(1)
     {}
 
     ~ModuleCompiler() {
         if (errorString_)
-            tokenStream_.reportAsmJSError(errorNode_, JSMSG_USE_ASM_TYPE_FAIL, errorString_);
+            tokenStream_.reportAsmJSError(errorNode_->pn_pos.begin, JSMSG_USE_ASM_TYPE_FAIL,
+                                          errorString_);
 
         // Avoid spurious Label assertions on compilation failure.
         if (!stackOverflowLabel_.bound())
             stackOverflowLabel_.bind(0);
         if (!operationCallbackLabel_.bound())
             operationCallbackLabel_.bind(0);
     }
 
--- a/js/src/jsfun.cpp
+++ b/js/src/jsfun.cpp
@@ -596,17 +596,17 @@ FindBody(JSContext *cx, HandleFunction f
     } while (onward);
     TokenKind tt = ts.getToken();
     if (tt == TOK_ARROW)
         tt = ts.getToken();
     if (tt == TOK_ERROR)
         return false;
     bool braced = tt == TOK_LC;
     JS_ASSERT_IF(fun->isExprClosure(), !braced);
-    *bodyStart = ts.offsetOfToken(ts.currentToken());
+    *bodyStart = ts.currentToken().pos.begin;
     if (braced)
         *bodyStart += 1;
     StableCharPtr end(chars.get() + length, chars.get(), length);
     if (end[-1] == '}') {
         end--;
     } else {
         JS_ASSERT(!braced);
         for (; unicode::IsSpaceOrBOM2(end[-1]); end--)
--- a/js/src/jsprvtd.h
+++ b/js/src/jsprvtd.h
@@ -153,17 +153,16 @@ typedef JSPropertyDescriptor PropertyDes
 namespace frontend {
 
 struct BytecodeEmitter;
 struct Definition;
 class FunctionBox;
 class ObjectBox;
 struct Token;
 struct TokenPos;
-struct TokenPtr;
 class TokenStream;
 class ParseMapPool;
 struct ParseNode;
 
 template <typename ParseHandler>
 struct Parser;
 
 } /* namespace frontend */
--- a/js/src/jsreflect.cpp
+++ b/js/src/jsreflect.cpp
@@ -129,27 +129,28 @@ typedef AutoValueVector NodeVector;
  *
  *     https://developer.mozilla.org/en/SpiderMonkey/Parser_API
  *
  * Bug 569487: generalize builder interface
  */
 class NodeBuilder
 {
     JSContext   *cx;
+    TokenStream *tokenStream;
     bool        saveLoc;               /* save source location information?     */
     char const  *src;                  /* source filename or null               */
     RootedValue srcval;                /* source filename JS value or null      */
     Value       callbacks[AST_LIMIT];  /* user-specified callbacks              */
     AutoValueArray callbacksRoots;     /* for rooting |callbacks|               */
     RootedValue userv;                 /* user-specified builder object or null */
     RootedValue undefinedVal;          /* a rooted undefined val, used by opt() */
 
   public:
     NodeBuilder(JSContext *c, bool l, char const *s)
-        : cx(c), saveLoc(l), src(s), srcval(c),
+        : cx(c), tokenStream(NULL), saveLoc(l), src(s), srcval(c),
           callbacksRoots(c, callbacks, AST_LIMIT), userv(c), undefinedVal(c, UndefinedValue())
     {
         MakeRangeGCSafe(callbacks, mozilla::ArrayLength(callbacks));
     }
 
     bool init(HandleObject userobj = NullPtr()) {
         if (src) {
             if (!atomValue(src, &srcval))
@@ -191,16 +192,20 @@ class NodeBuilder
             }
 
             callbacks[i] = funv;
         }
 
         return true;
     }
 
+    void setTokenStream(TokenStream *ts) {
+        tokenStream = ts;
+    }
+
   private:
     bool callback(HandleValue fun, TokenPos *pos, MutableHandleValue dst) {
         if (saveLoc) {
             RootedValue loc(cx);
             if (!newNodeLoc(pos, &loc))
                 return false;
             Value argv[] = { loc };
             AutoValueArray ava(cx, argv, 1);
@@ -671,37 +676,42 @@ NodeBuilder::newNodeLoc(TokenPos *pos, M
     RootedObject to(cx);
     RootedValue val(cx);
 
     if (!newObject(&loc))
         return false;
 
     dst.setObject(*loc);
 
+    uint32_t startLineNum, startColumnIndex;
+    uint32_t endLineNum, endColumnIndex;
+    tokenStream->srcCoords.lineNumAndColumnIndex(pos->begin, &startLineNum, &startColumnIndex);
+    tokenStream->srcCoords.lineNumAndColumnIndex(pos->end, &endLineNum, &endColumnIndex);
+
     if (!newObject(&to))
         return false;
     val.setObject(*to);
     if (!setProperty(loc, "start", val))
         return false;
-    val.setNumber(pos->begin.lineno);
+    val.setNumber(startLineNum);
     if (!setProperty(to, "line", val))
         return false;
-    val.setNumber(pos->begin.index);
+    val.setNumber(startColumnIndex);
     if (!setProperty(to, "column", val))
         return false;
 
     if (!newObject(&to))
         return false;
     val.setObject(*to);
     if (!setProperty(loc, "end", val))
         return false;
-    val.setNumber(pos->end.lineno);
+    val.setNumber(endLineNum);
     if (!setProperty(to, "line", val))
         return false;
-    val.setNumber(pos->end.index);
+    val.setNumber(endColumnIndex);
     if (!setProperty(to, "column", val))
         return false;
 
     if (!setProperty(loc, "source", srcval))
         return false;
 
     return true;
 }
@@ -1548,16 +1558,17 @@ class ASTSerializer
     {}
 
     bool init(HandleObject userobj) {
         return builder.init(userobj);
     }
 
     void setParser(Parser<FullParseHandler> *p) {
         parser = p;
+        builder.setTokenStream(&p->tokenStream);
     }
 
     bool program(ParseNode *pn, MutableHandleValue dst);
 };
 
 AssignmentOperator
 ASTSerializer::aop(JSOp op)
 {
@@ -1714,17 +1725,17 @@ ASTSerializer::blockStatement(ParseNode 
     NodeVector stmts(cx);
     return statements(pn, stmts) &&
            builder.blockStatement(stmts, &pn->pn_pos, dst);
 }
 
 bool
 ASTSerializer::program(ParseNode *pn, MutableHandleValue dst)
 {
-    JS_ASSERT(pn->pn_pos.begin.lineno == lineno);
+    JS_ASSERT(parser->tokenStream.srcCoords.lineNum(pn->pn_pos.begin) == lineno);
 
     NodeVector stmts(cx);
     return statements(pn, stmts) &&
            builder.program(stmts, &pn->pn_pos, dst);
 }
 
 bool
 ASTSerializer::sourceElement(ParseNode *pn, MutableHandleValue dst)