Bug 1424394 - Decouple TokenStream and TokenStreamBase from being related only through inheritance, but rather through a generalized static system. r=arai
authorJeff Walden <jwalden@mit.edu>
Tue, 05 Dec 2017 23:58:47 -0800
changeset 396283 7496d9ff6fa0ae03545ce206caba5490a26c9226
parent 396282 b2f626567ad247bf40c9bf4cea23026e1c92fa90
child 396284 5d3190b2b1bd77bdf476fa9c075be1829d8447d9
push id56975
push userdluca@mozilla.com
push dateThu, 14 Dec 2017 09:59:07 +0000
treeherderautoland@16bcfaad13e1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersarai
bugs1424394
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1424394 - Decouple TokenStream and TokenStreamBase from being related only through inheritance, but rather through a generalized static system. r=arai
js/src/devtools/rootAnalysis/annotations.js
js/src/frontend/TokenStream.cpp
js/src/frontend/TokenStream.h
--- a/js/src/devtools/rootAnalysis/annotations.js
+++ b/js/src/devtools/rootAnalysis/annotations.js
@@ -327,17 +327,16 @@ function extraRootedPointers()
     return [
         'ModuleValidator',
         'JSErrorResult',
         'WrappableJSErrorResult',
 
         // These are not actually rooted, but are only used in the context of
         // AutoKeepAtoms.
         'js::frontend::TokenStream',
-        'js::frontend::TokenStream::Position',
 
         'mozilla::ErrorResult',
         'mozilla::IgnoredErrorResult',
         'mozilla::dom::binding_detail::FastErrorResult',
     ];
 }
 
 function isRootedGCPointerTypeName(name)
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@@ -28,16 +28,17 @@
 #include "js/CharacterEncoding.h"
 #include "js/UniquePtr.h"
 #include "vm/HelperThreads.h"
 #include "vm/StringBuffer.h"
 #include "vm/Unicode.h"
 
 using mozilla::ArrayLength;
 using mozilla::Maybe;
+using mozilla::PodArrayZero;
 using mozilla::PodAssign;
 using mozilla::PodCopy;
 using mozilla::PodZero;
 
 struct ReservedWordInfo
 {
     const char* chars;         // C string with reserved word text
     js::frontend::TokenKind tokentype;
@@ -243,40 +244,40 @@ TokenStreamAnyChars::reservedWordToPrope
       FOR_EACH_JAVASCRIPT_RESERVED_WORD(EMIT_CASE)
 #undef EMIT_CASE
       default:
         MOZ_ASSERT_UNREACHABLE("Not a reserved word TokenKind.");
     }
     return nullptr;
 }
 
-TokenStream::SourceCoords::SourceCoords(JSContext* cx, uint32_t ln, uint32_t col,
-                                        uint32_t initialLineOffset)
+TokenStreamAnyChars::SourceCoords::SourceCoords(JSContext* cx, uint32_t ln, uint32_t col,
+                                                uint32_t initialLineOffset)
   : lineStartOffsets_(cx), initialLineNum_(ln), initialColumn_(col), lastLineIndex_(0)
 {
     // This is actually necessary!  Removing it causes compile errors on
     // GCC and clang.  You could try declaring this:
     //
-    //   const uint32_t TokenStream::SourceCoords::MAX_PTR;
+    //   const uint32_t TokenStreamAnyChars::SourceCoords::MAX_PTR;
     //
     // which fixes the GCC/clang error, but causes bustage on Windows.  Sigh.
     //
     uint32_t maxPtr = MAX_PTR;
 
     // The first line begins at buffer offset |initialLineOffset|.  MAX_PTR is
     // the sentinel.  The appends cannot fail because |lineStartOffsets_| has
     // statically-allocated elements.
     MOZ_ASSERT(lineStartOffsets_.capacity() >= 2);
     MOZ_ALWAYS_TRUE(lineStartOffsets_.reserve(2));
     lineStartOffsets_.infallibleAppend(initialLineOffset);
     lineStartOffsets_.infallibleAppend(maxPtr);
 }
 
 MOZ_ALWAYS_INLINE bool
-TokenStream::SourceCoords::add(uint32_t lineNum, uint32_t lineStartOffset)
+TokenStreamAnyChars::SourceCoords::add(uint32_t lineNum, uint32_t lineStartOffset)
 {
     uint32_t lineIndex = lineNumToIndex(lineNum);
     uint32_t sentinelIndex = lineStartOffsets_.length() - 1;
 
     MOZ_ASSERT(lineStartOffsets_[0] <= lineStartOffset &&
                lineStartOffsets_[sentinelIndex] == MAX_PTR);
 
     if (lineIndex == sentinelIndex) {
@@ -406,45 +407,58 @@ TokenStreamAnyChars::TokenStreamAnyChars
     options_(options),
     tokens(),
     cursor(),
     lookahead(),
     lineno(options.lineno),
     flags(),
     linebase(0),
     prevLinebase(size_t(-1)),
-    filename(options.filename()),
+    filename_(options.filename()),
     displayURL_(nullptr),
     sourceMapURL_(nullptr),
     cx(cx),
     mutedErrors(options.mutedErrors()),
     strictModeGetter(smg)
 {
-}
-
-TokenStream::TokenStream(JSContext* cx, const ReadOnlyCompileOptions& options,
-                         const CharT* base, size_t length, StrictModeGetter* smg)
-  : TokenStreamAnyChars(cx, options, smg),
-    userbuf(base, length, options.scriptSourceOffset),
-    tokenbuf(cx)
-{
     // Nb: the following tables could be static, but initializing them here is
     // much easier.  Don't worry, the time to initialize them for each
     // TokenStream is trivial.  See bug 639420.
 
     // See Parser::assignExpr() for an explanation of isExprEnding[].
-    memset(isExprEnding, 0, sizeof(isExprEnding));
+    PodArrayZero(isExprEnding);
     isExprEnding[TOK_COMMA] = 1;
-    isExprEnding[TOK_SEMI]  = 1;
+    isExprEnding[TOK_SEMI] = 1;
     isExprEnding[TOK_COLON] = 1;
-    isExprEnding[TOK_RP]    = 1;
-    isExprEnding[TOK_RB]    = 1;
-    isExprEnding[TOK_RC]    = 1;
+    isExprEnding[TOK_RP] = 1;
+    isExprEnding[TOK_RB] = 1;
+    isExprEnding[TOK_RC] = 1;
 }
 
+template<typename CharT>
+TokenStreamCharsBase<CharT>::TokenStreamCharsBase(JSContext* cx, const CharT* chars, size_t length,
+                                                  size_t startOffset)
+  : userbuf(chars, length, startOffset),
+    tokenbuf(cx)
+{}
+
+template<class AnyCharsAccess>
+TokenStreamChars<char16_t, AnyCharsAccess>::TokenStreamChars(JSContext* cx,
+                                                             const char16_t* base, size_t length,
+                                                             size_t startOffset)
+  : TokenStreamCharsBase<char16_t>(cx, base, length, startOffset)
+{}
+
+template<typename CharT, class AnyCharsAccess>
+TokenStreamSpecific<CharT, AnyCharsAccess>::TokenStreamSpecific(JSContext* cx,
+                                                                const ReadOnlyCompileOptions& options,
+                                                                const CharT* base, size_t length)
+  : TokenStreamChars<CharT, AnyCharsAccess>(cx, base, length, options.scriptSourceOffset)
+{}
+
 #ifdef _MSC_VER
 #pragma warning(pop)
 #endif
 
 bool
 TokenStreamAnyChars::checkOptions()
 {
     // Constrain starting columns to half of the range of a signed 32-bit value,
@@ -462,36 +476,46 @@ TokenStreamAnyChars::checkOptions()
 # define fast_getc getc_unlocked
 #elif defined(HAVE__GETC_NOLOCK)
 # define fast_getc _getc_nolock
 #else
 # define fast_getc getc
 #endif
 
 MOZ_MUST_USE MOZ_ALWAYS_INLINE bool
-TokenStream::updateLineInfoForEOL()
+TokenStreamAnyChars::internalUpdateLineInfoForEOL(uint32_t lineStartOffset)
 {
     prevLinebase = linebase;
-    linebase = userbuf.offset();
+    linebase = lineStartOffset;
     lineno++;
     return srcCoords.add(lineno, linebase);
 }
 
+template<typename CharT, class AnyCharsAccess>
+MOZ_MUST_USE MOZ_ALWAYS_INLINE bool
+TokenStreamSpecific<CharT, AnyCharsAccess>::updateLineInfoForEOL()
+{
+    return anyCharsAccess().internalUpdateLineInfoForEOL(userbuf.offset());
+}
+
 MOZ_ALWAYS_INLINE void
 TokenStreamAnyChars::updateFlagsForEOL()
 {
     flags.isDirtyLine = false;
 }
 
 // This gets the next char, normalizing all EOL sequences to '\n' as it goes.
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::getChar(int32_t* cp)
+TokenStreamSpecific<CharT, AnyCharsAccess>::getChar(int32_t* cp)
 {
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
+
     if (MOZ_UNLIKELY(!userbuf.hasRawChars())) {
-        flags.isEOF = true;
+        anyChars.flags.isEOF = true;
         *cp = EOF;
         return true;
     }
 
     int32_t c = userbuf.getRawChar();
 
     do {
         // Normalize the char16_t if it was a newline.
@@ -520,177 +544,209 @@ TokenStream::getChar(int32_t* cp)
     return true;
 }
 
 // This gets the next char. It does nothing special with EOL sequences, not
 // even updating the line counters.  It can be used safely if (a) the
 // resulting char is guaranteed to be ungotten (by ungetCharIgnoreEOL()) if
 // it's an EOL, and (b) the line-related state (lineno, linebase) is not used
 // before it's ungotten.
+template<typename CharT, class AnyCharsAccess>
 int32_t
-TokenStream::getCharIgnoreEOL()
+TokenStreamSpecific<CharT, AnyCharsAccess>::getCharIgnoreEOL()
 {
     if (MOZ_LIKELY(userbuf.hasRawChars()))
         return userbuf.getRawChar();
 
-    flags.isEOF = true;
+    anyCharsAccess().flags.isEOF = true;
     return EOF;
 }
 
 void
-TokenStream::ungetChar(int32_t c)
+TokenStreamAnyChars::undoGetChar()
+{
+    MOZ_ASSERT(prevLinebase != size_t(-1)); // we should never get more than one EOL
+    linebase = prevLinebase;
+    prevLinebase = size_t(-1);
+    lineno--;
+}
+
+template<typename CharT, class AnyCharsAccess>
+void
+TokenStreamSpecific<CharT, AnyCharsAccess>::ungetChar(int32_t c)
 {
     if (c == EOF)
         return;
+
     MOZ_ASSERT(!userbuf.atStart());
     userbuf.ungetRawChar();
     if (c == '\n') {
 #ifdef DEBUG
         int32_t c2 = userbuf.peekRawChar();
         MOZ_ASSERT(TokenBuf::isRawEOLChar(c2));
 #endif
 
         // If it's a \r\n sequence, also unget the \r.
         if (!userbuf.atStart())
             userbuf.matchRawCharBackwards('\r');
 
-        MOZ_ASSERT(prevLinebase != size_t(-1));    // we should never get more than one EOL char
-        linebase = prevLinebase;
-        prevLinebase = size_t(-1);
-        lineno--;
+        anyCharsAccess().undoGetChar();
     } else {
         MOZ_ASSERT(userbuf.peekRawChar() == c);
     }
 }
 
+template<typename CharT, class AnyCharsAccess>
 void
-TokenStream::ungetCharIgnoreEOL(int32_t c)
+TokenStreamSpecific<CharT, AnyCharsAccess>::ungetCharIgnoreEOL(int32_t c)
 {
     if (c == EOF)
         return;
+
     MOZ_ASSERT(!userbuf.atStart());
     userbuf.ungetRawChar();
 }
 
 // Return true iff |n| raw characters can be read from this without reading past
 // EOF or a newline, and copy those characters into |cp| if so.  The characters
 // are not consumed: use skipChars(n) to do so after checking that the consumed
 // characters had appropriate values.
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::peekChars(int n, char16_t* cp)
+TokenStreamSpecific<CharT, AnyCharsAccess>::peekChars(int n, CharT* cp)
 {
-    int i, j;
-    int32_t c;
-
+    int i;
     for (i = 0; i < n; i++) {
-        c = getCharIgnoreEOL();
+        int32_t c = getCharIgnoreEOL();
         if (c == EOF)
             break;
+
         if (c == '\n') {
             ungetCharIgnoreEOL(c);
             break;
         }
+
         cp[i] = char16_t(c);
     }
-    for (j = i - 1; j >= 0; j--)
+
+    for (int j = i - 1; j >= 0; j--)
         ungetCharIgnoreEOL(cp[j]);
+
     return i == n;
 }
 
+template<typename CharT>
 size_t
-TokenStream::TokenBuf::findEOLMax(size_t start, size_t max)
+TokenStreamCharsBase<CharT>::TokenBuf::findEOLMax(size_t start, size_t max)
 {
     const CharT* p = rawCharPtrAt(start);
 
     size_t n = 0;
     while (true) {
         if (p >= limit_)
             break;
         if (n >= max)
             break;
         n++;
         if (TokenBuf::isRawEOLChar(*p++))
             break;
     }
     return start + n;
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::advance(size_t position)
+TokenStreamSpecific<CharT, AnyCharsAccess>::advance(size_t position)
 {
     const CharT* end = userbuf.rawCharPtrAt(position);
     while (userbuf.addressOfNextRawChar() < end) {
         int32_t c;
         if (!getChar(&c))
             return false;
     }
 
-    Token* cur = &tokens[cursor];
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
+    Token* cur = &anyChars.tokens[anyChars.cursor];
     cur->pos.begin = userbuf.offset();
     MOZ_MAKE_MEM_UNDEFINED(&cur->type, sizeof(cur->type));
-    lookahead = 0;
+    anyChars.lookahead = 0;
     return true;
 }
 
+template<typename CharT, class AnyCharsAccess>
 void
-TokenStream::tell(Position* pos)
+TokenStreamSpecific<CharT, AnyCharsAccess>::tell(Position* pos)
 {
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
+
     pos->buf = userbuf.addressOfNextRawChar(/* allowPoisoned = */ true);
-    pos->flags = flags;
-    pos->lineno = lineno;
-    pos->linebase = linebase;
-    pos->prevLinebase = prevLinebase;
-    pos->lookahead = lookahead;
-    pos->currentToken = currentToken();
-    for (unsigned i = 0; i < lookahead; i++)
-        pos->lookaheadTokens[i] = tokens[(cursor + 1 + i) & ntokensMask];
+    pos->flags = anyChars.flags;
+    pos->lineno = anyChars.lineno;
+    pos->linebase = anyChars.linebase;
+    pos->prevLinebase = anyChars.prevLinebase;
+    pos->lookahead = anyChars.lookahead;
+    pos->currentToken = anyChars.currentToken();
+    for (unsigned i = 0; i < anyChars.lookahead; i++)
+        pos->lookaheadTokens[i] = anyChars.tokens[(anyChars.cursor + 1 + i) & ntokensMask];
 }
 
+template<typename CharT, class AnyCharsAccess>
 void
-TokenStream::seek(const Position& pos)
+TokenStreamSpecific<CharT, AnyCharsAccess>::seek(const Position& pos)
 {
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
+
     userbuf.setAddressOfNextRawChar(pos.buf, /* allowPoisoned = */ true);
-    flags = pos.flags;
-    lineno = pos.lineno;
-    linebase = pos.linebase;
-    prevLinebase = pos.prevLinebase;
-    lookahead = pos.lookahead;
+    anyChars.flags = pos.flags;
+    anyChars.lineno = pos.lineno;
+    anyChars.linebase = pos.linebase;
+    anyChars.prevLinebase = pos.prevLinebase;
+    anyChars.lookahead = pos.lookahead;
 
-    tokens[cursor] = pos.currentToken;
-    for (unsigned i = 0; i < lookahead; i++)
-        tokens[(cursor + 1 + i) & ntokensMask] = pos.lookaheadTokens[i];
+    anyChars.tokens[anyChars.cursor] = pos.currentToken;
+    for (unsigned i = 0; i < anyChars.lookahead; i++)
+        anyChars.tokens[(anyChars.cursor + 1 + i) & ntokensMask] = pos.lookaheadTokens[i];
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::seek(const Position& pos, const TokenStream& other)
+TokenStreamSpecific<CharT, AnyCharsAccess>::seek(const Position& pos,
+                                                 const TokenStreamSpecific& other)
 {
-    if (!srcCoords.fill(other.srcCoords))
+    if (!anyCharsAccess().srcCoords.fill(other.anyCharsAccess().srcCoords))
         return false;
+
     seek(pos);
     return true;
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::reportStrictModeErrorNumberVA(UniquePtr<JSErrorNotes> notes, uint32_t offset,
-                                           bool strictMode, unsigned errorNumber, va_list* args)
+TokenStreamSpecific<CharT, AnyCharsAccess>::reportStrictModeErrorNumberVA(UniquePtr<JSErrorNotes> notes,
+                                                                          uint32_t offset,
+                                                                          bool strictMode,
+                                                                          unsigned errorNumber,
+                                                                          va_list* args)
 {
-    if (!strictMode && !options().extraWarningsOption)
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
+    if (!strictMode && !anyChars.options().extraWarningsOption)
         return true;
 
     ErrorMetadata metadata;
     if (!computeErrorMetadata(&metadata, offset))
         return false;
 
     if (strictMode) {
-        ReportCompileError(cx, Move(metadata), Move(notes), JSREPORT_ERROR, errorNumber, *args);
+        ReportCompileError(anyChars.cx, Move(metadata), Move(notes), JSREPORT_ERROR, errorNumber,
+                           *args);
         return false;
     }
 
-    return compileWarning(Move(metadata), Move(notes), JSREPORT_WARNING | JSREPORT_STRICT,
-                          errorNumber, *args);
+    return anyChars.compileWarning(Move(metadata), Move(notes), JSREPORT_WARNING | JSREPORT_STRICT,
+                                   errorNumber, *args);
 }
 
 bool
 TokenStreamAnyChars::compileWarning(ErrorMetadata&& metadata, UniquePtr<JSErrorNotes> notes,
                                     unsigned flags, unsigned errorNumber, va_list args)
 {
     if (options().werrorOption) {
         flags &= ~JSREPORT_WARNING;
@@ -700,43 +756,43 @@ TokenStreamAnyChars::compileWarning(Erro
 
     return ReportCompileWarning(cx, Move(metadata), Move(notes), flags, errorNumber, args);
 }
 
 void
 TokenStreamAnyChars::computeErrorMetadataNoOffset(ErrorMetadata* err)
 {
     err->isMuted = mutedErrors;
-    err->filename = filename;
+    err->filename = filename_;
     err->lineNumber = 0;
     err->columnNumber = 0;
 
     MOZ_ASSERT(err->lineOfContext == nullptr);
 }
 
 bool
 TokenStreamAnyChars::fillExcludingContext(ErrorMetadata* err, uint32_t offset)
 {
     err->isMuted = mutedErrors;
 
     // If this TokenStreamAnyChars doesn't have location information, try to
     // get it from the caller.
-    if (!filename && !cx->helperThread()) {
+    if (!filename_ && !cx->helperThread()) {
         NonBuiltinFrameIter iter(cx,
                                  FrameIter::FOLLOW_DEBUGGER_EVAL_PREV_LINK,
                                  cx->compartment()->principals());
         if (!iter.done() && iter.filename()) {
             err->filename = iter.filename();
             err->lineNumber = iter.computeLine(&err->columnNumber);
             return false;
         }
     }
 
     // Otherwise use this TokenStreamAnyChars's location information.
-    err->filename = filename;
+    err->filename = filename_;
     srcCoords.lineNumAndColumnIndex(offset, &err->lineNumber, &err->columnNumber);
     return true;
 }
 
 bool
 TokenStreamAnyChars::hasTokenizationStarted() const
 {
     return isCurrentTokenType(TOK_EOF) && !isEOF();
@@ -750,183 +806,209 @@ TokenStreamAnyChars::lineAndColumnAt(siz
 
 void
 TokenStreamAnyChars::currentLineAndColumn(uint32_t* line, uint32_t* column) const
 {
     uint32_t offset = currentToken().pos.begin;
     srcCoords.lineNumAndColumnIndex(offset, line, column);
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::computeErrorMetadata(ErrorMetadata* err, uint32_t offset)
+TokenStreamSpecific<CharT, AnyCharsAccess>::computeErrorMetadata(ErrorMetadata* err,
+                                                                 uint32_t offset)
 {
     if (offset == NoOffset) {
-        computeErrorMetadataNoOffset(err);
+        anyCharsAccess().computeErrorMetadataNoOffset(err);
         return true;
     }
 
     // This function's return value isn't a success/failure indication: it
     // returns true if this TokenStream's location information could be used,
     // and it returns false when that information can't be used (and so we
     // can't provide a line of context).
-    if (!fillExcludingContext(err, offset))
+    if (!anyCharsAccess().fillExcludingContext(err, offset))
         return true;
 
     // Add a line of context from this TokenStream to help with debugging.
     return computeLineOfContext(err, offset);
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::computeLineOfContext(ErrorMetadata* err, uint32_t offset)
+TokenStreamSpecific<CharT, AnyCharsAccess>::computeLineOfContext(ErrorMetadata* err,
+                                                                 uint32_t offset)
 {
     // This function presumes |err| is filled in *except* for line-of-context
-    // fields.  It exists to make |TokenStream::computeErrorMetadata|, above,
-    // more readable.
+    // fields.  It exists to make |TokenStreamSpecific::computeErrorMetadata|,
+    // above, more readable.
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
 
     // We only have line-start information for the current line.  If the error
     // is on a different line, we can't easily provide context.  (This means
     // any error in a multi-line token, e.g. an unterminated multiline string
     // literal, won't have context.)
-    if (err->lineNumber != lineno)
+    if (err->lineNumber != anyChars.lineno)
         return true;
 
     constexpr size_t windowRadius = ErrorMetadata::lineOfContextRadius;
 
     // The window must start within the current line, no earlier than
     // |windowRadius| characters before |offset|.
-    MOZ_ASSERT(offset >= linebase);
-    size_t windowStart = (offset - linebase > windowRadius) ?
+    MOZ_ASSERT(offset >= anyChars.linebase);
+    size_t windowStart = (offset - anyChars.linebase > windowRadius) ?
                          offset - windowRadius :
-                         linebase;
+                         anyChars.linebase;
 
     // The window must start within the portion of the current line that we
     // actually have in our buffer.
     if (windowStart < userbuf.startOffset())
         windowStart = userbuf.startOffset();
 
     // The window must end within the current line, no later than
     // windowRadius after offset.
     size_t windowEnd = userbuf.findEOLMax(offset, windowRadius);
     size_t windowLength = windowEnd - windowStart;
     MOZ_ASSERT(windowLength <= windowRadius * 2);
 
     // Create the windowed string, not including the potential line
     // terminator.
-    StringBuffer windowBuf(cx);
-    if (!windowBuf.append(userbuf.rawCharPtrAt(windowStart), windowLength) ||
+    StringBuffer windowBuf(anyChars.cx);
+    if (!windowBuf.append(rawCharPtrAt(windowStart), windowLength) ||
         !windowBuf.append('\0'))
     {
         return false;
     }
 
     err->lineOfContext.reset(windowBuf.stealChars());
     if (!err->lineOfContext)
         return false;
 
     err->lineLength = windowLength;
     err->tokenOffset = offset - windowStart;
     return true;
 }
 
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::reportStrictModeError(unsigned errorNumber, ...)
+TokenStreamSpecific<CharT, AnyCharsAccess>::reportStrictModeError(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
-    bool result = reportStrictModeErrorNumberVA(nullptr, currentToken().pos.begin, strictMode(),
-                                                errorNumber, &args);
+
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
+    bool result = reportStrictModeErrorNumberVA(nullptr, anyChars.currentToken().pos.begin,
+                                                anyChars.strictMode(), errorNumber, &args);
+
     va_end(args);
     return result;
 }
 
+template<typename CharT, class AnyCharsAccess>
 void
-TokenStream::reportError(unsigned errorNumber, ...)
+TokenStreamSpecific<CharT, AnyCharsAccess>::reportError(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
 
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
     ErrorMetadata metadata;
-    if (computeErrorMetadata(&metadata, currentToken().pos.begin))
-        ReportCompileError(cx, Move(metadata), nullptr, JSREPORT_ERROR, errorNumber, args);
+    if (computeErrorMetadata(&metadata, anyChars.currentToken().pos.begin)) {
+        ReportCompileError(anyChars.cx, Move(metadata), nullptr, JSREPORT_ERROR, errorNumber,
+                           args);
+    }
 
     va_end(args);
 }
 
 void
 TokenStreamAnyChars::reportErrorNoOffsetVA(unsigned errorNumber, va_list args)
 {
     ErrorMetadata metadata;
     computeErrorMetadataNoOffset(&metadata);
 
     ReportCompileError(cx, Move(metadata), nullptr, JSREPORT_ERROR, errorNumber, args);
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::warning(unsigned errorNumber, ...)
+TokenStreamSpecific<CharT, AnyCharsAccess>::warning(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
 
     ErrorMetadata metadata;
     bool result =
-        computeErrorMetadata(&metadata, currentToken().pos.begin) &&
-        compileWarning(Move(metadata), nullptr, JSREPORT_WARNING, errorNumber, args);
+        computeErrorMetadata(&metadata, anyCharsAccess().currentToken().pos.begin) &&
+        anyCharsAccess().compileWarning(Move(metadata), nullptr, JSREPORT_WARNING, errorNumber,
+                                        args);
 
     va_end(args);
     return result;
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::reportExtraWarningErrorNumberVA(UniquePtr<JSErrorNotes> notes, uint32_t offset,
-                                             unsigned errorNumber, va_list* args)
+TokenStreamSpecific<CharT, AnyCharsAccess>::reportExtraWarningErrorNumberVA(UniquePtr<JSErrorNotes> notes,
+                                                                            uint32_t offset,
+                                                                            unsigned errorNumber,
+                                                                            va_list* args)
 {
-    if (!options().extraWarningsOption)
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
+    if (!anyChars.options().extraWarningsOption)
         return true;
 
     ErrorMetadata metadata;
     if (!computeErrorMetadata(&metadata, offset))
         return false;
 
-    return compileWarning(Move(metadata), Move(notes), JSREPORT_STRICT | JSREPORT_WARNING,
-                          errorNumber, *args);
+    return anyChars.compileWarning(Move(metadata), Move(notes), JSREPORT_STRICT | JSREPORT_WARNING,
+                                   errorNumber, *args);
 }
 
+template<typename CharT, class AnyCharsAccess>
 void
-TokenStream::error(unsigned errorNumber, ...)
+TokenStreamSpecific<CharT, AnyCharsAccess>::error(unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
 
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
     ErrorMetadata metadata;
-    if (computeErrorMetadata(&metadata, currentToken().pos.begin))
-        ReportCompileError(cx, Move(metadata), nullptr, JSREPORT_ERROR, errorNumber, args);
+    if (computeErrorMetadata(&metadata, anyChars.currentToken().pos.begin))
+        ReportCompileError(anyChars.cx, Move(metadata), nullptr, JSREPORT_ERROR, errorNumber,
+                           args);
 
     va_end(args);
 }
 
+template<typename CharT, class AnyCharsAccess>
 void
-TokenStream::errorAt(uint32_t offset, unsigned errorNumber, ...)
+TokenStreamSpecific<CharT, AnyCharsAccess>::errorAt(uint32_t offset, unsigned errorNumber, ...)
 {
     va_list args;
     va_start(args, errorNumber);
 
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
     ErrorMetadata metadata;
     if (computeErrorMetadata(&metadata, offset))
-        ReportCompileError(cx, Move(metadata), nullptr, JSREPORT_ERROR, errorNumber, args);
+        ReportCompileError(anyChars.cx, Move(metadata), nullptr, JSREPORT_ERROR, errorNumber,
+                           args);
 
     va_end(args);
 }
 
 // We have encountered a '\': check for a Unicode escape sequence after it.
 // Return the length of the escape sequence and the character code point (by
 // value) if we found a Unicode escape sequence.  Otherwise, return 0.  In both
 // cases, do not advance along the buffer.
+template<typename CharT, class AnyCharsAccess>
 uint32_t
-TokenStream::peekUnicodeEscape(uint32_t* codePoint)
+TokenStreamSpecific<CharT, AnyCharsAccess>::peekUnicodeEscape(uint32_t* codePoint)
 {
     int32_t c = getCharIgnoreEOL();
     if (c != 'u') {
         ungetCharIgnoreEOL(c);
         return 0;
     }
 
     CharT cp[3];
@@ -946,18 +1028,19 @@ TokenStream::peekUnicodeEscape(uint32_t*
         length = 0;
     }
 
     ungetCharIgnoreEOL(c);
     ungetCharIgnoreEOL('u');
     return length;
 }
 
+template<typename CharT, class AnyCharsAccess>
 uint32_t
-TokenStream::peekExtendedUnicodeEscape(uint32_t* codePoint)
+TokenStreamSpecific<CharT, AnyCharsAccess>::peekExtendedUnicodeEscape(uint32_t* codePoint)
 {
     // The opening brace character was already read.
     int32_t c = getCharIgnoreEOL();
 
     // Skip leading zeros.
     uint32_t leadingZeros = 0;
     while (c == '0') {
         leadingZeros++;
@@ -985,29 +1068,31 @@ TokenStream::peekExtendedUnicodeEscape(u
     while (i--)
         ungetCharIgnoreEOL(cp[i]);
     while (leadingZeros--)
         ungetCharIgnoreEOL('0');
 
     return length;
 }
 
+template<typename CharT, class AnyCharsAccess>
 uint32_t
-TokenStream::matchUnicodeEscapeIdStart(uint32_t* codePoint)
+TokenStreamSpecific<CharT, AnyCharsAccess>::matchUnicodeEscapeIdStart(uint32_t* codePoint)
 {
     uint32_t length = peekUnicodeEscape(codePoint);
     if (length > 0 && unicode::IsIdentifierStart(*codePoint)) {
         skipChars(length);
         return length;
     }
     return 0;
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::matchUnicodeEscapeIdent(uint32_t* codePoint)
+TokenStreamSpecific<CharT, AnyCharsAccess>::matchUnicodeEscapeIdent(uint32_t* codePoint)
 {
     uint32_t length = peekUnicodeEscape(codePoint);
     if (length > 0 && unicode::IsIdentifierPart(*codePoint)) {
         skipChars(length);
         return true;
     }
     return false;
 }
@@ -1021,40 +1106,41 @@ CharsMatch(const CharT* p, const char* q
     while (*q) {
         if (*p++ != *q++)
             return false;
     }
 
     return true;
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::getDirectives(bool isMultiline, bool shouldWarnDeprecated)
+TokenStreamSpecific<CharT, AnyCharsAccess>::getDirectives(bool isMultiline,
+                                                          bool shouldWarnDeprecated)
 {
     // Match directive comments used in debugging, such as "//# sourceURL" and
     // "//# sourceMappingURL". Use of "//@" instead of "//#" is deprecated.
     //
     // To avoid a crashing bug in IE, several JavaScript transpilers wrap single
     // line comments containing a source mapping URL inside a multiline
     // comment. To avoid potentially expensive lookahead and backtracking, we
     // only check for this case if we encounter a '#' character.
 
-    if (!getDisplayURL(isMultiline, shouldWarnDeprecated))
-        return false;
-    if (!getSourceMappingURL(isMultiline, shouldWarnDeprecated))
-        return false;
-
-    return true;
+    return getDisplayURL(isMultiline, shouldWarnDeprecated) &&
+           getSourceMappingURL(isMultiline, shouldWarnDeprecated);
 }
 
-bool
-TokenStream::getDirective(bool isMultiline, bool shouldWarnDeprecated,
-                          const char* directive, uint8_t directiveLength,
-                          const char* errorMsgPragma,
-                          UniqueTwoByteChars* destination)
+template<typename CharT, class AnyCharsAccess>
+MOZ_MUST_USE bool
+TokenStreamSpecific<CharT, AnyCharsAccess>::getDirective(bool isMultiline,
+                                                         bool shouldWarnDeprecated,
+                                                         const char* directive,
+                                                         uint8_t directiveLength,
+                                                         const char* errorMsgPragma,
+                                                         UniquePtr<char16_t[], JS::FreePolicy>* destination)
 {
     MOZ_ASSERT(directiveLength <= 18);
     char16_t peeked[18];
 
     if (peekChars(directiveLength, peeked) && CharsMatch(peeked, directive)) {
         if (shouldWarnDeprecated) {
             if (!warning(JSMSG_DEPRECATED_PRAGMA, errorMsgPragma))
                 return false;
@@ -1094,136 +1180,150 @@ TokenStream::getDirective(bool isMultili
         if (tokenbuf.empty()) {
             // The directive's URL was missing, but this is not quite an
             // exception that we should stop and drop everything for.
             return true;
         }
 
         size_t length = tokenbuf.length();
 
-        *destination = cx->make_pod_array<char16_t>(length + 1);
+        *destination = anyCharsAccess().cx->template make_pod_array<char16_t>(length + 1);
         if (!*destination)
             return false;
 
         PodCopy(destination->get(), tokenbuf.begin(), length);
         (*destination)[length] = '\0';
     }
 
     return true;
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::getDisplayURL(bool isMultiline, bool shouldWarnDeprecated)
+TokenStreamSpecific<CharT, AnyCharsAccess>::getDisplayURL(bool isMultiline,
+                                                          bool shouldWarnDeprecated)
 {
     // Match comments of the form "//# sourceURL=<url>" or
     // "/\* //# sourceURL=<url> *\/"
     //
     // Note that while these are labeled "sourceURL" in the source text,
     // internally we refer to it as a "displayURL" to distinguish what the
     // developer would like to refer to the source as from the source's actual
     // URL.
 
     static const char sourceURLDirective[] = " sourceURL=";
     constexpr uint8_t sourceURLDirectiveLength = ArrayLength(sourceURLDirective) - 1;
     return getDirective(isMultiline, shouldWarnDeprecated,
                         sourceURLDirective, sourceURLDirectiveLength,
-                        "sourceURL", &displayURL_);
+                        "sourceURL", &anyCharsAccess().displayURL_);
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::getSourceMappingURL(bool isMultiline, bool shouldWarnDeprecated)
+TokenStreamSpecific<CharT, AnyCharsAccess>::getSourceMappingURL(bool isMultiline,
+                                                                bool shouldWarnDeprecated)
 {
     // Match comments of the form "//# sourceMappingURL=<url>" or
     // "/\* //# sourceMappingURL=<url> *\/"
 
     static const char sourceMappingURLDirective[] = " sourceMappingURL=";
     constexpr uint8_t sourceMappingURLDirectiveLength = ArrayLength(sourceMappingURLDirective) - 1;
     return getDirective(isMultiline, shouldWarnDeprecated,
                         sourceMappingURLDirective, sourceMappingURLDirectiveLength,
-                        "sourceMappingURL", &sourceMapURL_);
+                        "sourceMappingURL", &anyCharsAccess().sourceMapURL_);
 }
 
+template<typename CharT, class AnyCharsAccess>
 MOZ_ALWAYS_INLINE Token*
-TokenStream::newToken(ptrdiff_t adjust)
+TokenStreamSpecific<CharT, AnyCharsAccess>::newToken(ptrdiff_t adjust)
 {
-    cursor = (cursor + 1) & ntokensMask;
-    Token* tp = &tokens[cursor];
+    TokenStreamAnyChars& anyChars = anyCharsAccess();
+
+    anyChars.cursor = (anyChars.cursor + 1) & ntokensMask;
+    Token* tp = &anyChars.tokens[anyChars.cursor];
     tp->pos.begin = userbuf.offset() + adjust;
 
     // NOTE: tp->pos.end is not set until the very end of getTokenInternal().
     MOZ_MAKE_MEM_UNDEFINED(&tp->pos.end, sizeof(tp->pos.end));
 
     return tp;
 }
 
-MOZ_ALWAYS_INLINE JSAtom*
-TokenStream::atomize(JSContext* cx, CharBuffer& cb)
-{
-    return AtomizeChars(cx, cb.begin(), cb.length());
-}
-
 #ifdef DEBUG
 static bool
 IsTokenSane(Token* tp)
 {
     // Nb: TOK_EOL should never be used in an actual Token;  it should only be
     // returned as a TokenKind from peekTokenSameLine().
     if (tp->type >= TOK_LIMIT || tp->type == TOK_EOL)
         return false;
 
     if (tp->pos.end < tp->pos.begin)
         return false;
 
     return true;
 }
 #endif
 
+template<class AnyCharsAccess>
 bool
-TokenStream::matchTrailForLeadSurrogate(char16_t lead, char16_t* trail, uint32_t* codePoint)
+TokenStreamChars<char16_t, AnyCharsAccess>::matchTrailForLeadSurrogate(char16_t lead,
+                                                                       uint32_t* codePoint)
 {
-    int32_t maybeTrail = getCharIgnoreEOL();
+    static_assert(mozilla::IsBaseOf<TokenStreamChars<char16_t, AnyCharsAccess>,
+                                    TokenStreamSpecific<char16_t, AnyCharsAccess>>::value,
+                  "static_cast below presumes an inheritance relationship");
+    auto* ts = static_cast<TokenStreamSpecific<char16_t, AnyCharsAccess>*>(this);
+
+    int32_t maybeTrail = ts->getCharIgnoreEOL();
     if (!unicode::IsTrailSurrogate(maybeTrail)) {
-        ungetCharIgnoreEOL(maybeTrail);
+        ts->ungetCharIgnoreEOL(maybeTrail);
         return false;
     }
 
-    if (trail)
-        *trail = maybeTrail;
     *codePoint = unicode::UTF16Decode(lead, maybeTrail);
     return true;
 }
 
-bool
-TokenStream::putIdentInTokenbuf(const char16_t* identStart)
+template<>
+MOZ_MUST_USE bool
+TokenStreamCharsBase<char16_t>::appendMultiUnitCodepointToTokenbuf(uint32_t codepoint)
 {
-    int32_t c;
-    uint32_t qc;
+    char16_t lead, trail;
+    unicode::UTF16Encode(codepoint, &lead, &trail);
+
+    return tokenbuf.append(lead) && tokenbuf.append(trail);
+}
+
+template<typename CharT, class AnyCharsAccess>
+bool
+TokenStreamSpecific<CharT, AnyCharsAccess>::putIdentInTokenbuf(const CharT* identStart)
+{
     const CharT* tmp = userbuf.addressOfNextRawChar();
     userbuf.setAddressOfNextRawChar(identStart);
 
     tokenbuf.clear();
     for (;;) {
-        c = getCharIgnoreEOL();
+        int32_t c = getCharIgnoreEOL();
+
+        uint32_t codePoint;
+        if (isMultiUnitCodepoint(c, &codePoint)) {
+            if (!unicode::IsIdentifierPart(codePoint))
+                break;
 
-        if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
-            char16_t trail;
-            uint32_t codePoint;
-            if (matchTrailForLeadSurrogate(c, &trail, &codePoint)) {
-                if (!unicode::IsIdentifierPart(codePoint))
-                    break;
+            if (!appendMultiUnitCodepointToTokenbuf(codePoint)) {
+                userbuf.setAddressOfNextRawChar(tmp);
+                return false;
+            }
 
-                if (!tokenbuf.append(c) || !tokenbuf.append(trail)) {
-                    userbuf.setAddressOfNextRawChar(tmp);
-                    return false;
-                }
-                continue;
-            }
+            continue;
         }
 
         if (!unicode::IsIdentifierPart(char16_t(c))) {
+            uint32_t qc;
             if (c != '\\' || !matchUnicodeEscapeIdent(&qc))
                 break;
 
             if (MOZ_UNLIKELY(unicode::IsSupplementary(qc))) {
                 char16_t lead, trail;
                 unicode::UTF16Encode(qc, &lead, &trail);
                 if (!tokenbuf.append(lead) || !tokenbuf.append(trail)) {
                     userbuf.setAddressOfNextRawChar(tmp);
@@ -1235,16 +1335,17 @@ TokenStream::putIdentInTokenbuf(const ch
             c = qc;
         }
 
         if (!tokenbuf.append(c)) {
             userbuf.setAddressOfNextRawChar(tmp);
             return false;
         }
     }
+
     userbuf.setAddressOfNextRawChar(tmp);
     return true;
 }
 
 enum FirstCharKind {
     // A char16_t has the 'OneChar' kind if it, by itself, constitutes a valid
     // token that cannot also be a prefix of a longer token.  E.g. ';' has the
     // OneChar kind, but '+' does not, because '++' and '+=' are valid longer tokens
@@ -1308,18 +1409,19 @@ static const uint8_t firstCharKinds[] = 
 #undef T_COLON
 #undef T_BITNOT
 #undef Templat
 #undef _______
 
 static_assert(LastCharKind < (1 << (sizeof(firstCharKinds[0]) * 8)),
               "Elements of firstCharKinds[] are too small");
 
-bool
-TokenStream::getTokenInternal(TokenKind* ttp, Modifier modifier)
+template<typename CharT, class AnyCharsAccess>
+MOZ_MUST_USE bool
+TokenStreamSpecific<CharT, AnyCharsAccess>::getTokenInternal(TokenKind* ttp, Modifier modifier)
 {
     int c;
     uint32_t qc;
     Token* tp;
     FirstCharKind c1kind;
     const CharT* numStart;
     bool hasExp;
     DecimalPoint decimalPoint;
@@ -1333,60 +1435,59 @@ TokenStream::getTokenInternal(TokenKind*
             goto error;
         goto out;
     }
 
   retry:
     if (MOZ_UNLIKELY(!userbuf.hasRawChars())) {
         tp = newToken(0);
         tp->type = TOK_EOF;
-        flags.isEOF = true;
+        anyCharsAccess().flags.isEOF = true;
         goto out;
     }
 
     c = userbuf.getRawChar();
     MOZ_ASSERT(c != EOF);
 
     // Chars not in the range 0..127 are rare.  Getting them out of the way
     // early allows subsequent checking to be faster.
     if (MOZ_UNLIKELY(c >= 128)) {
         if (unicode::IsSpaceOrBOM2(c)) {
             if (c == unicode::LINE_SEPARATOR || c == unicode::PARA_SEPARATOR) {
                 if (!updateLineInfoForEOL())
                     goto error;
 
-                updateFlagsForEOL();
+                anyCharsAccess().updateFlagsForEOL();
             }
 
             goto retry;
         }
 
         tp = newToken(-1);
 
+        // If the first codepoint is really the start of an identifier, the
+        // identifier starts at the previous raw char.  If it isn't, it's a bad
+        // char and this assignment won't be examined anyway.
+        identStart = userbuf.addressOfNextRawChar() - 1;
+
         static_assert('$' < 128,
                       "IdentifierStart contains '$', but as !IsUnicodeIDStart('$'), "
                       "ensure that '$' is never handled here");
         static_assert('_' < 128,
                       "IdentifierStart contains '_', but as !IsUnicodeIDStart('_'), "
                       "ensure that '_' is never handled here");
         if (unicode::IsUnicodeIDStart(char16_t(c))) {
-            identStart = userbuf.addressOfNextRawChar() - 1;
             hadUnicodeEscape = false;
             goto identifier;
         }
 
-        if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
-            uint32_t codePoint;
-            if (matchTrailForLeadSurrogate(c, nullptr, &codePoint) &&
-                unicode::IsUnicodeIDStart(codePoint))
-            {
-                identStart = userbuf.addressOfNextRawChar() - 2;
-                hadUnicodeEscape = false;
-                goto identifier;
-            }
+        uint32_t codePoint;
+        if (isMultiUnitCodepoint(c, &codePoint) && unicode::IsUnicodeIDStart(codePoint)) {
+            hadUnicodeEscape = false;
+            goto identifier;
         }
 
         goto badchar;
     }
 
     // Get the token kind, based on the first char.  The ordering of c1kind
     // comparison is based on the frequency of tokens in real code -- Parsemark
     // (which represents typical JS code on the web) and the Unreal demo (which
@@ -1430,24 +1531,22 @@ TokenStream::getTokenInternal(TokenKind*
         hadUnicodeEscape = false;
 
       identifier:
         for (;;) {
             c = getCharIgnoreEOL();
             if (c == EOF)
                 break;
 
-            if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
-                uint32_t codePoint;
-                if (matchTrailForLeadSurrogate(c, nullptr, &codePoint)) {
-                    if (!unicode::IsIdentifierPart(codePoint))
-                        break;
+            uint32_t codePoint;
+            if (isMultiUnitCodepoint(c, &codePoint)) {
+                if (!unicode::IsIdentifierPart(codePoint))
+                    break;
 
-                    continue;
-                }
+                continue;
             }
 
             if (!unicode::IsIdentifierPart(char16_t(c))) {
                 if (c != '\\' || !matchUnicodeEscapeIdent(&qc))
                     break;
                 hadUnicodeEscape = true;
             }
         }
@@ -1472,17 +1571,17 @@ TokenStream::getTokenInternal(TokenKind*
         // Represent reserved words as reserved word tokens.
         if (!hadUnicodeEscape) {
             if (const ReservedWordInfo* rw = FindReservedWord(chars, length)) {
                 tp->type = rw->tokentype;
                 goto out;
             }
         }
 
-        JSAtom* atom = AtomizeChars(cx, chars, length);
+        JSAtom* atom = atomizeChars(anyCharsAccess().cx, chars, length);
         if (!atom)
             goto error;
         tp->type = TOK_NAME;
         tp->setName(atom->asPropertyName());
         goto out;
     }
 
     // Look for a decimal number.
@@ -1521,38 +1620,42 @@ TokenStream::getTokenInternal(TokenKind*
         ungetCharIgnoreEOL(c);
 
         if (c != EOF) {
             if (unicode::IsIdentifierStart(char16_t(c))) {
                 reportError(JSMSG_IDSTART_AFTER_NUMBER);
                 goto error;
             }
 
-            if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
-                uint32_t codePoint;
-                if (matchTrailForLeadSurrogate(c, nullptr, &codePoint) &&
-                    unicode::IsIdentifierStart(codePoint))
-                {
-                    reportError(JSMSG_IDSTART_AFTER_NUMBER);
-                    goto error;
-                }
+            uint32_t codePoint;
+            if (isMultiUnitCodepoint(c, &codePoint) &&
+                unicode::IsIdentifierStart(codePoint))
+            {
+                reportError(JSMSG_IDSTART_AFTER_NUMBER);
+                goto error;
             }
         }
 
         // Unlike identifiers and strings, numbers cannot contain escaped
         // chars, so we don't need to use tokenbuf.  Instead we can just
         // convert the char16_t characters in userbuf to the numeric value.
         double dval;
         if (!((decimalPoint == HasDecimal) || hasExp)) {
-            if (!GetDecimalInteger(cx, numStart, userbuf.addressOfNextRawChar(), &dval))
+            if (!GetDecimalInteger(anyCharsAccess().cx, numStart,
+                                   userbuf.addressOfNextRawChar(), &dval))
+            {
                 goto error;
+            }
         } else {
             const CharT* dummy;
-            if (!js_strtod(cx, numStart, userbuf.addressOfNextRawChar(), &dummy, &dval))
+            if (!js_strtod(anyCharsAccess().cx, numStart, userbuf.addressOfNextRawChar(),
+                           &dummy, &dval))
+            {
                 goto error;
+            }
         }
         tp->type = TOK_NUMBER;
         tp->setNumber(dval, decimalPoint);
         goto out;
     }
 
     // Look for a string or a template string.
     //
@@ -1565,17 +1668,17 @@ TokenStream::getTokenInternal(TokenKind*
     // Skip over EOL chars, updating line state along the way.
     //
     if (c1kind == EOL) {
         // If it's a \r\n sequence: treat as a single EOL, skip over the \n.
         if (c == '\r' && userbuf.hasRawChars())
             userbuf.matchRawChar('\n');
         if (!updateLineInfoForEOL())
             goto error;
-        updateFlagsForEOL();
+        anyCharsAccess().updateFlagsForEOL();
         goto retry;
     }
 
     // Look for a hexadecimal, octal, or binary number.
     //
     if (c1kind == BasePrefix) {
         tp = newToken(-1);
         int radix;
@@ -1642,31 +1745,33 @@ TokenStream::getTokenInternal(TokenKind*
         ungetCharIgnoreEOL(c);
 
         if (c != EOF) {
             if (unicode::IsIdentifierStart(char16_t(c))) {
                 reportError(JSMSG_IDSTART_AFTER_NUMBER);
                 goto error;
             }
 
-            if (MOZ_UNLIKELY(unicode::IsLeadSurrogate(c))) {
-                uint32_t codePoint;
-                if (matchTrailForLeadSurrogate(c, nullptr, &codePoint) &&
-                    unicode::IsIdentifierStart(codePoint))
-                {
-                    reportError(JSMSG_IDSTART_AFTER_NUMBER);
-                    goto error;
-                }
+            uint32_t codePoint;
+            if (isMultiUnitCodepoint(c, &codePoint) &&
+                unicode::IsIdentifierStart(codePoint))
+            {
+                reportError(JSMSG_IDSTART_AFTER_NUMBER);
+                goto error;
             }
         }
 
         double dval;
         const char16_t* dummy;
-        if (!GetPrefixInteger(cx, numStart, userbuf.addressOfNextRawChar(), radix, &dummy, &dval))
+        if (!GetPrefixInteger(anyCharsAccess().cx, numStart, userbuf.addressOfNextRawChar(),
+                              radix, &dummy, &dval))
+        {
             goto error;
+        }
+
         tp->type = TOK_NUMBER;
         tp->setNumber(dval, NoDecimal);
         goto out;
     }
 
     // This handles everything else.
     //
     MOZ_ASSERT(c1kind == Other);
@@ -1741,17 +1846,17 @@ TokenStream::getTokenInternal(TokenKind*
       case '!':
         if (matchChar('='))
             tp->type = matchChar('=') ? TOK_STRICTNE : TOK_NE;
         else
             tp->type = TOK_NOT;
         goto out;
 
       case '<':
-        if (options().allowHTMLComments) {
+        if (anyCharsAccess().options().allowHTMLComments) {
             // Treat HTML begin-comment as comment-till-end-of-line.
             if (matchChar('!')) {
                 if (matchChar('-')) {
                     if (matchChar('-'))
                         goto skipline;
                     ungetChar('-');
                 }
                 ungetChar('!');
@@ -1797,23 +1902,24 @@ TokenStream::getTokenInternal(TokenKind*
 
         skipline:
             do {
                 if (!getChar(&c))
                     goto error;
             } while (c != EOF && c != '\n');
 
             ungetChar(c);
-            cursor = (cursor - 1) & ntokensMask;
+            anyCharsAccess().cursor = (anyCharsAccess().cursor - 1) & ntokensMask;
             goto retry;
         }
 
         // Look for a multi-line comment.
         if (matchChar('*')) {
-            unsigned linenoBefore = lineno;
+            TokenStreamAnyChars& anyChars = anyCharsAccess();
+            unsigned linenoBefore = anyChars.lineno;
 
             do {
                 if (!getChar(&c))
                     return false;
 
                 if (c == EOF) {
                     reportError(JSMSG_UNTERMINATED_COMMENT);
                     goto error;
@@ -1824,19 +1930,19 @@ TokenStream::getTokenInternal(TokenKind*
 
                 if (c == '@' || c == '#') {
                     bool shouldWarn = c == '@';
                     if (!getDirectives(true, shouldWarn))
                         goto error;
                 }
             } while (true);
 
-            if (linenoBefore != lineno)
-                updateFlagsForEOL();
-            cursor = (cursor - 1) & ntokensMask;
+            if (linenoBefore != anyChars.lineno)
+                anyChars.updateFlagsForEOL();
+            anyChars.cursor = (anyChars.cursor - 1) & ntokensMask;
             goto retry;
         }
 
         // Look for a regexp.
         if (modifier == Operand) {
             tokenbuf.clear();
 
             bool inCharClass = false;
@@ -1907,17 +2013,19 @@ TokenStream::getTokenInternal(TokenKind*
         goto out;
 
       case '%':
         tp->type = matchChar('=') ? TOK_MODASSIGN : TOK_MOD;
         goto out;
 
       case '-':
         if (matchChar('-')) {
-            if (options().allowHTMLComments && !flags.isDirtyLine) {
+            if (anyCharsAccess().options().allowHTMLComments &&
+                !anyCharsAccess().flags.isDirtyLine)
+            {
                 int32_t c2;
                 if (!peekChar(&c2))
                     goto error;
 
                 if (c2 == '>')
                     goto skipline;
             }
 
@@ -1931,47 +2039,48 @@ TokenStream::getTokenInternal(TokenKind*
       default:
         reportError(JSMSG_ILLEGAL_CHARACTER);
         goto error;
     }
 
     MOZ_CRASH("should have jumped to |out| or |error|");
 
   out:
-    flags.isDirtyLine = true;
+    anyCharsAccess().flags.isDirtyLine = true;
     tp->pos.end = userbuf.offset();
 #ifdef DEBUG
     // Save the modifier used to get this token, so that if an ungetToken()
     // occurs and then the token is re-gotten (or peeked, etc.), we can assert
     // that both gets have used the same modifiers.
     tp->modifier = modifier;
     tp->modifierException = NoException;
 #endif
     MOZ_ASSERT(IsTokenSane(tp));
     *ttp = tp->type;
     return true;
 
   error:
     // We didn't get a token, so don't set |flags.isDirtyLine|.  And don't
     // poison any of |*tp|: if we haven't allocated a token, |tp| could be
     // uninitialized.
-    flags.hadError = true;
+    anyCharsAccess().flags.hadError = true;
 #ifdef DEBUG
     // Poisoning userbuf on error establishes an invariant: once an erroneous
     // token has been seen, userbuf will not be consulted again.  This is true
     // because the parser will deal with the illegal token by aborting parsing
     // immediately.
     userbuf.poison();
 #endif
     MOZ_MAKE_MEM_UNDEFINED(ttp, sizeof(*ttp));
     return false;
 }
 
+template<typename CharT, class AnyCharsAccess>
 bool
-TokenStream::getStringOrTemplateToken(int untilChar, Token** tp)
+TokenStreamSpecific<CharT, AnyCharsAccess>::getStringOrTemplateToken(int untilChar, Token** tp)
 {
     int c;
     int nc = -1;
 
     bool parsingTemplate = (untilChar == '`');
 
     *tp = newToken(-1);
     tokenbuf.clear();
@@ -2021,54 +2130,63 @@ TokenStream::getStringOrTemplateToken(in
                     consumeKnownChar('{');
 
                     bool first = true;
                     bool valid = true;
                     do {
                         int32_t c = getCharIgnoreEOL();
                         if (c == EOF) {
                             if (parsingTemplate) {
-                                setInvalidTemplateEscape(start, InvalidEscapeType::Unicode);
+                                TokenStreamAnyChars& anyChars = anyCharsAccess();
+                                anyChars.setInvalidTemplateEscape(start,
+                                                                  InvalidEscapeType::Unicode);
                                 valid = false;
                                 break;
                             }
                             reportInvalidEscapeError(start, InvalidEscapeType::Unicode);
                             return false;
                         }
                         if (c == '}') {
                             if (first) {
                                 if (parsingTemplate) {
-                                    setInvalidTemplateEscape(start, InvalidEscapeType::Unicode);
+                                    TokenStreamAnyChars& anyChars = anyCharsAccess();
+                                    anyChars.setInvalidTemplateEscape(start,
+                                                                      InvalidEscapeType::Unicode);
                                     valid = false;
                                     break;
                                 }
                                 reportInvalidEscapeError(start, InvalidEscapeType::Unicode);
                                 return false;
                             }
                             break;
                         }
 
                         if (!JS7_ISHEX(c)) {
                             if (parsingTemplate) {
                                 // We put the character back so that we read
                                 // it on the next pass, which matters if it
                                 // was '`' or '\'.
                                 ungetCharIgnoreEOL(c);
-                                setInvalidTemplateEscape(start, InvalidEscapeType::Unicode);
+
+                                TokenStreamAnyChars& anyChars = anyCharsAccess();
+                                anyChars.setInvalidTemplateEscape(start,
+                                                                  InvalidEscapeType::Unicode);
                                 valid = false;
                                 break;
                             }
                             reportInvalidEscapeError(start, InvalidEscapeType::Unicode);
                             return false;
                         }
 
                         code = (code << 4) | JS7_UNHEX(c);
                         if (code > unicode::NonBMPMax) {
                             if (parsingTemplate) {
-                                setInvalidTemplateEscape(start + 3, InvalidEscapeType::UnicodeOverflow);
+                                TokenStreamAnyChars& anyChars = anyCharsAccess();
+                                anyChars.setInvalidTemplateEscape(start + 3,
+                                                                  InvalidEscapeType::UnicodeOverflow);
                                 valid = false;
                                 break;
                             }
                             reportInvalidEscapeError(start + 3, InvalidEscapeType::UnicodeOverflow);
                             return false;
                         }
 
                         first = false;
@@ -2094,17 +2212,18 @@ TokenStream::getStringOrTemplateToken(in
                 {
                     c = JS7_UNHEX(cp[0]);
                     c = (c << 4) + JS7_UNHEX(cp[1]);
                     c = (c << 4) + JS7_UNHEX(cp[2]);
                     c = (c << 4) + JS7_UNHEX(cp[3]);
                     skipChars(4);
                 } else {
                     if (parsingTemplate) {
-                        setInvalidTemplateEscape(start, InvalidEscapeType::Unicode);
+                        TokenStreamAnyChars& anyChars = anyCharsAccess();
+                        anyChars.setInvalidTemplateEscape(start, InvalidEscapeType::Unicode);
                         continue;
                     }
                     reportInvalidEscapeError(start, InvalidEscapeType::Unicode);
                     return false;
                 }
                 break;
               }
 
@@ -2112,17 +2231,18 @@ TokenStream::getStringOrTemplateToken(in
               case 'x': {
                 CharT cp[2];
                 if (peekChars(2, cp) && JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1])) {
                     c = (JS7_UNHEX(cp[0]) << 4) + JS7_UNHEX(cp[1]);
                     skipChars(2);
                 } else {
                     uint32_t start = userbuf.offset() - 2;
                     if (parsingTemplate) {
-                        setInvalidTemplateEscape(start, InvalidEscapeType::Hexadecimal);
+                        TokenStreamAnyChars& anyChars = anyCharsAccess();
+                        anyChars.setInvalidTemplateEscape(start, InvalidEscapeType::Hexadecimal);
                         continue;
                     }
                     reportInvalidEscapeError(start, InvalidEscapeType::Hexadecimal);
                     return false;
                 }
                 break;
               }
 
@@ -2131,23 +2251,25 @@ TokenStream::getStringOrTemplateToken(in
                 if (JS7_ISOCT(c)) {
                     int32_t val = JS7_UNOCT(c);
 
                     if (!peekChar(&c))
                         return false;
 
                     // Strict mode code allows only \0, then a non-digit.
                     if (val != 0 || JS7_ISDEC(c)) {
+                        TokenStreamAnyChars& anyChars = anyCharsAccess();
                         if (parsingTemplate) {
-                            setInvalidTemplateEscape(userbuf.offset() - 2, InvalidEscapeType::Octal);
+                            anyChars.setInvalidTemplateEscape(userbuf.offset() - 2,
+                                                              InvalidEscapeType::Octal);
                             continue;
                         }
                         if (!reportStrictModeError(JSMSG_DEPRECATED_OCTAL))
                             return false;
-                        flags.sawOctalEscape = true;
+                        anyChars.flags.sawOctalEscape = true;
                     }
 
                     if (JS7_ISOCT(c)) {
                         val = 8 * val + JS7_UNOCT(c);
                         consumeKnownChar(c);
                         if (!peekChar(&c))
                             return false;
                         if (JS7_ISOCT(c)) {
@@ -2174,30 +2296,30 @@ TokenStream::getStringOrTemplateToken(in
                 c = '\n';
                 if (userbuf.peekRawChar() == '\n')
                     skipCharsIgnoreEOL(1);
             }
 
             if (!updateLineInfoForEOL())
                 return false;
 
-            updateFlagsForEOL();
+            anyCharsAccess().updateFlagsForEOL();
         } else if (parsingTemplate && c == '$') {
             if ((nc = getCharIgnoreEOL()) == '{')
                 break;
             ungetCharIgnoreEOL(nc);
         }
 
         if (!tokenbuf.append(c)) {
-            ReportOutOfMemory(cx);
+            ReportOutOfMemory(anyCharsAccess().cx);
             return false;
         }
     }
 
-    JSAtom* atom = atomize(cx, tokenbuf);
+    JSAtom* atom = atomizeChars(anyCharsAccess().cx, tokenbuf.begin(), tokenbuf.length());
     if (!atom)
         return false;
 
     if (!parsingTemplate) {
         (*tp)->type = TOK_STRING;
     } else {
         if (c == '$' && nc == '{')
             (*tp)->type = TOK_TEMPLATE_HEAD;
@@ -2234,16 +2356,20 @@ TokenKindToString(TokenKind tt)
 #undef EMIT_CASE
       case TOK_LIMIT: break;
     }
 
     return "<bad TokenKind>";
 }
 #endif
 
+template class frontend::TokenStreamCharsBase<char16_t>;
+template class frontend::TokenStreamChars<char16_t, frontend::TokenStreamAnyCharsAccess>;
+template class frontend::TokenStreamSpecific<char16_t, frontend::TokenStreamAnyCharsAccess>;
+
 } // namespace frontend
 
 } // namespace js
 
 
 JS_FRIEND_API(int)
 js_fgets(char* buf, int size, FILE* file)
 {
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -1,18 +1,137 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
  * vim: set ts=8 sts=4 et sw=4 tw=99:
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+/*
+ * Streaming access to the raw tokens of JavaScript source.
+ *
+ * Because JS tokenization is context-sensitive -- a '/' could be either a
+ * regular expression *or* a division operator depending on context -- the
+ * various token stream classes are mostly not useful outside of the Parser
+ * where they reside.  We should probably eventually merge the two concepts.
+ */
 #ifndef frontend_TokenStream_h
 #define frontend_TokenStream_h
 
-// JS lexical scanner interface.
+/*
+ * A token stream exposes the raw tokens -- operators, names, numbers,
+ * keywords, and so on -- of JavaScript source code.
+ *
+ * These are the components of the overall token stream concept:
+ * TokenStreamShared, TokenStreamAnyChars, TokenStreamCharsBase<CharT>,
+ * TokenStreamChars<CharT>, and TokenStreamSpecific<CharT, AnyCharsAccess>.
+ *
+ * == TokenStreamShared → ∅ ==
+ *
+ * Certain aspects of tokenizing are used everywhere:
+ *
+ *   * modifiers (used to select which context-sensitive interpretation of a
+ *     character should be used to decide what token it is), modifier
+ *     exceptions, and modifier assertion handling;
+ *   * flags on the overall stream (have we encountered any characters on this
+ *     line?  have we hit a syntax error?  and so on);
+ *   * and certain token-count constants.
+ *
+ * These are all defined in TokenStreamShared.  (They could be namespace-
+ * scoped, but it seems tentatively better not to clutter the namespace.)
+ *
+ * == TokenStreamAnyChars → TokenStreamShared ==
+ *
+ * Certain aspects of tokenizing have meaning independent of the character type
+ * of the source text being tokenized: line/column number information, tokens
+ * in lookahead from determining the meaning of a prior token, compilation
+ * options, the filename, flags, source map URL, access to details of the
+ * current and next tokens (is the token of the given type?  what name or
+ * number is contained in the token?  and other queries), and others.
+ *
+ * All this data/functionality *could* be duplicated for both single-byte and
+ * double-byte tokenizing, but there are two problems.  First, it's potentially
+ * wasteful if the compiler doesnt recognize it can unify the concepts.  (And
+ * if any-character concepts are intermixed with character-specific concepts,
+ * potentially the compiler *can't* unify them because offsets into the
+ * hypothetical TokenStream<CharT>s would differ.)  Second, some of this stuff
+ * needs to be accessible in ParserBase, the aspects of JS language parsing
+ * that have meaning independent of the character type of the source text being
+ * parsed.  So we need a separate data structure that ParserBase can hold on to
+ * for it.  (ParserBase isn't the only instance of this, but it's certainly the
+ * biggest case of it.)  Ergo, TokenStreamAnyChars.
+ *
+ * == TokenStreamCharsBase<CharT> → ∅ ==
+ *
+ * Certain data structures in tokenizing are character-type-specific:
+ * the various pointers identifying the source text (including current offset
+ * and end) , and the temporary vector into which characters are read/written
+ * in certain cases (think writing out the actual codepoints identified by an
+ * identifier containing a Unicode escape, to create the atom for the
+ * identifier: |a\u0062c| versus |abc|, for example).
+ *
+ * Additionally, some functions operating on this data are defined the same way
+ * no matter what character type you have -- the offset being |offset - start|
+ * no matter whether those two variables are single- or double-byte pointers.
+ *
+ * All such functionality lives in TokenStreamCharsBase<CharT>.
+ *
+ * == TokenStreamChars<CharT, AnyCharsAccess> → TokenStreamCharsBase<CharT> ==
+ *
+ * Some functionality operates at a very low level upon character-type-specific
+ * data, but in distinct ways.  For example, "is this character the start of a
+ * multi-character codepoint?"  Consider how such functionality would work on
+ * various encodings (hypothetically -- we haven't fully implemented any
+ * particular single-byte encoding support yet):
+ *
+ *   * For two-byte text, the character must pass |unicode::IsLeadSurrogate|.
+ *   * For single-byte Latin-1 text, there are no multi-character codepoints.
+ *   * For single-byte UTF-8 text, the answer depends on how many high bits of
+ *     the character are set.
+ *
+ * Such functionality all lives in TokenStreamChars<CharT, AnyCharsAccess>, a
+ * declared-but-not-defined template class whose specializations have a common
+ * public interface (plus whatever private helper functions are desirable).
+ *
+ * Why the AnyCharsAccess parameter?  Some functionality along these lines
+ * really wants TokenStreamSpecific, below, e.g. to report an error.  Providing
+ * this parameter allows TokenStreamChars functions to statically cast to this
+ * presumed superclass to access its functionality.
+ *
+ * TokenStreamChars<CharT, AnyCharsAccess> is just functionality, no actual
+ * member data.
+ *
+ * == TokenStreamSpecific<CharT, AnyCharsAccess> →
+ *    TokenStreamChars<CharT, AnyCharsAccess>, TokenStreamShared ==
+ *
+ * TokenStreamSpecific is operations that are parametrized on character type
+ * but implement the *general* idea of tokenizing, without being intrinsically
+ * tied to character type.  Notably, this includes all operations that can
+ * report warnings or errors at particular offsets, because we include a line
+ * of context with such errors -- and that necessarily accesses the raw
+ * characters of their specific type.
+ *
+ * Much TokenStreamSpecific operation depends on functionality in
+ * TokenStreamAnyChars.  The obvious solution is to inherit it -- but this
+ * doesn't work in Parser: its ParserBase base class needs some
+ * TokenStreamAnyChars functionality without knowing character type.
+ *
+ * The AnyCharsAccess type parameter is a class that statically converts from a
+ * TokenStreamSpecific* to its corresponding TokenStreamAnyChars.  The
+ * TokenStreamSpecific in Parser<ParseHandler, CharT> can then specify a class
+ * that properly converts from TokenStreamSpecific Parser::tokenStream to
+ * TokenStreamAnyChars ParserBase::anyChars.
+ *
+ * Could we hardcode one set of offset calculations for this and eliminate
+ * AnyCharsAccess?  No.  Offset calculations possibly could be hardcoded if
+ * TokenStreamSpecific were present in Parser before Parser::handler, assuring
+ * the same offsets in all Parser-related cases.  But there's still a separate
+ * TokenStream class, that requires different offset calculations.  So even if
+ * we wanted to hardcode this (it's not clear we would, because forcing the
+ * TokenStreamSpecific declarer to specify this is more explicit), we couldn't.
+ */
 
 #include "mozilla/ArrayUtils.h"
 #include "mozilla/Assertions.h"
 #include "mozilla/Attributes.h"
 #include "mozilla/DebugOnly.h"
 #include "mozilla/PodOperations.h"
 #include "mozilla/Unused.h"
 
@@ -92,17 +211,17 @@ enum class InvalidEscapeType {
     Unicode,
     // An otherwise well-formed \u escape which represents a
     // codepoint > 10FFFF.
     UnicodeOverflow,
     // An octal escape in a template token.
     Octal
 };
 
-class TokenStreamAnyChars;
+class TokenStreamShared;
 
 struct Token
 {
   private:
     // Sometimes the parser needs to inform the tokenizer to interpret
     // subsequent text in a particular manner: for example, to tokenize a
     // keyword as an identifier, not as the actual keyword, on the right-hand
     // side of a dotted property access.  Such information is communicated to
@@ -160,19 +279,23 @@ struct Token
         // operand before try getting colon/comma/semicolon.
         // See also the comment in Parser::assignExpr().
         NoneIsOperand,
 
         // If a semicolon is inserted automatically, the next token is already
         // gotten with None, but we expect Operand.
         OperandIsNone,
     };
-    friend class TokenStreamAnyChars;
+    friend class TokenStreamShared;
 
   public:
+    // WARNING: TokenStreamSpecific::Position assumes that the only GC things
+    //          a Token includes are atoms.  DON'T ADD NON-ATOM GC THING
+    //          POINTERS HERE UNLESS YOU ADD ADDITIONAL ROOTING TO THAT CLASS.
+
     TokenKind           type;           // char value or above enumerator
     TokenPos            pos;            // token position in file
     union {
       private:
         friend struct Token;
         PropertyName*   name;          // non-numeric atom
         JSAtom*         atom;          // potentially-numeric atom
         struct {
@@ -260,40 +383,114 @@ ReservedWordToCharZ(TokenKind tt);
 // This class is a tiny back-channel from TokenStream to the strict mode flag
 // that avoids exposing the rest of SharedContext to TokenStream.
 //
 class StrictModeGetter {
   public:
     virtual bool strictMode() = 0;
 };
 
-class TokenStreamAnyChars: public ErrorReporter
+/**
+ * TokenStream types and constants that are used in both TokenStreamAnyChars
+ * and TokenStreamSpecific.  Do not add any non-static data members to this
+ * class!
+ */
+class TokenStreamShared
 {
   protected:
-    TokenStreamAnyChars(JSContext* cx, const ReadOnlyCompileOptions& options, StrictModeGetter* smg);
+    static constexpr size_t ntokens = 4; // 1 current + 2 lookahead, rounded
+                                         // to power of 2 to avoid divmod by 3
+
+    static constexpr unsigned maxLookahead = 2;
+    static constexpr unsigned ntokensMask = ntokens - 1;
 
-    static const size_t ntokens = 4;                // 1 current + 2 lookahead, rounded
-                                                    // to power of 2 to avoid divmod by 3
-    static const unsigned maxLookahead = 2;
-    static const unsigned ntokensMask = ntokens - 1;
+    struct Flags
+    {
+        bool isEOF:1;           // Hit end of file.
+        bool isDirtyLine:1;     // Non-whitespace since start of line.
+        bool sawOctalEscape:1;  // Saw an octal character escape.
+        bool hadError:1;        // Hit a syntax error, at start or during a
+                                // token.
+
+        Flags()
+          : isEOF(), isDirtyLine(), sawOctalEscape(), hadError()
+        {}
+    };
 
   public:
+    static constexpr uint32_t NoOffset = UINT32_MAX;
+
+    using Modifier = Token::Modifier;
+    static constexpr Modifier None = Token::None;
+    static constexpr Modifier Operand = Token::Operand;
+    static constexpr Modifier TemplateTail = Token::TemplateTail;
+
+    using ModifierException = Token::ModifierException;
+    static constexpr ModifierException NoException = Token::NoException;
+    static constexpr ModifierException NoneIsOperand = Token::NoneIsOperand;
+    static constexpr ModifierException OperandIsNone = Token::OperandIsNone;
+
+    static void
+    verifyConsistentModifier(Modifier modifier, Token lookaheadToken)
+    {
+#ifdef DEBUG
+        // Easy case: modifiers match.
+        if (modifier == lookaheadToken.modifier)
+            return;
+
+        if (lookaheadToken.modifierException == OperandIsNone) {
+            // getToken(Operand) permissibly following getToken().
+            if (modifier == Operand && lookaheadToken.modifier == None)
+                return;
+        }
+
+        if (lookaheadToken.modifierException == NoneIsOperand) {
+            // getToken() permissibly following getToken(Operand).
+            if (modifier == None && lookaheadToken.modifier == Operand)
+                return;
+        }
+
+        MOZ_ASSERT_UNREACHABLE("this token was previously looked up with a "
+                               "different modifier, potentially making "
+                               "tokenization non-deterministic");
+#endif
+    }
+};
+
+static_assert(mozilla::IsEmpty<TokenStreamShared>::value,
+              "TokenStreamShared shouldn't bloat classes that inherit from it");
+
+template<typename CharT, class AnyCharsAccess>
+class TokenStreamSpecific;
+
+class TokenStreamAnyChars
+  : public TokenStreamShared,
+    public ErrorReporter
+{
+  public:
+    TokenStreamAnyChars(JSContext* cx, const ReadOnlyCompileOptions& options,
+                        StrictModeGetter* smg);
+
+    template<typename CharT, class AnyCharsAccess> friend class TokenStreamSpecific;
+
     // Accessors.
     const Token& currentToken() const { return tokens[cursor]; }
     bool isCurrentTokenType(TokenKind type) const {
         return currentToken().type == type;
     }
 
     bool getMutedErrors() const { return mutedErrors; }
 
     MOZ_MUST_USE bool checkOptions();
 
-  protected:
+  private:
     PropertyName* reservedWordToPropertyName(TokenKind tt) const;
 
+    void undoGetChar();
+
   public:
     PropertyName* currentName() const {
         if (isCurrentTokenType(TOK_NAME))
             return currentToken().name();
 
         MOZ_ASSERT(TokenKindIsPossibleIdentifierName(currentToken().type));
         return reservedWordToPropertyName(currentToken().type);
     }
@@ -328,59 +525,33 @@ class TokenStreamAnyChars: public ErrorR
 
     bool hasInvalidTemplateEscape() const {
         return invalidTemplateEscapeType != InvalidEscapeType::None;
     }
     void clearInvalidTemplateEscape() {
         invalidTemplateEscapeType = InvalidEscapeType::None;
     }
 
-    static const uint32_t NoOffset = UINT32_MAX;
-
-  protected:
-    // This is protected because it should only be called by the tokenizer
-    // while tokenizing not by, for example, BytecodeEmitter.
+  private:
+    // This is private because it should only be called by the tokenizer while
+    // tokenizing not by, for example, BytecodeEmitter.
     bool strictMode() const { return strictModeGetter && strictModeGetter->strictMode(); }
 
     void setInvalidTemplateEscape(uint32_t offset, InvalidEscapeType type) {
         MOZ_ASSERT(type != InvalidEscapeType::None);
         if (invalidTemplateEscapeType != InvalidEscapeType::None)
             return;
         invalidTemplateEscapeOffset = offset;
         invalidTemplateEscapeType = type;
     }
 
     uint32_t invalidTemplateEscapeOffset = 0;
     InvalidEscapeType invalidTemplateEscapeType = InvalidEscapeType::None;
 
-  protected:
-    struct Flags
-    {
-        bool isEOF:1;           // Hit end of file.
-        bool isDirtyLine:1;     // Non-whitespace since start of line.
-        bool sawOctalEscape:1;  // Saw an octal character escape.
-        bool hadError:1;        // Hit a syntax error, at start or during a
-                                // token.
-
-        Flags()
-          : isEOF(), isDirtyLine(), sawOctalEscape(), hadError()
-        {}
-    };
-
   public:
-    typedef Token::Modifier Modifier;
-    static constexpr Modifier None = Token::None;
-    static constexpr Modifier Operand = Token::Operand;
-    static constexpr Modifier TemplateTail = Token::TemplateTail;
-
-    typedef Token::ModifierException ModifierException;
-    static constexpr ModifierException NoException = Token::NoException;
-    static constexpr ModifierException NoneIsOperand = Token::NoneIsOperand;
-    static constexpr ModifierException OperandIsNone = Token::OperandIsNone;
-
     void addModifierException(ModifierException modifierException) {
 #ifdef DEBUG
         const Token& next = nextToken();
 
         // Permit adding the same exception multiple times.  This is important
         // particularly for Parser::assignExpr's early fast-path cases and
         // arrow function parsing: we want to add modifier exceptions in the
         // fast paths, then potentially (but not necessarily) duplicate them
@@ -413,41 +584,16 @@ class TokenStreamAnyChars: public ErrorR
             break;
           default:
             MOZ_CRASH("unexpected modifier exception");
         }
         tokens[(cursor + 1) & ntokensMask].modifierException = modifierException;
 #endif
     }
 
-    void
-    verifyConsistentModifier(Modifier modifier, Token lookaheadToken) {
-#ifdef DEBUG
-        // Easy case: modifiers match.
-        if (modifier == lookaheadToken.modifier)
-            return;
-
-        if (lookaheadToken.modifierException == OperandIsNone) {
-            // getToken(Operand) permissibly following getToken().
-            if (modifier == Operand && lookaheadToken.modifier == None)
-                return;
-        }
-
-        if (lookaheadToken.modifierException == NoneIsOperand) {
-            // getToken() permissibly following getToken(Operand).
-            if (modifier == None && lookaheadToken.modifier == Operand)
-                return;
-        }
-
-        MOZ_ASSERT_UNREACHABLE("this token was previously looked up with a "
-                               "different modifier, potentially making "
-                               "tokenization non-deterministic");
-#endif
-    }
-
 #ifdef DEBUG
     inline bool debugHasNoLookahead() const {
         return lookahead == 0;
     }
 #endif
 
     bool hasDisplayURL() const {
         return displayURL_ != nullptr;
@@ -550,418 +696,104 @@ class TokenStreamAnyChars: public ErrorR
     JSAtomState& names() const {
         return cx->names();
     }
 
     JSContext* context() const {
         return cx;
     }
 
-    virtual const ReadOnlyCompileOptions& options() const override final {
-        return options_;
-    }
-
     /**
      * Fill in |err|, excepting line-of-context-related fields.  If the token
      * stream has location information, use that and return true.  If it does
      * not, use the caller's location information and return false.
      */
     bool fillExcludingContext(ErrorMetadata* err, uint32_t offset);
 
     void updateFlagsForEOL();
 
+  private:
+    MOZ_MUST_USE MOZ_ALWAYS_INLINE bool internalUpdateLineInfoForEOL(uint32_t lineStartOffset);
+
+  public:
     const Token& nextToken() const {
         MOZ_ASSERT(hasLookahead());
         return tokens[(cursor + 1) & ntokensMask];
     }
 
     bool hasLookahead() const { return lookahead > 0; }
 
   public:
     MOZ_MUST_USE bool compileWarning(ErrorMetadata&& metadata, UniquePtr<JSErrorNotes> notes,
                                      unsigned flags, unsigned errorNumber, va_list args);
 
     // Compute error metadata for an error at no offset.
     void computeErrorMetadataNoOffset(ErrorMetadata* err);
 
-    virtual const char* getFilename() const override { return filename; }
+  public:
+    // ErrorReporter API.
+
+    virtual const JS::ReadOnlyCompileOptions& options() const override final {
+        return options_;
+    }
 
-    virtual void lineAndColumnAt(size_t offset, uint32_t* line, uint32_t* column) const override;
-    virtual void currentLineAndColumn(uint32_t* line, uint32_t* column) const override;
+    virtual void
+    lineAndColumnAt(size_t offset, uint32_t* line, uint32_t* column) const override final;
+
+    virtual void currentLineAndColumn(uint32_t* line, uint32_t* column) const override final;
 
-    virtual bool hasTokenizationStarted() const override;
-    virtual void reportErrorNoOffsetVA(unsigned errorNumber, va_list args) override;
+    virtual bool hasTokenizationStarted() const override final;
+    virtual void reportErrorNoOffsetVA(unsigned errorNumber, va_list args) override final;
+
+    virtual const char* getFilename() const override final {
+        return filename_;
+    }
 
   protected:
     // Options used for parsing/tokenizing.
     const ReadOnlyCompileOptions& options_;
 
     Token               tokens[ntokens];    // circular token buffer
     unsigned            cursor;             // index of last parsed token
     unsigned            lookahead;          // count of lookahead tokens
     unsigned            lineno;             // current line number
     Flags               flags;              // flags -- see above
     size_t              linebase;           // start of current line
     size_t              prevLinebase;       // start of previous line;  size_t(-1) if on the first line
-    const char*         filename;           // input filename or null
+    const char*         filename_;          // input filename or null
     UniqueTwoByteChars  displayURL_;        // the user's requested source URL or null
     UniqueTwoByteChars  sourceMapURL_;      // source map's filename or null
     uint8_t             isExprEnding[TOK_LIMIT];// which tokens definitely terminate exprs?
     JSContext* const    cx;
     bool                mutedErrors;
     StrictModeGetter*   strictModeGetter;  // used to test for strict mode
 };
 
-// TokenStream is the lexical scanner for Javascript source text.
-//
-// It takes a buffer of char16_t characters and linearly scans it into |Token|s.
-// Internally the class uses a four element circular buffer |tokens| of
-// |Token|s. As an index for |tokens|, the member |cursor| points to the
-// current token.
-// Calls to getToken() increase |cursor| by one and return the new current
-// token. If a TokenStream was just created, the current token is initialized
-// with random data (i.e. not initialized). It is therefore important that
-// one of the first four member functions listed below is called first.
-// The circular buffer lets us go back up to two tokens from the last
-// scanned token. Internally, the relative number of backward steps that were
-// taken (via ungetToken()) after the last token was scanned is stored in
-// |lookahead|.
-//
-// The following table lists in which situations it is safe to call each listed
-// function. No checks are made by the functions in non-debug builds.
-//
-// Function Name     | Precondition; changes to |lookahead|
-// ------------------+---------------------------------------------------------
-// getToken          | none; if |lookahead > 0| then |lookahead--|
-// peekToken         | none; if |lookahead == 0| then |lookahead == 1|
-// peekTokenSameLine | none; if |lookahead == 0| then |lookahead == 1|
-// matchToken        | none; if |lookahead > 0| and the match succeeds then
-//                   |       |lookahead--|
-// consumeKnownToken | none; if |lookahead > 0| then |lookahead--|
-// ungetToken        | 0 <= |lookahead| <= |maxLookahead - 1|; |lookahead++|
-//
-// The behavior of the token scanning process (see getTokenInternal()) can be
-// modified by calling one of the first four above listed member functions with
-// an optional argument of type Modifier.  However, the modifier will be
-// ignored unless |lookahead == 0| holds.  Due to constraints of the grammar,
-// this turns out not to be a problem in practice. See the
-// mozilla.dev.tech.js-engine.internals thread entitled 'Bug in the scanner?'
-// for more details:
-// https://groups.google.com/forum/?fromgroups=#!topic/mozilla.dev.tech.js-engine.internals/2JLH5jRcr7E).
-//
-// The methods seek() and tell() allow to rescan from a previous visited
-// location of the buffer.
-//
-class MOZ_STACK_CLASS TokenStream final : public TokenStreamAnyChars
+template<typename CharT>
+class TokenStreamCharsBase
 {
   public:
-    using CharT = char16_t;
     using CharBuffer = Vector<CharT, 32>;
 
-    TokenStream(JSContext* cx, const ReadOnlyCompileOptions& options,
-                const CharT* base, size_t length, StrictModeGetter* smg);
+    TokenStreamCharsBase(JSContext* cx, const CharT* chars, size_t length, size_t startOffset);
 
     const CharBuffer& getTokenbuf() const { return tokenbuf; }
 
-    // If there is an invalid escape in a template, report it and return false,
-    // otherwise return true.
-    bool checkForInvalidTemplateEscapeError() {
-        if (invalidTemplateEscapeType == InvalidEscapeType::None)
-            return true;
-
-        reportInvalidEscapeError(invalidTemplateEscapeOffset, invalidTemplateEscapeType);
-        return false;
-    }
-
-    // TokenStream-specific error reporters.
-    void reportError(unsigned errorNumber, ...);
-
-    // Report the given error at the current offset.
-    void error(unsigned errorNumber, ...);
-
-    // Report the given error at the given offset.
-    void errorAt(uint32_t offset, unsigned errorNumber, ...);
-
-    // Warn at the current offset.
-    MOZ_MUST_USE bool warning(unsigned errorNumber, ...);
-
-  private:
-    // Compute a line of context for an otherwise-filled-in |err| at the given
-    // offset in this token stream.  (This function basically exists to make
-    // |computeErrorMetadata| more readable and shouldn't be called elsewhere.)
-    MOZ_MUST_USE bool computeLineOfContext(ErrorMetadata* err, uint32_t offset);
-
-  public:
-    // Compute error metadata for an error at the given offset.
-    MOZ_MUST_USE bool computeErrorMetadata(ErrorMetadata* err, uint32_t offset);
-
-    // General-purpose error reporters.  You should avoid calling these
-    // directly, and instead use the more succinct alternatives (error(),
-    // warning(), &c.) in TokenStream, Parser, and BytecodeEmitter.
-    //
-    // These functions take a |va_list*| parameter, not a |va_list| parameter,
-    // to hack around bug 1363116.  (Longer-term, the right fix is of course to
-    // not use ellipsis functions or |va_list| at all in error reporting.)
-    bool reportStrictModeErrorNumberVA(UniquePtr<JSErrorNotes> notes, uint32_t offset,
-                                       bool strictMode, unsigned errorNumber, va_list* args);
-    bool reportExtraWarningErrorNumberVA(UniquePtr<JSErrorNotes> notes, uint32_t offset,
-                                         unsigned errorNumber, va_list* args);
-
-    JSAtom* getRawTemplateStringAtom() {
-        MOZ_ASSERT(currentToken().type == TOK_TEMPLATE_HEAD ||
-                   currentToken().type == TOK_NO_SUBS_TEMPLATE);
-        const CharT* cur = userbuf.rawCharPtrAt(currentToken().pos.begin + 1);
-        const CharT* end;
-        if (currentToken().type == TOK_TEMPLATE_HEAD) {
-            // Of the form    |`...${|   or   |}...${|
-            end = userbuf.rawCharPtrAt(currentToken().pos.end - 2);
-        } else {
-            // NO_SUBS_TEMPLATE is of the form   |`...`|   or   |}...`|
-            end = userbuf.rawCharPtrAt(currentToken().pos.end - 1);
-        }
-
-        CharBuffer charbuf(cx);
-        while (cur < end) {
-            CharT ch = *cur;
-            if (ch == '\r') {
-                ch = '\n';
-                if ((cur + 1 < end) && (*(cur + 1) == '\n'))
-                    cur++;
-            }
-            if (!charbuf.append(ch))
-                return nullptr;
-            cur++;
-        }
-        return AtomizeChars(cx, charbuf.begin(), charbuf.length());
-    }
-
-  private:
-    // This is private because it should only be called by the tokenizer while
-    // tokenizing not by, for example, BytecodeEmitter.
-    bool reportStrictModeError(unsigned errorNumber, ...);
-
-    void reportInvalidEscapeError(uint32_t offset, InvalidEscapeType type) {
-        switch (type) {
-            case InvalidEscapeType::None:
-                MOZ_ASSERT_UNREACHABLE("unexpected InvalidEscapeType");
-                return;
-            case InvalidEscapeType::Hexadecimal:
-                errorAt(offset, JSMSG_MALFORMED_ESCAPE, "hexadecimal");
-                return;
-            case InvalidEscapeType::Unicode:
-                errorAt(offset, JSMSG_MALFORMED_ESCAPE, "Unicode");
-                return;
-            case InvalidEscapeType::UnicodeOverflow:
-                errorAt(offset, JSMSG_UNICODE_OVERFLOW, "escape sequence");
-                return;
-            case InvalidEscapeType::Octal:
-                errorAt(offset, JSMSG_DEPRECATED_OCTAL);
-                return;
-        }
-    }
-
-    static JSAtom* atomize(JSContext* cx, CharBuffer& cb);
-    MOZ_MUST_USE bool putIdentInTokenbuf(const CharT* identStart);
-
-  public:
-    // Advance to the next token.  If the token stream encountered an error,
-    // return false.  Otherwise return true and store the token kind in |*ttp|.
-    MOZ_MUST_USE bool getToken(TokenKind* ttp, Modifier modifier = None) {
-        // Check for a pushed-back token resulting from mismatching lookahead.
-        if (lookahead != 0) {
-            MOZ_ASSERT(!flags.hadError);
-            lookahead--;
-            cursor = (cursor + 1) & ntokensMask;
-            TokenKind tt = currentToken().type;
-            MOZ_ASSERT(tt != TOK_EOL);
-            verifyConsistentModifier(modifier, currentToken());
-            *ttp = tt;
-            return true;
-        }
-
-        return getTokenInternal(ttp, modifier);
-    }
-
-    // Push the last scanned token back into the stream.
-    void ungetToken() {
-        MOZ_ASSERT(lookahead < maxLookahead);
-        lookahead++;
-        cursor = (cursor - 1) & ntokensMask;
-    }
-
-    MOZ_MUST_USE bool peekToken(TokenKind* ttp, Modifier modifier = None) {
-        if (lookahead > 0) {
-            MOZ_ASSERT(!flags.hadError);
-            verifyConsistentModifier(modifier, nextToken());
-            *ttp = nextToken().type;
-            return true;
-        }
-        if (!getTokenInternal(ttp, modifier))
-            return false;
-        ungetToken();
-        return true;
-    }
-
-    MOZ_MUST_USE bool peekTokenPos(TokenPos* posp, Modifier modifier = None) {
-        if (lookahead == 0) {
-            TokenKind tt;
-            if (!getTokenInternal(&tt, modifier))
-                return false;
-            ungetToken();
-            MOZ_ASSERT(hasLookahead());
-        } else {
-            MOZ_ASSERT(!flags.hadError);
-            verifyConsistentModifier(modifier, nextToken());
-        }
-        *posp = nextToken().pos;
-        return true;
-    }
-
-    MOZ_MUST_USE bool peekOffset(uint32_t* offset, Modifier modifier = None) {
-        TokenPos pos;
-        if (!peekTokenPos(&pos, modifier))
-            return false;
-        *offset = pos.begin;
-        return true;
-    }
-
-    // This is like peekToken(), with one exception:  if there is an EOL
-    // between the end of the current token and the start of the next token, it
-    // return true and store TOK_EOL in |*ttp|.  In that case, no token with
-    // TOK_EOL is actually created, just a TOK_EOL TokenKind is returned, and
-    // currentToken() shouldn't be consulted.  (This is the only place TOK_EOL
-    // is produced.)
-    MOZ_ALWAYS_INLINE MOZ_MUST_USE bool
-    peekTokenSameLine(TokenKind* ttp, Modifier modifier = None) {
-        const Token& curr = currentToken();
-
-        // If lookahead != 0, we have scanned ahead at least one token, and
-        // |lineno| is the line that the furthest-scanned token ends on.  If
-        // it's the same as the line that the current token ends on, that's a
-        // stronger condition than what we are looking for, and we don't need
-        // to return TOK_EOL.
-        if (lookahead != 0) {
-            bool onThisLine;
-            if (!srcCoords.isOnThisLine(curr.pos.end, lineno, &onThisLine)) {
-                reportError(JSMSG_OUT_OF_MEMORY);
-                return false;
-            }
-
-            if (onThisLine) {
-                MOZ_ASSERT(!flags.hadError);
-                verifyConsistentModifier(modifier, nextToken());
-                *ttp = nextToken().type;
-                return true;
-            }
-        }
-
-        // The above check misses two cases where we don't have to return
-        // TOK_EOL.
-        // - The next token starts on the same line, but is a multi-line token.
-        // - The next token starts on the same line, but lookahead==2 and there
-        //   is a newline between the next token and the one after that.
-        // The following test is somewhat expensive but gets these cases (and
-        // all others) right.
-        TokenKind tmp;
-        if (!getToken(&tmp, modifier))
-            return false;
-        const Token& next = currentToken();
-        ungetToken();
-
-        *ttp = srcCoords.lineNum(curr.pos.end) == srcCoords.lineNum(next.pos.begin)
-             ? next.type
-             : TOK_EOL;
-        return true;
-    }
-
-    // Get the next token from the stream if its kind is |tt|.
-    MOZ_MUST_USE bool matchToken(bool* matchedp, TokenKind tt, Modifier modifier = None) {
-        TokenKind token;
-        if (!getToken(&token, modifier))
-            return false;
-        if (token == tt) {
-            *matchedp = true;
-        } else {
-            ungetToken();
-            *matchedp = false;
-        }
-        return true;
-    }
-
-    void consumeKnownToken(TokenKind tt, Modifier modifier = None) {
-        bool matched;
-        MOZ_ASSERT(hasLookahead());
-        MOZ_ALWAYS_TRUE(matchToken(&matched, tt, modifier));
-        MOZ_ALWAYS_TRUE(matched);
-    }
-
-    MOZ_MUST_USE bool nextTokenEndsExpr(bool* endsExpr) {
-        TokenKind tt;
-        if (!peekToken(&tt))
-            return false;
-
-        *endsExpr = isExprEnding[tt];
-        if (*endsExpr) {
-            // If the next token ends an overall Expression, we'll parse this
-            // Expression without ever invoking Parser::orExpr().  But we need
-            // that function's side effect of adding this modifier exception,
-            // so we have to do it manually here.
-            addModifierException(OperandIsNone);
-        }
-        return true;
-    }
-
-    class MOZ_STACK_CLASS Position {
-      public:
-        // The Token fields may contain pointers to atoms, so for correct
-        // rooting we must ensure collection of atoms is disabled while objects
-        // of this class are live.  Do this by requiring a dummy AutoKeepAtoms
-        // reference in the constructor.
-        //
-        // This class is explicity ignored by the analysis, so don't add any
-        // more pointers to GC things here!
-        explicit Position(AutoKeepAtoms&) { }
-      private:
-        Position(const Position&) = delete;
-        friend class TokenStream;
-        const CharT* buf;
-        Flags flags;
-        unsigned lineno;
-        size_t linebase;
-        size_t prevLinebase;
-        Token currentToken;
-        unsigned lookahead;
-        Token lookaheadTokens[maxLookahead];
-    };
-
-    MOZ_MUST_USE bool advance(size_t position);
-    void tell(Position*);
-    void seek(const Position& pos);
-    MOZ_MUST_USE bool seek(const Position& pos, const TokenStream& other);
-
-    const CharT* rawCharPtrAt(size_t offset) const {
-        return userbuf.rawCharPtrAt(offset);
-    }
-
-    const CharT* rawLimit() const {
-        return userbuf.limit();
-    }
-
-  private:
     // This is the low-level interface to the JS source code buffer.  It just
     // gets raw chars, basically.  TokenStreams functions are layered on top
     // and do some extra stuff like converting all EOL sequences to '\n',
     // tracking the line number, and setting |flags.isEOF|.  (The "raw" in "raw
     // chars" refers to the lack of EOL sequence normalization.)
     //
     // buf[0..length-1] often represents a substring of some larger source,
     // where we have only the substring in memory. The |startOffset| argument
     // indicates the offset within this larger string at which our string
     // begins, the offset of |buf[0]|.
-    class TokenBuf {
+    class TokenBuf
+    {
       public:
         TokenBuf(const CharT* buf, size_t length, size_t startOffset)
           : base_(buf),
             startOffset_(startOffset),
             limit_(buf + length),
             ptr(buf)
         { }
 
@@ -1046,22 +878,454 @@ class MOZ_STACK_CLASS TokenStream final 
                    c == unicode::PARA_SEPARATOR;
         }
 
         // Returns the offset of the next EOL, but stops once 'max' characters
         // have been scanned (*including* the char at startOffset_).
         size_t findEOLMax(size_t start, size_t max);
 
       private:
-        const CharT* base_;          // base of buffer
-        uint32_t startOffset_;          // offset of base_[0]
-        const CharT* limit_;         // limit for quick bounds check
-        const CharT* ptr;            // next char to get
+        /** Base of buffer. */
+        const CharT* base_;
+
+        /** Offset of base_[0]. */
+        uint32_t startOffset_;
+
+        /** Limit for quick bounds check. */
+        const CharT* limit_;
+
+        /** Next char to get. */
+        const CharT* ptr;
     };
 
+    MOZ_MUST_USE bool appendMultiUnitCodepointToTokenbuf(uint32_t codepoint);
+
+  protected:
+    /** User input buffer. */
+    TokenBuf userbuf;
+
+    /** Current token string buffer. */
+    CharBuffer tokenbuf;
+};
+
+template<typename CharT, class AnyCharsAccess> class TokenStreamChars;
+
+template<class AnyCharsAccess>
+class TokenStreamChars<char16_t, AnyCharsAccess>
+  : public TokenStreamCharsBase<char16_t>
+{
+    using CharsBase = TokenStreamCharsBase<char16_t>;
+
+    bool matchTrailForLeadSurrogate(char16_t lead, uint32_t* codePoint);
+
+  public:
+    TokenStreamChars(JSContext* cx, const char16_t* chars, size_t length, size_t startOffset);
+
+    TokenStreamAnyChars& anyChars() {
+        return AnyCharsAccess::anyChars(this);
+    }
+
+    const TokenStreamAnyChars& anyChars() const {
+        return AnyCharsAccess::anyChars(this);
+    }
+
+    MOZ_ALWAYS_INLINE bool isMultiUnitCodepoint(char16_t c, uint32_t* codepoint) {
+        if (MOZ_LIKELY(!unicode::IsLeadSurrogate(c)))
+            return false;
+
+        return matchTrailForLeadSurrogate(c, codepoint);
+    }
+
+    static MOZ_ALWAYS_INLINE JSAtom*
+    atomizeChars(JSContext* cx, const char16_t* chars, size_t length) {
+        return AtomizeChars(cx, chars, length);
+    }
+};
+
+// TokenStream is the lexical scanner for JavaScript source text.
+//
+// It takes a buffer of CharT characters (currently only char16_t encoding
+// UTF-16, but we're adding either UTF-8 or Latin-1 single-byte text soon) and
+// linearly scans it into |Token|s.
+//
+// Internally the class uses a four element circular buffer |tokens| of
+// |Token|s. As an index for |tokens|, the member |cursor| points to the
+// current token. Calls to getToken() increase |cursor| by one and return the
+// new current token. If a TokenStream was just created, the current token is
+// uninitialized. It's therefore important that one of the first four member
+// functions listed below is called first. The circular buffer lets us go back
+// up to two tokens from the last scanned token. Internally, the relative
+// number of backward steps that were taken (via ungetToken()) after the last
+// token was scanned is stored in |lookahead|.
+//
+// The following table lists in which situations it is safe to call each listed
+// function. No checks are made by the functions in non-debug builds.
+//
+// Function Name     | Precondition; changes to |lookahead|
+// ------------------+---------------------------------------------------------
+// getToken          | none; if |lookahead > 0| then |lookahead--|
+// peekToken         | none; if |lookahead == 0| then |lookahead == 1|
+// peekTokenSameLine | none; if |lookahead == 0| then |lookahead == 1|
+// matchToken        | none; if |lookahead > 0| and the match succeeds then
+//                   |       |lookahead--|
+// consumeKnownToken | none; if |lookahead > 0| then |lookahead--|
+// ungetToken        | 0 <= |lookahead| <= |maxLookahead - 1|; |lookahead++|
+//
+// The behavior of the token scanning process (see getTokenInternal()) can be
+// modified by calling one of the first four above listed member functions with
+// an optional argument of type Modifier.  However, the modifier will be
+// ignored unless |lookahead == 0| holds.  Due to constraints of the grammar,
+// this turns out not to be a problem in practice. See the
+// mozilla.dev.tech.js-engine.internals thread entitled 'Bug in the scanner?'
+// for more details:
+// https://groups.google.com/forum/?fromgroups=#!topic/mozilla.dev.tech.js-engine.internals/2JLH5jRcr7E).
+//
+// The methods seek() and tell() allow to rescan from a previous visited
+// location of the buffer.
+//
+template<typename CharT, class AnyCharsAccess>
+class MOZ_STACK_CLASS TokenStreamSpecific
+  : public TokenStreamChars<CharT, AnyCharsAccess>,
+    public TokenStreamShared
+{
+    using CharsBase = TokenStreamChars<CharT, AnyCharsAccess>;
+    using CharsSharedBase = TokenStreamCharsBase<CharT>;
+
+  public:
+    // Anything inherited through a base class whose type depends upon this
+    // class's template parameters can only be accessed through a dependent
+    // name: prefixed with |this|, by explicit qualification, and so on.  (This
+    // is so that references to inherited fields are statically distinguishable
+    // from references to names outside of the class.)  This is tedious and
+    // onerous.
+    //
+    // As an alternative, we directly add every one of these functions to this
+    // class, using explicit qualification to address the dependent-name
+    // problem.  |this| or other qualification is no longer necessary -- at
+    // cost of this ever-changing laundry list of |using|s.  So it goes.
+    using CharsBase::isMultiUnitCodepoint;
+    using CharsBase::atomizeChars;
+
+    using typename CharsSharedBase::CharBuffer;
+
+    using CharsSharedBase::getTokenbuf;
+
+    using typename CharsSharedBase::TokenBuf;
+
+    using CharsSharedBase::appendMultiUnitCodepointToTokenbuf;
+
+    using CharsSharedBase::userbuf;
+    using CharsSharedBase::tokenbuf;
+
+  public:
+    TokenStreamSpecific(JSContext* cx, const ReadOnlyCompileOptions& options,
+                        const CharT* base, size_t length);
+
+    TokenStreamAnyChars& anyCharsAccess() {
+        return CharsBase::anyChars();
+    }
+
+    const TokenStreamAnyChars& anyCharsAccess() const {
+        return CharsBase::anyChars();
+    }
+
+    // If there is an invalid escape in a template, report it and return false,
+    // otherwise return true.
+    bool checkForInvalidTemplateEscapeError() {
+        if (anyCharsAccess().invalidTemplateEscapeType == InvalidEscapeType::None)
+            return true;
+
+        reportInvalidEscapeError(anyCharsAccess().invalidTemplateEscapeOffset,
+                                 anyCharsAccess().invalidTemplateEscapeType);
+        return false;
+    }
+
+    // TokenStream-specific error reporters.
+    void reportError(unsigned errorNumber, ...);
+
+    // Report the given error at the current offset.
+    void error(unsigned errorNumber, ...);
+
+    // Report the given error at the given offset.
+    void errorAt(uint32_t offset, unsigned errorNumber, ...);
+
+    // Warn at the current offset.
+    MOZ_MUST_USE bool warning(unsigned errorNumber, ...);
+
+  private:
+    // Compute a line of context for an otherwise-filled-in |err| at the given
+    // offset in this token stream.  (This function basically exists to make
+    // |computeErrorMetadata| more readable and shouldn't be called elsewhere.)
+    MOZ_MUST_USE bool computeLineOfContext(ErrorMetadata* err, uint32_t offset);
+
+  public:
+    // Compute error metadata for an error at the given offset.
+    MOZ_MUST_USE bool computeErrorMetadata(ErrorMetadata* err, uint32_t offset);
+
+    // General-purpose error reporters.  You should avoid calling these
+    // directly, and instead use the more succinct alternatives (error(),
+    // warning(), &c.) in TokenStream, Parser, and BytecodeEmitter.
+    //
+    // These functions take a |va_list*| parameter, not a |va_list| parameter,
+    // to hack around bug 1363116.  (Longer-term, the right fix is of course to
+    // not use ellipsis functions or |va_list| at all in error reporting.)
+    bool reportStrictModeErrorNumberVA(UniquePtr<JSErrorNotes> notes, uint32_t offset,
+                                       bool strictMode, unsigned errorNumber, va_list* args);
+    bool reportExtraWarningErrorNumberVA(UniquePtr<JSErrorNotes> notes, uint32_t offset,
+                                         unsigned errorNumber, va_list* args);
+
+    JSAtom* getRawTemplateStringAtom() {
+        TokenStreamAnyChars& anyChars = anyCharsAccess();
+
+        MOZ_ASSERT(anyChars.currentToken().type == TOK_TEMPLATE_HEAD ||
+                   anyChars.currentToken().type == TOK_NO_SUBS_TEMPLATE);
+        const CharT* cur = userbuf.rawCharPtrAt(anyChars.currentToken().pos.begin + 1);
+        const CharT* end;
+        if (anyChars.currentToken().type == TOK_TEMPLATE_HEAD) {
+            // Of the form    |`...${|   or   |}...${|
+            end = userbuf.rawCharPtrAt(anyChars.currentToken().pos.end - 2);
+        } else {
+            // NO_SUBS_TEMPLATE is of the form   |`...`|   or   |}...`|
+            end = userbuf.rawCharPtrAt(anyChars.currentToken().pos.end - 1);
+        }
+
+        CharBuffer charbuf(anyChars.cx);
+        while (cur < end) {
+            CharT ch = *cur;
+            if (ch == '\r') {
+                ch = '\n';
+                if ((cur + 1 < end) && (*(cur + 1) == '\n'))
+                    cur++;
+            }
+            if (!charbuf.append(ch))
+                return nullptr;
+            cur++;
+        }
+        return CharsBase::atomizeChars(anyChars.cx, charbuf.begin(), charbuf.length());
+    }
+
+  private:
+    // This is private because it should only be called by the tokenizer while
+    // tokenizing not by, for example, BytecodeEmitter.
+    bool reportStrictModeError(unsigned errorNumber, ...);
+
+    void reportInvalidEscapeError(uint32_t offset, InvalidEscapeType type) {
+        switch (type) {
+            case InvalidEscapeType::None:
+                MOZ_ASSERT_UNREACHABLE("unexpected InvalidEscapeType");
+                return;
+            case InvalidEscapeType::Hexadecimal:
+                errorAt(offset, JSMSG_MALFORMED_ESCAPE, "hexadecimal");
+                return;
+            case InvalidEscapeType::Unicode:
+                errorAt(offset, JSMSG_MALFORMED_ESCAPE, "Unicode");
+                return;
+            case InvalidEscapeType::UnicodeOverflow:
+                errorAt(offset, JSMSG_UNICODE_OVERFLOW, "escape sequence");
+                return;
+            case InvalidEscapeType::Octal:
+                errorAt(offset, JSMSG_DEPRECATED_OCTAL);
+                return;
+        }
+    }
+
+    MOZ_MUST_USE bool putIdentInTokenbuf(const CharT* identStart);
+
+  public:
+    // Advance to the next token.  If the token stream encountered an error,
+    // return false.  Otherwise return true and store the token kind in |*ttp|.
+    MOZ_MUST_USE bool getToken(TokenKind* ttp, Modifier modifier = None) {
+        // Check for a pushed-back token resulting from mismatching lookahead.
+        TokenStreamAnyChars& anyChars = anyCharsAccess();
+        if (anyChars.lookahead != 0) {
+            MOZ_ASSERT(!anyChars.flags.hadError);
+            anyChars.lookahead--;
+            anyChars.cursor = (anyChars.cursor + 1) & ntokensMask;
+            TokenKind tt = anyChars.currentToken().type;
+            MOZ_ASSERT(tt != TOK_EOL);
+            verifyConsistentModifier(modifier, anyChars.currentToken());
+            *ttp = tt;
+            return true;
+        }
+
+        return getTokenInternal(ttp, modifier);
+    }
+
+    // Push the last scanned token back into the stream.
+    void ungetToken() {
+        TokenStreamAnyChars& anyChars = anyCharsAccess();
+
+        MOZ_ASSERT(anyChars.lookahead < maxLookahead);
+        anyChars.lookahead++;
+        anyChars.cursor = (anyChars.cursor - 1) & ntokensMask;
+    }
+
+    MOZ_MUST_USE bool peekToken(TokenKind* ttp, Modifier modifier = None) {
+        TokenStreamAnyChars& anyChars = anyCharsAccess();
+        if (anyChars.lookahead > 0) {
+            MOZ_ASSERT(!anyChars.flags.hadError);
+            verifyConsistentModifier(modifier, anyChars.nextToken());
+            *ttp = anyChars.nextToken().type;
+            return true;
+        }
+        if (!getTokenInternal(ttp, modifier))
+            return false;
+        ungetToken();
+        return true;
+    }
+
+    MOZ_MUST_USE bool peekTokenPos(TokenPos* posp, Modifier modifier = None) {
+        TokenStreamAnyChars& anyChars = anyCharsAccess();
+        if (anyChars.lookahead == 0) {
+            TokenKind tt;
+            if (!getTokenInternal(&tt, modifier))
+                return false;
+            ungetToken();
+            MOZ_ASSERT(anyChars.hasLookahead());
+        } else {
+            MOZ_ASSERT(!anyChars.flags.hadError);
+            verifyConsistentModifier(modifier, anyChars.nextToken());
+        }
+        *posp = anyChars.nextToken().pos;
+        return true;
+    }
+
+    MOZ_MUST_USE bool peekOffset(uint32_t* offset, Modifier modifier = None) {
+        TokenPos pos;
+        if (!peekTokenPos(&pos, modifier))
+            return false;
+        *offset = pos.begin;
+        return true;
+    }
+
+    // This is like peekToken(), with one exception:  if there is an EOL
+    // between the end of the current token and the start of the next token, it
+    // return true and store TOK_EOL in |*ttp|.  In that case, no token with
+    // TOK_EOL is actually created, just a TOK_EOL TokenKind is returned, and
+    // currentToken() shouldn't be consulted.  (This is the only place TOK_EOL
+    // is produced.)
+    MOZ_ALWAYS_INLINE MOZ_MUST_USE bool
+    peekTokenSameLine(TokenKind* ttp, Modifier modifier = None) {
+        TokenStreamAnyChars& anyChars = anyCharsAccess();
+        const Token& curr = anyChars.currentToken();
+
+        // If lookahead != 0, we have scanned ahead at least one token, and
+        // |lineno| is the line that the furthest-scanned token ends on.  If
+        // it's the same as the line that the current token ends on, that's a
+        // stronger condition than what we are looking for, and we don't need
+        // to return TOK_EOL.
+        if (anyChars.lookahead != 0) {
+            bool onThisLine;
+            if (!anyChars.srcCoords.isOnThisLine(curr.pos.end, anyChars.lineno, &onThisLine)) {
+                reportError(JSMSG_OUT_OF_MEMORY);
+                return false;
+            }
+
+            if (onThisLine) {
+                MOZ_ASSERT(!anyChars.flags.hadError);
+                verifyConsistentModifier(modifier, anyChars.nextToken());
+                *ttp = anyChars.nextToken().type;
+                return true;
+            }
+        }
+
+        // The above check misses two cases where we don't have to return
+        // TOK_EOL.
+        // - The next token starts on the same line, but is a multi-line token.
+        // - The next token starts on the same line, but lookahead==2 and there
+        //   is a newline between the next token and the one after that.
+        // The following test is somewhat expensive but gets these cases (and
+        // all others) right.
+        TokenKind tmp;
+        if (!getToken(&tmp, modifier))
+            return false;
+        const Token& next = anyChars.currentToken();
+        ungetToken();
+
+        const auto& srcCoords = anyChars.srcCoords;
+        *ttp = srcCoords.lineNum(curr.pos.end) == srcCoords.lineNum(next.pos.begin)
+             ? next.type
+             : TOK_EOL;
+        return true;
+    }
+
+    // Get the next token from the stream if its kind is |tt|.
+    MOZ_MUST_USE bool matchToken(bool* matchedp, TokenKind tt, Modifier modifier = None) {
+        TokenKind token;
+        if (!getToken(&token, modifier))
+            return false;
+        if (token == tt) {
+            *matchedp = true;
+        } else {
+            ungetToken();
+            *matchedp = false;
+        }
+        return true;
+    }
+
+    void consumeKnownToken(TokenKind tt, Modifier modifier = None) {
+        bool matched;
+        MOZ_ASSERT(anyCharsAccess().hasLookahead());
+        MOZ_ALWAYS_TRUE(matchToken(&matched, tt, modifier));
+        MOZ_ALWAYS_TRUE(matched);
+    }
+
+    MOZ_MUST_USE bool nextTokenEndsExpr(bool* endsExpr) {
+        TokenKind tt;
+        if (!peekToken(&tt))
+            return false;
+
+        *endsExpr = anyCharsAccess().isExprEnding[tt];
+        if (*endsExpr) {
+            // If the next token ends an overall Expression, we'll parse this
+            // Expression without ever invoking Parser::orExpr().  But we need
+            // that function's side effect of adding this modifier exception,
+            // so we have to do it manually here.
+            anyCharsAccess().addModifierException(OperandIsNone);
+        }
+        return true;
+    }
+
+    class MOZ_STACK_CLASS Position {
+      public:
+        // The JS_HAZ_ROOTED is permissible below because: 1) the only field in
+        // Position that can keep GC things alive is Token, 2) the only GC
+        // things Token can keep alive are atoms, and 3) the AutoKeepAtoms&
+        // passed to the constructor here represents that collection of atoms
+        // is disabled while atoms in Tokens in this Position are alive.  DON'T
+        // ADD NON-ATOM GC THING POINTERS HERE!  They would create a rooting
+        // hazard that JS_HAZ_ROOTED will cause to be ignored.
+        explicit Position(AutoKeepAtoms&) { }
+
+      private:
+        Position(const Position&) = delete;
+        friend class TokenStreamSpecific;
+        const CharT* buf;
+        Flags flags;
+        unsigned lineno;
+        size_t linebase;
+        size_t prevLinebase;
+        Token currentToken;
+        unsigned lookahead;
+        Token lookaheadTokens[maxLookahead];
+    } JS_HAZ_ROOTED;
+
+    MOZ_MUST_USE bool advance(size_t position);
+    void tell(Position*);
+    void seek(const Position& pos);
+    MOZ_MUST_USE bool seek(const Position& pos, const TokenStreamSpecific& other);
+
+    const CharT* rawCharPtrAt(size_t offset) const {
+        return userbuf.rawCharPtrAt(offset);
+    }
+
+    const CharT* rawLimit() const {
+        return userbuf.limit();
+    }
+
     MOZ_MUST_USE bool getTokenInternal(TokenKind* ttp, Modifier modifier);
 
     MOZ_MUST_USE bool getStringOrTemplateToken(int untilChar, Token** tp);
 
     // Try to get the next character, normalizing '\r', '\r\n', and '\n' into
     // '\n'.  Also updates internal line-counter state.  Return true on success
     // and store the character in |*c|.  Return false and leave |*c| undefined
     // on failure.
@@ -1070,32 +1334,30 @@ class MOZ_STACK_CLASS TokenStream final 
 
     void ungetChar(int32_t c);
     void ungetCharIgnoreEOL(int32_t c);
     Token* newToken(ptrdiff_t adjust);
     uint32_t peekUnicodeEscape(uint32_t* codePoint);
     uint32_t peekExtendedUnicodeEscape(uint32_t* codePoint);
     uint32_t matchUnicodeEscapeIdStart(uint32_t* codePoint);
     bool matchUnicodeEscapeIdent(uint32_t* codePoint);
-    bool matchTrailForLeadSurrogate(char16_t lead, char16_t* trail, uint32_t* codePoint);
     bool peekChars(int n, CharT* cp);
 
     MOZ_MUST_USE bool getDirectives(bool isMultiline, bool shouldWarnDeprecated);
     MOZ_MUST_USE bool getDirective(bool isMultiline, bool shouldWarnDeprecated,
                                    const char* directive, uint8_t directiveLength,
                                    const char* errorMsgPragma,
-                                   UniquePtr<CharT[], JS::FreePolicy>* destination);
+                                   UniquePtr<char16_t[], JS::FreePolicy>* destination);
     MOZ_MUST_USE bool getDisplayURL(bool isMultiline, bool shouldWarnDeprecated);
     MOZ_MUST_USE bool getSourceMappingURL(bool isMultiline, bool shouldWarnDeprecated);
 
     // |expect| cannot be an EOL char.
     bool matchChar(int32_t expect) {
         MOZ_ASSERT(!TokenBuf::isRawEOLChar(expect));
-        return MOZ_LIKELY(userbuf.hasRawChars()) &&
-               userbuf.matchRawChar(expect);
+        return MOZ_LIKELY(userbuf.hasRawChars()) && userbuf.matchRawChar(expect);
     }
 
     void consumeKnownChar(int32_t expect) {
         int32_t c;
         MOZ_ALWAYS_TRUE(getChar(&c));
         MOZ_ASSERT(c == expect);
     }
 
@@ -1117,21 +1379,58 @@ class MOZ_STACK_CLASS TokenStream final 
     void skipCharsIgnoreEOL(uint8_t n) {
         while (n-- > 0) {
             MOZ_ASSERT(userbuf.hasRawChars());
             getCharIgnoreEOL();
         }
     }
 
     MOZ_MUST_USE MOZ_ALWAYS_INLINE bool updateLineInfoForEOL();
+};
 
-    TokenBuf            userbuf;            // user input buffer
-    CharBuffer          tokenbuf;           // current token string buffer
+class TokenStreamAnyCharsAccess
+{
+  public:
+    template<class TokenStreamSpecific>
+    static inline TokenStreamAnyChars& anyChars(TokenStreamSpecific* tss);
+
+    template<class TokenStreamSpecific>
+    static inline const TokenStreamAnyChars& anyChars(const TokenStreamSpecific* tss);
 };
 
+class MOZ_STACK_CLASS TokenStream final
+  : public TokenStreamAnyChars,
+    public TokenStreamSpecific<char16_t, TokenStreamAnyCharsAccess>
+{
+    using CharT = char16_t;
+
+  public:
+    TokenStream(JSContext* cx, const ReadOnlyCompileOptions& options,
+                const CharT* base, size_t length, StrictModeGetter* smg)
+    : TokenStreamAnyChars(cx, options, smg),
+      TokenStreamSpecific<CharT, TokenStreamAnyCharsAccess>(cx, options, base, length)
+    {}
+};
+
+template<class TokenStreamSpecific>
+/* static */ inline TokenStreamAnyChars&
+TokenStreamAnyCharsAccess::anyChars(TokenStreamSpecific* tss)
+{
+    auto* ts = static_cast<TokenStream*>(tss);
+    return *static_cast<TokenStreamAnyChars*>(ts);
+}
+
+template<class TokenStreamSpecific>
+/* static */ inline const TokenStreamAnyChars&
+TokenStreamAnyCharsAccess::anyChars(const TokenStreamSpecific* tss)
+{
+    const auto* ts = static_cast<const TokenStream*>(tss);
+    return *static_cast<const TokenStreamAnyChars*>(ts);
+}
+
 extern const char*
 TokenKindToDesc(TokenKind tt);
 
 } // namespace frontend
 } // namespace js
 
 extern JS_FRIEND_API(int)
 js_fgets(char* buf, int size, FILE* file);