Bug 1377007 - Scaffolding Token Reader;r?jorendorff,arai draft
authorDavid Teller <dteller@mozilla.com>
Fri, 08 Sep 2017 17:41:11 +0200
changeset 663862 4c912c2bd0e8cc941e6bdd6a94f785e5b6e51ac6
parent 655774 ab2d700fda2b4934d24227216972dce9fac19b74
child 663863 664572bb0e3f7bae3965dd6f3b49895873eb9aee
push id79549
push userdteller@mozilla.com
push dateWed, 13 Sep 2017 13:20:37 +0000
reviewersjorendorff, arai
bugs1377007
milestone57.0a1
Bug 1377007 - Scaffolding Token Reader;r?jorendorff,arai This patch ports to SpiderMonkey the tokenizer currently implemented in the external binjs-ref tool. While this tokenizer will clearly not be the tokenizer eventually shipped to end-users (whether in Firefox or in binjs-ref), the plan is to keep it both in binjs-ref and in SpiderMonkey (Nightly only) as a tool for helping test the higher layers of Binjs. MozReview-Commit-ID: 1i6XnVIf8p5
js/src/frontend/BinToken.h
js/src/frontend/BinTokenReaderTester.cpp
js/src/frontend/BinTokenReaderTester.h
js/src/jsapi-tests/binast/tokenizer/tester/test-empty-list.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-empty-untagged-tuple.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-nested-lists.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-simple-string.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-simple-tagged-tuple.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-string-with-escapes.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-trivial-list.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-trivial-untagged-tuple.binjs
js/src/jsapi-tests/moz.build
js/src/jsapi-tests/testBinTokenReaderTester.cpp
js/src/moz.build
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinToken.h
@@ -0,0 +1,178 @@
+#ifndef frontend_BinToken_h
+#define frontend_BinToken_h
+
+/**
+ * Definition of Binary AST tokens.
+ *
+ * In the Binary AST world, an AST is composed of nodes, where a node is
+ * defined by:
+ * - a Kind (see `BinKind`);
+ * - a list of fields, where each field is:
+ *    - a Name (see `BinField`);
+ *    - a Value, which may be either a node or a primitive value.
+ *
+ * The mapping between Kind and list of fields is determined entirely by
+ * the grammar of Binary AST. The mapping between (Kind, Name) and the structure
+ * of Value is also determined entirely by the grammar of Binary AST.
+ *
+ * As per the specifications of Binary AST, kinds may be added as the language
+ * grows, but never removed. The mapping between Kind and list of fields may
+ * also change to add new fields or make some fields optional, but may never
+ * remove a field. Finally, the mapping between (Kind, Name) and the structure
+ * of Value may be modified to add new possible values, but never to remove a
+ * value.
+ *
+ * A Binary AST parser must be able to fail gracefully when confronted with
+ * unknown Kinds or Names.
+ *
+ * At the time of this writing, the Binary AST defined from the Babylon AST
+ * (see https://github.com/babel/babylon/blob/master/ast/spec.md) restricted
+ * to ES5, with a few amendments to store additional scoping data and to
+ * represent the empty AST.
+ *
+ * Future versions of the Binary AST will progressively grow to encompass ES6
+ * and beyond.
+ */
+
+namespace js {
+namespace frontend {
+/**
+ * The different kinds of Binary AST nodes, as per the specifications of
+ * Binary AST.
+ *
+ * These kinds match roughly with the `ParseNodeKind` used internally.
+ *
+ * (sorted by alphabetical order)
+ */
+#define FOR_EACH_BIN_KIND(F) \
+    F(ARRAY_EXPRESSION, ArrayExpression) \
+    F(ASSIGNMENT_EXPRESSION, AssignmentExpression) \
+    F(ASSIGNMENT_OPERATOR, AssignmentOperator) \
+    F(BINARY_EXPRESSION, BinaryExpression) \
+    F(BINARY_OPERATOR, BinaryOperator) \
+    F(BINJS_NULL, BINJS:Null) \
+    F(BINJS_SCOPE, BINJS:Scope) \
+    F(BLOCK_STATEMENT, BlockStatement) \
+    F(BOOLEAN_LITERAL, BooleanLiteral) \
+    F(BREAK_STATEMENT, BreakStatement) \
+    F(CALL_EXPRESSION, CallExpression) \
+    F(CATCH_CLAUSE, CatchClause) \
+    F(CONDITIONAL_EXPRESSION, ConditionalExpression) \
+    F(CONTINUE_STATEMENT, ContinueStatement) \
+    F(DEBUGGER_STATEMENT, DebuggerStatement) \
+    F(DECLARATION, Declaration) \
+    F(DIRECTIVE, Directive) \
+    F(DIRECTIVE_LITERAL, DirectiveLiteral) \
+    F(DO_WHILE_STATEMENT, DoWhileStatement) \
+    F(EMPTY_STATEMENT, EmptyStatement) \
+    F(EXPRESSION, Expression) \
+    F(EXPRESSION_STATEMENT, ExpressionStatement) \
+    F(FOR_STATEMENT, ForStatement) \
+    F(FOR_IN_STATEMENT, ForInStatement) \
+    F(FUNCTION_EXPRESSION, FunctionExpression) \
+    F(FUNCTION_DECLARATION, FunctionDeclaration) \
+    F(IDENTIFIER, Identifier) \
+    F(IF_STATEMENT, IfStatement) \
+    F(LABELED_STATEMENT, LabeledStatement) \
+    F(LITERAL, Literal) \
+    F(LOGICAL_EXPRESSION, LogicalExpression) \
+    F(LOGICAL_OPERATOR, LogicalOperator) \
+    F(MEMBER_EXPRESSION, MemberExpression) \
+    F(NEW_EXPRESSION, NewExpression) \
+    F(NULL_LITERAL, NullLiteral) \
+    F(NUMERIC_LITERAL, NumericLiteral) \
+    F(OBJECT_EXPRESSION, ObjectExpression) \
+    F(OBJECT_METHOD, ObjectMethod) \
+    F(OBJECT_PROPERTY, ObjectProperty) \
+    F(PATTERN, Pattern) \
+    F(PROGRAM, Program) \
+    F(PROPERTY_KIND, PropertyKind) \
+    F(REGEXP_LITERAL, RegExpLiteral) \
+    F(RETURN_STATEMENT, ReturnStatement) \
+    F(SEQUENCE_EXPRESSION, SequenceExpression) \
+    F(STRING_LITERAL, StringLiteral) \
+    F(STATEMENT, Statement) \
+    F(SWITCH_CASE, SwitchCase) \
+    F(SWITCH_STATEMENT, SwitchStatement) \
+    F(THIS_EXPRESSION, ThisExpression) \
+    F(THROW_STATEMENT, ThrowStatement) \
+    F(TRY_STATEMENT, TryStatement) \
+    F(UNARY_EXPRESSION, UnaryExpression) \
+    F(UNARY_OPERATOR, UnaryOperator) \
+    F(UPDATE_EXPRESSION, UpdateExpression) \
+    F(UPDATE_OPERATOR, UpdateOperator) \
+    F(VARIABLE_DECLARATION, VariableDeclaration) \
+    F(VARIABLE_DECLARATOR, VariableDeclarator) \
+    F(VARIABLE_KIND, VariableKind) \
+    F(WHILE_STATEMENT, WhileStatement) \
+    F(WITH_STATEMENT, WithStatement)
+
+enum class BinKind {
+#define EMIT_ENUM(name, _) name,
+    FOR_EACH_BIN_KIND(EMIT_ENUM)
+#undef EMIT_ENUM
+    BINKIND_LIMIT /* domain size */
+};
+
+/**
+ * The different fields of Binary AST nodes, as per the specifications of
+ * Binary AST.
+ *
+ * (sorted by alphabetical order)
+ */
+ #define FOR_EACH_BIN_FIELD(F) \
+    F(ALTERNATE, alternate) \
+    F(ARGUMENT, argument) \
+    F(ARGUMENTS, arguments) \
+    F(BINJS_CAPTURED_NAMES, BINJS:CapturedNames) \
+    F(BINJS_CONST_DECL_NAMES, BINJS:ConstDeclaredNames) \
+    F(BINJS_HAS_DIRECT_EVAL, BINJS:HasDirectEval) \
+    F(BINJS_LET_DECL_NAMES, BINJS:LetDeclaredNames) \
+    F(BINJS_VAR_DECL_NAMES, BINJS:VarDeclaredNames) \
+    F(BINJS_SCOPE, BINJS:Scope) \
+    F(BLOCK, block) \
+    F(CALLEE, callee) \
+    F(CASES, cases) \
+    F(CONSEQUENT, consequent) \
+    F(COMPUTED, computed) \
+    F(BODY, body) \
+    F(DECLARATIONS, declarations) \
+    F(DIRECTIVES, directives) \
+    F(DISCRIMINANT, discriminant) \
+    F(ELEMENTS, elements) \
+    F(EXPRESSION, expression) \
+    F(EXPRESSIONS, expressions) \
+    F(FINALIZER, finalizer) \
+    F(FLAGS, flags) \
+    F(HANDLER, handler) \
+    F(ID, id) \
+    F(INIT, init) \
+    F(KEY, key) \
+    F(KIND, kind) \
+    F(LABEL, label) \
+    F(LEFT, left) \
+    F(NAME, name) \
+    F(OBJECT, object) \
+    F(OPERATOR, operator) \
+    F(PARAM, param) \
+    F(PARAMS, params) \
+    F(PATTERN, pattern) \
+    F(PREFIX, prefix) \
+    F(PROPERTIES, properties) \
+    F(PROPERTY, property) \
+    F(RIGHT, right) \
+    F(TEST, test) \
+    F(UPDATE, update) \
+    F(VALUE, value)
+
+enum class BinField {
+#define EMIT_ENUM(name, _) name,
+    FOR_EACH_BIN_FIELD(EMIT_ENUM)
+#undef EMIT_ENUM
+    BINFIELD_LIMIT /* domain size */
+};
+
+} // namespace frontend
+} // namespace js
+
+#endif // frontend_BinToken_h
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinTokenReaderTester.cpp
@@ -0,0 +1,541 @@
+#include "frontend/BinTokenReaderTester.h"
+
+namespace js {
+namespace frontend {
+
+BinTokenReaderTester::BinTokenReaderTester(JSContext* cx, const char* start, const size_t length_)
+    : cx_(cx)
+    , poisoned_(false)
+    , pendingError_(false)
+    , start_(start)
+    , current_(start)
+    , stop_(start + length_)
+    , latestKnownGoodPos_(0)
+{ }
+
+BinTokenReaderTester::BinTokenReaderTester(JSContext* cx, const Vector<char>& chars)
+    : cx_(cx)
+    , poisoned_(false)
+    , pendingError_(false)
+    , start_(chars.begin())
+    , current_(chars.begin())
+    , stop_(chars.end())
+    , latestKnownGoodPos_(0)
+{ }
+
+bool
+BinTokenReaderTester::raiseError(const char* description)
+{
+    poisoned_ = true;
+    pendingError_ = false;
+    TokenPos pos;
+    latestTokenPos(pos);
+    JS_ReportErrorASCII(cx_, "BinAST token error: %s at offsets %u => %u",
+                        description, pos.begin, pos.end);
+    return false;
+}
+
+void
+BinTokenReaderTester::AutoBase::postPendingError()
+{
+    reader_.pendingError_ = true;
+}
+
+bool
+BinTokenReaderTester::checkStatus()
+{
+    if (pendingError_)
+        return raiseError("Invalid format in subset (tuple, tagged tuple or list)");
+
+    return true;
+}
+
+bool
+BinTokenReaderTester::readBuf(char* bytes, uint32_t len)
+{
+    if (!checkStatus())
+        return false;
+
+    MOZ_ASSERT(!poisoned_);
+    MOZ_ASSERT(len > 0);
+
+    if (stop_ - current_ < len)
+        return raiseError("Buffer exceeds length");
+
+    for (uint32_t i = 0; i < len; ++i)
+        *bytes++ = *current_++;
+
+    return true;
+}
+
+bool
+BinTokenReaderTester::readByte(char* byte)
+{
+    return readBuf(byte, 1);
+}
+
+
+// Nullable booleans:
+//
+// 0 => false
+// 1 => true
+// 2 => null
+bool
+BinTokenReaderTester::readMaybeBool(Maybe<bool>& result)
+{
+    updateLatestKnownGood();
+    char byte;
+    if (!readByte(&byte))
+        return false;
+
+    switch (byte) {
+        case 0:
+            result = Some(false);
+            break;
+        case 1:
+            result = Some(true);
+            break;
+        case 2:
+            result = Nothing();
+            break;
+        default:
+            return raiseError("Invalid boolean value");
+    }
+    return true;
+}
+
+
+// Nullable doubles (little-endian)
+//
+// 0x7FF0000000000001 (signaling NaN) => null
+// anything other 64 bit sequence => IEEE-764 64-bit floating point number
+bool
+BinTokenReaderTester::readMaybeDouble(Maybe<double>& result)
+{
+    updateLatestKnownGood();
+
+    uint8_t bytes[8];
+    MOZ_ASSERT(sizeof(bytes) == sizeof(double));
+    if (!readBuf(reinterpret_cast<char*>(bytes), ArrayLength(bytes)))
+        return false;
+
+    // Decode little-endian.
+    const uint64_t asInt =
+          ((uint64_t)(bytes[0])<<0)
+        | ((uint64_t)(bytes[1])<<8)
+        | ((uint64_t)(bytes[2])<<16)
+        | ((uint64_t)(bytes[3])<<24)
+        | ((uint64_t)(bytes[4])<<32)
+        | ((uint64_t)(bytes[5])<<40)
+        | ((uint64_t)(bytes[6])<<48)
+        | ((uint64_t)(bytes[7])<<56);
+
+    if (asInt == 0x7FF0000000000001) {
+        result = Nothing();
+    } else {
+        const double asDouble = BitwiseCast<double>(asInt);
+        result = Some(asDouble);
+    }
+
+    return true;
+}
+
+
+// Internal uint32_t
+//
+// Encoded as 4 bytes, little-endian.
+bool
+BinTokenReaderTester::readInternalUint32(uint32_t* result)
+{
+    uint8_t bytes[4];
+    MOZ_ASSERT(sizeof(bytes) == sizeof(uint32_t));
+    if (!readBuf((char*)(void*)bytes, 4))
+        return false;
+
+    // Decode little-endian.
+    *result = ((uint32_t)(bytes[0]) << 0)
+            | ((uint32_t)(bytes[1]) << 8)
+            | ((uint32_t)(bytes[2]) << 16)
+            | ((uint32_t)(bytes[3]) << 24);
+
+    return true;
+}
+
+
+
+// Nullable strings:
+// - "<string>" (not counted in byte length)
+// - byte length (not counted in byte length)
+// - bytes (UTF-8)
+// - "</string>" (not counted in byte length)
+//
+// The special sequence of bytes `[255, 0]` (which is an invalid UTF-8 sequence)
+// is reserved to `null`.
+bool
+BinTokenReaderTester::readMaybeChars(Maybe<Chars>& result)
+{
+    updateLatestKnownGood();
+
+    if (!readConst("<string>"))
+        return false;
+
+    // 1. Read byteLength
+    uint32_t byteLen;
+    if (!readInternalUint32(&byteLen))
+        return false;
+
+    // 2. Reject if we can't read
+    if (current_ + byteLen > stop_)
+        return raiseError("Not enough bytes to read chars");
+
+    // 3. Check null string (no allocation)
+    if (byteLen == 2 && *current_ == -1 && *(current_ + 1) == 0) {
+        // Special case: null string.
+        result = Nothing();
+        current_ += byteLen;
+        return true;
+    }
+
+    // 4. Other strings (bytes are copied)
+    result.emplace(cx_);
+    if (!result->resize(byteLen)) {
+        poisoned_ = true;
+        ReportOutOfMemory(cx_);
+        return false;
+    }
+    PodCopy(result->begin(), current_, byteLen);
+    current_ += byteLen;
+
+    if (!readConst("</string>"))
+        return false;
+
+    return true;
+}
+
+bool
+BinTokenReaderTester::matchConst(const char* value, const size_t length)
+{
+    if (!checkStatus())
+        return false;
+    MOZ_ASSERT(!poisoned_);
+
+    // Perform lookup, without side-effects.
+    const char* lookupPtr = value;
+    const char* contentPtr = current_;
+
+    if (contentPtr + length >= stop_) {
+        return false;
+    }
+
+    for (size_t i = 0; i < length; ++i) {
+        if (*lookupPtr != *contentPtr) {
+            // No match, strings differ.
+            return false;
+        }
+        ++contentPtr;
+        ++lookupPtr;
+    }
+
+    // Looks like we have a match. Now perform side-effects
+    current_ = contentPtr;
+    updateLatestKnownGood();
+    return true;
+}
+
+bool
+BinTokenReaderTester::matchConst(const char* value)
+{
+    if (!checkStatus())
+        return false;
+    MOZ_ASSERT(!poisoned_);
+
+    // Perform lookup, without side-effects.
+    const char* lookupPtr = value;
+    const char* contentPtr = current_;
+
+    while (true) {
+        if (*lookupPtr == 0) {
+            // Full match.
+            break;
+        }
+        if (contentPtr >= stop_) {
+            // No match, end of buffer reached.
+            return false;
+        }
+        if (*lookupPtr != *contentPtr) {
+            // No match, strings differ.
+            return false;
+        }
+        ++contentPtr;
+        ++lookupPtr;
+    }
+
+    // Looks like we have a match. Now perform side-effects
+    current_ = contentPtr;
+    updateLatestKnownGood();
+    return true;
+}
+
+
+// Untagged tuple:
+// - "<tuple>";
+// - contents (specified by the higher-level grammar);
+// - "</tuple>"
+bool
+BinTokenReaderTester::readUntaggedTuple(AutoTuple& guard)
+{
+    if (!readConst("<tuple>"))
+        return false;
+
+    guard.init();
+    return true;
+}
+
+bool
+BinTokenReaderTester::readConst(const char* value)
+{
+    updateLatestKnownGood();
+    if (!matchConst(value))
+        return raiseError("Could not find expected literal");
+
+    return true;
+}
+
+void
+BinTokenReaderTester::AutoBase::readConst(const char* value)
+{
+    reader_.updateLatestKnownGood();
+    if (!reader_.matchConst(value))
+        return postPendingError();
+
+}
+
+// Tagged tuples:
+// - "<tuple>"
+// - "<head>"
+// - non-null string `name` (see `readString()`);
+// - uint32_t number of fields;
+// - array of `number of fields` non-null strings (see `readString()`);
+// - "</head>"
+// - content (specified by the higher-level grammar);
+// - "</tuple>"
+bool
+BinTokenReaderTester::readTaggedTuple(BinKind& tag, BinTokenReaderTester::BinFields& fields, BinTokenReaderTester::AutoTaggedTuple& guard)
+{
+    // Header
+    if (!readConst("<tuple>"))
+        return false;
+
+    if (!readConst("<head>"))
+        return false;
+
+    // This would probably be much faster with a HashTable, but we don't
+    // really care about the speed of BinTokenReaderTester.
+    do {
+
+#define FIND_MATCH(CONSTRUCTOR, NAME)     \
+        if (matchConst(#NAME, strlen(#NAME) + 1)) {        \
+            tag = BinKind::CONSTRUCTOR;   \
+            break;                        \
+        } // else
+
+        FOR_EACH_BIN_KIND(FIND_MATCH)
+#undef FIND_MATCH
+
+        // else
+        return raiseError("Invalid tag");
+    } while(false);
+
+    // Now fields.
+    uint32_t fieldNum;
+    if (!readInternalUint32(&fieldNum))
+        return false;
+
+    fields.clear();
+    if (!fields.reserve(fieldNum))
+        return raiseError("Out of memory");
+
+    for (uint32_t i = 0; i < fieldNum; ++i) {
+        // This would probably be much faster with a HashTable, but we don't
+        // really care about the speed of BinTokenReaderTester.
+        BinField field;
+        do {
+
+#define FIND_MATCH(CONSTRUCTOR, NAME)         \
+            if (matchConst(#NAME, strlen(#NAME) + 1)) {        \
+                field = BinField::CONSTRUCTOR;\
+                break;                        \
+            } // else
+
+            FOR_EACH_BIN_FIELD(FIND_MATCH)
+#undef FIND_MATCH
+
+            // else
+            return raiseError("Invalid field");
+        } while (false);
+
+        MOZ_ALWAYS_TRUE(fields.append(field)); // Already checked.
+    }
+
+    // End of header
+
+    if (!readConst("</head>"))
+        return false;
+
+    // Enter the body.
+    guard.init();
+    return true;
+}
+
+// List:
+//
+// - "<list>" (not counted in byte length);
+// - uint32_t byte length (not counted in byte length);
+// - uint32_t number of items;
+// - contents (specified by higher-level grammar);
+// - "</list>" (not counted in byte length)
+//
+// The total byte length of `number of items` + `contents` must be `byte length`.
+bool
+BinTokenReaderTester::readList(uint32_t& items, AutoList& guard)
+{
+    if (!readConst("<list>"))
+        return false;
+
+    uint32_t byteLen;
+    if (!readInternalUint32(&byteLen))
+        return false;
+
+    const char* stop_ = current_ + byteLen;
+    if (stop_ > this->stop_)
+        return raiseError("Incorrect list length");
+
+    guard.init(stop_);
+
+    if (!readInternalUint32(&items))
+        return false;
+
+    return true;
+}
+
+void
+BinTokenReaderTester::updateLatestKnownGood()
+{
+    MOZ_ASSERT(current_ >= start_);
+    const size_t update = current_ - start_;
+    MOZ_ASSERT(update >= latestKnownGoodPos_);
+    latestKnownGoodPos_ = update;
+}
+
+size_t
+BinTokenReaderTester::offset() const
+{
+    return latestKnownGoodPos_;
+}
+
+void
+BinTokenReaderTester::latestTokenPos(TokenPos& pos)
+{
+    pos.begin = latestKnownGoodPos_;
+    pos.end = current_ - start_;
+    MOZ_ASSERT(pos.end >= pos.begin);
+}
+
+void
+BinTokenReaderTester::AutoBase::init()
+{
+    initialized_ = true;
+}
+
+BinTokenReaderTester::AutoBase::AutoBase(BinTokenReaderTester& reader)
+    : reader_(reader)
+{ }
+
+BinTokenReaderTester::AutoBase::~AutoBase()
+{
+    if (!initialized_) {
+        // We failed before initialization.
+        return;
+    }
+    if (reader_.poisoned_) {
+        // Already poisoned, no need to add more checks.
+        return;
+    }
+    if (reader_.pendingError_) {
+        // Don't update good position.
+        return;
+    }
+    reader_.updateLatestKnownGood();
+}
+
+BinTokenReaderTester::AutoList::AutoList(BinTokenReaderTester& reader)
+    : AutoBase(reader)
+{ }
+
+void
+BinTokenReaderTester::AutoList::init(const char* expectedEnd)
+{
+    AutoBase::init();
+    this->expectedEnd_ = expectedEnd;
+}
+
+BinTokenReaderTester::AutoList::~AutoList()
+{
+    if (!initialized_) {
+        // We failed before initialization.
+        return;
+    }
+    if (reader_.poisoned_) {
+        // Already poisoned, no need to add more checks.
+        return;
+    }
+
+    if (reader_.current_ != expectedEnd_) {
+        // We did not consume all bytes.
+        postPendingError();
+    }
+
+    // Check suffix, poisoning the reader if the suffix is absent.
+    readConst("</list>");
+}
+
+BinTokenReaderTester::AutoTaggedTuple::AutoTaggedTuple(BinTokenReaderTester& reader)
+    : AutoBase(reader)
+{ }
+
+BinTokenReaderTester::AutoTaggedTuple::~AutoTaggedTuple()
+{
+    if (!initialized_) {
+        // We failed before initialization.
+        return;
+    }
+    if (reader_.poisoned_) {
+        // Already poisoned, no need to add more checks.
+        return;
+    }
+
+    // Check suffix, poisoning the reader if the suffix is absent.
+    readConst("</tuple>");
+}
+
+BinTokenReaderTester::AutoTuple::AutoTuple(BinTokenReaderTester& reader)
+    : AutoBase(reader)
+{ }
+
+BinTokenReaderTester::AutoTuple::~AutoTuple()
+{
+    if (!initialized_) {
+        // We failed before initialization.
+        return;
+    }
+    if (reader_.poisoned_) {
+        // Already poisoned, no need to add more checks.
+        return;
+    }
+
+    // Check suffix, poisoning the reader if the suffix is absent.
+    readConst("</tuple>");
+}
+
+} // namespace frontend
+} // namespace js
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinTokenReaderTester.h
@@ -0,0 +1,378 @@
+#ifndef frontend_BinTokenReaderTester_h
+#define frontend_BinTokenReaderTester_h
+
+#include "mozilla/Maybe.h"
+
+#include "jsapi.h"
+
+#include "frontend/BinToken.h"
+#include "frontend/TokenStream.h"
+
+#include "js/TypeDecls.h"
+
+#if !defined(NIGHTLY_BUILD)
+#error "BinTokenReaderTester.* is designed to help test implementations of successive versions of JS BinaryAST. It is available only on Nightly."
+#endif // !defined(NIGHTLY_BUILD)
+
+namespace js {
+namespace frontend {
+
+using namespace mozilla;
+using namespace JS;
+
+/**
+ * A token reader for a simple, alternative serialization format for BinAST.
+ *
+ * This serialization format, which is also supported by the reference
+ * implementation of the BinAST compression suite, is designed to be
+ * mostly human-readable and easy to check for all sorts of deserialization
+ * errors. While this format is NOT designed to be shipped to end-users, it
+ * is nevertheless a very useful tool for implementing and testing parsers.
+ *
+ * Both the format and the implementation are ridiculously inefficient:
+ *
+ * - the underlying format tags almost all its data with e.g. `<tuple>`, `</tuple>`
+ *   to aid with detecting offset errors or format error;
+ * - the underlying format copies list of fields into every single node, instead
+ *   of keeping them once in the header;
+ * - every kind/field extraction requires memory allocation and plenty of string
+ *   comparisons;
+ * - ...
+ *
+ * This token reader is designed to be API-compatible with the standard, shipped,
+ * token reader. For these reasons:
+ *
+ * - it does not support any form of look ahead, push back;
+ * - it does not support any form of error recovery.
+ */
+class BinTokenReaderTester MOZ_STACK_CLASS
+{
+  public:
+    // A list of fields, in the order in which they appear in the stream.
+    using BinFields = Vector<BinField, 8>;
+
+    // A bunch of characters. At this stage, there is no guarantee on whether
+    // they are valid UTF-8. Future versions may replace this by slice into
+    // the buffer.
+    using Chars     = Vector<char, 32>;
+
+    class AutoList;
+    class AutoTuple;
+    class AutoTaggedTuple;
+
+  public:
+    /**
+     * Construct a token reader.
+     *
+     * Does NOT copy the buffer.
+     */
+    BinTokenReaderTester(JSContext* cx, const char* start, const size_t length);
+    /**
+     * Construct a token reader.
+     *
+     * Does NOT copy the buffer.
+     */
+    BinTokenReaderTester(JSContext* cx, const Vector<char>& chars);
+
+    /**
+     * If an error is pending, return `false` and report an error.
+     *
+     * Otherwise, return `true` and do nothing.
+     *
+     * This method MUST be called at the end of parsing, once all
+     * BinTokenReaderTester::Auto* destructors have had the opportunity to
+     * perform their sanity-checks.
+     */
+    MOZ_MUST_USE bool checkStatus();
+
+
+    // --- Primitive values.
+    //
+    // Note that the underlying format allows for a `null` value for primitive
+    // values.
+    //
+    // Reading will return an error either in case of I/O error, in case of
+    // a format problem or if the tokenizer has been poisoned by a previous
+    // error.
+
+    /**
+     * Read a single `true | false | null` value.
+     *
+     * @param out Set to `Nothing` if the data specifies that the value is `null`.
+     * Otherwise, `Some(true)` or `Some(false)`.
+     *
+     * @return false If a boolean could not be read. In this case, an error
+     * has been raised.
+     */
+    MOZ_MUST_USE bool readMaybeBool(Maybe<bool>& out);
+
+    /**
+     * Read a single `number | null` value.
+     *
+     * @param out Set to `Nothing` if the data specifies that the value is `null`.
+     * Otherwise, `Some(x)`, where `x` is a valid `double`.
+     *
+     * @return false If a double could not be read. In this case, an error
+     * has been raised.
+     */
+    MOZ_MUST_USE bool readMaybeDouble(Maybe<double>& out);
+
+    /**
+     * Read a single `string | null` value.
+     *
+     * @param out Set to `Nothing` if the data specifies that the value is `null`.
+     * Otherwise, `Some(x)`, where `x` is a `string`.
+     *
+     * WARNING: At this stage, the `string` encoding has NOT been validated.
+     *
+     * @return false If a string could not be read. In this case, an error
+     * has been raised.
+     */
+    MOZ_MUST_USE bool readMaybeChars(Maybe<Chars>& out);
+
+    // --- Composite values.
+    //
+    // The underlying format does NOT allows for a `null` composite value.
+    //
+    // Reading a composite value returns a sub-`BinTokenReaderTester` dedicated to
+    // the value.
+    //
+    // Reading will return an error either in case of I/O error, in case of
+    // a format problem or if the tokenizer has been poisoned by a previous error.
+
+    /**
+     * Start reading a list.
+     *
+     * @param length (OUT) The number of elements in the list.
+     * @param guard (OUT) A guard, ensuring that we read the list correctly.
+     *
+     * Before destructing `guard`, callers MUST
+     * reach the end of the list. Failure
+     * to do so will raise an error.
+     *
+     * @return out If the header of the list is invalid.
+     */
+    MOZ_MUST_USE bool readList(uint32_t& length, AutoList& guard);
+
+    /**
+     * Start reading a tagged tuple.
+     *
+     * @param tag (OUT) The tag of the tuple.
+     * @param fields (OUT) The ORDERED list of fields encoded in this tuple.
+     * @param guard (OUT) A guard, ensuring that we read the tagged tuple correctly.
+     *
+     * Before destructing `guard`, callers MUST
+     * reach the end of the tuple. Failure
+     * to do so will raise an error.
+     *
+     * @return out If the header of the tuple is invalid.
+     */
+    MOZ_MUST_USE bool readTaggedTuple(BinKind& tag, BinTokenReaderTester::BinFields& fields, AutoTaggedTuple& guard);
+
+    /**
+     * Start reading an untagged tuple.
+     * @param guard (OUT) A guard, ensuring that we read the tuple correctly.
+     *
+     * Before destructing `guard`, callers MUST
+     * reach the end of the tuple. Failure
+     * to do so will raise an error.
+     *
+     * @return out If the header of the tuple is invalid.
+     */
+    MOZ_MUST_USE bool readUntaggedTuple(AutoTuple& guard);
+
+    /**
+     * Return the position of the latest token.
+     */
+    void latestTokenPos(TokenPos& out);
+    size_t offset() const;
+
+  private:
+    /**
+     * Read a single byte.
+     */
+    MOZ_MUST_USE bool readByte(char* byte);
+
+    /**
+     * Read several bytes.
+     *
+     * If there is not enough data, or if the tokenizer has previously been
+     * poisoned, return `false` and report an exception.
+     */
+    MOZ_MUST_USE bool readBuf(char* bytes, uint32_t len);
+
+    /**
+     * Read a single uint32_t.
+     */
+    MOZ_MUST_USE bool readInternalUint32(uint32_t*);
+
+    /**
+     * Read a sequence of chars, ensuring that they match an expected
+     * sequence of chars.
+     *
+     * @param data The sequence of chars to expect, NUL-terminated. The NUL
+     * is not expected in the stream.
+     */
+    MOZ_MUST_USE bool readConst(const char* data);
+
+    /**
+     * Read a sequence of chars, consuming the bytes only if they match an expected
+     * sequence of chars.
+     *
+     * @param data The sequence of chars to expect, NUL-terminated. The NUL
+     * is not expected in the stream.
+     * @return true if `data` (minus NUL) represents the next few chars in the
+     * internal buffer, false otherwise. If `true`, the chars are consumed,
+     * otherwise there is no side-effect.
+     */
+    MOZ_MUST_USE bool matchConst(const char* data);
+    MOZ_MUST_USE bool matchConst(const char* data, const size_t length);
+
+    /**
+     * Read a single `string | null` value.
+     *
+     * @param result Set to `Nothing` if the data specifies that the value is `null`.
+     * Otherwise, `Some(x)`, where `x` is a `string`.
+     *
+     * WARNING: At this stage, the `string` encoding has NOT been validated.
+     *
+     * @return false If a string could not be read. In this case, an error
+     * has been raised.
+     */
+    MOZ_MUST_USE bool readMaybeCharsData(Maybe<std::string>& result);
+
+
+    /**
+     * Raise an error.
+     */
+    MOZ_MUST_USE bool raiseError(const char* description);
+
+    /**
+     * Update the "latest known good" position, which is used during error
+     * reporting.
+     */
+    void updateLatestKnownGood();
+
+  private:
+    JSContext* cx_;
+
+    // `true` if we have encountered an error. Errors are non recoverable.
+    // Attempting to read from a poisoned tokenizer will cause assertion errors.
+    bool poisoned_;
+
+    // `true` if an error has been detected during a `Auto*` destructor call and will
+    // be reported during the next operation or call to `checkStatus()`.
+    // Once the error has been reported, the tokenizer is poisoned.
+    bool pendingError_;
+
+    // The first byte of the buffer. Not owned.
+    const char* start_;
+
+    // The current position.
+    const char* current_;
+
+    // The last+1 byte of the buffer.
+    const char* stop_;
+
+
+    // Latest known good position. Used for error reporting.
+    size_t latestKnownGoodPos_;
+
+    BinTokenReaderTester(const BinTokenReaderTester&) = delete;
+    BinTokenReaderTester(BinTokenReaderTester&&) = delete;
+    BinTokenReaderTester& operator=(BinTokenReaderTester&) = delete;
+
+  public:
+    // The following classes are used whenever we encounter a tuple/tagged tuple/list
+    // to make sure that:
+    //
+    // - if the construct "knows" its byte length, we have exactly consumed all
+    //   the bytes (otherwise, this means that the file is corrupted, perhaps on
+    //   purpose, so we need to reject the stream);
+    // - if the construct has a footer, once we are done reading it, we have
+    //   reached the footer (this is to aid with debugging).
+    //
+    // In either case, we perform the check when the guard is destructed. Failure
+    // is reported on the next call to `checkStatus()` or to any of the reading
+    // methods.
+
+    // Base class used by other Auto* classes.
+    class MOZ_STACK_CLASS AutoBase
+    {
+      protected:
+        AutoBase(BinTokenReaderTester& reader);
+        ~AutoBase();
+
+        // A variant of BinTokenReaderTester::raiseError that causes the
+        // error to be raised at the next call of BinTokenReaderTester::read*
+        // or BinTokenReaderTester::checkStatus.
+        void postPendingError();
+
+        // A variant of BinTokenReaderTester::readConst that uses AutoBase::raiseError
+        // instead of the regular raiseError.
+        void readConst(const char* data);
+
+        friend BinTokenReaderTester;
+        void init();
+
+        bool initialized_;
+        BinTokenReaderTester& reader_;
+    };
+
+    // Guard class used to ensure that `readList` is used properly.
+    class MOZ_STACK_CLASS AutoList : public AutoBase
+    {
+      public:
+        AutoList(BinTokenReaderTester& reader);
+        ~AutoList();
+      protected:
+        friend BinTokenReaderTester;
+        void init(const char* expectedEnd);
+      private:
+        const char* expectedEnd_;
+    };
+
+    // Guard class used to ensure that `readTaggedTuple` is used properly.
+    class MOZ_STACK_CLASS AutoTaggedTuple : public AutoBase
+    {
+      public:
+        AutoTaggedTuple(BinTokenReaderTester& reader);
+        ~AutoTaggedTuple();
+    };
+
+    // Guard class used to ensure that `readTuple` is used properly.
+    class MOZ_STACK_CLASS AutoTuple : public AutoBase
+    {
+      public:
+        AutoTuple(BinTokenReaderTester& reader);
+        ~AutoTuple();
+    };
+
+    // Compare a `Chars` and a string literal (ONLY a string literal).
+    template <size_t N>
+    static bool equals(const Chars& left, const char (&right)[N]) {
+        MOZ_ASSERT(N > 0);
+        MOZ_ASSERT(right[N - 1] == 0);
+        if (left.length() + 1 /* implicit NUL */ != N) {
+            return false;
+        }
+
+        const char* leftIter;
+        const char* rightIter;
+        for (leftIter = left.begin(), rightIter = right;
+             leftIter < left.end();
+             ++leftIter, ++rightIter) {
+            if (*leftIter != *rightIter)
+                return false;
+        }
+
+        MOZ_ASSERT(*rightIter == 0);
+        MOZ_ASSERT(leftIter = left.end());
+        return true;
+    }
+};
+
+} // namespace frontend
+} // namespace js
+
+#endif // frontend_BinTokenReaderTester_h
new file mode 100644
index 0000000000000000000000000000000000000000..da01d11ad0fa16a02f162d329fb3cd7ba3977eb1
GIT binary patch
literal 21
Uc$~Az$t*6hV_|>*8+|Yr05Ts0pa1{>
new file mode 100644
--- /dev/null
+++ b/js/src/jsapi-tests/binast/tokenizer/tester/test-empty-untagged-tuple.binjs
@@ -0,0 +1,1 @@
+<tuple></tuple>
\ No newline at end of file
new file mode 100644
index 0000000000000000000000000000000000000000..d20681c3cc1dd283dee8568cfaa854af41335d5f
GIT binary patch
literal 90
zc$~Az$t*6h^JHLPU<6_tFwX?YVgm7sONuh{((Ram?6mxR8+|ArSv)DR2qCTy)()ir
D@ZuQQ
new file mode 100644
index 0000000000000000000000000000000000000000..4dd9869685ebb23c52fa3f61a299fb94df82428b
GIT binary patch
literal 34
fc$~8-E-A{)OSj`?U|=ZD%q_@CRe*@t=tE@y!tV>9
new file mode 100644
index 0000000000000000000000000000000000000000..395fdb4476b5632e0445b8e17aa76c3f91c31a57
GIT binary patch
literal 81
zc$~8-DJ{rJwX?}cO-!*1NGvHyEy`nHVqjp%OkpTX%qdM}u+az0*%X%)W#*;ZF$3k&
X^7C!<q5OQI<^!h`IUQ{DAtnI;9z+@$
new file mode 100644
index 0000000000000000000000000000000000000000..b895cf16b7e053f6e50dff48f37d62b6f9d30126
GIT binary patch
literal 44
pc$~8-E-A{)OScneU|@hS70NS9G89salM@S4ixn6c8Eo{SY5+ZI4gmlF
new file mode 100644
index 0000000000000000000000000000000000000000..b92d59df5ea358770349b9bd2cb1f0b1c14eb9d8
GIT binary patch
literal 69
vc$~Az$t*6hGhtw0U;<*B;*z4wymUKeAUiET-$ozGM;1>?EJBFugS7(yf`AkA
new file mode 100644
index 0000000000000000000000000000000000000000..7100fae76b620b7ecce0e6b7304c10ecaccd0823
GIT binary patch
literal 63
qc$~8-DJ{rJwX-QMDay=Cw_|2tU`Wf)x6z04k;Rh|ixA@a5PblXG8CBr
--- a/js/src/jsapi-tests/moz.build
+++ b/js/src/jsapi-tests/moz.build
@@ -124,16 +124,21 @@ if CONFIG['ENABLE_ION']:
         'testJitRValueAlloc.cpp',
     ]
 
 if CONFIG['ENABLE_STREAMS']:
     UNIFIED_SOURCES += [
         'testReadableStream.cpp',
     ]
 
+if CONFIG['NIGHTLY_BUILD']:
+    UNIFIED_SOURCES += [
+        'testBinTokenReaderTester.cpp'
+    ]
+
 DEFINES['EXPORT_JS_API'] = True
 
 LOCAL_INCLUDES += [
     '!..',
     '..',
 ]
 
 if CONFIG['ENABLE_INTL_API'] and CONFIG['MOZ_ICU_DATA_ARCHIVE']:
new file mode 100644
--- /dev/null
+++ b/js/src/jsapi-tests/testBinTokenReaderTester.cpp
@@ -0,0 +1,239 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Vector.h"
+ 
+#include <sys/stat.h>
+
+#include "frontend/BinTokenReaderTester.h"
+
+#include "jsapi-tests/tests.h"
+
+using Tokenizer = js::frontend::BinTokenReaderTester;
+using Chars = Tokenizer::Chars;
+
+void readFull(const char* path, Vector<char>& buf) {
+    buf.shrinkTo(0);
+    FILE* in = fopen(path, "r");
+    if (!in) {
+        fprintf(stderr, "Could not open %s: %s", path, strerror(errno));
+        MOZ_CRASH();
+    }
+
+    struct stat info;
+    if (stat(path, &info) < 0)
+        MOZ_CRASH();
+
+    if (!buf.growBy(info.st_size))
+        MOZ_CRASH("OOM");
+
+    int result = fread(buf.begin(), 1, info.st_size, in);
+    MOZ_ASSERT(result == info.st_size);
+}
+
+
+// Reading a simple string.
+BEGIN_TEST(testBinTokenReaderTesterSimpleString)
+{
+    Vector<char> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-simple-string.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    Maybe<Chars> found;
+    CHECK(tokenizer.readMaybeChars(found));
+
+    CHECK(Tokenizer::equals(*found, "simple string")); // FIXME: Find a way to make CHECK_EQUAL use `Tokenizer::equals`.
+
+    CHECK(tokenizer.checkStatus());
+    return true;
+}
+END_TEST(testBinTokenReaderTesterSimpleString)
+
+// Reading a string with embedded 0.
+BEGIN_TEST(testBinTokenReaderTesterStringWithEscapes)
+{
+    Vector<char> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-string-with-escapes.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    Maybe<Chars> found;
+    CHECK(tokenizer.readMaybeChars(found));
+
+    CHECK(Tokenizer::equals(*found, "string with escapes \0\1\0")); // FIXME: Find a way to make CHECK_EQUAL use `Tokenizer::equals`.
+
+    CHECK(tokenizer.checkStatus());
+    return true;
+}
+END_TEST(testBinTokenReaderTesterStringWithEscapes)
+
+// Reading an empty untagged tuple
+BEGIN_TEST(testBinTokenReaderTesterEmptyUntaggedTuple)
+{
+    Vector<char> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-empty-untagged-tuple.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        Tokenizer::AutoTuple guard(tokenizer);
+        CHECK(tokenizer.readUntaggedTuple(guard));
+    }
+
+    CHECK(tokenizer.checkStatus());
+    return true;
+}
+END_TEST(testBinTokenReaderTesterEmptyUntaggedTuple)
+
+// Reading a untagged tuple with two strings
+BEGIN_TEST(testBinTokenReaderTesterTwoStringsInTuple)
+{
+    Vector<char> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-trivial-untagged-tuple.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        Tokenizer::AutoTuple guard(tokenizer);
+        CHECK(tokenizer.readUntaggedTuple(guard));
+
+        Maybe<Chars> found_0;
+        CHECK(tokenizer.readMaybeChars(found_0));
+        CHECK(Tokenizer::equals(*found_0, "foo")); // FIXME: Find a way to make CHECK_EQUAL use `Tokenizer::equals`.
+
+        Maybe<Chars> found_1;
+        CHECK(tokenizer.readMaybeChars(found_1));
+        CHECK(Tokenizer::equals(*found_1, "bar")); // FIXME: Find a way to make CHECK_EQUAL use `Tokenizer::equals`.
+    }
+
+    CHECK(tokenizer.checkStatus());
+    return true;
+}
+END_TEST(testBinTokenReaderTesterTwoStringsInTuple)
+
+// Reading a tagged tuple `Pattern { id: "foo", value: 3.1415}`
+BEGIN_TEST(testBinTokenReaderTesterSimpleTaggedTuple)
+{
+    Vector<char> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-simple-tagged-tuple.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        js::frontend::BinKind tag;
+        Tokenizer::BinFields fields(cx);
+        Tokenizer::AutoTaggedTuple guard(tokenizer);
+        CHECK(tokenizer.readTaggedTuple(tag, fields, guard));
+
+        CHECK(tag == js::frontend::BinKind::PATTERN);
+
+        Maybe<Chars> found_id;
+        const double EXPECTED_value = 3.1415;
+        Maybe<double> found_value;
+
+        // Order of fields is not deterministic in the encoder (we could make
+        // it deterministic for the test, though, since we already know the binary).
+        if (fields[0] == js::frontend::BinField::ID) {
+            CHECK(fields[1] == js::frontend::BinField::VALUE);
+            CHECK(tokenizer.readMaybeChars(found_id));
+            CHECK(tokenizer.readMaybeDouble(found_value));
+        } else if (fields[0] == js::frontend::BinField::VALUE) {
+            CHECK(fields[1] == js::frontend::BinField::ID);
+            CHECK(tokenizer.readMaybeDouble(found_value));
+            CHECK(tokenizer.readMaybeChars(found_id));
+        } else {
+            CHECK(false);
+        }
+
+        CHECK(EXPECTED_value == *found_value); // Apparently, CHECK_EQUAL doesn't work on `double`.
+        CHECK(Tokenizer::equals(*found_id, "foo"));
+    }
+    CHECK(tokenizer.checkStatus());
+    return true;
+}
+END_TEST(testBinTokenReaderTesterSimpleTaggedTuple)
+
+
+// Reading an empty list
+BEGIN_TEST(testBinTokenReaderTesterEmptyList)
+{
+    Vector<char> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-empty-list.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        uint32_t length;
+        Tokenizer::AutoList guard(tokenizer);
+        CHECK(tokenizer.readList(length, guard));
+
+        CHECK(length == 0);
+    }
+
+    CHECK(tokenizer.checkStatus());
+    return true;
+}
+END_TEST(testBinTokenReaderTesterEmptyList)
+
+// Reading `["foo", "bar"]`
+BEGIN_TEST(testBinTokenReaderTesterSimpleList)
+{
+    Vector<char> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-trivial-list.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        uint32_t length;
+        Tokenizer::AutoList guard(tokenizer);
+        CHECK(tokenizer.readList(length, guard));
+
+        CHECK(length == 2);
+
+        Maybe<Chars> found_0;
+        CHECK(tokenizer.readMaybeChars(found_0));
+        CHECK(Tokenizer::equals(*found_0, "foo"));
+
+        Maybe<Chars> found_1;
+        CHECK(tokenizer.readMaybeChars(found_1));
+        CHECK(Tokenizer::equals(*found_1, "bar"));
+    }
+
+    CHECK(tokenizer.checkStatus());
+    return true;
+}
+END_TEST(testBinTokenReaderTesterSimpleList)
+
+
+// Reading `[["foo", "bar"]]`
+BEGIN_TEST(testBinTokenReaderTesterNestedList)
+{
+    Vector<char> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-nested-lists.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        uint32_t outerLength;
+        Tokenizer::AutoList outerGuard(tokenizer);
+        CHECK(tokenizer.readList(outerLength, outerGuard));
+        CHECK(outerLength == 1);
+
+        {
+            uint32_t innerLength;
+            Tokenizer::AutoList innerGuard(tokenizer);
+            CHECK(tokenizer.readList(innerLength, innerGuard));
+            CHECK(innerLength == 2);
+
+            Maybe<Chars> found_0;
+            CHECK(tokenizer.readMaybeChars(found_0));
+            CHECK(Tokenizer::equals(*found_0, "foo"));
+
+            Maybe<Chars> found_1;
+            CHECK(tokenizer.readMaybeChars(found_1));
+            CHECK(Tokenizer::equals(*found_1, "bar"));
+
+        }
+    }
+
+    CHECK(tokenizer.checkStatus());
+    return true;
+}
+END_TEST(testBinTokenReaderTesterNestedList)
--- a/js/src/moz.build
+++ b/js/src/moz.build
@@ -641,16 +641,21 @@ if CONFIG['NIGHTLY_BUILD']:
     DEFINES['ENABLE_SIMD'] = True
 
 # In-flight WebAssembly atomic operations, sign extension operations,
 # and shared memory objects proposal:
 # https://github.com/WebAssembly/threads
 if CONFIG['NIGHTLY_BUILD']:
     DEFINES['ENABLE_WASM_THREAD_OPS'] = True
 
+# Some parts of BinAST are designed only to test evolutions of the
+# specification:
+if CONFIG['NIGHTLY_BUILD']:
+    UNIFIED_SOURCES += ['frontend/BinTokenReaderTester.cpp']
+
 if CONFIG['MOZ_DEBUG'] or CONFIG['NIGHTLY_BUILD']:
     DEFINES['JS_CACHEIR_SPEW'] = True
 
 # Also set in shell/moz.build
 DEFINES['ENABLE_SHARED_ARRAY_BUFFER'] = True
 
 DEFINES['EXPORT_JS_API'] = True