Bug 1377007 - Implementation of the Token Reader dedicated to testing;r?jorendorff,arai draft
authorDavid Teller <dteller@mozilla.com>
Wed, 13 Sep 2017 15:30:19 +0200
changeset 706119 d32d1f218d6ff445b30c60a15648700f3e057762
parent 706118 774db547caf1708ea5aaefaec8cccb42c8897049
child 706120 c595daf03ffec77e3211ccc8f090f546d1a76c94
push id91725
push userdteller@mozilla.com
push dateFri, 01 Dec 2017 11:07:07 +0000
reviewersjorendorff, arai
bugs1377007
milestone59.0a1
Bug 1377007 - Implementation of the Token Reader dedicated to testing;r?jorendorff,arai This patch ports to SpiderMonkey the tokenizer currently implemented in the external binjs-ref tool. While this tokenizer will clearly not be the tokenizer eventually shipped to end-users (whether in Firefox or in binjs-ref), the plan is to keep it both in binjs-ref and in SpiderMonkey (Nightly only) as a tool for helping test the higher layers of Binjs. MozReview-Commit-ID: 1i6XnVIf8p5
js/src/frontend/BinToken.cpp
js/src/frontend/BinToken.h
js/src/frontend/BinTokenReaderTester.cpp
js/src/frontend/BinTokenReaderTester.h
js/src/jsapi-tests/binast/tokenizer/tester/test-empty-list.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-empty-untagged-tuple.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-nested-lists.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-simple-string.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-simple-tagged-tuple.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-string-with-escapes.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-trivial-list.binjs
js/src/jsapi-tests/binast/tokenizer/tester/test-trivial-untagged-tuple.binjs
js/src/jsapi-tests/moz.build
js/src/jsapi-tests/testBinASTReader.cpp
js/src/jsapi-tests/testBinTokenReaderTester.cpp
js/src/moz.build
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinToken.cpp
@@ -0,0 +1,29 @@
+#include "frontend/BinToken.h"
+
+#include <sys/types.h>
+
+namespace js {
+namespace frontend {
+
+const char* BINKIND_DESCRIPTIONS[] = {
+#define WITH_KIND(_, SPEC_NAME) #SPEC_NAME,
+    FOR_EACH_BIN_KIND(WITH_KIND)
+#undef WITH_KIND
+};
+
+const char* BINFIELD_DESCRIPTIONS[] = {
+    #define WITH_FIELD(_, SPEC_NAME) #SPEC_NAME,
+        FOR_EACH_BIN_FIELD(WITH_FIELD)
+    #undef WITH_FIELD
+};
+
+const char* describeBinKind(const BinKind& kind) {
+    return BINKIND_DESCRIPTIONS[static_cast<size_t>(kind)];
+}
+
+const char* describeBinField(const BinField& field) {
+    return BINFIELD_DESCRIPTIONS[static_cast<size_t>(field)];
+}
+
+} // namespace frontend
+} // namespace js
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinToken.h
@@ -0,0 +1,201 @@
+#ifndef frontend_BinToken_h
+#define frontend_BinToken_h
+
+/**
+ * Definition of Binary AST tokens.
+ *
+ * In the Binary AST world, an AST is composed of nodes, where a node is
+ * defined by:
+ * - a Kind (see `BinKind`);
+ * - a list of fields, where each field is:
+ *    - a Name (see `BinField`);
+ *    - a Value, which may be either a node or a primitive value.
+ *
+ * The mapping between Kind and list of fields is determined entirely by
+ * the grammar of Binary AST. The mapping between (Kind, Name) and the structure
+ * of Value is also determined entirely by the grammar of Binary AST.
+ *
+ * As per the specifications of Binary AST, kinds may be added as the language
+ * grows, but never removed. The mapping between Kind and list of fields may
+ * also change to add new fields or make some fields optional, but may never
+ * remove a field. Finally, the mapping between (Kind, Name) and the structure
+ * of Value may be modified to add new possible values, but never to remove a
+ * value.
+ *
+ * A Binary AST parser must be able to fail gracefully when confronted with
+ * unknown Kinds or Names.
+ *
+ * At the time of this writing, the Binary AST defined from the Babylon AST
+ * (see https://github.com/babel/babylon/blob/master/ast/spec.md) restricted
+ * to ES5, with a few amendments to store additional scoping data and to
+ * represent the empty AST.
+ *
+ * Future versions of the Binary AST will progressively grow to encompass ES6
+ * and beyond.
+ */
+
+namespace js {
+namespace frontend {
+
+ /**
+ * The different kinds of Binary AST nodes, as per the specifications of
+ * Binary AST.
+ *
+ * These kinds match roughly with the `ParseNodeKind` used internally.
+ *
+ * Usage:
+ *
+ * ```c++
+ * #define WITH_KIND(CPP_NAME, SPEC_NAME) ...
+ * FOR_EACH_BIN_KIND(WITH_KIND)
+ * ```
+ *
+ *
+ * (sorted by alphabetical order)
+ */
+#define FOR_EACH_BIN_KIND(F) \
+    F(ArrayExpression, ArrayExpression) \
+    F(AssignmentExpression, AssignmentExpression) \
+    F(AssignmentOperator, AssignmentOperator) \
+    F(BinaryExpression, BinaryExpression) \
+    F(BinaryOperator, BinaryOperator) \
+    F(BINJS_Scope, BINJS:Scope) \
+    F(BlockStatement, BlockStatement) \
+    F(BooleanLiteral, BooleanLiteral) \
+    F(BracketExpression, BracketExpression) \
+    F(BreakStatement, BreakStatement) \
+    F(CallExpression, CallExpression) \
+    F(CatchClause, CatchClause) \
+    F(ComputedPropertyName, ComputedPropertyName) \
+    F(ConditionalExpression, ConditionalExpression) \
+    F(ContinueStatement, ContinueStatement) \
+    F(DebuggerStatement, DebuggerStatement) \
+    F(Declaration, Declaration) \
+    F(Directive, Directive) \
+    F(DirectiveLiteral, DirectiveLiteral) \
+    F(DotExpression, DotExpression) \
+    F(DoWhileStatement, DoWhileStatement) \
+    F(Elision, Elision) \
+    F(EmptyStatement, EmptyStatement) \
+    F(Expression, Expression) \
+    F(ExpressionStatement, ExpressionStatement) \
+    F(ForStatement, ForStatement) \
+    F(ForInStatement, ForInStatement) \
+    F(FunctionExpression, FunctionExpression) \
+    F(FunctionDeclaration, FunctionDeclaration) \
+    F(Identifier, Identifier) \
+    F(IfStatement, IfStatement) \
+    F(LabeledStatement, LabeledStatement) \
+    F(Literal, Literal) \
+    F(LogicalExpression, LogicalExpression) \
+    F(LogicalOperator, LogicalOperator) \
+    F(NewExpression, NewExpression) \
+    F(NullLiteral, NullLiteral) \
+    F(NumericLiteral, NumericLiteral) \
+    F(ObjectExpression, ObjectExpression) \
+    F(ObjectGetter, ObjectGetter) \
+    F(ObjectMethod, ObjectMethod) \
+    F(ObjectSetter, ObjectSetter) \
+    F(ObjectProperty, ObjectProperty) \
+    F(Pattern, Pattern) \
+    F(Program, Program) \
+    F(PropertyKind, PropertyKind) \
+    F(RegExpLiteral, RegExpLiteral) \
+    F(ReturnStatement, ReturnStatement) \
+    F(SequenceExpression, SequenceExpression) \
+    F(StringLiteral, StringLiteral) \
+    F(Statement, Statement) \
+    F(SwitchCase, SwitchCase) \
+    F(SwitchStatement, SwitchStatement) \
+    F(ThisExpression, ThisExpression) \
+    F(ThrowStatement, ThrowStatement) \
+    F(TryStatement, TryStatement) \
+    F(UnaryExpression, UnaryExpression) \
+    F(UnaryOperator, UnaryOperator) \
+    F(UpdateExpression, UpdateExpression) \
+    F(UpdateOperator, UpdateOperator) \
+    F(VariableDeclaration, VariableDeclaration) \
+    F(VariableDeclarator, VariableDeclarator) \
+    F(VariableKind, VariableKind) \
+    F(WhileStatement, WhileStatement) \
+    F(WithStatement, WithStatement)
+
+enum class BinKind {
+#define EMIT_ENUM(name, _) name,
+    FOR_EACH_BIN_KIND(EMIT_ENUM)
+#undef EMIT_ENUM
+    BINKIND_LIMIT /* domain size */
+};
+
+const char* describeBinKind(const BinKind& kind);
+
+/**
+ * The different fields of Binary AST nodes, as per the specifications of
+ * Binary AST.
+ *
+ * Usage:
+ *
+ * ```c++
+ * #define WITH_FIELD(CPP_NAME, SPEC_NAME) ...
+ * FOR_EACH_BIN_FIELD(WITH_FIELD)
+ * ```
+ *
+ * (sorted by alphabetical order)
+ */
+ #define FOR_EACH_BIN_FIELD(F) \
+    F(Alternate, alternate) \
+    F(Argument, argument) \
+    F(Arguments, arguments) \
+    F(BINJS_CapturedNames, BINJS:CapturedNames) \
+    F(BINJS_ConstDeclaredNames, BINJS:ConstDeclaredNames) \
+    F(BINJS_HasDirectEval, BINJS:HasDirectEval) \
+    F(BINJS_LetDeclaredNames, BINJS:LetDeclaredNames) \
+    F(BINJS_VarDeclaredNames, BINJS:VarDeclaredNames) \
+    F(BINJS_Scope, BINJS:Scope) \
+    F(Block, block) \
+    F(Callee, callee) \
+    F(Cases, cases) \
+    F(Consequent, consequent) \
+    F(Body, body) \
+    F(Declarations, declarations) \
+    F(Directives, directives) \
+    F(Discriminant, discriminant) \
+    F(Elements, elements) \
+    F(Expression, expression) \
+    F(Expressions, expressions) \
+    F(Finalizer, finalizer) \
+    F(Flags, flags) \
+    F(Handler, handler) \
+    F(Id, id) \
+    F(Init, init) \
+    F(Key, key) \
+    F(Kind, kind) \
+    F(Label, label) \
+    F(Left, left) \
+    F(Name, name) \
+    F(Object, object) \
+    F(Operator, operator) \
+    F(Param, param) \
+    F(Params, params) \
+    F(Pattern, pattern) \
+    F(Prefix, prefix) \
+    F(Properties, properties) \
+    F(Property, property) \
+    F(Right, right) \
+    F(Test, test) \
+    F(Update, update) \
+    F(Value, value)
+
+enum class BinField {
+#define EMIT_ENUM(name, _) name,
+    FOR_EACH_BIN_FIELD(EMIT_ENUM)
+#undef EMIT_ENUM
+    BINFIELD_LIMIT /* domain size */
+};
+
+const char* describeBinField(const BinField& kind);
+
+} // namespace frontend
+} // namespace js
+
+#endif // frontend_BinToken_h
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinTokenReaderTester.cpp
@@ -0,0 +1,529 @@
+#include "frontend/BinTokenReaderTester.h"
+
+#include "mozilla/EndianUtils.h"
+#include "gc/Zone.h"
+
+namespace js {
+namespace frontend {
+
+using BinFields = BinTokenReaderTester::BinFields;
+using AutoList = BinTokenReaderTester::AutoList;
+using AutoTaggedTuple = BinTokenReaderTester::AutoTaggedTuple;
+using AutoTuple = BinTokenReaderTester::AutoTuple;
+
+BinTokenReaderTester::BinTokenReaderTester(JSContext* cx, const uint8_t* start, const size_t length)
+    : cx_(cx)
+    , start_(start)
+    , current_(start)
+    , stop_(start + length)
+    , latestKnownGoodPos_(0)
+{ }
+
+BinTokenReaderTester::BinTokenReaderTester(JSContext* cx, const Vector<uint8_t>& chars)
+    : cx_(cx)
+    , start_(chars.begin())
+    , current_(chars.begin())
+    , stop_(chars.end())
+    , latestKnownGoodPos_(0)
+{ }
+
+bool
+BinTokenReaderTester::raiseError(const char* description)
+{
+    MOZ_ASSERT(!cx_->isExceptionPending());
+    TokenPos pos;
+    latestTokenPos(pos);
+    JS_ReportErrorASCII(cx_, "BinAST parsing error: %s at offsets %u => %u",
+                        description, pos.begin, pos.end);
+    return false;
+}
+
+bool
+BinTokenReaderTester::readBuf(uint8_t* bytes, uint32_t len)
+{
+    MOZ_ASSERT(!cx_->isExceptionPending());
+    MOZ_ASSERT(len > 0);
+
+    if (stop_ < current_ + len)
+        return raiseError("Buffer exceeds length");
+
+    for (uint32_t i = 0; i < len; ++i)
+        *bytes++ = *current_++;
+
+    return true;
+}
+
+bool
+BinTokenReaderTester::readByte(uint8_t* byte)
+{
+    return readBuf(byte, 1);
+}
+
+
+// Nullable booleans:
+//
+// 0 => false
+// 1 => true
+// 2 => null
+bool
+BinTokenReaderTester::readMaybeBool(Maybe<bool>& result)
+{
+    updateLatestKnownGood();
+    uint8_t byte;
+    if (!readByte(&byte))
+        return false;
+
+    switch (byte) {
+      case 0:
+        result = Some(false);
+        break;
+      case 1:
+        result = Some(true);
+        break;
+      case 2:
+        result = Nothing();
+        break;
+      default:
+        return raiseError("Invalid boolean value");
+    }
+    return true;
+}
+
+bool
+BinTokenReaderTester::readBool(bool& out)
+{
+    Maybe<bool> result;
+
+    if (!readMaybeBool(result))
+        return false;
+
+    if (result.isNothing())
+        return raiseError("Empty boolean value");
+
+    out = *result;
+    return true;
+}
+
+// Nullable doubles (little-endian)
+//
+// 0x7FF0000000000001 (signaling NaN) => null
+// anything other 64 bit sequence => IEEE-764 64-bit floating point number
+bool
+BinTokenReaderTester::readMaybeDouble(Maybe<double>& result)
+{
+    updateLatestKnownGood();
+
+    uint8_t bytes[8];
+    MOZ_ASSERT(sizeof(bytes) == sizeof(double));
+    if (!readBuf(reinterpret_cast<uint8_t*>(bytes), ArrayLength(bytes)))
+        return false;
+
+    // Decode little-endian.
+    const uint64_t asInt = LittleEndian::readUint64(bytes);
+
+    if (asInt == 0x7FF0000000000001) {
+        result = Nothing();
+    } else {
+        // Canonicalize NaN, just to make sure another form of signalling NaN
+        // doesn't slip past us.
+        const double asDouble = CanonicalizeNaN(BitwiseCast<double>(asInt));
+        result = Some(asDouble);
+    }
+
+    return true;
+}
+
+bool
+BinTokenReaderTester::readDouble(double& out)
+{
+    Maybe<double> result;
+
+    if (!readMaybeDouble(result))
+        return false;
+
+    if (result.isNothing())
+        return raiseError("Empty double value");
+
+    out = *result;
+    return true;
+}
+
+// Internal uint32_t
+//
+// Encoded as 4 bytes, little-endian.
+bool
+BinTokenReaderTester::readInternalUint32(uint32_t* result)
+{
+    uint8_t bytes[4];
+    MOZ_ASSERT(sizeof(bytes) == sizeof(uint32_t));
+    if (!readBuf(bytes, 4))
+        return false;
+
+    // Decode little-endian.
+    *result = LittleEndian::readUint32(bytes);
+
+    return true;
+}
+
+
+
+// Nullable strings:
+// - "<string>" (not counted in byte length)
+// - byte length (not counted in byte length)
+// - bytes (UTF-8)
+// - "</string>" (not counted in byte length)
+//
+// The special sequence of bytes `[255, 0]` (which is an invalid UTF-8 sequence)
+// is reserved to `null`.
+bool
+BinTokenReaderTester::readMaybeChars(Maybe<Chars>& out)
+{
+    updateLatestKnownGood();
+
+    if (!readConst("<string>"))
+        return false;
+
+    // 1. Read byteLength
+    uint32_t byteLen;
+    if (!readInternalUint32(&byteLen))
+        return false;
+
+    // 2. Reject if we can't read
+    if (current_ + byteLen < current_) // Check for overflows
+        return raiseError("Arithmetics overflow: string is too long");
+
+    if (current_ + byteLen > stop_)
+        return raiseError("Not enough bytes to read chars");
+
+    // 3. Check null string (no allocation)
+    if (byteLen == 2 && *current_ == 255 && *(current_ + 1) == 0) {
+        // Special case: null string.
+        out = Nothing();
+        current_ += byteLen;
+        return true;
+    }
+
+    // 4. Other strings (bytes are copied)
+    out.emplace(cx_);
+    if (!out->resize(byteLen)) {
+        ReportOutOfMemory(cx_);
+        return false;
+    }
+    PodCopy(out->begin(), current_, byteLen);
+    current_ += byteLen;
+
+    if (!readConst("</string>"))
+        return false;
+
+    return true;
+}
+
+bool
+BinTokenReaderTester::readChars(Chars& out)
+{
+    Maybe<Chars> result;
+
+    if (!readMaybeChars(result))
+        return false;
+
+    if (result.isNothing())
+        return raiseError("Empty string");
+
+    out = Move(*result);
+    return true;
+}
+
+template <size_t N>
+bool
+BinTokenReaderTester::matchConst(const char (&value)[N])
+{
+    MOZ_ASSERT(N > 0);
+    MOZ_ASSERT(value[N - 1] == 0);
+    MOZ_ASSERT(!cx_->isExceptionPending());
+
+    if (current_ + N - 1 > stop_)
+        return false;
+
+    // Perform lookup, without side-effects.
+    if (!std::equal(current_, current_ + N - 1 /*implicit NUL*/, value))
+        return false;
+
+    // Looks like we have a match. Now perform side-effects
+    current_ += N - 1;
+    updateLatestKnownGood();
+    return true;
+}
+
+
+// Untagged tuple:
+// - "<tuple>";
+// - contents (specified by the higher-level grammar);
+// - "</tuple>"
+bool
+BinTokenReaderTester::enterUntaggedTuple(AutoTuple& guard)
+{
+    if (!readConst("<tuple>"))
+        return false;
+
+    guard.init();
+    return true;
+}
+
+template <size_t N>
+bool
+BinTokenReaderTester::readConst(const char (&value)[N])
+{
+    updateLatestKnownGood();
+    if (!matchConst(value))
+        return raiseError("Could not find expected literal");
+
+    return true;
+}
+
+// Tagged tuples:
+// - "<tuple>"
+// - "<head>"
+// - non-null string `name`, followed by \0 (see `readString()`);
+// - uint32_t number of fields;
+// - array of `number of fields` non-null strings followed each by \0 (see `readString()`);
+// - "</head>"
+// - content (specified by the higher-level grammar);
+// - "</tuple>"
+bool
+BinTokenReaderTester::enterTaggedTuple(BinKind& tag, BinFields& fields, AutoTaggedTuple& guard)
+{
+    // Header
+    if (!readConst("<tuple>"))
+        return false;
+
+    if (!readConst("<head>"))
+        return false;
+
+    // This would probably be much faster with a HashTable, but we don't
+    // really care about the speed of BinTokenReaderTester.
+    do {
+
+#define FIND_MATCH(CONSTRUCTOR, NAME) \
+        if (matchConst(#NAME "\0")) { \
+            tag = BinKind::CONSTRUCTOR; \
+            break; \
+        } // else
+
+        FOR_EACH_BIN_KIND(FIND_MATCH)
+#undef FIND_MATCH
+
+        // else
+        return raiseError("Invalid tag");
+    } while(false);
+
+    // Now fields.
+    uint32_t fieldNum;
+    if (!readInternalUint32(&fieldNum))
+        return false;
+
+    fields.clear();
+    if (!fields.reserve(fieldNum))
+        return raiseError("Out of memory");
+
+    for (uint32_t i = 0; i < fieldNum; ++i) {
+        // This would probably be much faster with a HashTable, but we don't
+        // really care about the speed of BinTokenReaderTester.
+        BinField field;
+        do {
+
+#define FIND_MATCH(CONSTRUCTOR, NAME) \
+            if (matchConst(#NAME "\0")) { \
+                field = BinField::CONSTRUCTOR; \
+                break; \
+            } // else
+
+            FOR_EACH_BIN_FIELD(FIND_MATCH)
+#undef FIND_MATCH
+
+            // else
+            return raiseError("Invalid field");
+        } while (false);
+
+        // Make sure that we do not have duplicate fields.
+        // Search is linear, but again, we don't really care
+        // in this implementation.
+        for (uint32_t j = 0; j < i; ++j) {
+            if (fields[j] == field) {
+                return raiseError("Duplicate field");
+            }
+        }
+
+        fields.infallibleAppend(field); // Already checked.
+    }
+
+    // End of header
+
+    if (!readConst("</head>"))
+        return false;
+
+    // Enter the body.
+    guard.init();
+    return true;
+}
+
+// List:
+//
+// - "<list>" (not counted in byte length);
+// - uint32_t byte length (not counted in byte length);
+// - uint32_t number of items;
+// - contents (specified by higher-level grammar);
+// - "</list>" (not counted in byte length)
+//
+// The total byte length of `number of items` + `contents` must be `byte length`.
+bool
+BinTokenReaderTester::enterList(uint32_t& items, AutoList& guard)
+{
+    if (!readConst("<list>"))
+        return false;
+
+    uint32_t byteLen;
+    if (!readInternalUint32(&byteLen))
+        return false;
+
+    const uint8_t* stop = current_ + byteLen;
+
+    if (stop < current_) // Check for overflows
+        return raiseError("Arithmetics overflow: list is too long");
+
+    if (stop > this->stop_)
+        return raiseError("Incorrect list length");
+
+    guard.init(stop);
+
+    if (!readInternalUint32(&items))
+        return false;
+
+    return true;
+}
+
+void
+BinTokenReaderTester::updateLatestKnownGood()
+{
+    MOZ_ASSERT(current_ >= start_);
+    const size_t update = current_ - start_;
+    MOZ_ASSERT(update >= latestKnownGoodPos_);
+    latestKnownGoodPos_ = update;
+}
+
+size_t
+BinTokenReaderTester::offset() const
+{
+    return latestKnownGoodPos_;
+}
+
+void
+BinTokenReaderTester::latestTokenPos(TokenPos& pos)
+{
+    pos.begin = latestKnownGoodPos_;
+    pos.end = current_ - start_;
+    MOZ_ASSERT(pos.end >= pos.begin);
+}
+
+void
+BinTokenReaderTester::AutoBase::init()
+{
+    initialized_ = true;
+}
+
+BinTokenReaderTester::AutoBase::AutoBase(BinTokenReaderTester& reader)
+    : reader_(reader)
+{ }
+
+BinTokenReaderTester::AutoBase::~AutoBase()
+{
+    // By now, the `AutoBase` must have been deinitialized by calling `done()`.
+    // The only case in which we can accept not calling `done()` is if we have
+    // bailed out because of an error.
+    MOZ_ASSERT_IF(initialized_, reader_.cx_->isExceptionPending());
+}
+
+bool
+BinTokenReaderTester::AutoBase::checkPosition(const uint8_t* expectedEnd)
+{
+    if (reader_.current_ != expectedEnd)
+        return reader_.raiseError("Caller did not consume the expected set of bytes");
+
+    return true;
+}
+
+BinTokenReaderTester::AutoList::AutoList(BinTokenReaderTester& reader)
+    : AutoBase(reader)
+{ }
+
+void
+BinTokenReaderTester::AutoList::init(const uint8_t* expectedEnd)
+{
+    AutoBase::init();
+    this->expectedEnd_ = expectedEnd;
+}
+
+bool
+BinTokenReaderTester::AutoList::done()
+{
+    MOZ_ASSERT(initialized_);
+    initialized_ = false;
+    if (reader_.cx_->isExceptionPending()) {
+        // Already errored, no need to check further.
+        return false;
+    }
+
+    // Check that we have consumed the exact number of bytes.
+    if (!checkPosition(expectedEnd_))
+        return false;
+
+    // Check suffix.
+    if (!reader_.readConst("</list>"))
+        return false;
+
+    return true;
+}
+
+BinTokenReaderTester::AutoTaggedTuple::AutoTaggedTuple(BinTokenReaderTester& reader)
+    : AutoBase(reader)
+{ }
+
+bool
+BinTokenReaderTester::AutoTaggedTuple::done()
+{
+    MOZ_ASSERT(initialized_);
+    initialized_ = false;
+    if (reader_.cx_->isExceptionPending()) {
+        // Already errored, no need to check further.
+        return false;
+    }
+
+    // Check suffix.
+    if (!reader_.readConst("</tuple>"))
+        return false;
+
+    return true;
+}
+
+BinTokenReaderTester::AutoTuple::AutoTuple(BinTokenReaderTester& reader)
+    : AutoBase(reader)
+{ }
+
+bool
+BinTokenReaderTester::AutoTuple::done()
+{
+    MOZ_ASSERT(initialized_);
+    initialized_ = false;
+    if (reader_.cx_->isExceptionPending()) {
+        // Already errored, no need to check further.
+        return false;
+    }
+
+    // Check suffix.
+    if (!reader_.readConst("</tuple>"))
+        return false;
+
+    return true;
+}
+
+} // namespace frontend
+} // namespace js
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinTokenReaderTester.h
@@ -0,0 +1,356 @@
+#ifndef frontend_BinTokenReaderTester_h
+#define frontend_BinTokenReaderTester_h
+
+#include "mozilla/Maybe.h"
+
+#include "frontend/BinToken.h"
+#include "frontend/TokenStream.h"
+
+#include "js/TypeDecls.h"
+
+#if !defined(NIGHTLY_BUILD)
+#error "BinTokenReaderTester.* is designed to help test implementations of successive versions of JS BinaryAST. It is available only on Nightly."
+#endif // !defined(NIGHTLY_BUILD)
+
+namespace js {
+namespace frontend {
+
+using namespace mozilla;
+using namespace JS;
+
+/**
+ * A token reader for a simple, alternative serialization format for BinAST.
+ *
+ * This serialization format, which is also supported by the reference
+ * implementation of the BinAST compression suite, is designed to be
+ * mostly human-readable and easy to check for all sorts of deserialization
+ * errors. While this format is NOT designed to be shipped to end-users, it
+ * is nevertheless a very useful tool for implementing and testing parsers.
+ *
+ * Both the format and the implementation are ridiculously inefficient:
+ *
+ * - the underlying format tags almost all its data with e.g. `<tuple>`, `</tuple>`
+ *   to aid with detecting offset errors or format error;
+ * - the underlying format copies list of fields into every single node, instead
+ *   of keeping them once in the header;
+ * - every kind/field extraction requires memory allocation and plenty of string
+ *   comparisons;
+ * - ...
+ *
+ * This token reader is designed to be API-compatible with the standard, shipped,
+ * token reader. For these reasons:
+ *
+ * - it does not support any form of look ahead, push back;
+ * - it does not support any form of error recovery.
+ */
+class MOZ_STACK_CLASS BinTokenReaderTester
+{
+  public:
+    // A list of fields, in the order in which they appear in the stream.
+    using BinFields = Vector<BinField, 8>;
+
+    // A bunch of characters. At this stage, there is no guarantee on whether
+    // they are valid UTF-8. Future versions may replace this by slice into
+    // the buffer.
+    using Chars     = Vector<uint8_t, 32>;
+
+    class AutoList;
+    class AutoTuple;
+    class AutoTaggedTuple;
+
+  public:
+    /**
+     * Construct a token reader.
+     *
+     * Does NOT copy the buffer.
+     */
+    BinTokenReaderTester(JSContext* cx, const uint8_t* start, const size_t length);
+
+    /**
+     * Construct a token reader.
+     *
+     * Does NOT copy the buffer.
+     */
+    BinTokenReaderTester(JSContext* cx, const Vector<uint8_t>& chars);
+
+    // --- Primitive values.
+    //
+    // Note that the underlying format allows for a `null` value for primitive
+    // values.
+    //
+    // Reading will return an error either in case of I/O error or in case of
+    // a format problem. Reading if an exception in pending is an error and
+    // will cause assertion failures. Do NOT attempt to read once an exception
+    // has been cleared: the token reader does NOT support recovery, by design.
+
+    /**
+     * Read a single `true | false | null` value.
+     *
+     * @param out Set to `Nothing` if the data specifies that the value is `null`.
+     * Otherwise, `Some(true)` or `Some(false)`.
+     *
+     * @return false If a boolean could not be read. In this case, an error
+     * has been raised.
+     */
+    MOZ_MUST_USE bool readMaybeBool(Maybe<bool>& out);
+    MOZ_MUST_USE bool readBool(bool& out);
+
+    /**
+     * Read a single `number | null` value.
+     *
+     * @param out Set to `Nothing` if the data specifies that the value is `null`.
+     * Otherwise, `Some(x)`, where `x` is a valid `double` (i.e. either a non-NaN
+     * or a canonical NaN).
+     *
+     * @return false If a double could not be read. In this case, an error
+     * has been raised.
+     */
+    MOZ_MUST_USE bool readMaybeDouble(Maybe<double>& out);
+    MOZ_MUST_USE bool readDouble(double& out);
+
+    /**
+     * Read a single `string | null` value.
+     *
+     * @param out Set to `Nothing` if the data specifies that the value is `null`.
+     * Otherwise, `Some(x)`, where `x` is a `string`.
+     *
+     * WARNING: At this stage, the `string` encoding has NOT been validated.
+     *
+     * @return false If a string could not be read. In this case, an error
+     * has been raised.
+     */
+    MOZ_MUST_USE bool readMaybeChars(Maybe<Chars>& out);
+    MOZ_MUST_USE bool readChars(Chars& out);
+
+    // --- Composite values.
+    //
+    // The underlying format does NOT allows for a `null` composite value.
+    //
+    // Reading will return an error either in case of I/O error or in case of
+    // a format problem. Reading from a poisoned tokenizer is an error and
+    // will cause assertion failures.
+
+    /**
+     * Start reading a list.
+     *
+     * @param length (OUT) The number of elements in the list.
+     * @param guard (OUT) A guard, ensuring that we read the list correctly.
+     *
+     * The `guard` is dedicated to ensuring that reading the list has consumed
+     * exactly all the bytes from that list. The `guard` MUST therefore be
+     * destroyed at the point where the caller has reached the end of the list.
+     * If the caller has consumed too few/too many bytes, this will be reported
+     * in the call go `guard.done()`.
+     *
+     * @return out If the header of the list is invalid.
+     */
+    MOZ_MUST_USE bool enterList(uint32_t& length, AutoList& guard);
+
+    /**
+     * Start reading a tagged tuple.
+     *
+     * @param tag (OUT) The tag of the tuple.
+     * @param fields (OUT) The ORDERED list of fields encoded in this tuple.
+     * @param guard (OUT) A guard, ensuring that we read the tagged tuple correctly.
+     *
+     * The `guard` is dedicated to ensuring that reading the list has consumed
+     * exactly all the bytes from that tuple. The `guard` MUST therefore be
+     * destroyed at the point where the caller has reached the end of the tuple.
+     * If the caller has consumed too few/too many bytes, this will be reported
+     * in the call go `guard.done()`.
+     *
+     * @return out If the header of the tuple is invalid.
+     */
+    MOZ_MUST_USE bool enterTaggedTuple(BinKind& tag, BinTokenReaderTester::BinFields& fields, AutoTaggedTuple& guard);
+
+    /**
+     * Start reading an untagged tuple.
+     *
+     * @param guard (OUT) A guard, ensuring that we read the tuple correctly.
+     *
+     * The `guard` is dedicated to ensuring that reading the list has consumed
+     * exactly all the bytes from that tuple. The `guard` MUST therefore be
+     * destroyed at the point where the caller has reached the end of the tuple.
+     * If the caller has consumed too few/too many bytes, this will be reported
+     * in the call go `guard.done()`.
+     *
+     * @return out If the header of the tuple is invalid.
+     */
+    MOZ_MUST_USE bool enterUntaggedTuple(AutoTuple& guard);
+
+    /**
+     * Return the position of the latest token.
+     */
+    void latestTokenPos(TokenPos& out);
+    size_t offset() const;
+
+    /**
+     * Raise an error.
+     *
+     * Once `raiseError` has been called, the tokenizer is poisoned.
+     */
+    MOZ_MUST_USE bool raiseError(const char* description);
+
+     /**
+      * Poison this tokenizer.
+      */
+    void poison();
+
+  private:
+    /**
+     * Read a single byte.
+     */
+    MOZ_MUST_USE bool readByte(uint8_t* byte);
+
+    /**
+     * Read several bytes.
+     *
+     * If there is not enough data, or if the tokenizer has previously been
+     * poisoned, return `false` and report an exception.
+     */
+    MOZ_MUST_USE bool readBuf(uint8_t* bytes, uint32_t len);
+
+    /**
+     * Read a single uint32_t.
+     */
+    MOZ_MUST_USE bool readInternalUint32(uint32_t*);
+
+    /**
+     * Read a sequence of chars, ensuring that they match an expected
+     * sequence of chars.
+     *
+     * @param value The sequence of chars to expect, NUL-terminated. The NUL
+     * is not expected in the stream.
+     */
+     template <size_t N>
+     MOZ_MUST_USE bool readConst(const char (&value)[N]);
+
+     /**
+     * Read a sequence of chars, consuming the bytes only if they match an expected
+     * sequence of chars.
+     *
+     * @param value The sequence of chars to expect, NUL-terminated. The NUL
+     * is not expected in the stream.
+     * @return true if `value` (minus NUL) represents the next few chars in the
+     * internal buffer, false otherwise. If `true`, the chars are consumed,
+     * otherwise there is no side-effect.
+     */
+    template <size_t N>
+    MOZ_MUST_USE bool matchConst(const char (&value)[N]);
+
+    /**
+     * Update the "latest known good" position, which is used during error
+     * reporting.
+     */
+    void updateLatestKnownGood();
+
+  private:
+    JSContext* cx_;
+
+    // `true` if we have encountered an error. Errors are non recoverable.
+    // Attempting to read from a poisoned tokenizer will cause assertion errors.
+    bool poisoned_;
+
+    // The first byte of the buffer. Not owned.
+    const uint8_t* start_;
+
+    // The current position.
+    const uint8_t* current_;
+
+    // The last+1 byte of the buffer.
+    const uint8_t* stop_;
+
+
+    // Latest known good position. Used for error reporting.
+    size_t latestKnownGoodPos_;
+
+    BinTokenReaderTester(const BinTokenReaderTester&) = delete;
+    BinTokenReaderTester(BinTokenReaderTester&&) = delete;
+    BinTokenReaderTester& operator=(BinTokenReaderTester&) = delete;
+
+  public:
+    // The following classes are used whenever we encounter a tuple/tagged tuple/list
+    // to make sure that:
+    //
+    // - if the construct "knows" its byte length, we have exactly consumed all
+    //   the bytes (otherwise, this means that the file is corrupted, perhaps on
+    //   purpose, so we need to reject the stream);
+    // - if the construct has a footer, once we are done reading it, we have
+    //   reached the footer (this is to aid with debugging).
+    //
+    // In either case, the caller MUST call method `done()` of the guard once
+    // it is done reading the tuple/tagged tuple/list, to report any pending error.
+
+    // Base class used by other Auto* classes.
+    class MOZ_STACK_CLASS AutoBase
+    {
+      protected:
+        explicit AutoBase(BinTokenReaderTester& reader);
+        ~AutoBase();
+
+        // Raise an error if we are not in the expected position.
+        MOZ_MUST_USE bool checkPosition(const uint8_t* expectedPosition);
+
+        friend BinTokenReaderTester;
+        void init();
+
+        // Set to `true` if `init()` has been called. Reset to `false` once
+        // all conditions have been checked.
+        bool initialized_;
+        BinTokenReaderTester& reader_;
+    };
+
+    // Guard class used to ensure that `enterList` is used properly.
+    class MOZ_STACK_CLASS AutoList : public AutoBase
+    {
+      public:
+        explicit AutoList(BinTokenReaderTester& reader);
+
+        // Check that we have properly read to the end of the list.
+        MOZ_MUST_USE bool done();
+      protected:
+        friend BinTokenReaderTester;
+        void init(const uint8_t* expectedEnd);
+      private:
+        const uint8_t* expectedEnd_;
+    };
+
+    // Guard class used to ensure that `enterTaggedTuple` is used properly.
+    class MOZ_STACK_CLASS AutoTaggedTuple : public AutoBase
+    {
+      public:
+        explicit AutoTaggedTuple(BinTokenReaderTester& reader);
+
+        // Check that we have properly read to the end of the tuple.
+        MOZ_MUST_USE bool done();
+    };
+
+    // Guard class used to ensure that `readTuple` is used properly.
+    class MOZ_STACK_CLASS AutoTuple : public AutoBase
+    {
+      public:
+        explicit AutoTuple(BinTokenReaderTester& reader);
+
+        // Check that we have properly read to the end of the tuple.
+        MOZ_MUST_USE bool done();
+    };
+
+    // Compare a `Chars` and a string literal (ONLY a string literal).
+    template <size_t N>
+    static bool equals(const Chars& left, const char (&right)[N]) {
+        MOZ_ASSERT(N > 0);
+        MOZ_ASSERT(right[N - 1] == 0);
+        if (left.length() + 1 /* implicit NUL */ != N)
+            return false;
+
+        if (!std::equal(left.begin(), left.end(), right))
+          return false;
+
+        return true;
+    }
+};
+
+} // namespace frontend
+} // namespace js
+
+#endif // frontend_BinTokenReaderTester_h
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..da01d11ad0fa16a02f162d329fb3cd7ba3977eb1
GIT binary patch
literal 21
Uc$~Az$t*6hV_|>*8+|Yr05Ts0pa1{>
new file mode 100644
--- /dev/null
+++ b/js/src/jsapi-tests/binast/tokenizer/tester/test-empty-untagged-tuple.binjs
@@ -0,0 +1,1 @@
+<tuple></tuple>
\ No newline at end of file
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..d20681c3cc1dd283dee8568cfaa854af41335d5f
GIT binary patch
literal 90
zc$~Az$t*6h^JHLPU<6_tFwX?YVgm7sONuh{((Ram?6mxR8+|ArSv)DR2qCTy)()ir
D@ZuQQ
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4dd9869685ebb23c52fa3f61a299fb94df82428b
GIT binary patch
literal 34
fc$~8-E-A{)OSj`?U|=ZD%q_@CRe*@t=tE@y!tV>9
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..395fdb4476b5632e0445b8e17aa76c3f91c31a57
GIT binary patch
literal 81
zc$~8-DJ{rJwX?}cO-!*1NGvHyEy`nHVqjp%OkpTX%qdM}u+az0*%X%)W#*;ZF$3k&
X^7C!<q5OQI<^!h`IUQ{DAtnI;9z+@$
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b895cf16b7e053f6e50dff48f37d62b6f9d30126
GIT binary patch
literal 44
pc$~8-E-A{)OScneU|@hS70NS9G89salM@S4ixn6c8Eo{SY5+ZI4gmlF
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b92d59df5ea358770349b9bd2cb1f0b1c14eb9d8
GIT binary patch
literal 69
vc$~Az$t*6hGhtw0U;<*B;*z4wymUKeAUiET-$ozGM;1>?EJBFugS7(yf`AkA
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..7100fae76b620b7ecce0e6b7304c10ecaccd0823
GIT binary patch
literal 63
qc$~8-DJ{rJwX-QMDay=Cw_|2tU`Wf)x6z04k;Rh|ixA@a5PblXG8CBr
--- a/js/src/jsapi-tests/moz.build
+++ b/js/src/jsapi-tests/moz.build
@@ -132,16 +132,21 @@ if CONFIG['ENABLE_ION']:
         'testJitRValueAlloc.cpp',
     ]
 
 if CONFIG['ENABLE_STREAMS']:
     UNIFIED_SOURCES += [
         'testReadableStream.cpp',
     ]
 
+if CONFIG['JS_BUILD_BINAST']:
+    UNIFIED_SOURCES += [
+        'testBinTokenReaderTester.cpp'
+    ]
+
 DEFINES['EXPORT_JS_API'] = True
 
 LOCAL_INCLUDES += [
     '!..',
     '..',
 ]
 
 if CONFIG['ENABLE_INTL_API'] and CONFIG['MOZ_ICU_DATA_ARCHIVE']:
new file mode 100644
--- /dev/null
+++ b/js/src/jsapi-tests/testBinASTReader.cpp
@@ -0,0 +1,164 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+
+#include "mozilla/Vector.h"
+
+#include <dirent.h>
+#include <sys/stat.h>
+
+#include "frontend/BinSource.h"
+#include "frontend/FullParseHandler.h"
+#include "frontend/ParseContext.h"
+#include "frontend/Parser.h"
+
+#include "jsapi-tests/tests.h"
+
+using UsedNameTracker = js::frontend::UsedNameTracker;
+
+extern void readFull(const char* path, Vector<char>& buf);
+
+void readFull(JSContext* cx, const char* path, Vector<char16_t>& buf) {
+    buf.shrinkTo(0);
+
+    Vector<char> intermediate(cx);
+    readFull(path, intermediate);
+
+    if (!buf.appendAll(intermediate))
+        MOZ_CRASH();
+}
+
+BEGIN_TEST(testBinASTReaderECMAScript2)
+{
+    const char PATH[] = "jsapi-tests/binast/parser/tester/";
+
+    // Read the list of files in the directory.
+    DIR* dir = opendir(PATH);
+    if (!dir)
+        MOZ_CRASH();
+
+    const char BIN_SUFFIX[] = ".binjs";
+    const char TXT_SUFFIX[] = ".js";
+
+    CompileOptions options(cx);
+    options.setIntroductionType("unit test parse")
+           .setFileAndLine("<string>", 1);
+
+    while (auto entry = readdir(dir)) {
+        // Find files whose name ends with ".binjs".
+        if (entry->d_namlen < sizeof(BIN_SUFFIX))
+            continue;
+        if (strncmp(entry->d_name + entry->d_namlen - (sizeof(BIN_SUFFIX) - 1), BIN_SUFFIX, sizeof(BIN_SUFFIX)) != 0)
+            continue;
+
+        // Find text file.
+        UniqueChars txtPath(static_cast<char*>(js_malloc(entry->d_namlen + sizeof(PATH) + 1)));
+        strncpy(txtPath.get(), PATH, sizeof(PATH));
+        strncpy(txtPath.get() + sizeof(PATH) - 1, entry->d_name, entry->d_namlen);
+        strncpy(txtPath.get() + sizeof(PATH) + entry->d_namlen - sizeof(BIN_SUFFIX), TXT_SUFFIX, sizeof(TXT_SUFFIX));
+        txtPath[sizeof(PATH) + entry->d_namlen - sizeof(BIN_SUFFIX) + sizeof(TXT_SUFFIX) - 1] = 0;
+        fprintf(stderr, "Testing %s\n", txtPath.get());
+
+        // Read text file.
+        Vector<char16_t> txtSource(cx);
+        readFull(cx, txtPath.get(), txtSource);
+
+        // Parse text file.
+        UsedNameTracker txtUsedNames(cx);
+        if (!txtUsedNames.init())
+            MOZ_CRASH();
+        js::frontend::Parser<js::frontend::FullParseHandler, char16_t> parser(cx, cx->tempLifoAlloc(), options, txtSource.begin(), txtSource.length(),
+                                                  /* foldConstants = */ false, txtUsedNames, nullptr,
+                                                  nullptr);
+        if (!parser.checkOptions())
+            MOZ_CRASH();
+
+        auto txtParsed = parser.parse(); // Will be deallocated once `parser` goes out of scope.
+        RootedValue txtExn(cx);
+        if (!txtParsed) {
+            // Save exception for more detailed error message, if necessary.
+            if (!js::GetAndClearException(cx, &txtExn))
+                MOZ_CRASH();
+        }
+
+        // Read binary file.
+        UniqueChars binPath(static_cast<char*>(js_malloc(entry->d_namlen + sizeof(PATH) + 1)));
+        strncpy(binPath.get(), PATH, sizeof(PATH));
+        strncpy(binPath.get() + sizeof(PATH) - 1, entry->d_name, entry->d_namlen);
+        binPath[entry->d_namlen + sizeof(PATH) - 1] = 0;
+
+        Vector<char> binSource(cx);
+        readFull(binPath.get(), binSource);
+
+        // Parse binary file.
+        js::frontend::UsedNameTracker binUsedNames(cx);
+        if (!binUsedNames.init())
+            MOZ_CRASH();
+
+        js::frontend::BinASTParser reader(cx, cx->tempLifoAlloc(), binUsedNames, options);
+
+        auto binParsed = reader.parse(binSource); // Will be deallocated once `reader` goes out of scope.
+        RootedValue binExn(cx);
+        if (binParsed.isErr()) {
+            // Save exception for more detailed error message, if necessary.
+            if (!js::GetAndClearException(cx, &binExn))
+                MOZ_CRASH();
+        }
+
+        // The binary parser should accept the file iff the text parser has.
+        if (binParsed.isOk() && !txtParsed) {
+            fprintf(stderr, "Text file parsing failed: ");
+
+            js::ErrorReport report(cx);
+            if (!report.init(cx, txtExn, js::ErrorReport::WithSideEffects))
+                MOZ_CRASH();
+
+            PrintError(cx, stderr, report.toStringResult(), report.report(), /* reportWarnings */ true);
+            MOZ_CRASH("Binary parser accepted a file that text parser rejected");
+        }
+
+        if (binParsed.isErr() && txtParsed) {
+            fprintf(stderr, "Binary file parsing failed: ");
+
+            js::ErrorReport report(cx);
+            if (!report.init(cx, binExn, js::ErrorReport::WithSideEffects))
+                MOZ_CRASH();
+
+            PrintError(cx, stderr, report.toStringResult(), report.report(), /* reportWarnings */ true);
+            MOZ_CRASH("Binary parser rejected a file that text parser accepted");
+        }
+
+        if (binParsed.isErr()) {
+            fprintf(stderr, "Binary parser and text parser agree that %s is invalid\n", txtPath.get());
+            continue;
+        }
+
+#if defined(DEBUG) // Dumping an AST is only defined in DEBUG builds
+        // Compare ASTs.
+        Sprinter binPrinter(cx);
+        if (!binPrinter.init())
+            MOZ_CRASH();
+        DumpParseTree(binParsed.unwrap(), binPrinter);
+
+        Sprinter txtPrinter(cx);
+        if (!txtPrinter.init())
+            MOZ_CRASH();
+        DumpParseTree(txtParsed, txtPrinter);
+
+        if (strcmp(binPrinter.string(), txtPrinter.string()) != 0) {
+            fprintf(stderr, "Got distinct ASTs when parsing %s:\n\tBINARY\n%s\n\n\tTEXT\n%s\n", txtPath.get(), binPrinter.string(), txtPrinter.string());            
+            MOZ_CRASH();
+        }
+        fprintf(stderr, "Got the same AST when parsing %s\n", txtPath.get());
+
+#endif // defined(DEBUG)
+    }
+
+    return true;
+}
+END_TEST(testBinASTReaderECMAScript2)
+
new file mode 100644
--- /dev/null
+++ b/js/src/jsapi-tests/testBinTokenReaderTester.cpp
@@ -0,0 +1,318 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <string.h>
+#include <sys/stat.h>
+
+#if defined (XP_WIN)
+#include <windows.h>
+#elif defined(XP_UNIX)
+#include <fcntl.h>
+#include <unistd.h>
+#endif // defined (XP_WIN) || defined (XP_UNIX)
+
+#include "frontend/BinTokenReaderTester.h"
+#include "gc/Zone.h"
+
+#include "js/Vector.h"
+
+#include "jsapi-tests/tests.h"
+
+using Tokenizer = js::frontend::BinTokenReaderTester;
+using Chars = Tokenizer::Chars;
+
+// Hack: These tests need access to resources, which are present in the source dir
+// but not copied by our build system. To simplify things, we chdir to the source
+// dir at the start of each test and return to the previous directory afterwards.
+
+#if defined(XP_UNIX)
+
+static int gJsDirectory(0);
+void enterJsDirectory() {
+// Save current directory.
+    MOZ_ASSERT(gJsDirectory == 0);
+    gJsDirectory = open(".", O_RDONLY);
+    MOZ_ASSERT(gJsDirectory != 0, "Could not open directory '.'");
+// Go to the directory provided by autospider, if any.
+    const char* destination = getenv("AUTOSPIDER_DIR");
+    if (destination) {
+        if (chdir(destination) == -1)
+            MOZ_CRASH_UNSAFE_PRINTF("Could not chdir to %s", destination);
+    }
+}
+void exitJsDirectory() {
+    MOZ_ASSERT(gJsDirectory);
+    if (fchdir(gJsDirectory) == -1)
+        MOZ_CRASH("Could not return to original directory");
+    if (close(gJsDirectory) != 0)
+        MOZ_CRASH("Could not close js directory");
+    gJsDirectory = 0;
+}
+
+#else
+
+char gJsDirectory[MAX_PATH] = { 0 };
+
+void enterJsDirectory() {
+    // Save current directory.
+    MOZ_ASSERT(strlen(gJsDirectory) == 0);
+    auto result = GetCurrentDirectory(MAX_PATH, gJsDirectory);
+    if (result <= 0)
+        MOZ_CRASH("Could not get current directory");
+    if (result > MAX_PATH)
+        MOZ_CRASH_UNSAFE_PRINTF("Could not get current directory: needed %ld bytes, got %ld\n", result, MAX_PATH);
+
+    // Find destination directory, if any.
+    char destination[MAX_PATH];
+    if (!GetEnvironmentVariable("AUTOSPIDER_DIR", destination, MAX_PATH)) {
+        if (GetLastError() != ERROR_ENVVAR_NOT_FOUND)
+            MOZ_CRASH("Could not get AUTOSPIDER_DIR");
+        else
+            return;
+    }
+
+    // Go to the directory.
+    if (SetCurrentDirectory(destination) != 0)
+        MOZ_CRASH_UNSAFE_PRINTF("Could not chdir to %s", destination);
+}
+
+void exitJsDirectory() {
+    MOZ_ASSERT(strlen(gJsDirectory) > 0);
+    if (SetCurrentDirectory(gJsDirectory) != 0)
+        MOZ_CRASH("Could not return to original directory");
+    gJsDirectory[0] = 0;
+}
+
+#endif // defined(XP_UNIX) || defined(XP_WIN)
+
+void readFull(const char* path, js::Vector<uint8_t>& buf) {
+    enterJsDirectory();
+    buf.shrinkTo(0);
+    FILE* in = fopen(path, "r");
+    if (!in)
+        MOZ_CRASH_UNSAFE_PRINTF("Could not open %s: %s", path, strerror(errno));
+
+    struct stat info;
+    if (stat(path, &info) < 0)
+        MOZ_CRASH_UNSAFE_PRINTF("Could not get stat on %s", path);
+
+    if (!buf.growBy(info.st_size))
+        MOZ_CRASH("OOM");
+
+    int result = fread(buf.begin(), 1, info.st_size, in);
+    if (fclose(in) != 0)
+        MOZ_CRASH("Could not close input file");
+    if (result != info.st_size)
+        MOZ_CRASH("Read error");
+    exitJsDirectory();
+}
+
+
+// Reading a simple string.
+BEGIN_TEST(testBinTokenReaderTesterSimpleString)
+{
+    js::Vector<uint8_t> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-simple-string.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    Maybe<Chars> found;
+    CHECK(tokenizer.readMaybeChars(found));
+
+    CHECK(Tokenizer::equals(*found, "simple string")); // FIXME: Find a way to make CHECK_EQUAL use `Tokenizer::equals`.
+
+    return true;
+}
+END_TEST(testBinTokenReaderTesterSimpleString)
+
+// Reading a string with embedded 0.
+BEGIN_TEST(testBinTokenReaderTesterStringWithEscapes)
+{
+    js::Vector<uint8_t> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-string-with-escapes.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    Maybe<Chars> found;
+    CHECK(tokenizer.readMaybeChars(found));
+
+    CHECK(Tokenizer::equals(*found, "string with escapes \0\1\0")); // FIXME: Find a way to make CHECK_EQUAL use `Tokenizer::equals`.
+
+    return true;
+}
+END_TEST(testBinTokenReaderTesterStringWithEscapes)
+
+// Reading an empty untagged tuple
+BEGIN_TEST(testBinTokenReaderTesterEmptyUntaggedTuple)
+{
+    js::Vector<uint8_t> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-empty-untagged-tuple.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        Tokenizer::AutoTuple guard(tokenizer);
+        CHECK(tokenizer.enterUntaggedTuple(guard));
+        CHECK(guard.done());
+    }
+
+    return true;
+}
+END_TEST(testBinTokenReaderTesterEmptyUntaggedTuple)
+
+// Reading a untagged tuple with two strings
+BEGIN_TEST(testBinTokenReaderTesterTwoStringsInTuple)
+{
+    js::Vector<uint8_t> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-trivial-untagged-tuple.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        Tokenizer::AutoTuple guard(tokenizer);
+        CHECK(tokenizer.enterUntaggedTuple(guard));
+
+        Maybe<Chars> found_0;
+        CHECK(tokenizer.readMaybeChars(found_0));
+        CHECK(Tokenizer::equals(*found_0, "foo")); // FIXME: Find a way to make CHECK_EQUAL use `Tokenizer::equals`.
+
+        Maybe<Chars> found_1;
+        CHECK(tokenizer.readMaybeChars(found_1));
+        CHECK(Tokenizer::equals(*found_1, "bar")); // FIXME: Find a way to make CHECK_EQUAL use `Tokenizer::equals`.
+
+        CHECK(guard.done());
+    }
+
+    return true;
+}
+END_TEST(testBinTokenReaderTesterTwoStringsInTuple)
+
+// Reading a tagged tuple `Pattern { id: "foo", value: 3.1415}`
+BEGIN_TEST(testBinTokenReaderTesterSimpleTaggedTuple)
+{
+    js::Vector<uint8_t> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-simple-tagged-tuple.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        js::frontend::BinKind tag;
+        Tokenizer::BinFields fields(cx);
+        Tokenizer::AutoTaggedTuple guard(tokenizer);
+        CHECK(tokenizer.enterTaggedTuple(tag, fields, guard));
+
+        CHECK(tag == js::frontend::BinKind::Pattern);
+
+        Maybe<Chars> found_id;
+        const double EXPECTED_value = 3.1415;
+        Maybe<double> found_value;
+
+        // Order of fields is not deterministic in the encoder (we could make
+        // it deterministic for the test, though, since we already know the binary).
+        if (fields[0] == js::frontend::BinField::Id) {
+            CHECK(fields[1] == js::frontend::BinField::Value);
+            CHECK(tokenizer.readMaybeChars(found_id));
+            CHECK(tokenizer.readMaybeDouble(found_value));
+        } else if (fields[0] == js::frontend::BinField::Value) {
+            CHECK(fields[1] == js::frontend::BinField::Id);
+            CHECK(tokenizer.readMaybeDouble(found_value));
+            CHECK(tokenizer.readMaybeChars(found_id));
+        } else {
+            CHECK(false);
+        }
+
+        CHECK(EXPECTED_value == *found_value); // Apparently, CHECK_EQUAL doesn't work on `double`.
+        CHECK(Tokenizer::equals(*found_id, "foo"));
+        CHECK(guard.done());
+    }
+
+    return true;
+}
+END_TEST(testBinTokenReaderTesterSimpleTaggedTuple)
+
+
+// Reading an empty list
+BEGIN_TEST(testBinTokenReaderTesterEmptyList)
+{
+    js::Vector<uint8_t> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-empty-list.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        uint32_t length;
+        Tokenizer::AutoList guard(tokenizer);
+        CHECK(tokenizer.enterList(length, guard));
+
+        CHECK(length == 0);
+        CHECK(guard.done());
+    }
+
+    return true;
+}
+END_TEST(testBinTokenReaderTesterEmptyList)
+
+// Reading `["foo", "bar"]`
+BEGIN_TEST(testBinTokenReaderTesterSimpleList)
+{
+    js::Vector<uint8_t> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-trivial-list.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        uint32_t length;
+        Tokenizer::AutoList guard(tokenizer);
+        CHECK(tokenizer.enterList(length, guard));
+
+        CHECK(length == 2);
+
+        Maybe<Chars> found_0;
+        CHECK(tokenizer.readMaybeChars(found_0));
+        CHECK(Tokenizer::equals(*found_0, "foo"));
+
+        Maybe<Chars> found_1;
+        CHECK(tokenizer.readMaybeChars(found_1));
+        CHECK(Tokenizer::equals(*found_1, "bar"));
+
+        CHECK(guard.done());
+    }
+
+    return true;
+}
+END_TEST(testBinTokenReaderTesterSimpleList)
+
+
+// Reading `[["foo", "bar"]]`
+BEGIN_TEST(testBinTokenReaderTesterNestedList)
+{
+    js::Vector<uint8_t> contents(cx);
+    readFull("jsapi-tests/binast/tokenizer/tester/test-nested-lists.binjs", contents);
+    Tokenizer tokenizer(cx, contents);
+
+    {
+        uint32_t outerLength;
+        Tokenizer::AutoList outerGuard(tokenizer);
+        CHECK(tokenizer.enterList(outerLength, outerGuard));
+        CHECK(outerLength == 1);
+
+        {
+            uint32_t innerLength;
+            Tokenizer::AutoList innerGuard(tokenizer);
+            CHECK(tokenizer.enterList(innerLength, innerGuard));
+            CHECK(innerLength == 2);
+
+            Maybe<Chars> found_0;
+            CHECK(tokenizer.readMaybeChars(found_0));
+            CHECK(Tokenizer::equals(*found_0, "foo"));
+
+            Maybe<Chars> found_1;
+            CHECK(tokenizer.readMaybeChars(found_1));
+            CHECK(Tokenizer::equals(*found_1, "bar"));
+
+            CHECK(innerGuard.done());
+        }
+
+        CHECK(outerGuard.done());
+    }
+
+    return true;
+}
+END_TEST(testBinTokenReaderTesterNestedList)
--- a/js/src/moz.build
+++ b/js/src/moz.build
@@ -637,16 +637,23 @@ DIRS += [
 
 FINAL_LIBRARY = 'js'
 
 if CONFIG['NIGHTLY_BUILD']:
     DEFINES['ENABLE_BINARYDATA'] = True
     DEFINES['ENABLE_SIMD'] = True
     DEFINES['ENABLE_WASM_THREAD_OPS'] = True
 
+if CONFIG['JS_BUILD_BINAST']:
+    # Some parts of BinAST are designed only to test evolutions of the
+    # specification:
+    UNIFIED_SOURCES += ['frontend/BinTokenReaderTester.cpp']
+    # The rest of BinAST should eventually move to release.
+    UNIFIED_SOURCES += ['frontend/BinToken.cpp']
+
 # Wasm code should use WASM_HUGE_MEMORY instead of JS_CODEGEN_X64
 # so that it is easy to use the huge-mapping optimization for other
 # 64-bit platforms in the future.
 
 if CONFIG['JS_CODEGEN_X64']:
     DEFINES['WASM_HUGE_MEMORY'] = True
 
 if CONFIG['MOZ_DEBUG'] or CONFIG['NIGHTLY_BUILD']: