Bug 1459067 - Part 3: Store BinTokenReaderMultipart metadata in ScriptSource. (r=arai)
authorEric Faust <efaustbmo@gmail.com>
Tue, 02 Oct 2018 01:16:51 -0700
changeset 494894 fad6ee2d0675e9609bd7d5165787efdcaa861163
parent 494893 932d8da50a8ff509b36f0ba3e0e291af226eea4b
child 494895 4e1f31f8ec6d66067c4d2b3b03f4221d63ef20e8
push id9984
push userffxbld-merge
push dateMon, 15 Oct 2018 21:07:35 +0000
treeherdermozilla-beta@183d27ea8570 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersarai
bugs1459067
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1459067 - Part 3: Store BinTokenReaderMultipart metadata in ScriptSource. (r=arai)
js/src/frontend/BinSource.cpp
js/src/frontend/BinSource.h
js/src/frontend/BinSourceRuntimeSupport.cpp
js/src/frontend/BinSourceRuntimeSupport.h
js/src/frontend/BinTokenReaderMultipart.cpp
js/src/frontend/BinTokenReaderMultipart.h
js/src/frontend/BinTokenReaderTester.h
js/src/frontend/BytecodeCompiler.cpp
js/src/moz.build
js/src/vm/JSScript.cpp
js/src/vm/JSScript.h
--- a/js/src/frontend/BinSource.cpp
+++ b/js/src/frontend/BinSource.cpp
@@ -107,33 +107,36 @@ BinASTParserBase::~BinASTParserBase()
     alloc_.freeAllIfHugeAndUnused();
 
     cx_->frontendCollectionPool().removeActiveCompilation();
 }
 
 // ------------- Toplevel constructions
 
 template<typename Tok> JS::Result<ParseNode*>
-BinASTParser<Tok>::parse(GlobalSharedContext* globalsc, const Vector<uint8_t>& data)
+BinASTParser<Tok>::parse(GlobalSharedContext* globalsc, const Vector<uint8_t>& data,
+                         BinASTSourceMetadata** metadataPtr)
 {
-    return parse(globalsc, data.begin(), data.length());
+    return parse(globalsc, data.begin(), data.length(), metadataPtr);
 }
 
 template<typename Tok> JS::Result<ParseNode*>
-BinASTParser<Tok>::parse(GlobalSharedContext* globalsc, const uint8_t* start, const size_t length)
+BinASTParser<Tok>::parse(GlobalSharedContext* globalsc, const uint8_t* start, const size_t length,
+                         BinASTSourceMetadata** metadataPtr)
 {
-    auto result = parseAux(globalsc, start, length);
+    auto result = parseAux(globalsc, start, length, metadataPtr);
     poison(); // Make sure that the parser is never used again accidentally.
     return result;
 }
 
 
 template<typename Tok> JS::Result<ParseNode*>
 BinASTParser<Tok>::parseAux(GlobalSharedContext* globalsc,
-                            const uint8_t* start, const size_t length)
+                            const uint8_t* start, const size_t length,
+                            BinASTSourceMetadata** metadataPtr)
 {
     MOZ_ASSERT(globalsc);
 
     tokenizer_.emplace(cx_, this, start, length);
 
     BinParseContext globalpc(cx_, this, globalsc, /* newDirectives = */ nullptr);
     if (!globalpc.init()) {
         return cx_->alreadyReportedError();
@@ -151,29 +154,32 @@ BinASTParser<Tok>::parseAux(GlobalShared
 
     mozilla::Maybe<GlobalScope::Data*> bindings = NewGlobalScopeData(cx_, varScope, alloc_,
                                                                      parseContext_);
     if (!bindings) {
         return cx_->alreadyReportedError();
     }
     globalsc->bindings = *bindings;
 
+    if (metadataPtr) {
+        *metadataPtr = tokenizer_->takeMetadata();
+    }
+
     return result; // Magic conversion to Ok.
 }
 
 template<typename Tok> JS::Result<ParseNode*>
-BinASTParser<Tok>::parseLazyFunction(const uint8_t* start, const size_t firstOffset, const size_t len)
+BinASTParser<Tok>::parseLazyFunction(ScriptSource* scriptSource, const size_t firstOffset)
 {
     MOZ_ASSERT(lazyScript_);
-
-    tokenizer_.emplace(cx_, this, start, len);
+    MOZ_ASSERT(scriptSource->length() > firstOffset);
 
-    // Re-initialize the tokenizer
-    mozilla::DebugOnly<bool> success = tokenizer_->readHeader().isOk();
-    MOZ_ASSERT(success);
+    tokenizer_.emplace(cx_, this, scriptSource->binASTSource(), scriptSource->length());
+
+    MOZ_TRY(tokenizer_->initFromScriptSource(scriptSource));
 
     tokenizer_->seek(firstOffset);
 
     // For now, only function declarations and function expression are supported.
     RootedFunction func(cx_, lazyScript_->functionNonDelazifying());
     bool isExpr = func->isLambda();
     MOZ_ASSERT(func->kind() == JSFunction::FunctionKind::NormalFunction);
 
@@ -731,16 +737,23 @@ BinASTParserBase::hasUsedName(HandleProp
 }
 
 void
 TraceBinParser(JSTracer* trc, JS::AutoGCRooter* parser)
 {
     static_cast<BinASTParserBase*>(parser)->trace(trc);
 }
 
+template<typename Tok>
+void
+BinASTParser<Tok>::doTrace(JSTracer* trc)
+{
+    if (tokenizer_)
+        tokenizer_->traceMetadata(trc);
+}
 
 // Force class instantiation.
 // This ensures that the symbols are built, without having to export all our
 // code (and its baggage of #include and macros) in the header.
 template class BinASTParser<BinTokenReaderMultipart>;
 template class BinASTParser<BinTokenReaderTester>;
 
 } // namespace frontend
--- a/js/src/frontend/BinSource.h
+++ b/js/src/frontend/BinSource.h
@@ -43,18 +43,21 @@ class BinASTParserBase: private JS::Auto
   public:
     // Names
 
 
     bool hasUsedName(HandlePropertyName name);
 
     // --- GC.
 
+    virtual void doTrace(JSTracer* trc) {}
+
     void trace(JSTracer* trc) {
         ObjectBox::TraceList(trc, traceListHead_);
+        doTrace(trc);
     }
 
 
   public:
     ParseNode* allocParseNode(size_t size) {
         MOZ_ASSERT(size == sizeof(ParseNode));
         return static_cast<ParseNode*>(nodeAlloc_.allocNode());
     }
@@ -124,24 +127,27 @@ class BinASTParser : public BinASTParser
      * or Nothing() in case of error.
      *
      * The instance of `ParseNode` MAY NOT survive the `BinASTParser`. Indeed,
      * destruction of the `BinASTParser` will also destroy the `ParseNode`.
      *
      * In case of error, the parser reports the JS error.
      */
     JS::Result<ParseNode*> parse(GlobalSharedContext* globalsc,
-                                 const uint8_t* start, const size_t length);
-    JS::Result<ParseNode*> parse(GlobalSharedContext* globalsc, const Vector<uint8_t>& data);
+                                 const uint8_t* start, const size_t length,
+                                 BinASTSourceMetadata** metadataPtr = nullptr);
+    JS::Result<ParseNode*> parse(GlobalSharedContext* globalsc, const Vector<uint8_t>& data,
+                                 BinASTSourceMetadata** metadataPtr = nullptr);
 
-    JS::Result<ParseNode*> parseLazyFunction(const uint8_t* start, const size_t firstOffset, const size_t length);
+    JS::Result<ParseNode*> parseLazyFunction(ScriptSource* src, const size_t firstOffset);
 
   private:
     MOZ_MUST_USE JS::Result<ParseNode*> parseAux(GlobalSharedContext* globalsc,
-                                                 const uint8_t* start, const size_t length);
+                                                 const uint8_t* start, const size_t length,
+                                                 BinASTSourceMetadata** metadataPtr = nullptr);
 
     // --- Raise errors.
     //
     // These methods return a (failed) JS::Result for convenience.
 
     MOZ_MUST_USE mozilla::GenericErrorResult<JS::Error&> raiseInvalidClosedVar(JSAtom* name);
     MOZ_MUST_USE mozilla::GenericErrorResult<JS::Error&> raiseMissingVariableInAssertedScope(JSAtom* name);
     MOZ_MUST_USE mozilla::GenericErrorResult<JS::Error&> raiseMissingDirectEvalInAssertedScope();
@@ -227,16 +233,18 @@ class BinASTParser : public BinASTParser
 
   private: // Implement ErrorReporter
     const JS::ReadOnlyCompileOptions& options_;
 
     const JS::ReadOnlyCompileOptions& options() const override {
         return this->options_;
     }
 
+    void doTrace(JSTracer* trc) final;
+
   public:
     virtual ObjectBox* newObjectBox(JSObject* obj) override {
         MOZ_ASSERT(obj);
 
         /*
          * We use JSContext.tempLifoAlloc to allocate parsed objects and place them
          * on a list in this Parser to ensure GC safety. Thus the tempLifoAlloc
          * arenas containing the entries must be alive until we are done with
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinSourceRuntimeSupport.cpp
@@ -0,0 +1,44 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "frontend/BinSourceRuntimeSupport.h"
+
+#include "gc/Tracer.h"
+#include "js/Vector.h"
+#include "vm/StringType.h"
+
+namespace js {
+namespace frontend {
+
+/* static */ BinASTSourceMetadata*
+BinASTSourceMetadata::Create(const Vector<BinKind>& binKinds, uint32_t numStrings)
+{
+    uint32_t numBinKinds = binKinds.length();
+
+    BinASTSourceMetadata* data = static_cast<BinASTSourceMetadata*>(js_malloc(BinASTSourceMetadata::totalSize(numBinKinds, numStrings)));
+    if (!data) {
+        return nullptr;
+    }
+
+    new (data) BinASTSourceMetadata(numBinKinds, numStrings);
+    memcpy(data->binKindBase(), binKinds.begin(), binKinds.length() * sizeof(BinKind));
+
+    return data;
+}
+
+void
+BinASTSourceMetadata::trace(JSTracer* tracer)
+{
+    JSAtom** base = atomsBase();
+    for (uint32_t i = 0; i < numStrings_; i++) {
+        if (base[i]) {
+            TraceManuallyBarrieredEdge(tracer, &base[i], "BinAST Strings");
+        }
+    }
+}
+
+} // namespace frontend
+} // namespace js
--- a/js/src/frontend/BinSourceRuntimeSupport.h
+++ b/js/src/frontend/BinSourceRuntimeSupport.h
@@ -5,23 +5,28 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef frontend_BinSourceSupport_h
 #define frontend_BinSourceSupport_h
 
 #include "mozilla/HashFunctions.h"
 
 #include "frontend/BinToken.h"
+#include "gc/DeletePolicy.h"
 
 #include "js/AllocPolicy.h"
 #include "js/HashTable.h"
 #include "js/Result.h"
+#include "js/UniquePtr.h"
+#include "js/Vector.h"
 
 namespace js {
 
+class ScriptSource;
+
 // Support for parsing JS Binary ASTs.
 struct BinaryASTSupport {
     using BinVariant = js::frontend::BinVariant;
     using BinField = js::frontend::BinField;
     using BinKind = js::frontend::BinKind;
 
     // A structure designed to perform fast char* + length lookup
     // without copies.
@@ -31,16 +36,19 @@ struct BinaryASTSupport {
         CharSlice(const CharSlice& other)
             : start_(other.start_)
             , byteLen_(other.byteLen_)
         {  }
         CharSlice(const char* start, const uint32_t byteLen)
             : start_(start)
             , byteLen_(byteLen)
         { }
+        explicit CharSlice(JSContext*)
+          : CharSlice(nullptr, 0)
+        { }
         const char* begin() const {
             return start_;
         }
         const char* end() const {
             return start_ + byteLen_;
         }
 #ifdef DEBUG
         void dump() const {
@@ -84,11 +92,78 @@ struct BinaryASTSupport {
     using BinFieldMap = js::HashMap<const CharSlice, BinField, CharSlice, js::SystemAllocPolicy>;
     BinFieldMap binFieldMap_;
 
     using BinVariantMap = js::HashMap<const CharSlice, BinVariant, CharSlice, js::SystemAllocPolicy>;
     BinVariantMap binVariantMap_;
 
 };
 
+namespace frontend {
+
+class BinASTSourceMetadata
+{
+    using CharSlice = BinaryASTSupport::CharSlice;
+
+    const uint32_t numStrings_;
+    const uint32_t numBinKinds_;
+
+    // The data lives inline in the allocation, after this class.
+    inline JSAtom** atomsBase() {
+        return reinterpret_cast<JSAtom**>(reinterpret_cast<uintptr_t>(this + 1));
+    }
+    inline CharSlice* sliceBase() {
+        return reinterpret_cast<CharSlice*>(reinterpret_cast<uintptr_t>(atomsBase()) + numStrings_ * sizeof(JSAtom*));
+    }
+    inline BinKind* binKindBase() {
+        return reinterpret_cast<BinKind*>(reinterpret_cast<uintptr_t>(sliceBase()) + numStrings_ * sizeof(CharSlice));
+    }
+
+    static inline size_t totalSize(uint32_t numBinKinds, uint32_t numStrings) {
+        return sizeof(BinASTSourceMetadata) +
+               numStrings * sizeof(JSAtom*) +
+               numStrings * sizeof(CharSlice) +
+               numBinKinds * sizeof(BinKind);
+    }
+
+    BinASTSourceMetadata(uint32_t numBinKinds, uint32_t numStrings)
+      : numStrings_(numStrings),
+        numBinKinds_(numBinKinds)
+    { }
+
+    friend class js::ScriptSource;
+
+  public:
+    static BinASTSourceMetadata* Create(const Vector<BinKind>& binKinds, uint32_t numStrings);
+
+    inline uint32_t numBinKinds() {
+        return numBinKinds_;
+    }
+
+    inline uint32_t numStrings() {
+        return numStrings_;
+    }
+
+    inline BinKind& getBinKind(uint32_t index) {
+        MOZ_ASSERT(index < numBinKinds_);
+        return binKindBase()[index];
+    }
+
+    inline CharSlice& getSlice(uint32_t index) {
+        MOZ_ASSERT(index < numStrings_);
+        return sliceBase()[index];
+    }
+
+    inline JSAtom*& getAtom(uint32_t index) {
+        MOZ_ASSERT(index < numStrings_);
+        return atomsBase()[index];
+    }
+
+    void trace(JSTracer* tracer);
+};
+
+}
+
+typedef UniquePtr<frontend::BinASTSourceMetadata, GCManagedDeletePolicy<frontend::BinASTSourceMetadata>> UniqueBinASTSourceMetadataPtr;
+
 } // namespace js
 
 #endif // frontend_BinSourceSupport_h
--- a/js/src/frontend/BinTokenReaderMultipart.cpp
+++ b/js/src/frontend/BinTokenReaderMultipart.cpp
@@ -5,16 +5,17 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "frontend/BinTokenReaderMultipart.h"
 
 #include "mozilla/ArrayUtils.h"
 #include "mozilla/Casting.h"
 #include "mozilla/EndianUtils.h"
 #include "mozilla/Maybe.h"
+#include "mozilla/ScopeExit.h"
 
 #include <utility>
 
 #include "frontend/BinSource-macros.h"
 #include "frontend/BinSourceRuntimeSupport.h"
 
 #include "js/Result.h"
 
@@ -42,24 +43,46 @@ const uint32_t MAX_NUMBER_OF_STRINGS = 3
 using AutoList = BinTokenReaderMultipart::AutoList;
 using AutoTaggedTuple = BinTokenReaderMultipart::AutoTaggedTuple;
 using AutoTuple = BinTokenReaderMultipart::AutoTuple;
 using CharSlice = BinaryASTSupport::CharSlice;
 using Chars = BinTokenReaderMultipart::Chars;
 
 BinTokenReaderMultipart::BinTokenReaderMultipart(JSContext* cx, ErrorReporter* er, const uint8_t* start, const size_t length)
   : BinTokenReaderBase(cx, er, start, length)
-  , grammarTable_(cx)
-  , atomsTable_(cx, AtomVector(cx))
-  , slicesTable_(cx)
+  , metadata_(nullptr)
   , posBeforeTree_(nullptr)
 {
     MOZ_ASSERT(er);
 }
 
+BinTokenReaderMultipart::~BinTokenReaderMultipart()
+{
+    if (metadata_ && metadataOwned_ == MetadataOwnership::Owned) {
+        UniqueBinASTSourceMetadataPtr ptr(metadata_);
+    }
+}
+
+BinASTSourceMetadata*
+BinTokenReaderMultipart::takeMetadata()
+{
+    MOZ_ASSERT(metadataOwned_ == MetadataOwnership::Owned);
+    metadataOwned_ = MetadataOwnership::Unowned;
+    return metadata_;
+}
+
+JS::Result<Ok>
+BinTokenReaderMultipart::initFromScriptSource(ScriptSource* scriptSource)
+{
+    metadata_ = scriptSource->binASTSourceMetadata();
+    metadataOwned_ = MetadataOwnership::Unowned;
+
+    return Ok();
+}
+
 JS::Result<Ok>
 BinTokenReaderMultipart::readHeader()
 {
     // Check that we don't call this function twice.
     MOZ_ASSERT(!posBeforeTree_);
 
     // Read global headers.
     MOZ_TRY(readConst(MAGIC_HEADER));
@@ -84,16 +107,17 @@ BinTokenReaderMultipart::readHeader()
 
     BINJS_MOZ_TRY_DECL(grammarNumberOfEntries, readInternalUint32());
     if (grammarNumberOfEntries > BINKIND_LIMIT) { // Sanity check.
         return raiseError("Invalid number of entries in grammar table");
     }
 
     // This table maps BinKind index -> BinKind.
     // Initialize and populate.
+    Vector<BinKind> grammarTable_(cx_);
     if (!grammarTable_.reserve(grammarNumberOfEntries)) {
         return raiseOOM();
     }
 
     for (uint32_t i = 0; i < grammarNumberOfEntries; ++i) {
         BINJS_MOZ_TRY_DECL(byteLen, readInternalUint32());
         if (current_ + byteLen > stop_) {
             return raiseError("Invalid byte length in grammar table");
@@ -125,68 +149,79 @@ BinTokenReaderMultipart::readHeader()
         return raiseError("Invalid byte length in strings table");
     }
 
     BINJS_MOZ_TRY_DECL(stringsNumberOfEntries, readInternalUint32());
     if (stringsNumberOfEntries > MAX_NUMBER_OF_STRINGS) { // Sanity check.
         return raiseError("Too many entries in strings table");
     }
 
-    // This table maps String index -> String.
-    // Initialize and populate.
-    if (!atomsTable_.reserve(stringsNumberOfEntries)) {
+    BinASTSourceMetadata* metadata = BinASTSourceMetadata::Create(grammarTable_, stringsNumberOfEntries);
+    if (!metadata) {
         return raiseOOM();
     }
-    if (!slicesTable_.reserve(stringsNumberOfEntries)) {
-        return raiseOOM();
-    }
+
+    // Free it if we don't make it out of here alive. Since we don't want to calloc(), we
+    // need to avoid marking atoms that might not be there.
+    auto se = mozilla::MakeScopeExit([metadata](){ js_free(metadata); });
 
     RootedAtom atom(cx_);
     for (uint32_t i = 0; i < stringsNumberOfEntries; ++i) {
         BINJS_MOZ_TRY_DECL(byteLen, readInternalUint32());
         if (current_ + byteLen > stop_ || current_ + byteLen < current_) {
             return raiseError("Invalid byte length in individual string");
         }
 
         // Check null string.
         if (byteLen == 2 && *current_ == 255 && *(current_ + 1) == 0) {
             atom = nullptr;
         } else {
             BINJS_TRY_VAR(atom, AtomizeUTF8Chars(cx_, (const char*)current_, byteLen));
         }
 
-        // Populate `atomsTable_`: i => atom.
-        atomsTable_.infallibleAppend(atom); // We have reserved before entering the loop.
+        metadata->getAtom(i) = atom;
 
         // Populate `slicesTable_`: i => slice
-        Chars slice((const char*)current_, byteLen);
-        slicesTable_.infallibleAppend(std::move(slice)); // We have reserved before entering the loop.
+        new (&metadata->getSlice(i)) Chars((const char*)current_, byteLen);
 
         current_ += byteLen;
     }
 
     if (posBeforeStrings + stringsByteLen != current_) {
         return raiseError("The length of the strings table didn't match its contents.");
     }
 
+    MOZ_ASSERT(!metadata_);
+    se.release();
+    metadata_ = metadata;
+    metadataOwned_ = MetadataOwnership::Owned;
+
     // Start reading AST.
     MOZ_TRY(readConst(SECTION_HEADER_TREE));
     MOZ_TRY(readConst(COMPRESSION_IDENTITY)); // For the moment, we only support identity compression.
     posBeforeTree_ = current_;
 
     BINJS_MOZ_TRY_DECL(treeByteLen, readInternalUint32());
 
     if (posBeforeTree_ + treeByteLen > stop_ || posBeforeTree_ + treeByteLen < posBeforeTree_) { // Sanity check.
         return raiseError("Invalid byte length in tree table");
     }
 
     // At this stage, we're ready to start reading the tree.
     return Ok();
 }
 
+void
+BinTokenReaderMultipart::traceMetadata(JSTracer* trc)
+{
+    if (metadata_) {
+        metadata_->trace(trc);
+    }
+}
+
 JS::Result<bool>
 BinTokenReaderMultipart::readBool()
 {
     updateLatestKnownGood();
     BINJS_MOZ_TRY_DECL(byte, readByte());
 
     switch (byte) {
       case 0:
@@ -228,20 +263,20 @@ BinTokenReaderMultipart::readDouble()
 
 // A single atom is represented as an index into the table of strings.
 JS::Result<JSAtom*>
 BinTokenReaderMultipart::readMaybeAtom()
 {
     updateLatestKnownGood();
     BINJS_MOZ_TRY_DECL(index, readInternalUint32());
 
-    if (index >= atomsTable_.length()) {
+    if (index >= metadata_->numStrings()) {
         return raiseError("Invalid index to strings table");
     }
-    return atomsTable_[index].get();
+    return metadata_->getAtom(index);
 }
 
 JS::Result<JSAtom*>
 BinTokenReaderMultipart::readAtom()
 {
     BINJS_MOZ_TRY_DECL(maybe, readMaybeAtom());
 
     if (!maybe) {
@@ -252,46 +287,46 @@ BinTokenReaderMultipart::readAtom()
 }
 
 JS::Result<Ok>
 BinTokenReaderMultipart::readChars(Chars& out)
 {
     updateLatestKnownGood();
     BINJS_MOZ_TRY_DECL(index, readInternalUint32());
 
-    if (index >= slicesTable_.length()) {
+    if (index >= metadata_->numStrings()) {
         return raiseError("Invalid index to strings table for string enum");
     }
 
-    out = slicesTable_[index];
+    out = metadata_->getSlice(index);
     return Ok();
 }
 
 JS::Result<BinVariant>
 BinTokenReaderMultipart::readVariant()
 {
     updateLatestKnownGood();
     BINJS_MOZ_TRY_DECL(index, readInternalUint32());
 
-    if (index >= slicesTable_.length()) {
+    if (index >= metadata_->numStrings()) {
         return raiseError("Invalid index to strings table for string enum");
     }
 
     auto variantsPtr = variantsTable_.lookupForAdd(index);
     if (variantsPtr) {
         return variantsPtr->value();
     }
 
     // Either we haven't cached the result yet or this is not a variant.
     // Check in the slices table and, in case of success, cache the result.
 
     // Note that we stop parsing if we attempt to readVariant() with an
     // ill-formed variant, so we don't run the risk of feching an ill-variant
     // more than once.
-    Chars slice = slicesTable_[index]; // We have checked `index` above.
+    Chars slice = metadata_->getSlice(index); // We have checked `index` above.
     BINJS_MOZ_TRY_DECL(variant, cx_->runtime()->binast().binVariant(cx_, slice));
 
     if (!variant) {
         return raiseError("Invalid string enum variant");
     }
 
     if (!variantsTable_.add(variantsPtr, index, *variant)) {
         return raiseOOM();
@@ -329,21 +364,21 @@ BinTokenReaderMultipart::enterUntaggedTu
 
 // Tagged tuples:
 // - uint32_t index in table [grammar];
 // - content (specified by the higher-level grammar);
 JS::Result<Ok>
 BinTokenReaderMultipart::enterTaggedTuple(BinKind& tag, BinTokenReaderMultipart::BinFields&, AutoTaggedTuple& guard)
 {
     BINJS_MOZ_TRY_DECL(index, readInternalUint32());
-    if (index >= grammarTable_.length()) {
+    if (index >= metadata_->numBinKinds()) {
         return raiseError("Invalid index to grammar table");
     }
 
-    tag = grammarTable_[index];
+    tag = metadata_->getBinKind(index);
 
     // Enter the body.
     guard.init();
     return Ok();
 }
 
 // List:
 //
--- a/js/src/frontend/BinTokenReaderMultipart.h
+++ b/js/src/frontend/BinTokenReaderMultipart.h
@@ -40,41 +40,35 @@ class MOZ_STACK_CLASS BinTokenReaderMult
     using CharSlice = BinaryASTSupport::CharSlice;
 
     // This implementation of `BinFields` is effectively `void`, as the format
     // does not embed field information.
     class BinFields {
       public:
         explicit BinFields(JSContext*) {}
     };
-    struct Chars: public CharSlice {
-        explicit Chars(JSContext*)
-          : CharSlice(nullptr, 0)
-        { }
-        Chars(const char* start, const uint32_t byteLen)
-          : CharSlice(start, byteLen)
-        { }
-        Chars(const Chars& other) = default;
-    };
+    using Chars = CharSlice;
 
   public:
     /**
      * Construct a token reader.
      *
      * Does NOT copy the buffer.
      */
     BinTokenReaderMultipart(JSContext* cx, ErrorReporter* er, const uint8_t* start, const size_t length);
 
     /**
      * Construct a token reader.
      *
      * Does NOT copy the buffer.
      */
     BinTokenReaderMultipart(JSContext* cx, ErrorReporter* er, const Vector<uint8_t>& chars);
 
+    ~BinTokenReaderMultipart();
+
     /**
      * Read the header of the file.
      */
     MOZ_MUST_USE JS::Result<Ok> readHeader();
 
     // --- Primitive values.
     //
     // Note that the underlying format allows for a `null` value for primitive
@@ -187,40 +181,39 @@ class MOZ_STACK_CLASS BinTokenReaderMult
 
   private:
     /**
      * Read a single uint32_t.
      */
     MOZ_MUST_USE JS::Result<uint32_t> readInternalUint32();
 
   private:
-    // A mapping grammar index => BinKind, as defined by the [GRAMMAR]
-    // section of the file. Populated during readHeader().
-    Vector<BinKind> grammarTable_;
-
     // A mapping string index => BinVariant as extracted from the [STRINGS]
     // section of the file. Populated lazily.
     js::HashMap<uint32_t, BinVariant, DefaultHasher<uint32_t>, SystemAllocPolicy> variantsTable_;
 
-    // A mapping index => JSAtom, as defined by the [STRINGS]
-    // section of the file. Populated during readHeader().
-    using AtomVector = GCVector<JSAtom*>;
-    JS::Rooted<AtomVector> atomsTable_;
-
-    // The same mapping, but as CharSlice. Populated during readHeader().
-    // The slices are into the source buffer.
-    Vector<Chars> slicesTable_;
+    enum class MetadataOwnership {
+        Owned,
+        Unowned
+    };
+    MetadataOwnership metadataOwned_;
+    BinASTSourceMetadata* metadata_;
 
     const uint8_t* posBeforeTree_;
 
     BinTokenReaderMultipart(const BinTokenReaderMultipart&) = delete;
     BinTokenReaderMultipart(BinTokenReaderMultipart&&) = delete;
     BinTokenReaderMultipart& operator=(BinTokenReaderMultipart&) = delete;
 
   public:
+    void traceMetadata(JSTracer* trc);
+    BinASTSourceMetadata* takeMetadata();
+    MOZ_MUST_USE JS::Result<Ok> initFromScriptSource(ScriptSource* scriptSource);
+
+  public:
     // The following classes are used whenever we encounter a tuple/tagged tuple/list
     // to make sure that:
     //
     // - if the construct "knows" its byte length, we have exactly consumed all
     //   the bytes (otherwise, this means that the file is corrupted, perhaps on
     //   purpose, so we need to reject the stream);
     // - if the construct has a footer, once we are done reading it, we have
     //   reached the footer (this is to aid with debugging).
--- a/js/src/frontend/BinTokenReaderTester.h
+++ b/js/src/frontend/BinTokenReaderTester.h
@@ -17,16 +17,18 @@
 
 #if !defined(NIGHTLY_BUILD)
 #error "BinTokenReaderTester.* is designed to help test implementations of successive versions of JS BinaryAST. It is available only on Nightly."
 #endif // !defined(NIGHTLY_BUILD)
 
 namespace js {
 namespace frontend {
 
+class BinASTSourceMetadata;
+
 /**
  * A token reader for a simple, alternative serialization format for BinAST.
  *
  * This serialization format, which is also supported by the reference
  * implementation of the BinAST compression suite, is designed to be
  * mostly human-readable and easy to check for all sorts of deserialization
  * errors. While this format is NOT designed to be shipped to end-users, it
  * is nevertheless a very useful tool for implementing and testing parsers.
@@ -209,16 +211,21 @@ class MOZ_STACK_CLASS BinTokenReaderTest
     }
 
     /**
      * Read a single uint32_t.
      */
     MOZ_MUST_USE JS::Result<uint32_t> readInternalUint32();
 
   public:
+    void traceMetadata(JSTracer*) {}
+    BinASTSourceMetadata* takeMetadata() { MOZ_CRASH("Tester format has no metadata to take!"); }
+    MOZ_MUST_USE JS::Result<Ok> initFromScriptSource(ScriptSource*) { MOZ_CRASH("Tester format not for dynamic use"); }
+
+  public:
     // The following classes are used whenever we encounter a tuple/tagged tuple/list
     // to make sure that:
     //
     // - if the construct "knows" its byte length, we have exactly consumed all
     //   the bytes (otherwise, this means that the file is corrupted, perhaps on
     //   purpose, so we need to reject the stream);
     // - if the construct has a footer, once we are done reading it, we have
     //   reached the footer (this is to aid with debugging).
--- a/js/src/frontend/BytecodeCompiler.cpp
+++ b/js/src/frontend/BytecodeCompiler.cpp
@@ -685,22 +685,27 @@ frontend::CompileGlobalBinASTScript(JSCo
         return nullptr;
     }
 
     Directives directives(options.strictOption);
     GlobalSharedContext globalsc(cx, ScopeKind::Global, directives, options.extraWarningsOption);
 
     frontend::BinASTParser<BinTokenReaderMultipart> parser(cx, alloc, usedNames, options, sourceObj);
 
-    auto parsed = parser.parse(&globalsc, src, len);
+    // Metadata stores internal pointers, so we must use the same buffer every time, including for lazy parses
+    ScriptSource* ss = sourceObj->source();
+    BinASTSourceMetadata* metadata = nullptr;
+    auto parsed = parser.parse(&globalsc, ss->binASTSource(), ss->length(), &metadata);
 
     if (parsed.isErr()) {
         return nullptr;
     }
 
+    sourceObj->source()->setBinASTSourceMetadata(metadata);
+
     BytecodeEmitter bce(nullptr, &parser, &globalsc, script, nullptr, 0);
 
     if (!bce.init()) {
         return nullptr;
     }
 
     ParseNode *pn = parsed.unwrap();
     if (!bce.emitScript(pn)) {
@@ -955,19 +960,17 @@ frontend::CompileLazyBinASTFunction(JSCo
 
     if (lazy->hasBeenCloned())
         script->setHasBeenCloned();
 
     frontend::BinASTParser<BinTokenReaderMultipart> parser(cx, cx->tempLifoAlloc(),
                                                            usedNames, options, sourceObj,
                                                            lazy);
 
-    auto parsed = parser.parseLazyFunction(lazy->scriptSource()->binASTSource(),
-                                           lazy->sourceStart(),
-                                           lazy->scriptSource()->length());
+    auto parsed = parser.parseLazyFunction(lazy->scriptSource(), lazy->sourceStart());
 
     if (parsed.isErr())
         return false;
 
     ParseNode *pn = parsed.unwrap();
 
     BytecodeEmitter bce(nullptr, &parser, pn->as<CodeNode>().funbox(), script,
                         lazy, pn->pn_pos, BytecodeEmitter::LazyFunction);
--- a/js/src/moz.build
+++ b/js/src/moz.build
@@ -704,16 +704,17 @@ if CONFIG['JS_BUILD_BINAST']:
     # Using SOURCES, as UNIFIED_SOURCES causes mysterious bugs on 32-bit platforms.
     # These parts of BinAST are designed only to test evolutions of the
     # specification.
     SOURCES += ['frontend/BinTokenReaderTester.cpp']
     # These parts of BinAST should eventually move to release.
     SOURCES += [
         'frontend/BinSource-auto.cpp',
         'frontend/BinSource.cpp',
+        'frontend/BinSourceRuntimeSupport.cpp',
         'frontend/BinToken.cpp',
         'frontend/BinTokenReaderBase.cpp',
         'frontend/BinTokenReaderMultipart.cpp',
     ]
 
     # Instrument BinAST files for fuzzing as we have a fuzzing target for BinAST.
     if CONFIG['FUZZING_INTERFACES'] and CONFIG['LIBFUZZER']:
         SOURCES['frontend/BinSource-auto.cpp'].flags += libfuzzer_flags
--- a/js/src/vm/JSScript.cpp
+++ b/js/src/vm/JSScript.cpp
@@ -1439,28 +1439,38 @@ JSScript::hasScriptName()
 void
 ScriptSourceObject::finalize(FreeOp* fop, JSObject* obj)
 {
     MOZ_ASSERT(fop->onMainThread());
     ScriptSourceObject* sso = &obj->as<ScriptSourceObject>();
     sso->source()->decref();
 }
 
+void
+ScriptSourceObject::trace(JSTracer* trc, JSObject* obj)
+{
+    // This can be invoked during allocation of the SSO itself, before we've had a chance
+    // to initialize things properly. In that case, there's nothing to trace.
+    if (obj->as<ScriptSourceObject>().hasSource()) {
+        obj->as<ScriptSourceObject>().source()->trace(trc);
+    }
+}
+
 static const ClassOps ScriptSourceObjectClassOps = {
     nullptr, /* addProperty */
     nullptr, /* delProperty */
     nullptr, /* enumerate */
     nullptr, /* newEnumerate */
     nullptr, /* resolve */
     nullptr, /* mayResolve */
     ScriptSourceObject::finalize,
     nullptr, /* call */
     nullptr, /* hasInstance */
     nullptr, /* construct */
-    nullptr  /* trace */
+    ScriptSourceObject::trace
 };
 
 const Class ScriptSourceObject::class_ = {
     "ScriptSource",
     JSCLASS_HAS_RESERVED_SLOTS(RESERVED_SLOTS) |
     JSCLASS_IS_ANONYMOUS |
     JSCLASS_FOREGROUND_FINALIZE,
     &ScriptSourceObjectClassOps
@@ -2073,16 +2083,28 @@ ScriptSource::setSourceCopy(JSContext* c
         ReportOutOfMemory(cx);
         return false;
     }
     setSource(std::move(*deduped));
 
     return true;
 }
 
+void
+ScriptSource::trace(JSTracer* trc)
+{
+#ifdef JS_BUILD_BINAST
+    if (binASTMetadata_) {
+        binASTMetadata_->trace(trc);
+    }
+#else
+    MOZ_ASSERT(!binASTMetadata_);
+#endif // JS_BUILD_BINAST
+}
+
 static MOZ_MUST_USE bool
 reallocUniquePtr(UniqueChars& unique, size_t size)
 {
     auto newPtr = static_cast<char*>(js_realloc(unique.get(), size));
     if (!newPtr) {
         return false;
     }
 
@@ -2359,16 +2381,88 @@ ScriptSource::performXDR(XDRState<mode>*
                     return xdr->fail(JS::TranscodeResult_Throw);
                 }
             }
         } else {
             RawDataMatcher rdm;
             void* p = data.match(rdm);
             MOZ_TRY(xdr->codeBytes(p, byteLen));
         }
+
+        uint8_t hasMetadata = !!binASTMetadata_;
+        MOZ_TRY(xdr->codeUint8(&hasMetadata));
+        if (hasMetadata) {
+#if defined(JS_BUILD_BINAST)
+            uint32_t numBinKinds;
+            uint32_t numStrings;
+            if (mode == XDR_ENCODE) {
+                numBinKinds = binASTMetadata_->numBinKinds();
+                numStrings = binASTMetadata_->numStrings();
+            }
+            MOZ_TRY(xdr->codeUint32(&numBinKinds));
+            MOZ_TRY(xdr->codeUint32(&numStrings));
+
+            if (mode == XDR_DECODE) {
+                // Use calloc, since we're storing this immediately, and filling it might GC, to
+                // avoid marking bogus atoms.
+                setBinASTSourceMetadata(
+                    static_cast<frontend::BinASTSourceMetadata*>(
+                        js_calloc(frontend::BinASTSourceMetadata::totalSize(numBinKinds, numStrings))));
+                if (!binASTMetadata_) {
+                    return xdr->fail(JS::TranscodeResult_Throw);
+                }
+            }
+
+            for (uint32_t i = 0; i < numBinKinds; i++) {
+                frontend::BinKind* binKindBase = binASTMetadata_->binKindBase();
+                MOZ_TRY(xdr->codeEnum32(&binKindBase[i]));
+            }
+
+            RootedAtom atom(xdr->cx());
+            JSAtom** atomsBase = binASTMetadata_->atomsBase();
+            auto slices = binASTMetadata_->sliceBase();
+            auto sourceBase = reinterpret_cast<const char*>(binASTSource());
+
+            for (uint32_t i = 0; i < numStrings; i++) {
+                uint8_t isNull;
+                if (mode == XDR_ENCODE) {
+                    atom = binASTMetadata_->getAtom(i);
+                    isNull = !atom;
+                }
+                MOZ_TRY(xdr->codeUint8(&isNull));
+                if (isNull) {
+                    atom = nullptr;
+                } else {
+                    MOZ_TRY(XDRAtom(xdr, &atom));
+                }
+                if (mode == XDR_DECODE) {
+                    atomsBase[i] = atom;
+                }
+
+                uint64_t sliceOffset;
+                uint32_t sliceLen;
+                if (mode == XDR_ENCODE) {
+                    auto &slice = binASTMetadata_->getSlice(i);
+                    sliceOffset = slice.begin()-sourceBase;
+                    sliceLen = slice.byteLen_;
+                }
+
+                MOZ_TRY(xdr->codeUint64(&sliceOffset));
+                MOZ_TRY(xdr->codeUint32(&sliceLen));
+
+                if (mode == XDR_DECODE) {
+                    new (&slices[i]) frontend::BinASTSourceMetadata::CharSlice(sourceBase + sliceOffset, sliceLen);
+                }
+            }
+#else
+            // No BinAST, no BinASTMetadata
+            MOZ_ASSERT(mode != XDR_ENCODE);
+            xdr->fail(JS::TranscodeResult_Throw);
+#endif // JS_BUILD_BINAST
+        }
     }
 
     uint8_t haveSourceMap = hasSourceMapURL();
     MOZ_TRY(xdr->codeUint8(&haveSourceMap));
 
     if (haveSourceMap) {
         uint32_t sourceMapURLLen = (mode == XDR_DECODE) ? 0 : js_strlen(sourceMapURL_.get());
         MOZ_TRY(xdr->codeUint32(&sourceMapURLLen));
--- a/js/src/vm/JSScript.h
+++ b/js/src/vm/JSScript.h
@@ -13,16 +13,17 @@
 #include "mozilla/Atomics.h"
 #include "mozilla/Maybe.h"
 #include "mozilla/MemoryReporting.h"
 #include "mozilla/Span.h"
 #include "mozilla/Variant.h"
 
 #include "jstypes.h"
 
+#include "frontend/BinSourceRuntimeSupport.h"
 #include "frontend/NameAnalysisTypes.h"
 #include "gc/Barrier.h"
 #include "gc/Rooting.h"
 #include "jit/IonCode.h"
 #include "js/CompileOptions.h"
 #include "js/UbiNode.h"
 #include "js/UniquePtr.h"
 #include "vm/BytecodeUtil.h"
@@ -504,16 +505,18 @@ class ScriptSource
 
     // True if we can call JSRuntime::sourceHook to load the source on
     // demand. If sourceRetrievable_ and hasSourceText() are false, it is not
     // possible to get source at all.
     bool sourceRetrievable_:1;
     bool hasIntroductionOffset_:1;
     bool containsAsmJS_:1;
 
+    UniquePtr<frontend::BinASTSourceMetadata> binASTMetadata_;
+
     const char16_t* chunkChars(JSContext* cx, UncompressedSourceCache::AutoHoldEntry& holder,
                                size_t chunk);
 
     // Return a string containing the chars starting at |begin| and ending at
     // |begin + len|.
     //
     // Warning: this is *not* GC-safe! Any chars to be handed out should use
     // PinnedChars. See comment below.
@@ -563,16 +566,25 @@ class ScriptSource
     MOZ_MUST_USE bool setSourceCopy(JSContext* cx, JS::SourceBufferHolder& srcBuf);
     void setSourceRetrievable() { sourceRetrievable_ = true; }
     bool sourceRetrievable() const { return sourceRetrievable_; }
     bool hasSourceText() const { return hasUncompressedSource() || hasCompressedSource(); }
     bool hasBinASTSource() const { return data.is<BinAST>(); }
     bool hasUncompressedSource() const { return data.is<Uncompressed>(); }
     bool hasCompressedSource() const { return data.is<Compressed>(); }
 
+    void setBinASTSourceMetadata(frontend::BinASTSourceMetadata* metadata) {
+        MOZ_ASSERT(hasBinASTSource());
+        binASTMetadata_.reset(metadata);
+    }
+    frontend::BinASTSourceMetadata* binASTSourceMetadata() const {
+        MOZ_ASSERT(hasBinASTSource());
+        return binASTMetadata_.get();
+    }
+
     size_t length() const {
         struct LengthMatcher
         {
             size_t match(const Uncompressed& u) {
                 return u.string.length();
             }
 
             size_t match(const Compressed& c) {
@@ -718,16 +730,18 @@ class ScriptSource
     const mozilla::TimeStamp parseEnded() const {
         return parseEnded_;
     }
     // Inform `this` source that it has been fully parsed.
     void recordParseEnded() {
         MOZ_ASSERT(parseEnded_.IsNull());
         parseEnded_ = ReallyNow();
     }
+
+    void trace(JSTracer* trc);
 };
 
 class ScriptSourceHolder
 {
     ScriptSource* ss;
   public:
     ScriptSourceHolder()
       : ss(nullptr)
@@ -772,16 +786,19 @@ class ScriptSourceObject : public Native
     // Initialize those properties of this ScriptSourceObject whose values
     // are provided by |options|, re-wrapping as necessary.
     static bool initFromOptions(JSContext* cx, HandleScriptSourceObject source,
                                 const JS::ReadOnlyCompileOptions& options);
 
     static bool initElementProperties(JSContext* cx, HandleScriptSourceObject source,
                                       HandleObject element, HandleString elementAttrName);
 
+    bool hasSource() const {
+        return !getReservedSlot(SOURCE_SLOT).isUndefined();
+    }
     ScriptSource* source() const {
         return static_cast<ScriptSource*>(getReservedSlot(SOURCE_SLOT).toPrivate());
     }
     JSObject* element() const {
         return getReservedSlot(ELEMENT_SLOT).toObjectOrNull();
     }
     const Value& elementAttributeName() const {
         MOZ_ASSERT(!getReservedSlot(ELEMENT_PROPERTY_SLOT).isMagic());