Bug 1549953 - Add BinASTTokenReaderContext without implementation. r=Yoric
authorTooru Fujisawa <arai_a@mac.com>
Mon, 13 May 2019 13:28:38 +0000
changeset 535488 7a303f34b6ce2df1f444b031cc947c01117bc0eb
parent 535487 e2b09eeb4cc18a3a4aeb66f26a1f2052aa3d9634
child 535489 3fc09d5a4f225b2164b2a4ade0c2b1545f999903
push id2082
push userffxbld-merge
push dateMon, 01 Jul 2019 08:34:18 +0000
treeherdermozilla-release@2fb19d0466d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersYoric
bugs1549953
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1549953 - Add BinASTTokenReaderContext without implementation. r=Yoric Differential Revision: https://phabricator.services.mozilla.com/D30342
js/src/frontend/BinASTTokenReaderContext.cpp
js/src/frontend/BinASTTokenReaderContext.h
js/src/frontend/moz.build
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinASTTokenReaderContext.cpp
@@ -0,0 +1,203 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "frontend/BinASTTokenReaderContext.h"
+
+#include "mozilla/Result.h"  // MOZ_TRY*
+
+#include <string.h>  // memcmp
+
+#include "frontend/BinAST-macros.h"  // BINJS_TRY*, BINJS_MOZ_TRY*
+#include "vm/JSScript.h"             // ScriptSource
+
+namespace js {
+namespace frontend {
+
+// The magic header, at the start of every binjs file.
+const char CX_MAGIC_HEADER[] =
+    "\x89"
+    "BJS\r\n\0\n";
+
+// The latest format version understood by this tokenizer.
+const uint32_t MAGIC_FORMAT_VERSION = 2;
+
+using AutoList = BinASTTokenReaderContext::AutoList;
+using AutoTaggedTuple = BinASTTokenReaderContext::AutoTaggedTuple;
+using CharSlice = BinaryASTSupport::CharSlice;
+using Chars = BinASTTokenReaderContext::Chars;
+
+BinASTTokenReaderContext::BinASTTokenReaderContext(JSContext* cx,
+                                                   ErrorReporter* er,
+                                                   const uint8_t* start,
+                                                   const size_t length)
+    : BinASTTokenReaderBase(cx, er, start, length),
+      metadata_(nullptr),
+      posBeforeTree_(nullptr) {
+  MOZ_ASSERT(er);
+}
+
+BinASTTokenReaderContext::~BinASTTokenReaderContext() {
+  if (metadata_ && metadataOwned_ == MetadataOwnership::Owned) {
+    UniqueBinASTSourceMetadataPtr ptr(metadata_);
+  }
+}
+
+BinASTSourceMetadata* BinASTTokenReaderContext::takeMetadata() {
+  MOZ_ASSERT(metadataOwned_ == MetadataOwnership::Owned);
+  metadataOwned_ = MetadataOwnership::Unowned;
+  return metadata_;
+}
+
+JS::Result<Ok> BinASTTokenReaderContext::initFromScriptSource(
+    ScriptSource* scriptSource) {
+  metadata_ = scriptSource->binASTSourceMetadata();
+  metadataOwned_ = MetadataOwnership::Unowned;
+
+  return Ok();
+}
+
+JS::Result<Ok> BinASTTokenReaderContext::readHeader() {
+  // Check that we don't call this function twice.
+  MOZ_ASSERT(!posBeforeTree_);
+
+  // Read global headers.
+  MOZ_TRY(readConst(CX_MAGIC_HEADER));
+  BINJS_MOZ_TRY_DECL(version, readVarU32());
+
+  if (version != MAGIC_FORMAT_VERSION) {
+    return raiseError("Format version not implemented");
+  }
+
+  // TODO: handle `LinkToSharedDictionary` and remaining things here.
+
+  return raiseError("Not Yet Implemented");
+}
+
+void BinASTTokenReaderContext::traceMetadata(JSTracer* trc) {
+  if (metadata_) {
+    metadata_->trace(trc);
+  }
+}
+
+JS::Result<bool> BinASTTokenReaderContext::readBool() {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<double> BinASTTokenReaderContext::readDouble() {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<JSAtom*> BinASTTokenReaderContext::readMaybeAtom() {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<JSAtom*> BinASTTokenReaderContext::readAtom() {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<JSAtom*> BinASTTokenReaderContext::readMaybeIdentifierName() {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<JSAtom*> BinASTTokenReaderContext::readIdentifierName() {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<JSAtom*> BinASTTokenReaderContext::readPropertyKey() {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<Ok> BinASTTokenReaderContext::readChars(Chars& out) {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<BinASTVariant> BinASTTokenReaderContext::readVariant() {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<BinASTTokenReaderBase::SkippableSubTree>
+BinASTTokenReaderContext::readSkippableSubTree() {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<Ok> BinASTTokenReaderContext::enterTaggedTuple(
+    BinASTKind& tag, BinASTTokenReaderContext::BinASTFields&,
+    AutoTaggedTuple& guard) {
+  return raiseError("Not Yet Implemented");
+}
+
+JS::Result<Ok> BinASTTokenReaderContext::enterList(uint32_t& items,
+                                                   AutoList& guard) {
+  return raiseError("Not Yet Implemented");
+}
+
+void BinASTTokenReaderContext::AutoBase::init() { initialized_ = true; }
+
+BinASTTokenReaderContext::AutoBase::AutoBase(BinASTTokenReaderContext& reader)
+    : initialized_(false), reader_(reader) {}
+
+BinASTTokenReaderContext::AutoBase::~AutoBase() {
+  // By now, the `AutoBase` must have been deinitialized by calling `done()`.
+  // The only case in which we can accept not calling `done()` is if we have
+  // bailed out because of an error.
+  MOZ_ASSERT_IF(initialized_, reader_.hasRaisedError());
+}
+
+JS::Result<Ok> BinASTTokenReaderContext::AutoBase::checkPosition(
+    const uint8_t* expectedEnd) {
+  return reader_.raiseError("Not Yet Implemented");
+}
+
+BinASTTokenReaderContext::AutoList::AutoList(BinASTTokenReaderContext& reader)
+    : AutoBase(reader) {}
+
+void BinASTTokenReaderContext::AutoList::init() { AutoBase::init(); }
+
+JS::Result<Ok> BinASTTokenReaderContext::AutoList::done() {
+  return reader_.raiseError("Not Yet Implemented");
+}
+
+// Internal uint32_t
+//
+// Encoded as variable length number.
+
+MOZ_MUST_USE JS::Result<uint32_t> BinASTTokenReaderContext::readVarU32() {
+  uint32_t result = 0;
+  uint32_t shift = 0;
+  while (true) {
+    MOZ_ASSERT(shift < 32);
+    uint32_t byte;
+    MOZ_TRY_VAR(byte, readByte());
+
+    const uint32_t newResult = result | (byte >> 1) << shift;
+    if (newResult < result) {
+      return raiseError("Overflow during readVarU32");
+    }
+
+    result = newResult;
+    shift += 7;
+
+    if ((byte & 1) == 0) {
+      return result;
+    }
+
+    if (shift >= 32) {
+      return raiseError("Overflow during readVarU32");
+    }
+  }
+}
+
+BinASTTokenReaderContext::AutoTaggedTuple::AutoTaggedTuple(
+    BinASTTokenReaderContext& reader)
+    : AutoBase(reader) {}
+
+JS::Result<Ok> BinASTTokenReaderContext::AutoTaggedTuple::done() {
+  return reader_.raiseError("Not Yet Implemented");
+}
+
+}  // namespace frontend
+
+}  // namespace js
new file mode 100644
--- /dev/null
+++ b/js/src/frontend/BinASTTokenReaderContext.h
@@ -0,0 +1,314 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef frontend_BinASTTokenReaderContext_h
+#define frontend_BinASTTokenReaderContext_h
+
+#include "mozilla/Assertions.h"  // MOZ_ASSERT
+#include "mozilla/Attributes.h"  // MOZ_MUST_USE, MOZ_STACK_CLASS
+
+#include "mozilla/Maybe.h"  // mozilla::Maybe
+
+#include <stddef.h>  // size_t
+#include <stdint.h>  // uint8_t, uint32_t
+
+#include "frontend/BinASTRuntimeSupport.h"  // CharSlice, BinASTVariant, BinASTKind, BinASTField, BinASTSourceMetadata
+#include "frontend/BinASTToken.h"
+#include "frontend/BinASTTokenReaderBase.h"  // BinASTTokenReaderBase, SkippableSubTree
+#include "js/AllocPolicy.h"                  // SystemAllocPolicy
+#include "js/HashTable.h"                    // HashMap, DefaultHasher
+#include "js/Result.h"                       // JS::Result, Ok, Error
+#include "js/Vector.h"                       // js::Vector
+
+class JSAtom;
+class JSTracer;
+struct JSContext;
+
+namespace js {
+
+class ScriptSource;
+
+namespace frontend {
+
+class ErrorReporter;
+
+/**
+ * A token reader implementing the "context" serialization format for BinAST.
+ *
+ * This serialization format, which is also supported by the reference
+ * implementation of the BinAST compression suite, is designed to be
+ * space- and time-efficient.
+ *
+ * As other token readers for the BinAST:
+ *
+ * - the reader does not support error recovery;
+ * - the reader does not support lookahead or pushback.
+ */
+class MOZ_STACK_CLASS BinASTTokenReaderContext : public BinASTTokenReaderBase {
+ public:
+  class AutoList;
+  class AutoTaggedTuple;
+
+  using CharSlice = BinaryASTSupport::CharSlice;
+
+  // This implementation of `BinASTFields` is effectively `void`, as the format
+  // does not embed field information.
+  class BinASTFields {
+   public:
+    explicit BinASTFields(JSContext*) {}
+  };
+  using Chars = CharSlice;
+
+ public:
+  /**
+   * Construct a token reader.
+   *
+   * Does NOT copy the buffer.
+   */
+  BinASTTokenReaderContext(JSContext* cx, ErrorReporter* er,
+                           const uint8_t* start, const size_t length);
+
+  /**
+   * Construct a token reader.
+   *
+   * Does NOT copy the buffer.
+   */
+  BinASTTokenReaderContext(JSContext* cx, ErrorReporter* er,
+                           const Vector<uint8_t>& chars);
+
+  ~BinASTTokenReaderContext();
+
+  /**
+   * Read the header of the file.
+   */
+  MOZ_MUST_USE JS::Result<Ok> readHeader();
+
+  // --- Primitive values.
+  //
+  // Note that the underlying format allows for a `null` value for primitive
+  // values.
+  //
+  // Reading will return an error either in case of I/O error or in case of
+  // a format problem. Reading if an exception in pending is an error and
+  // will cause assertion failures. Do NOT attempt to read once an exception
+  // has been cleared: the token reader does NOT support recovery, by design.
+
+  /**
+   * Read a single `true | false` value.
+   */
+  MOZ_MUST_USE JS::Result<bool> readBool();
+
+  /**
+   * Read a single `number` value.
+   */
+  MOZ_MUST_USE JS::Result<double> readDouble();
+
+  /**
+   * Read a single `string | null` value.
+   *
+   * Fails if that string is not valid UTF-8.
+   */
+  MOZ_MUST_USE JS::Result<JSAtom*> readMaybeAtom();
+  MOZ_MUST_USE JS::Result<JSAtom*> readAtom();
+
+  /**
+   * Read a single IdentifierName value.
+   */
+  MOZ_MUST_USE JS::Result<JSAtom*> readMaybeIdentifierName();
+  MOZ_MUST_USE JS::Result<JSAtom*> readIdentifierName();
+
+  /**
+   * Read a single PropertyKey value.
+   */
+  MOZ_MUST_USE JS::Result<JSAtom*> readPropertyKey();
+
+  /**
+   * Read a single `string | null` value.
+   *
+   * MAY check if that string is not valid UTF-8.
+   */
+  MOZ_MUST_USE JS::Result<Ok> readChars(Chars&);
+
+  /**
+   * Read a single `BinASTVariant | null` value.
+   */
+  MOZ_MUST_USE JS::Result<mozilla::Maybe<BinASTVariant>> readMaybeVariant();
+  MOZ_MUST_USE JS::Result<BinASTVariant> readVariant();
+
+  /**
+   * Read over a single `[Skippable]` subtree value.
+   *
+   * This does *not* attempt to parse the subtree itself. Rather, the
+   * returned `SkippableSubTree` contains the necessary information
+   * to parse/tokenize the subtree at a later stage
+   */
+  MOZ_MUST_USE JS::Result<SkippableSubTree> readSkippableSubTree();
+
+  // --- Composite values.
+  //
+  // The underlying format does NOT allows for a `null` composite value.
+  //
+  // Reading will return an error either in case of I/O error or in case of
+  // a format problem. Reading from a poisoned tokenizer is an error and
+  // will cause assertion failures.
+
+  /**
+   * Start reading a list.
+   *
+   * @param length (OUT) The number of elements in the list.
+   * @param guard (OUT) A guard, ensuring that we read the list correctly.
+   *
+   * The `guard` is dedicated to ensuring that reading the list has consumed
+   * exactly all the bytes from that list. The `guard` MUST therefore be
+   * destroyed at the point where the caller has reached the end of the list.
+   * If the caller has consumed too few/too many bytes, this will be reported
+   * in the call go `guard.done()`.
+   */
+  MOZ_MUST_USE JS::Result<Ok> enterList(uint32_t& length, AutoList& guard);
+
+  /**
+   * Start reading a tagged tuple.
+   *
+   * @param tag (OUT) The tag of the tuple.
+   * @param fields Ignored, provided for API compatibility.
+   * @param guard (OUT) A guard, ensuring that we read the tagged tuple
+   * correctly.
+   *
+   * The `guard` is dedicated to ensuring that reading the list has consumed
+   * exactly all the bytes from that tuple. The `guard` MUST therefore be
+   * destroyed at the point where the caller has reached the end of the tuple.
+   * If the caller has consumed too few/too many bytes, this will be reported
+   * in the call go `guard.done()`.
+   *
+   * @return out If the header of the tuple is invalid.
+   */
+  MOZ_MUST_USE JS::Result<Ok> enterTaggedTuple(
+      BinASTKind& tag, BinASTTokenReaderContext::BinASTFields& fields,
+      AutoTaggedTuple& guard);
+
+  /**
+   * Read a single unsigned long.
+   */
+  MOZ_MUST_USE JS::Result<uint32_t> readUnsignedLong() { return readVarU32(); }
+
+ private:
+  /**
+   * Read a single uint32_t.
+   */
+  MOZ_MUST_USE JS::Result<uint32_t> readVarU32();
+
+ private:
+  // A mapping string index => BinASTVariant as extracted from the [STRINGS]
+  // section of the file. Populated lazily.
+  js::HashMap<uint32_t, BinASTVariant, DefaultHasher<uint32_t>,
+              SystemAllocPolicy>
+      variantsTable_;
+
+  enum class MetadataOwnership { Owned, Unowned };
+  MetadataOwnership metadataOwned_ = MetadataOwnership::Owned;
+  BinASTSourceMetadata* metadata_;
+
+  const uint8_t* posBeforeTree_;
+
+ public:
+  BinASTTokenReaderContext(const BinASTTokenReaderContext&) = delete;
+  BinASTTokenReaderContext(BinASTTokenReaderContext&&) = delete;
+  BinASTTokenReaderContext& operator=(BinASTTokenReaderContext&) = delete;
+
+ public:
+  void traceMetadata(JSTracer* trc);
+  BinASTSourceMetadata* takeMetadata();
+  MOZ_MUST_USE JS::Result<Ok> initFromScriptSource(ScriptSource* scriptSource);
+
+ public:
+  // The following classes are used whenever we encounter a tuple/tagged
+  // tuple/list to make sure that:
+  //
+  // - if the construct "knows" its byte length, we have exactly consumed all
+  //   the bytes (otherwise, this means that the file is corrupted, perhaps on
+  //   purpose, so we need to reject the stream);
+  // - if the construct has a footer, once we are done reading it, we have
+  //   reached the footer (this is to aid with debugging).
+  //
+  // In either case, the caller MUST call method `done()` of the guard once
+  // it is done reading the tuple/tagged tuple/list, to report any pending
+  // error.
+
+  // Base class used by other Auto* classes.
+  class MOZ_STACK_CLASS AutoBase {
+   protected:
+    explicit AutoBase(BinASTTokenReaderContext& reader);
+    ~AutoBase();
+
+    // Raise an error if we are not in the expected position.
+    MOZ_MUST_USE JS::Result<Ok> checkPosition(const uint8_t* expectedPosition);
+
+    friend BinASTTokenReaderContext;
+    void init();
+
+    // Set to `true` if `init()` has been called. Reset to `false` once
+    // all conditions have been checked.
+    bool initialized_;
+    BinASTTokenReaderContext& reader_;
+  };
+
+  // Guard class used to ensure that `enterList` is used properly.
+  class MOZ_STACK_CLASS AutoList : public AutoBase {
+   public:
+    explicit AutoList(BinASTTokenReaderContext& reader);
+
+    // Check that we have properly read to the end of the list.
+    MOZ_MUST_USE JS::Result<Ok> done();
+
+   protected:
+    friend BinASTTokenReaderContext;
+    void init();
+  };
+
+  // Guard class used to ensure that `enterTaggedTuple` is used properly.
+  class MOZ_STACK_CLASS AutoTaggedTuple : public AutoBase {
+   public:
+    explicit AutoTaggedTuple(BinASTTokenReaderContext& reader);
+
+    // Check that we have properly read to the end of the tuple.
+    MOZ_MUST_USE JS::Result<Ok> done();
+  };
+
+  // Compare a `Chars` and a string literal (ONLY a string literal).
+  template <size_t N>
+  static bool equals(const Chars& left, const char (&right)[N]) {
+    MOZ_ASSERT(N > 0);
+    MOZ_ASSERT(right[N - 1] == 0);
+    if (left.byteLen_ + 1 /* implicit NUL */ != N) {
+      return false;
+    }
+
+    if (!std::equal(left.start_, left.start_ + left.byteLen_, right)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  template <size_t N>
+  static JS::Result<Ok, JS::Error&> checkFields(
+      const BinASTKind kind, const BinASTFields& actual,
+      const BinASTField (&expected)[N]) {
+    // Not implemented in this tokenizer.
+    return Ok();
+  }
+
+  static JS::Result<Ok, JS::Error&> checkFields0(const BinASTKind kind,
+                                                 const BinASTFields& actual) {
+    // Not implemented in this tokenizer.
+    return Ok();
+  }
+};
+
+}  // namespace frontend
+}  // namespace js
+
+#endif  // frontend_BinASTTokenReaderContext_h
--- a/js/src/frontend/moz.build
+++ b/js/src/frontend/moz.build
@@ -71,16 +71,17 @@ if CONFIG['JS_BUILD_BINAST']:
     # These parts of BinAST should eventually move to release.
     SOURCES += [
         'BinASTParser.cpp',
         'BinASTParserBase.cpp',
         'BinASTParserPerTokenizer.cpp',
         'BinASTRuntimeSupport.cpp',
         'BinASTToken.cpp',
         'BinASTTokenReaderBase.cpp',
+        'BinASTTokenReaderContext.cpp',
         'BinASTTokenReaderMultipart.cpp',
     ]
 
     DIRS += [
         'binast'
     ]
 
     # Instrument BinAST files for fuzzing as we have a fuzzing target for BinAST.