Bug 1215063 - Implement a simple module loader for the shell r=shu r=froydnj
authorJon Coppeard <jcoppeard@mozilla.com>
Tue, 10 Nov 2015 09:44:53 +0000
changeset 308027 26f84ff40db55f203fe0e5b322d5819fa1892671
parent 308026 548a09b1a4e6af5e098c4b560ee637b1cc6d624e
child 308028 e3d2cb194e984a2cb96486b9db92de60ae697ff1
push id7422
push userpehrsons@gmail.com
push dateWed, 11 Nov 2015 04:19:53 +0000
reviewersshu, froydnj
bugs1215063
milestone45.0a1
Bug 1215063 - Implement a simple module loader for the shell r=shu r=froydnj
config/check_spidermonkey_style.py
js/src/builtin/embedjs.py
js/src/shell/ModuleLoader.js
js/src/shell/js.cpp
js/src/shell/moz.build
--- a/config/check_spidermonkey_style.py
+++ b/config/check_spidermonkey_style.py
@@ -71,16 +71,17 @@ included_inclnames_to_ignore = set([
     'prerror.h',                # NSPR
     'prinit.h',                 # NSPR
     'prlink.h',                 # NSPR
     'prlock.h',                 # NSPR
     'prprf.h',                  # NSPR
     'prthread.h',               # NSPR
     'prtypes.h',                # NSPR
     'selfhosted.out.h',         # generated in $OBJDIR
+    'shellmoduleloader.out.h',  # generated in $OBJDIR
     'unicode/locid.h',          # ICU
     'unicode/numsys.h',         # ICU
     'unicode/timezone.h',       # ICU
     'unicode/ucal.h',           # ICU
     'unicode/uclean.h',         # ICU
     'unicode/ucol.h',           # ICU
     'unicode/udat.h',           # ICU
     'unicode/udatpg.h',         # ICU
--- a/js/src/builtin/embedjs.py
+++ b/js/src/builtin/embedjs.py
@@ -57,49 +57,50 @@ def ToCArray(lines):
   return ", ".join(result)
 
 HEADER_TEMPLATE = """\
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 namespace js {
-namespace selfhosted {
+namespace %(namespace)s {
     static const %(sources_type)s data[] = { %(sources_data)s };
 
     static const %(sources_type)s * const %(sources_name)s = reinterpret_cast<const %(sources_type)s *>(data);
 
     uint32_t GetCompressedSize() {
         return %(compressed_total_length)i;
     }
 
     uint32_t GetRawScriptsSize() {
         return %(raw_total_length)i;
     }
 } // selfhosted
 } // js
 """
 
-def embed(cxx, preprocessorOption, msgs, sources, c_out, js_out, env):
+def embed(cxx, preprocessorOption, msgs, sources, c_out, js_out, namespace, env):
   combinedSources = '\n'.join([msgs] + ['#include "%(s)s"' % { 's': source } for source in sources])
   args = ['-D%(k)s=%(v)s' % { 'k': k, 'v': env[k] } for k in env]
   preprocessed = preprocess(cxx, preprocessorOption, combinedSources, args)
   processed = '\n'.join([line for line in preprocessed.splitlines() if \
                          (line.strip() and not line.startswith('#'))])
 
   js_out.write(processed)
   import zlib
   compressed = zlib.compress(processed)
   data = ToCArray(compressed)
   c_out.write(HEADER_TEMPLATE % {
     'sources_type': 'unsigned char',
     'sources_data': data,
     'sources_name': 'compressedSources',
     'compressed_total_length': len(compressed),
-    'raw_total_length': len(processed)
+    'raw_total_length': len(processed),
+    'namespace': namespace
   })
 
 def preprocess(cxx, preprocessorOption, source, args = []):
   if (not os.path.exists(cxx[0])):
     cxx[0] = which.which(cxx[0])
   # Clang seems to complain and not output anything if the extension of the
   # input is not something it recognizes, so just fake a .cpp here.
   tmpIn = 'self-hosting-cpp-input.cpp';
@@ -136,20 +137,27 @@ def get_config_defines(buildconfig):
   for value in buildconfig.substs['MOZ_DEBUG_DEFINES'].split():
     assert value[:2] == "-D"
     pair = value[2:].split('=', 1)
     if len(pair) == 1:
       pair.append(1)
     env[pair[0]] = pair[1]
   return env
 
-def generate_selfhosted(c_out, msg_file, *inputs):
-  # Called from moz.build to embed selfhosted JS.
+def process_inputs(namespace, c_out, msg_file, inputs):
   deps = [path for path in inputs if path.endswith(".h")]
   sources = [path for path in inputs if path.endswith(".js")]
   assert len(deps) + len(sources) == len(inputs)
   cxx = shlex.split(buildconfig.substs['CXX'])
   cxx_option = buildconfig.substs['PREPROCESS_OPTION']
   env = get_config_defines(buildconfig)
   js_path = re.sub(r"\.out\.h$", "", c_out.name) + ".js"
   msgs = messages(msg_file)
   with open(js_path, 'w') as js_out:
-    embed(cxx, cxx_option, msgs, sources, c_out, js_out, env)
+    embed(cxx, cxx_option, msgs, sources, c_out, js_out, namespace, env)
+
+def generate_selfhosted(c_out, msg_file, *inputs):
+  # Called from moz.build to embed selfhosted JS.
+  process_inputs('selfhosted', c_out, msg_file, inputs)
+
+def generate_shellmoduleloader(c_out, msg_file, *inputs):
+  # Called from moz.build to embed shell module loader JS.
+  process_inputs('moduleloader', c_out, msg_file, inputs)
new file mode 100644
--- /dev/null
+++ b/js/src/shell/ModuleLoader.js
@@ -0,0 +1,43 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// A basic synchronous module loader for testing the shell.
+
+Reflect.Loader = new class {
+    constructor() {
+        this.registry = new Map();
+        this.loadPath = getModuleLoadPath();
+    }
+
+    resolve(name) {
+        if (os.path.isAbsolute(name))
+            return name;
+
+        return os.path.join(this.loadPath, name);
+    }
+
+    fetch(path) {
+        return os.file.readFile(path);
+    }
+
+    loadAndParse(name) {
+        let path = this.resolve(name);
+
+        if (this.registry.has(path))
+            return this.registry.get(path);
+
+        let source = this.fetch(path);
+        let module = parseModule(source, path);
+        this.registry.set(path, module);
+        return module;
+    }
+
+    ["import"](name, referrer) {
+        let module = this.loadAndParse(name);
+        module.declarationInstantiation();
+        return module.evaluation();
+    }
+};
+
+setModuleResolveHook((module, requestName) => Reflect.Loader.loadAndParse(requestName));
--- a/js/src/shell/js.cpp
+++ b/js/src/shell/js.cpp
@@ -50,16 +50,17 @@
 #include "jsprf.h"
 #include "jsscript.h"
 #include "jstypes.h"
 #include "jsutil.h"
 #ifdef XP_WIN
 # include "jswin.h"
 #endif
 #include "jswrapper.h"
+#include "shellmoduleloader.out.h"
 
 #include "builtin/ModuleObject.h"
 #include "builtin/TestingFunctions.h"
 #include "frontend/Parser.h"
 #include "gc/GCInternals.h"
 #include "jit/arm/Simulator-arm.h"
 #include "jit/InlinableNatives.h"
 #include "jit/Ion.h"
@@ -69,21 +70,23 @@
 #include "js/GCAPI.h"
 #include "js/Initialization.h"
 #include "js/StructuredClone.h"
 #include "js/TrackedOptimizationInfo.h"
 #include "perf/jsperf.h"
 #include "shell/jsoptparse.h"
 #include "shell/OSObject.h"
 #include "vm/ArgumentsObject.h"
+#include "vm/Compression.h"
 #include "vm/Debugger.h"
 #include "vm/HelperThreads.h"
 #include "vm/Monitor.h"
 #include "vm/Shape.h"
 #include "vm/SharedArrayObject.h"
+#include "vm/StringBuffer.h"
 #include "vm/Time.h"
 #include "vm/TypedArrayObject.h"
 #include "vm/WrapperObject.h"
 
 #include "jscompartmentinlines.h"
 #include "jsobjinlines.h"
 
 #include "vm/ErrorObject-inl.h"
@@ -172,16 +175,18 @@ static bool printTiming = false;
 static const char* jsCacheDir = nullptr;
 static const char* jsCacheAsmJSPath = nullptr;
 static FILE* gErrFile = nullptr;
 static FILE* gOutFile = nullptr;
 static bool reportWarnings = true;
 static bool compileOnly = false;
 static bool fuzzingSafe = false;
 static bool disableOOMFunctions = false;
+static const char* moduleLoadPath = ".";
+
 #ifdef DEBUG
 static bool dumpEntrainedVariables = false;
 static bool OOM_printAllocationCount = false;
 #endif
 
 // Shell state this is only accessed on the main thread.
 bool jsCachingEnabled = false;
 mozilla::Atomic<bool> jsCacheOpened(false);
@@ -513,16 +518,103 @@ RunFile(JSContext* cx, const char* filen
         }
         int64_t t2 = PRMJ_Now() - t1;
         if (printTiming)
             printf("runtime = %.3f ms\n", double(t2) / PRMJ_USEC_PER_MSEC);
     }
 }
 
 static bool
+InitModuleLoader(JSContext* cx)
+{
+    // Decompress and evaluate the embedded module loader source to initialize
+    // the module loader for the current compartment.
+
+    uint32_t srcLen = moduleloader::GetRawScriptsSize();
+    ScopedJSFreePtr<char> src(cx->pod_malloc<char>(srcLen));
+    if (!src || !DecompressString(moduleloader::compressedSources, moduleloader::GetCompressedSize(),
+                                  reinterpret_cast<unsigned char*>(src.get()), srcLen))
+    {
+        return false;
+    }
+
+    CompileOptions options(cx);
+    options.setIntroductionType("shell module loader");
+    options.setFileAndLine("shell/ModuleLoader.js", 1);
+    options.setSelfHostingMode(false);
+    options.setCanLazilyParse(false);
+    options.setVersion(JSVERSION_LATEST);
+    options.werrorOption = true;
+    options.strictOption = true;
+
+    RootedValue rv(cx);
+    return Evaluate(cx, options, src, srcLen, &rv);
+}
+
+static bool
+GetLoaderObject(JSContext* cx, MutableHandleObject resultOut)
+{
+    // Look up the |Reflect.Loader| object that has been defined by the module
+    // loader.
+
+    RootedObject object(cx, cx->global());
+    RootedValue value(cx);
+    if (!JS_GetProperty(cx, object, "Reflect", &value) || !value.isObject())
+        return false;
+
+    object = &value.toObject();
+    if (!JS_GetProperty(cx, object, "Loader", &value) || !value.isObject())
+        return false;
+
+    resultOut.set(&value.toObject());
+    return true;
+}
+
+static bool
+GetImportMethod(JSContext* cx, HandleObject loader, MutableHandleFunction resultOut)
+{
+    // Look up the module loader's |import| method.
+
+    RootedValue value(cx);
+    if (!JS_GetProperty(cx, loader, "import", &value) || !value.isObject())
+        return false;
+
+    RootedObject object(cx, &value.toObject());
+    if (!object->is<JSFunction>())
+        return false;
+
+    resultOut.set(&object->as<JSFunction>());
+    return true;
+}
+
+static void
+RunModule(JSContext* cx, const char* filename, FILE* file, bool compileOnly)
+{
+    // Exectute a module by calling |Reflect.Loader.import(filename)|.
+
+    ShellRuntime* sr = GetShellRuntime(cx);
+
+    RootedObject loaderObj(cx);
+    MOZ_ALWAYS_TRUE(GetLoaderObject(cx, &loaderObj));
+
+    RootedFunction importFun(cx);
+    MOZ_ALWAYS_TRUE(GetImportMethod(cx, loaderObj, &importFun));
+
+    AutoValueArray<2> args(cx);
+    args[0].setString(JS_NewStringCopyZ(cx, filename));
+    args[1].setUndefined();
+
+    RootedValue value(cx);
+    if (!JS_CallFunction(cx, loaderObj, importFun, args, &value)) {
+        sr->exitCode = EXITCODE_RUNTIME_ERROR;
+        return;
+    }
+}
+
+static bool
 EvalAndPrint(JSContext* cx, const char* bytes, size_t length,
              int lineno, bool compileOnly, FILE* out)
 {
     // Eval.
     JS::CompileOptions options(cx);
     options.setIntroductionType("js shell interactive")
            .setUTF8(true)
            .setIsRunOnce(true)
@@ -603,37 +695,47 @@ ReadEvalPrintLoop(JSContext* cx, FILE* i
             // Catch the error, report it, and keep going.
             JS_ReportPendingException(cx);
         }
     } while (!hitEOF && !sr->quitting);
 
     fprintf(out, "\n");
 }
 
+enum FileKind
+{
+    FileScript,
+    FileModule
+};
+
 static void
-Process(JSContext* cx, const char* filename, bool forceTTY)
+Process(JSContext* cx, const char* filename, bool forceTTY, FileKind kind = FileScript)
 {
     FILE* file;
     if (forceTTY || !filename || strcmp(filename, "-") == 0) {
         file = stdin;
     } else {
         file = fopen(filename, "r");
         if (!file) {
             JS_ReportErrorNumber(cx, my_GetErrorMessage, nullptr,
                                  JSSMSG_CANT_OPEN, filename, strerror(errno));
             return;
         }
     }
     AutoCloseFile autoClose(file);
 
     if (!forceTTY && !isatty(fileno(file))) {
         // It's not interactive - just execute it.
-        RunFile(cx, filename, file, compileOnly);
+        if (kind == FileScript)
+            RunFile(cx, filename, file, compileOnly);
+        else
+            RunModule(cx, filename, file, compileOnly);
     } else {
         // It's an interactive filehandle; drop into read-eval-print loop.
+        MOZ_ASSERT(kind == FileScript);
         ReadEvalPrintLoop(cx, file, gOutFile, compileOnly);
     }
 }
 
 static bool
 Version(JSContext* cx, unsigned argc, Value* vp)
 {
     CallArgs args = CallArgsFromVp(argc, vp);
@@ -3162,34 +3264,51 @@ Compile(JSContext* cx, unsigned argc, Va
     args.rval().setUndefined();
     return ok;
 }
 
 static bool
 ParseModule(JSContext* cx, unsigned argc, Value* vp)
 {
     CallArgs args = CallArgsFromVp(argc, vp);
-    if (args.length() < 1) {
+    if (args.length() == 0) {
         JS_ReportErrorNumber(cx, GetErrorMessage, nullptr,
                              JSMSG_MORE_ARGS_NEEDED, "parseModule", "0", "s");
         return false;
     }
+
     if (!args[0].isString()) {
         const char* typeName = InformalValueTypeName(args[0]);
         JS_ReportError(cx, "expected string to compile, got %s", typeName);
         return false;
     }
 
     RootedObject global(cx, JS::CurrentGlobalOrNull(cx));
     JSFlatString* scriptContents = args[0].toString()->ensureFlat(cx);
     if (!scriptContents)
         return false;
 
+    mozilla::UniquePtr<char, JS::FreePolicy> filename;
     CompileOptions options(cx);
-    options.setFileAndLine("<string>", 1);
+    if (args.length() > 1) {
+        if (!args[1].isString()) {
+            JS_ReportError(cx, "expected filename string, got %s",
+                           JS_TypeOfValue(cx, args[1]));
+            return false;
+        }
+
+        RootedString str(cx, args[1].toString());
+        filename.reset(JS_EncodeString(cx, str));
+        if (!filename)
+            return false;
+
+        options.setFileAndLine(filename.get(), 1);
+    } else {
+        options.setFileAndLine("<string>", 1);
+    }
 
     AutoStableStringChars stableChars(cx);
     if (!stableChars.initTwoByte(cx, scriptContents))
         return false;
 
     const char16_t* chars = stableChars.twoByteRange().start().get();
     SourceBufferHolder srcBuf(chars, scriptContents->length(),
                               SourceBufferHolder::NoOwnership);
@@ -3221,16 +3340,24 @@ SetModuleResolveHook(JSContext* cx, unsi
     RootedFunction hook(cx, &args[0].toObject().as<JSFunction>());
     Rooted<GlobalObject*> global(cx, cx->global());
     global->setModuleResolveHook(hook);
     args.rval().setUndefined();
     return true;
 }
 
 static bool
+GetModuleLoadPath(JSContext* cx, unsigned argc, Value* vp)
+{
+    CallArgs args = CallArgsFromVp(argc, vp);
+    args.rval().setString(JS_NewStringCopyZ(cx, moduleLoadPath));
+    return true;
+}
+
+static bool
 Parse(JSContext* cx, unsigned argc, Value* vp)
 {
     using namespace js::frontend;
 
     CallArgs args = CallArgsFromVp(argc, vp);
 
     if (args.length() < 1) {
         JS_ReportErrorNumber(cx, GetErrorMessage, nullptr, JSMSG_MORE_ARGS_NEEDED,
@@ -4792,16 +4919,21 @@ static const JSFunctionSpecWithHelp shel
 "  Parses source text as a module and returns a Module object."),
 
     JS_FN_HELP("setModuleResolveHook", SetModuleResolveHook, 1, 0,
 "setModuleResolveHook(function(module, specifier) {})",
 "  Set the HostResolveImportedModule hook to |function|.\n"
 "  This hook is used to look up a previously loaded module object.  It should\n"
 "  be implemented by the module loader."),
 
+    JS_FN_HELP("getModuleLoadPath", GetModuleLoadPath, 0, 0,
+"getModuleLoadPath()",
+"  Return any --module-load-path argument passed to the shell.  Used by the\n"
+"  module loader.\n"),
+
     JS_FN_HELP("parse", Parse, 1, 0,
 "parse(code)",
 "  Parses a string, potentially throwing."),
 
     JS_FN_HELP("syntaxParse", SyntaxParse, 1, 0,
 "syntaxParse(code)",
 "  Check the syntax of a string, returning success value"),
 
@@ -5977,41 +6109,61 @@ ProcessArgs(JSContext* cx, OptionParser*
         JS::RuntimeOptionsRef(cx).toggleExtraWarnings();
 
     /* |scriptArgs| gets bound on the global before any code is run. */
     if (!BindScriptArgs(cx, op))
         return EXIT_FAILURE;
 
     MultiStringRange filePaths = op->getMultiStringOption('f');
     MultiStringRange codeChunks = op->getMultiStringOption('e');
-
-    if (filePaths.empty() && codeChunks.empty() && !op->getStringArg("script")) {
+    MultiStringRange modulePaths = op->getMultiStringOption('m');
+
+    if (filePaths.empty() &&
+        codeChunks.empty() &&
+        modulePaths.empty() &&
+        !op->getStringArg("script"))
+    {
         Process(cx, nullptr, true); /* Interactive. */
         return sr->exitCode;
     }
 
-    while (!filePaths.empty() || !codeChunks.empty()) {
-        size_t fpArgno = filePaths.empty() ? -1 : filePaths.argno();
-        size_t ccArgno = codeChunks.empty() ? -1 : codeChunks.argno();
-        if (fpArgno < ccArgno) {
+    if (const char* path = op->getStringOption("module-load-path"))
+        moduleLoadPath = path;
+
+    if (!modulePaths.empty() && !InitModuleLoader(cx))
+        return EXIT_FAILURE;
+
+    while (!filePaths.empty() || !codeChunks.empty() || !modulePaths.empty()) {
+        size_t fpArgno = filePaths.empty() ? SIZE_MAX : filePaths.argno();
+        size_t ccArgno = codeChunks.empty() ? SIZE_MAX : codeChunks.argno();
+        size_t mpArgno = modulePaths.empty() ? SIZE_MAX : modulePaths.argno();
+
+        if (fpArgno < ccArgno && fpArgno < mpArgno) {
             char* path = filePaths.front();
-            Process(cx, path, false);
+            Process(cx, path, false, FileScript);
             if (sr->exitCode)
                 return sr->exitCode;
             filePaths.popFront();
-        } else {
+        } else if (ccArgno < fpArgno && ccArgno < mpArgno) {
             const char* code = codeChunks.front();
             RootedValue rval(cx);
             JS::CompileOptions opts(cx);
             opts.setFileAndLine("-e", 1);
             if (!JS::Evaluate(cx, opts, code, strlen(code), &rval))
                 return sr->exitCode ? sr->exitCode : EXITCODE_RUNTIME_ERROR;
             codeChunks.popFront();
             if (sr->quitting)
                 break;
+        } else {
+            MOZ_ASSERT(mpArgno < fpArgno && mpArgno < ccArgno);
+            char* path = modulePaths.front();
+            Process(cx, path, false, FileModule);
+            if (sr->exitCode)
+                return sr->exitCode;
+            modulePaths.popFront();
         }
     }
 
     if (sr->quitting)
         return sr->exitCode ? sr->exitCode : EXIT_SUCCESS;
 
     /* The |script| argument is processed after all options. */
     if (const char* path = op->getStringArg("script")) {
@@ -6396,16 +6548,17 @@ main(int argc, char** argv, char** envp)
         "run left to right. If provided, the optional script argument is run after "
         "all options have been processed. Just-In-Time compilation modes may be enabled via "
         "command line options.");
     op.setDescriptionWidth(72);
     op.setHelpWidth(80);
     op.setVersion(JS_GetImplementationVersion());
 
     if (!op.addMultiStringOption('f', "file", "PATH", "File path to run")
+        || !op.addMultiStringOption('m', "module", "PATH", "Module path to run")
         || !op.addMultiStringOption('e', "execute", "CODE", "Inline code to run")
         || !op.addBoolOption('i', "shell", "Enter prompt after running code")
         || !op.addBoolOption('c', "compileonly", "Only compile, don't run (syntax checking mode)")
         || !op.addBoolOption('w', "warnings", "Emit warnings")
         || !op.addBoolOption('W', "nowarnings", "Don't emit warnings")
         || !op.addBoolOption('s', "strict", "Check strictness")
         || !op.addBoolOption('D', "dump-bytecode", "Dump bytecode with exec count for all scripts")
         || !op.addBoolOption('b', "print-timing", "Print sub-ms runtime for each file that's run")
@@ -6532,16 +6685,17 @@ main(int argc, char** argv, char** envp)
                              "simulator.")
         || !op.addIntOption('\0', "mips-sim-stop-at", "NUMBER", "Stop the MIPS simulator after the given "
                             "NUMBER of instructions.", -1)
 #endif
         || !op.addIntOption('\0', "nursery-size", "SIZE-MB", "Set the maximum nursery size in MB", 16)
 #ifdef JS_GC_ZEAL
         || !op.addStringOption('z', "gc-zeal", "LEVEL[,N]", gc::ZealModeHelpText)
 #endif
+        || !op.addStringOption('\0', "module-load-path", "DIR", "Set directory to load modules from")
     )
     {
         return EXIT_FAILURE;
     }
 
     op.setArgTerminatesOptions("script", true);
     op.setArgCapturesRest("scriptArgs");
 
--- a/js/src/shell/moz.build
+++ b/js/src/shell/moz.build
@@ -29,8 +29,17 @@ LOCAL_INCLUDES += [
     '..',
 ]
 
 OS_LIBS += CONFIG['EDITLINE_LIBS']
 OS_LIBS += CONFIG['MOZ_ZLIB_LIBS']
 
 if not CONFIG['GNU_CXX']:
     ALLOW_COMPILER_WARNINGS = True
+
+# Prepare module loader JS code for embedding
+GENERATED_FILES += ['shellmoduleloader.out.h']
+shellmoduleloader = GENERATED_FILES['shellmoduleloader.out.h']
+shellmoduleloader.script = '../builtin/embedjs.py:generate_shellmoduleloader'
+shellmoduleloader.inputs = [
+    '../js.msg',
+    'ModuleLoader.js',
+]