Bug NODE - Add |mach node ...| draft
authorNick Alexander <nalexander@mozilla.com>
Tue, 20 Mar 2018 14:56:11 -0700
changeset 795814 9340bf883c3220667f4e47edbd550ca6cb34bfbc
parent 795813 d3be09fd0cb58e8c7348fd701f8e015c770d8ef9
child 795815 c7a10b005ada4da4707df34115e1d0afbcd2f773
push id110091
push userbmo:dmose@mozilla.org
push dateWed, 16 May 2018 16:58:05 +0000
milestone62.0a1
Bug NODE - Add |mach node ...|
build/mach_bootstrap.py
build/moz.configure/init.configure
package.json
python/mozbuild/mozbuild/node_manager.py
tools/node/mach_commands.py
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -58,16 +58,17 @@ MACH_MODULES = [
     'testing/tps/mach_commands.py',
     'testing/talos/mach_commands.py',
     'testing/web-platform/mach_commands.py',
     'testing/xpcshell/mach_commands.py',
     'tools/compare-locales/mach_commands.py',
     'tools/docs/mach_commands.py',
     'tools/lint/mach_commands.py',
     'tools/mach_commands.py',
+    'tools/node/mach_commands.py',
     'tools/power/mach_commands.py',
     'tools/tryselect/mach_commands.py',
     'mobile/android/mach_commands.py',
 ]
 
 
 CATEGORIES = {
     'build': {
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -450,16 +450,27 @@ def node_version(node):
     min_node_version = '8.9.0'
     if not node_version >= min_node_version:
         raise FatalCheckError('NODE must point to node %s or newer; '
                               '%s found' % (min_node_version, node_version))
 
     return node_version
 
 
+@depends(check_build_environment)
+@imports('os')
+def node_options(build_environment):
+    return ['--require',
+            os.path.join(build_environment.topsrcdir,
+                         'tools', 'node', 'mozilla-preload', 'index.js')]
+
+
+set_config('NODE_OPTIONS', node_options)
+
+
 option('--enable-node-environment',
        help='Opt-in to experimental Node development experience.')
 
 
 @depends('--enable-node-environment', node, '--help')
 def enable_node_environment(node_env, node, help):
     if node_env:
         if not node:
--- a/package.json
+++ b/package.json
@@ -1,11 +1,12 @@
 {
-  "name": "mozillaeslintsetup",
-  "description": "This package file is for setup of ESLint only for editor integration.",
+  "name": "firefox",
+  "description": "This package file is for setting up node tools used for developing/building/testing firefox",
+  "private": true,
   "repository": {},
   "license": "MPL-2.0",
   "dependencies": {
     "eslint": "4.19.1",
     "eslint-plugin-html": "4.0.3",
     "eslint-plugin-mozilla": "file:tools/lint/eslint/eslint-plugin-mozilla",
     "eslint-plugin-no-unsanitized": "3.0.0",
     "eslint-plugin-react": "7.1.0",
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/node_manager.py
@@ -0,0 +1,115 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains code for populating the virtualenv environment for
+# Mozilla's build system. It is typically called as part of configure.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import os
+import pipes
+import shutil
+import subprocess
+from mozbuild.util import (
+    ensureParentDir,
+)
+from mozpack.copier import (
+    FileCopier,
+    FileRegistry,
+)
+from mozpack.manifests import InstallManifest
+import mozpack.path as mozpath
+
+
+class NodeManager:
+    def __init__(self, topsrcdir, log_handle):
+        self.topsrcdir = topsrcdir
+        self.log_handle = log_handle
+
+    def _log_process_output(self, *args, **kwargs):
+        if hasattr(self.log_handle, 'fileno'):
+            return subprocess.call(*args, stdout=self.log_handle,
+                                   stderr=subprocess.STDOUT, **kwargs)
+
+        proc = subprocess.Popen(*args, stdout=subprocess.PIPE,
+                                stderr=subprocess.STDOUT, **kwargs)
+
+        for line in proc.stdout:
+            self.log_handle.write(line)
+
+        return proc.wait()
+
+    def populate(self, node, destdir, offline=True):
+        """
+XXX
+Create a new, empty virtualenv.
+
+        Receives the path to virtualenv's virtualenv.py script (which will be
+        called out to), the path to create the virtualenv in, and a handle to
+        write output to.
+        """
+        package_json = json.load(open(mozpath.join(self.topsrcdir, 'package.json'), 'rt'))
+        dependencies = package_json.get('dependencies', {})
+
+        install_manifest = InstallManifest()
+
+        for workspace in package_json.get('workspaces', []):
+            workspace_name = os.path.basename(os.path.abspath(workspace))
+            if workspace_name in dependencies:
+                raise ValueError('XXX 1')
+
+            src = mozpath.join(self.topsrcdir, workspace, 'package.json')
+            dst = mozpath.join(workspace, 'package.json')
+
+            install_manifest.add_link(src, dst)
+
+        copier = FileCopier()
+        install_manifest.populate_registry(
+            copier, link_policy="symlink"
+        )
+        result = copier.copy(destdir,
+                             remove_unaccounted=False,
+                             remove_all_directory_symlinks=False,
+                             remove_empty_directories=False)
+
+        # "workspaces": ["path/relative/to/topsrcdir"] correctly installs
+        # one layer of devDependencies, essential for (e.g.) webpack.
+        # However, this can write into topsrcdir because of symlinks.
+        # (Unclear what happens across drives, where symlinking is
+        # generally not supported.)  Solutions:
+        #
+        # 1) unroll writes into topsrcdir.  Violates read-only nature of
+        # topsrcdir.
+        #
+        # 2) create a shadow package hierarchy in object directory so that
+        # symlink targets are in the object directory.
+        #
+        # "dependencies": ["file:/absolute/path/into/topsrcdir"] and
+        # "yarn-link-file-dependencies false" does not handle
+        # devDependencies correctly.
+
+        json.dump(package_json, open(mozpath.join(destdir, 'package.json'), 'wt'))
+
+        print(json.dumps(package_json, indent=2))
+
+        s = open(mozpath.join(self.topsrcdir, '.yarnrc'), 'rt').read()
+
+        with open(mozpath.join(destdir, '.yarnrc'), 'wt') as f:
+            f.write(s.replace('"third_party/yarn-offline-mirror"',
+                              '"{}/third_party/yarn-offline-mirror"'.format(self.topsrcdir)))
+
+        yarn = mozpath.join(self.topsrcdir, 'third_party', 'yarn', 'yarn.js')
+        args = [node, yarn, 'install', '--production=false', '--ignore-engines', '--verbose']
+        if offline:
+            shutil.copy(mozpath.join(self.topsrcdir, 'yarn.lock'), destdir)
+            args.append('--offline')
+
+        cmd = ' '.join(pipes.quote(a) for a in args)
+        print('"{}" in {}'.format(cmd, destdir))
+        result = self._log_process_output(args, cwd=destdir)
+
+        if result:
+            node_modules = mozpath.join(destdir, 'node_modules')
+            raise Exception('Failed to create node_modules: {}'.format(node_modules))
new file mode 100644
--- /dev/null
+++ b/tools/node/mach_commands.py
@@ -0,0 +1,98 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import logging
+
+import mozpack.path as mozpath
+
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+from mozbuild.base import (
+    MachCommandBase,
+)
+
+
+@CommandProvider
+class MachCommands(MachCommandBase):
+
+    @Command('node', category='devenv',
+        description='Run Node.')
+    @CommandArgument('-v', '--verbose', action='store_true',
+        help='Verbose output for what commands the build is running.')
+    @CommandArgument('args', nargs=argparse.REMAINDER)
+    def node(self, args, verbose=False):
+        import pipes
+
+        if not verbose:
+            # Avoid logging the command
+            self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
+
+        # TODO: think about how we might message that NODE_OPTIONS is being
+        # set, since we are using it to change a significant part of the Node
+        # runtime environment.
+        append_env = {
+            'MOZ_DEVELOPER_REPO_DIR': self.topsrcdir,
+            'MOZ_DEVELOPER_OBJ_DIR': self.topobjdir,
+        }
+        node_options = self.substs.get('NODE_OPTIONS')
+        if node_options:
+            append_env['NODE_OPTIONS'] = ' '.join(pipes.quote(x) for x in node_options)
+
+        return self.run_process(
+            [self.substs['NODE']] + args,
+            append_env=append_env,
+            pass_thru=True,  # Allow user to run Node interactively.
+            ensure_exit_code=False,  # Don't throw on non-zero exit code.
+        )
+
+
+    @Command('yarn', category='devenv',
+        description='Run Yarn.')
+    @CommandArgument('-v', '--verbose', action='store_true',
+        help='Verbose output for what commands the build is running.')
+    @CommandArgument('args', nargs=argparse.REMAINDER)
+    def yarn(self, args, verbose=False):
+        if not verbose:
+            # Avoid logging the command
+            self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
+
+        yarn = mozpath.join(self.topsrcdir, 'third_party', 'yarn', 'yarn.js')
+        return self.node([yarn] + args, verbose=verbose)
+
+
+    @Command('node-install', category='devenv',
+        description='Install a node_modules directory tree.')
+    @CommandArgument('-d', '--dest', default=None,
+        help='Directory to write node_modules into (default: $topobjdir)')
+    @CommandArgument('-v', '--verbose', action='store_true',
+        help='Verbose output for what commands the build is running.')
+    @CommandArgument('--offline', action='store_true', default=False,
+        help='XXX offline.')
+    def node_install(self, dest=None, verbose=False, offline=False):
+        if not verbose:
+            # Avoid logging the command
+            self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
+
+        if not dest:
+            dest = self.topobjdir
+
+        from mozbuild.configure.util import (
+            LineIO,
+        )
+
+        log_handle = LineIO(lambda l: self.log(logging.INFO, 'node-install', {}, l), 'replace')
+
+        from mozbuild import node_manager
+        manager = node_manager.NodeManager(self.topsrcdir, log_handle)
+
+        # XXX -- change name.
+        manager.populate(self.substs['NODE'], dest, offline=offline)
+
+        return 0