Bug 1313099 - Put nodejs in tooltool (mozharness changes) r=jlund
authorNicholas Hurley <hurley@todesschaf.org>
Wed, 26 Oct 2016 14:05:27 -0700
changeset 351284 231272fa95c5c156601c2a121f050b3ad62d74a3
parent 351283 0d05d3e09e51f1fb71a1b37abc8dc19156dac723
child 351285 97bf9717631977f76a43fadd51c99dea1df770c6
push id6795
push userjlund@mozilla.com
push dateMon, 23 Jan 2017 14:19:46 +0000
treeherdermozilla-esr52@76101b503191 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlund
bugs1313099
milestone52.0a1
Bug 1313099 - Put nodejs in tooltool (mozharness changes) r=jlund MozReview-Commit-ID: 7ijnlF5D9B2
testing/mozharness/configs/remove_executables.py
testing/mozharness/configs/unittests/linux_unittest.py
testing/mozharness/configs/unittests/mac_unittest.py
testing/mozharness/configs/unittests/win_taskcluster_unittest.py
testing/mozharness/configs/unittests/win_unittest.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
testing/mozharness/scripts/desktop_unittest.py
--- a/testing/mozharness/configs/remove_executables.py
+++ b/testing/mozharness/configs/remove_executables.py
@@ -1,6 +1,8 @@
 config = {
     # We bake this directly into the tester image now...
     "download_minidump_stackwalk": False,
     "minidump_stackwalk_path": "/usr/local/bin/linux64-minidump_stackwalk",
+    "download_nodejs": False,
+    "nodejs_path": "/usr/local/bin/node",
     "exes": {}
 }
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -17,21 +17,25 @@ VALGRIND_SUPP_CROSS_ARCH = os.path.join(
                                         "cross-architecture.sup")
 VALGRIND_SUPP_ARCH = None
 
 if platform.architecture()[0] == "64bit":
     TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
     MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
     VALGRIND_SUPP_ARCH = os.path.join(VALGRIND_SUPP_DIR,
                                       "x86_64-redhat-linux-gnu.sup")
+    NODEJS_PATH = "node-linux-x64/bin/node"
+    NODEJS_TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/nodejs.manifest"
 else:
     TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
     MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
     VALGRIND_SUPP_ARCH = os.path.join(VALGRIND_SUPP_DIR,
                                       "i386-redhat-linux-gnu.sup")
+    NODEJS_PATH = "node-linux-x86/bin/node"
+    NODEJS_TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/nodejs.manifest"
 
 #####
 config = {
     "buildbot_json_path": "buildprops.json",
     "exes": {
         "python": "/tools/buildbot/bin/python",
         "virtualenv": ["/tools/buildbot/bin/python", "/tools/misc-python/virtualenv.py"],
         "tooltool.py": "/tools/tooltool.py",
@@ -314,9 +318,12 @@ config = {
                             "jittest": [],
                             "mozbase": [],
                             },
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "download_minidump_stackwalk": True,
     "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
     "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
     "tooltool_cache": "/builds/tooltool_cache",
+    "download_nodejs": True,
+    "nodejs_path": NODEJS_PATH,
+    "nodejs_tooltool_manifest_path": NODEJS_TOOLTOOL_MANIFEST_PATH,
 }
--- a/testing/mozharness/configs/unittests/mac_unittest.py
+++ b/testing/mozharness/configs/unittests/mac_unittest.py
@@ -262,9 +262,12 @@ config = {
                             "jittest": [],
                             "mozbase": [],
                             },
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "download_minidump_stackwalk": True,
     "minidump_stackwalk_path": "macosx64-minidump_stackwalk",
     "minidump_tooltool_manifest_path": "config/tooltool-manifests/macosx64/releng.manifest",
     "tooltool_cache": "/builds/tooltool_cache",
+    "download_nodejs": True,
+    "nodejs_path": "node-osx/bin/node",
+    "nodejs_tooltool_manifest_path": "config/tooltool-manifests/macosx64/nodejs.manifest",
 }
--- a/testing/mozharness/configs/unittests/win_taskcluster_unittest.py
+++ b/testing/mozharness/configs/unittests/win_taskcluster_unittest.py
@@ -274,9 +274,12 @@ config = {
     "default_blob_upload_servers": [
         "https://blobupload.elasticbeanstalk.com",
     ],
     "structured_suites": ["reftest"],
     'blob_uploader_auth_file': 'C:/builds/oauth.txt',
     "download_minidump_stackwalk": True,
     "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
     "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest"
+    "download_nodejs": True,
+    "nodejs_path": "node-win32.exe",
+    "nodejs_tooltool_manifest_path": "config/tooltool-manifests/win32/nodejs.manifest",
 }
--- a/testing/mozharness/configs/unittests/win_unittest.py
+++ b/testing/mozharness/configs/unittests/win_unittest.py
@@ -281,9 +281,12 @@ config = {
                             "cppunittest": [],
                             "jittest": [],
                             "mozbase": [],
                             },
     "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
     "download_minidump_stackwalk": True,
     "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
     "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
+    "download_nodejs": True,
+    "nodejs_path": "node-win32.exe",
+    "nodejs_tooltool_manifest_path": "config/tooltool-manifests/win32/nodejs.manifest",
 }
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -112,16 +112,17 @@ class TestingMixin(VirtualenvMixin, Buil
     installer_path = None
     binary_path = None
     test_url = None
     test_packages_url = None
     symbols_url = None
     symbols_path = None
     jsshell_url = None
     minidump_stackwalk_path = None
+    nodejs_path = None
     default_tools_repo = 'https://hg.mozilla.org/build/tools'
     proxxy = None
 
     def _query_proxxy(self):
         """manages the proxxy"""
         if not self.proxxy:
             self.proxxy = Proxxy(self.config, self.log_obj)
         return self.proxxy
@@ -658,16 +659,77 @@ Did you run with --create-virtualenv? Is
         if platform_name:
             minidump_filename = '%s-minidump_stackwalk' % TOOLTOOL_PLATFORM_DIR[platform_name]
             if platform_name in ('win32', 'win64'):
                 minidump_filename += '.exe'
             return minidump_filename
         else:
             self.fatal('We could not determine the minidump\'s filename.')
 
+    def query_nodejs_tooltool_manifest(self):
+        if self.config.get('nodejs_tooltool_manifest_path'):
+            return self.config['nodejs_tooltool_manifest_path']
+
+        self.info('NodeJS tooltool manifest unknown. Determining based upon '
+                  'platform and architecture.')
+        platform_name = self.platform_name()
+
+        if platform_name:
+            tooltool_path = "config/tooltool-manifests/%s/nodejs.manifest" % \
+                TOOLTOOL_PLATFORM_DIR[platform_name]
+            return tooltool_path
+        else:
+            self.fatal('Could not determine nodejs manifest filename')
+
+    def query_nodejs_filename(self):
+        if self.config.get('nodejs_path'):
+            return self.config['nodejs_path']
+
+        self.fatal('Could not determine nodejs filename')
+
+    def query_nodejs(self, manifest=None):
+        if self.nodejs_path:
+            return self.nodejs_path
+
+        c = self.config
+        dirs = self.query_abs_dirs();
+
+        nodejs_path = self.query_nodejs_filename()
+        if not self.config.get('download_nodejs'):
+            self.nodejs_path = nodejs_path
+            return self.nodejs_path
+
+        if not manifest:
+            tooltool_manifest_path = self.query_nodejs_tooltool_manifest()
+            manifest = os.path.join(dirs.get('abs_test_install_dir',
+                                             os.path.join(dirs['abs_work_dir'], 'tests')),
+                                    tooltool_manifest_path)
+
+        self.info('grabbing nodejs binary from tooltool')
+        try:
+            self.tooltool_fetch(
+                manifest=manifest,
+                output_dir=dirs['abs_work_dir'],
+                cache=c.get('tooltool_cache')
+            )
+        except KeyError:
+            self.error('missing a required key')
+
+        abs_nodejs_path = os.path.join(dirs['abs_work_dir'], nodejs_path)
+
+        if os.path.exists(abs_nodejs_path):
+            if self.platform_name() not in ('win32', 'win64'):
+                self.chmod(abs_nodejs_path, 0755)
+            self.nodejs_path = abs_nodejs_path
+        else:
+            self.warning("nodejs path was given but couldn't be found. Tried looking in '%s'" % abs_nodejs_path)
+            self.buildbot_status(TBPL_WARNING, WARNING)
+
+        return self.nodejs_path
+
     def query_minidump_stackwalk(self, manifest=None):
         if self.minidump_stackwalk_path:
             return self.minidump_stackwalk_path
 
         c = self.config
         dirs = self.query_abs_dirs()
 
         # This is the path where we either download to or is already on the host
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -688,16 +688,18 @@ class DesktopUnittest(TestingMixin, Merc
                     ref_formatter = imp.load_source(
                         "ReftestFormatter",
                         os.path.abspath(
                             os.path.join(dirs["abs_reftest_dir"], "output.py")))
                     parser.formatter = ref_formatter.ReftestFormatter()
 
                 if self.query_minidump_stackwalk():
                     env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
+                if self.query_nodejs():
+                    env['MOZ_NODE_PATH'] = self.nodejs_path
                 env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
                 env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
                 if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
                     self.mkdir_p(env['MOZ_UPLOAD_DIR'])
 
                 if self.config['allow_software_gl_layers']:
                     env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'