bug 1159960 - Update mozharness to use taskcluster-npm-cache. r=jgriffin
☠☠ backed out by d98eef5e2dbf ☠ ☠
authorAus Lacroix <aus@mozilla.com>
Tue, 12 May 2015 23:37:13 -0700 (2015-05-13)
changeset 4000 839eaca81ee21d5b38b2106094612b78ec5afd1f
parent 3999 c5fbe5b9de2c496cda00c4205733b7d547b0894c
child 4001 d98eef5e2dbf94535d111db87e273a5c0c9edb7f
push id3170
push userglacroix@mozilla.com
push dateWed, 13 May 2015 06:37:26 +0000 (2015-05-13)
reviewersjgriffin
bugs1159960
bug 1159960 - Update mozharness to use taskcluster-npm-cache. r=jgriffin
mozharness/mozilla/gaia.py
scripts/gaia_build_integration.py
scripts/gaia_build_unit.py
scripts/gaia_integration.py
scripts/gaia_linter.py
--- a/mozharness/mozilla/gaia.py
+++ b/mozharness/mozilla/gaia.py
@@ -350,47 +350,60 @@ class GaiaMixin(object):
         dirs = self.query_abs_dirs()
 
         def cleanup_node_modules():
             node_dir = os.path.join(dirs['abs_gaia_dir'], 'node_modules')
             self.rmtree(node_dir)
 
         self.run_command(['npm', 'cache', 'clean'])
 
-        # run 'make node_modules' first, so we can separately handle
+        # get the node modules first from the node module cache, if this fails it will 
+        # install the node modules from npmjs.org.
+        cmd = ['taskcluster-npm-cache-get',
+               '--namespace',
+               'gaia.npm_cache',
+               'package.json']
+        kwargs = {
+            'output_timeout': 300
+        }
+        code = self.retry(self.run_command, attempts=3, good_statuses=(0,),
+                          args=[cmd, dirs['abs_gaia_dir']], cleanup=cleanup_node_modules, kwargs=kwargs)
+        if code:
+            self.fatal('Error occurred fetching node modules from cache',
+                       exit_code=code)
+
+        # run 'make node_modules' second, so we can separately handle
         # errors that occur here
         cmd = ['make',
                'node_modules',
-               'NODE_MODULES_GIT_URL=https://git.mozilla.org/b2g/gaia-node-modules.git',
+               'NODE_MODULES_SRC=npm-cache',
                'VIRTUALENV_EXISTS=1']
         kwargs = {
-            'cwd': dirs['abs_gaia_dir'],
             'output_timeout': 300,
             'error_list': self.npm_error_list
         }
         code = self.retry(self.run_command, attempts=3, good_statuses=(0,),
-                          args=[cmd], cleanup=cleanup_node_modules, kwargs=kwargs)
+                          args=[cmd, dirs['abs_gaia_dir']], kwargs=kwargs)
         if code:
             # Dump npm-debug.log, if it exists
             npm_debug = os.path.join(dirs['abs_gaia_dir'], 'npm-debug.log')
             if os.access(npm_debug, os.F_OK):
                 self.info('dumping npm-debug.log')
                 self.run_command(['cat', npm_debug])
             else:
                 self.info('npm-debug.log doesn\'t exist, not dumping')
             self.fatal('Errors during \'npm install\'', exit_code=code)
 
         cmd = ['make',
                'update-common']
         kwargs = {
-            'cwd': dirs['abs_gaia_dir'],
             'output_timeout': 300
         }
         code = self.retry(self.run_command, attempts=3, good_statuses=(0,),
-                          args=[cmd], kwargs=kwargs)
+                          args=[cmd, dirs['abs_gaia_dir']], kwargs=kwargs)
         if code:
             self.fatal('Errors during make update-common')
 
     def node_setup(self):
         """
         Set up environment for node-based Gaia tests.
         """
         dirs = self.query_abs_dirs()
--- a/scripts/gaia_build_integration.py
+++ b/scripts/gaia_build_integration.py
@@ -29,18 +29,19 @@ class GaiaBuildIntegrationTest(GaiaTest)
         self.node_setup()
 
         output_parser = TestSummaryOutputParserHelper(
           config=self.config, log_obj=self.log_obj, error_list=self.error_list)
 
         code = self.run_command([
             'make',
             'build-test-integration',
-            'NPM_REGISTRY=' + self.config.get('npm_registry'),
             'REPORTER=mocha-tbpl-reporter',
+            'NODE_MODULES_SRC=npm-cache',
+            'VIRTUALENV_EXISTS=1',
             'TRY_ENV=1'
         ], cwd=dirs['abs_gaia_dir'],
            output_parser=output_parser,
            output_timeout=600)
 
         output_parser.print_summary('gaia-build-integration-tests')
         self.publish(code)
 
--- a/scripts/gaia_build_unit.py
+++ b/scripts/gaia_build_unit.py
@@ -29,18 +29,19 @@ class GaiaBuildUnitTest(GaiaTest):
         self.node_setup()
 
         output_parser = TestSummaryOutputParserHelper(
           config=self.config, log_obj=self.log_obj, error_list=self.error_list)
 
         code = self.run_command([
             'make',
             'build-test-unit',
-            'NPM_REGISTRY=' + self.config.get('npm_registry'),
             'REPORTER=mocha-tbpl-reporter',
+            'NODE_MODULES_SRC=npm-cache',
+            'VIRTUALENV_EXISTS=1',
             'TRY_ENV=1'
         ], cwd=dirs['abs_gaia_dir'],
            output_parser=output_parser,
            output_timeout=330)
 
         output_parser.print_summary('gaia-build-unit-tests')
         self.publish(code)
 
--- a/scripts/gaia_integration.py
+++ b/scripts/gaia_integration.py
@@ -48,17 +48,19 @@ class GaiaIntegrationTest(GaiaTest):
         if os.path.exists(gaia_runner_service):
             self.install_module('gaia-runner-service', gaia_runner_service)
         env['VIRTUALENV_PATH'] = self.query_virtualenv_path()
 
         cmd = [
             'make',
             'test-integration',
             'REPORTER=mocha-tbpl-reporter',
-            'TEST_MANIFEST=./shared/test/integration/tbpl-manifest.json'
+            'TEST_MANIFEST=./shared/test/integration/tbpl-manifest.json',
+            'NODE_MODULE_SRC=npm-cache',
+            'VIRTUALENV_EXISTS=1'
         ]
 
         # for Mulet
         if 'firefox' in self.binary_path:
             cmd += ['RUNTIME=%s' % self.binary_path]
 
         code = self.run_command(cmd, cwd=dirs['abs_gaia_dir'], env=env,
            output_parser=output_parser,
--- a/scripts/gaia_linter.py
+++ b/scripts/gaia_linter.py
@@ -122,17 +122,19 @@ class GaiaIntegrationTest(GaiaTest):
 
         output_parser = GaiaLinterOutputParser(
             base_dir=dirs['abs_gaia_dir'],
             config=self.config,
             log_obj=self.log_obj)
 
         code = self.run_command([
             'make',
-            'lint'
+            'lint',
+            'NODE_MODULES_SRC=npm-cache',
+            'VIRTUALENV_EXISTS=1'
         ], cwd=dirs['abs_gaia_dir'],
            output_parser=output_parser,
            output_timeout=330)
 
         output_parser.print_summary('gaia-lint')
         self.publish(code)
 
 if __name__ == '__main__':