Bug 1185539 - './mach talos-test' doesn't run. r=jmaher a=test-only
authorJulien Pagès <j.parkouss@gmail.com>
Wed, 22 Jul 2015 16:59:01 +0200
changeset 275445 26c44dd74c2298cbbe754327635c2eed73a00f10
parent 275440 5657e76d4ee58ed4eb1117529784352d3d51bb27
child 275446 7b60354f789266505020094c75f65caa4fd3dc9c
push id863
push userraliiev@mozilla.com
push dateMon, 03 Aug 2015 13:22:43 +0000
treeherdermozilla-release@f6321b14228d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher, test-only
bugs1185539
milestone40.0
Bug 1185539 - './mach talos-test' doesn't run. r=jmaher a=test-only
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/mach_commands.py
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -154,21 +154,16 @@ class Talos(TestingMixin, MercurialScrip
         kwargs.setdefault('default_actions', ['clobber',
                                               'download-and-extract',
                                               'clone-talos',
                                               'create-virtualenv',
                                               'install',
                                               'run-tests',
                                               ])
         kwargs.setdefault('config', {})
-        kwargs['config'].setdefault(
-            'virtualenv_modules', ["mozinstall", "mozdevice", "pyyaml", "mozversion", "datazilla",
-                                   "mozcrash", "mozhttpd", "mozprofile", "mozfile", "mozinfo",
-                                   "moznetwork", "mozprocess", "httplib2"]
-        )
         super(Talos, self).__init__(**kwargs)
 
         self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience
 
         # results output
         self.results_url = self.config.get('results_url')
         if self.results_url is None:
             # use a results_url by default based on the class name in the working directory
@@ -531,17 +526,26 @@ class Talos(TestingMixin, MercurialScrip
             self._populate_webroot()
 
     def create_virtualenv(self, **kwargs):
         """VirtualenvMixin.create_virtualenv() assuemes we're using
         self.config['virtualenv_modules']. Since we are installing
         talos from its source, we have to wrap that method here."""
         # XXX This method could likely be replaced with a PreScriptAction hook.
         if self.has_cloned_talos:
-            virtualenv_modules = list(self.config.get('virtualenv_modules', []))
+            requirements = self.read_from_file(
+                os.path.join(self.talos_path, 'requirements.txt'),
+                verbose=False
+            )
+            # talos in harness requires mozinstall
+            virtualenv_modules = ['mozinstall']
+            for requirement in requirements.splitlines():
+                requirement = requirement.strip()
+                if requirement:
+                    virtualenv_modules.append(requirement)
 
             # Bug 900015 - Silent warnings on osx when libyaml is not found
             pyyaml_module = {
                 'name': 'PyYAML',
                 'url': None,
                 'global_options': ['--without-libyaml']
             }
             virtualenv_modules.insert(0, pyyaml_module)
--- a/testing/talos/mach_commands.py
+++ b/testing/talos/mach_commands.py
@@ -19,70 +19,50 @@ from mozbuild.base import (
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
 class TalosRunner(MozbuildObject):
-    def run_test(self, suite, repo, rev, sps_profile):
+    def run_test(self, suite, sps_profile):
         """
         We want to do couple of things before running Talos
         1. Clone mozharness
         2. Make config for Talos Mozharness
         3. Run mozharness
         """
 
         print("Running Talos test suite %s" % suite)
-        self.init_variables(suite, repo, rev, sps_profile)
-        self.clone_mozharness()
+        self.init_variables(suite, sps_profile)
         self.make_config()
         self.write_config()
         self.make_args()
         return self.run_mozharness()
 
-    def init_variables(self, suite, repo, rev, sps_profile):
+    def init_variables(self, suite, sps_profile):
         self.suite = suite
         self.sps_profile = sps_profile
-        self.mozharness_repo = repo
-        self.mozharness_rev = rev
 
         self.talos_dir = os.path.join(self.topsrcdir, 'testing', 'talos')
-        self.mozharness_dir = os.path.join(self.topobjdir, 'mozharness')
+        self.talos_webroot = os.path.join(self.topobjdir, 'testing', 'talos')
+        self.mozharness_dir = os.path.join(self.topsrcdir, 'testing',
+                                           'mozharness')
         self.config_dir = os.path.join(self.mozharness_dir, 'configs', 'talos')
         self.talos_json = os.path.join(self.talos_dir, 'talos.json')
         self.config_filename = 'in_tree_conf.json'
         self.config_file_path = os.path.join(self.config_dir,
                                              self.config_filename)
         self.binary_path = self.get_binary_path()
         self.virtualenv_script = os.path.join(self.topsrcdir, 'python',
                                               'virtualenv', 'virtualenv.py')
         self.virtualenv_path = os.path.join(self.mozharness_dir, 'venv')
         self.python_interp = sys.executable
 
-    def clone_mozharness(self):
-        """Clones mozharness into topobjdir/mozharness
-           using mercurial. If mozharness is already cloned,
-           it updates it to the latest version"""
-        try:
-            mercurial = which.which('hg')
-        except which.WhichError as e:
-            print("You don't have hg in your PATH: {0}".format(e))
-            raise e
-        clone_cmd = [mercurial, 'clone', '-r', self.mozharness_rev,
-                     self.mozharness_repo, self.mozharness_dir]
-        pull_cmd = [mercurial, 'pull', '-r', self.mozharness_rev, '-u']
-
-        dot_hg = os.path.join(self.mozharness_dir, '.hg')
-        if os.path.exists(dot_hg):
-            self.run_process(args=pull_cmd, cwd=self.mozharness_dir)
-        else:
-            self.run_process(args=clone_cmd)
-
     def make_config(self):
         self.config = {
             'talos_json': self.talos_json,
             'binary_path': self.binary_path,
             'log_name': 'talos',
             'virtualenv_path': self.virtualenv_path,
             'pypi_url': 'http://pypi.python.org/simple',
             'use_talos_json': True,
@@ -92,25 +72,28 @@ class TalosRunner(MozbuildObject):
                 'virtualenv': [self.python_interp, self.virtualenv_script]
             },
             'title': socket.gethostname(),
             'default_actions': [
                 'clone-talos',
                 'create-virtualenv',
                 'run-tests',
             ],
-            'python_webserver': True
+            'python_webserver': False,
+            'populate_webroot': True,
+            'talos_extra_options': ['--develop'],
         }
 
     def make_args(self):
         self.args = {
             'config': {
                 'suite': self.suite,
                 'sps_profile': self.sps_profile,
-                'use_talos_json': True
+                'use_talos_json': True,
+                'webroot': self.talos_webroot,
             },
            'initial_config_file': self.config_file_path,
        }
 
     def write_config(self):
         try:
             config_file = open(self.config_file_path, 'wb')
             config_file.write(json.dumps(self.config))
@@ -123,34 +106,25 @@ class TalosRunner(MozbuildObject):
         sys.path.insert(0, self.mozharness_dir)
         from mozharness.mozilla.testing.talos import Talos
         talos_mh = Talos(config=self.args['config'],
                          initial_config_file=self.args['initial_config_file'])
         return talos_mh.run()
 
 @CommandProvider
 class MachCommands(MachCommandBase):
-    mozharness_repo = 'https://hg.mozilla.org/build/mozharness'
-    mozharness_rev = 'production'
-
     @Command('talos-test', category='testing',
              description='Run talos tests (performance testing).')
     @CommandArgument('suite', help='Talos test suite to run. Valid suites are '
                                    'chromez, dirtypaint, dromaeojs, other,'
                                    'svgr, rafx, tpn, tp5o, xperf.')
-    @CommandArgument('--repo', default=mozharness_repo,
-                     help='The mozharness repository to clone from. '
-                          'Defaults to http://hg.mozilla.org/build/mozharness')
-    @CommandArgument('--rev', default=mozharness_rev,
-                     help='The mozharness revision to clone. Defaults to '
-                          'production')
     @CommandArgument('--spsProfile', default=False,
                      help='Use the Gecko Profiler to capture profiles that can '
                           'then be displayed by Cleopatra.', action='store_true')
 
-    def run_talos_test(self, suite, repo=None, rev=None, spsProfile=False):
+    def run_talos_test(self, suite, spsProfile=False):
         talos = self._spawn(TalosRunner)
 
         try:
-            return talos.run_test(suite, repo, rev, spsProfile)
+            return talos.run_test(suite, spsProfile)
         except Exception as e:
             print(str(e))
             return 1