Bug 1334209 - Activate mozharness virtualenv in-process from one click loaner mach commands, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 31 Jan 2017 11:53:36 -0500
changeset 331986 d70a7e18fe84dbe3e35f282def60544897ce04dc
parent 331985 a8643fc321642ee3b4ceff2ea4b99508a5437560
child 331987 0c237089b3179a7a972294adbf72d9b0248bebb2
push id31293
push userkwierso@gmail.com
push dateThu, 02 Feb 2017 00:07:12 +0000
treeherdermozilla-central@8196774c6b8a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1334209
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1334209 - Activate mozharness virtualenv in-process from one click loaner mach commands, r=jmaher Previously the run-wizard script would add a command to source the virtualenv in ~/.bashrc after mozharness finished setting things up. This is fragile, assumes people are using bash, etc. Plus it appeared to intermittently fail for some users. Instead, this activates the virtualenv directly from individual mach commands that need it. This guarantees we will always be using the virtualenv if required (and won't be using it if not). The 'activate_this.py' script is invoked the same way that we do it for in-tree mach commands: https://dxr.mozilla.org/mozilla-central/rev/9c06e744b1befb3a2e2fdac7414ce18220774a1d/python/mozbuild/mozbuild/virtualenv.py#456 MozReview-Commit-ID: CfcoiVJXQTl
layout/tools/reftest/mach_test_package_commands.py
taskcluster/scripts/tester/run-wizard
testing/marionette/mach_test_package_commands.py
testing/mochitest/mach_test_package_commands.py
testing/tools/mach_test_package_bootstrap.py
testing/xpcshell/mach_test_package_commands.py
--- a/layout/tools/reftest/mach_test_package_commands.py
+++ b/layout/tools/reftest/mach_test_package_commands.py
@@ -83,10 +83,11 @@ class ReftestCommands(object):
 
     def __init__(self, context):
         self.context = context
 
     @Command('reftest', category='testing',
              description='Run the reftest harness.',
              parser=setup_argument_parser)
     def reftest(self, **kwargs):
+        self.context.activate_mozharness_venv()
         kwargs['suite'] = 'reftest'
         return run_reftest(self.context, **kwargs)
--- a/taskcluster/scripts/tester/run-wizard
+++ b/taskcluster/scripts/tester/run-wizard
@@ -53,23 +53,16 @@ def setup():
     build_dir = os.path.expanduser(os.path.join('~', 'workspace', 'build'))
     mach_src = os.path.join(build_dir, 'tests', 'mach')
     mach_dest = os.path.expanduser(os.path.join('~', 'bin', 'mach'))
 
     if os.path.exists(mach_dest):
         os.remove(mach_dest)
     os.symlink(mach_src, mach_dest)
 
-    activate = os.path.join(build_dir, 'venv', 'bin', 'activate')
-    if os.path.isfile(activate):
-        # TODO Support other shells
-        bashrc = os.path.expanduser(os.path.join('~', '.bashrc'))
-        with open(bashrc, 'ab') as f:
-            f.write(". {}".format(activate))
-
     print("""
 Mozharness has finished downloading the build and tests to:
 {}
 
 A limited mach environment has also been set up and added to the $PATH, but
 it may be missing the command you need. To see a list of commands, run:
     $ mach help
 """.lstrip().format(build_dir))
--- a/testing/marionette/mach_test_package_commands.py
+++ b/testing/marionette/mach_test_package_commands.py
@@ -59,9 +59,10 @@ class MachCommands(object):
         self.context = context
 
     @Command(
         'marionette-test', category='testing',
         description='Run a Marionette test (Check UI or the internal JavaScript '
                     'using marionette).',
         parser=setup_marionette_argument_parser)
     def run_marionette_test(self, **kwargs):
+        self.context.activate_mozharness_venv()
         return run_marionette(self.context, **kwargs)
--- a/testing/mochitest/mach_test_package_commands.py
+++ b/testing/mochitest/mach_test_package_commands.py
@@ -88,9 +88,10 @@ class MochitestCommands(object):
 
     def __init__(self, context):
         self.context = context
 
     @Command('mochitest', category='testing',
              description='Run the mochitest harness.',
              parser=setup_argument_parser)
     def mochitest(self, **kwargs):
+        self.context.activate_mozharness_venv()
         return run_mochitest(self.context, **kwargs)
--- a/testing/tools/mach_test_package_bootstrap.py
+++ b/testing/tools/mach_test_package_bootstrap.py
@@ -71,28 +71,57 @@ CATEGORIES = {
         'long': 'The disabled commands are hidden by default. Use -v to display them. '
                 'These commands are unavailable for your current context, '
                 'run "mach <command>" to see why.',
         'priority': 0,
     }
 }
 
 
+IS_WIN = sys.platform in ('win32', 'cygwin')
+
+
 def ancestors(path, depth=0):
     """Emit the parent directories of a path."""
     count = 1
     while path and count != depth:
         yield path
         newpath = os.path.dirname(path)
         if newpath == path:
             break
         path = newpath
         count += 1
 
 
+def activate_mozharness_venv(context):
+    """Activate the mozharness virtualenv in-process."""
+    venv = os.path.join(context.mozharness_workdir,
+                        context.mozharness_config.get('virtualenv_path', 'venv'))
+
+    if not os.path.isdir(venv):
+        print("No mozharness virtualenv detected at '{}'.".format(venv))
+        return 1
+
+    venv_bin = os.path.join(venv, 'Scripts' if IS_WIN else 'bin')
+    activate_path = os.path.join(venv_bin, 'activate_this.py')
+
+    execfile(activate_path, dict(__file__=activate_path))
+
+    if isinstance(os.environ['PATH'], unicode):
+        os.environ['PATH'] = os.environ['PATH'].encode('utf-8')
+
+    # sys.executable is used by mochitest-media to start the websocketprocessbridge,
+    # for some reason it doesn't get set when calling `activate_this.py` so set it
+    # here instead.
+    binary = 'python'
+    if IS_WIN:
+        binary += '.exe'
+    sys.executable = os.path.join(venv_bin, binary)
+
+
 def find_firefox(context):
     """Try to automagically find the firefox binary."""
     import mozinstall
     search_paths = []
 
     # Check for a mozharness setup
     config = context.mozharness_config
     if config and 'binary_path' in config:
@@ -171,16 +200,19 @@ def bootstrap(test_package_root):
                         return json.load(f)
             return {}
 
         if key == 'mozharness_workdir':
             config = context.mozharness_config
             if config:
                 return os.path.join(config['base_work_dir'], config['work_dir'])
 
+        if key == 'activate_mozharness_venv':
+            return types.MethodType(activate_mozharness_venv, context)
+
     mach = mach.main.Mach(os.getcwd())
     mach.populate_context_handler = populate_context
 
     for category, meta in CATEGORIES.items():
         mach.define_category(category, meta['short'], meta['long'],
                              meta['priority'])
 
     for path in MACH_MODULES:
--- a/testing/xpcshell/mach_test_package_commands.py
+++ b/testing/xpcshell/mach_test_package_commands.py
@@ -56,9 +56,10 @@ class MochitestCommands(object):
 
     def __init__(self, context):
         self.context = context
 
     @Command('xpcshell-test', category='testing',
              description='Run the xpcshell harness.',
              parser=parser_desktop)
     def xpcshell(self, **kwargs):
+        self.context.activate_mozharness_venv()
         return run_xpcshell(self.context, **kwargs)