Bug 1361732 - Mozharness support for a second virtualenv using python3; r=aki
authorGeoff Brown <gbrown@mozilla.com>
Fri, 12 May 2017 15:18:38 -0600
changeset 406366 f0f6bd0a45434bcd6159ae9f7c4f558367332fe7
parent 406365 2deaa53b7d9d1ae95388dc01f6cf6377d354f932
child 406367 f37e89ab37a874b41d388334239d2958cdf74071
push id7391
push usermtabara@mozilla.com
push dateMon, 12 Jun 2017 13:08:53 +0000
treeherdermozilla-beta@2191d7f87e2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1361732
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1361732 - Mozharness support for a second virtualenv using python3; r=aki Currently Mozharness creates a Python 2.7 virtualenvironment with all the packages needed for Mozharness to run plus any applications that need to execute within it. For Talos, we now need a second Python virtual environment for mitmproxy, however, this package is for Python 3 and not for Python 2 (perhaps it could have worked). In any case, having the ability to support create Python 3 virtual environments will be handy. Specially since the approach of creating virtual environments is not by using 'virtualenv /path/to/venv' but by calling the venv module like 'python3 -m venv /path/to/venv'. The initial implementation only supports a list of modules. The following iteration will support requirement files and other parameters needed for pip and internal pypi hosts. Originally authored by armenzg.
testing/mozharness/mozharness/base/python.py
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -792,14 +792,71 @@ class ResourceMonitoringMixin(Perfherder
             start_time, end_time = rm.phases[phase]
             cpu_percent, cpu_times, io, swap = resources(phase)
             log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
 
     def _tinderbox_print(self, message):
         self.info('TinderboxPrint: %s' % message)
 
 
+# This needs to be inherited only if you have already inherited ScriptMixin
+class Python3Virtualenv(object):
+    ''' Support Python3.5+ virtualenv creation.'''
+    py3_initialized_venv = False
+
+    def py3_venv_configuration(self, python_path, venv_path):
+        '''We don't use __init__ to allow integrating with other mixins.
+
+        python_path - Path to Python 3 binary.
+        venv_path - Path to virtual environment to be created.
+        '''
+        self.py3_initialized_venv = True
+        self.py3_python_path = os.path.abspath(python_path)
+        version = self.get_output_from_command(
+                    [self.py3_python_path, '--version']).split()[-1]
+        # Using -m venv is only used on 3.5+ versions
+        assert version > '3.5.0'
+        self.py3_venv_path = os.path.abspath(venv_path)
+        self.py3_pip_path = os.path.join(self.py3_path_to_executables(), 'pip')
+
+    def py3_path_to_executables(self):
+        platform = self.platform_name()
+        if platform.startswith('win'):
+            return os.path.join(self.py3_venv_path, 'Scripts')
+        else:
+            return os.path.join(self.py3_venv_path, 'bin')
+
+    def py3_venv_initialized(func):
+        def call(self, *args, **kwargs):
+            if not self.py3_initialized_venv:
+                raise Exception('You need to call py3_venv_configuration() '
+                                'before using this method.')
+            func(self, *args, **kwargs)
+        return call
+
+    @py3_venv_initialized
+    def py3_create_venv(self):
+        '''Create Python environment with python3 -m venv /path/to/venv.'''
+        if os.path.exists(self.py3_venv_path):
+            self.info("Virtualenv %s appears to already exist; skipping "
+                      "virtualenv creation." % self.py3_venv_path)
+        else:
+            self.info('Running command...')
+            self.run_command(
+                '%s -m venv %s' % (self.py3_python_path, self.py3_venv_path),
+                error_list=VirtualenvErrorList,
+                halt_on_failure=True)
+
+    @py3_venv_initialized
+    def py3_install_modules(self, modules):
+        if not os.path.exists(self.py3_venv_path):
+            raise Exception('You need to call py3_create_venv() first.')
+
+        for m in modules:
+            self.run_command('%s install %s' % (self.py3_pip_path, m))
+
+
 # __main__ {{{1
 
 if __name__ == '__main__':
     '''TODO: unit tests.
     '''
     pass