Bug 1595982 - make mozharness::base::python.py python3 compatible r=aki
authorEdwin Takahashi <egao@mozilla.com>
Mon, 18 Nov 2019 19:16:04 +0000
changeset 502461 9d2c4851e407f7f49d01549f295f44094ca0a525
parent 502460 d8bdee06190fa6b2df2625dcfb3241c69ac1914b
child 502462 a52abf84721ad2f0ff2f56b8f2667ab46cc716b3
push id36817
push userncsoregi@mozilla.com
push dateTue, 19 Nov 2019 04:39:02 +0000
treeherdermozilla-central@fdd07df83c87 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1595982
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1595982 - make mozharness::base::python.py python3 compatible r=aki Changes: Run `isort` to fix the import ordering. Run `autopep8` to fix code formatting issues. Import `six.string_types` to replace the `basestring` comparison. Depends on D52791 Differential Revision: https://phabricator.services.mozilla.com/D53093
testing/mozharness/mozharness/base/python.py
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -3,32 +3,33 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 '''Python usage, esp. virtualenv.
 '''
 
 import errno
+import json
 import os
+import socket
 import sys
-import json
-import socket
 import traceback
-import urlparse
+try:
+    import urlparse
+except ImportError:
+    import urllib.urlparse as urlparse
+
+from six import string_types
 
 import mozharness
-from mozharness.base.script import (
-    PostScriptAction,
-    PostScriptRun,
-    PreScriptAction,
-    ScriptMixin,
-)
 from mozharness.base.errors import VirtualenvErrorList
-from mozharness.base.log import WARNING, FATAL
+from mozharness.base.log import FATAL, WARNING
+from mozharness.base.script import (PostScriptAction, PostScriptRun,
+                                    PreScriptAction, ScriptMixin)
 
 external_tools_path = os.path.join(
     os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
     'external_tools',
 )
 
 
 def get_tlsv1_post():
@@ -156,36 +157,38 @@ class VirtualenvMixin(object):
         {package_name: 'version'}
         """
         packages = {}
 
         if pip_freeze_output is None:
             # get the output from `pip freeze`
             pip = self.query_python_path("pip")
             if not pip:
-                self.log("package_versions: Program pip not in path", level=error_level)
+                self.log("package_versions: Program pip not in path",
+                         level=error_level)
                 return {}
             pip_freeze_output = self.get_output_from_command(
                 [pip, "freeze"], silent=True, ignore_errors=True)
-            if not isinstance(pip_freeze_output, basestring):
+            if not isinstance(pip_freeze_output, string_types):
                 self.fatal(
                     "package_versions: Error encountered running `pip freeze`: "
                     + pip_freeze_output)
 
         for line in pip_freeze_output.splitlines():
             # parse the output into package, version
             line = line.strip()
             if not line:
                 # whitespace
                 continue
             if line.startswith('-'):
                 # not a package, probably like '-e http://example.com/path#egg=package-dev'
                 continue
             if '==' not in line:
-                self.fatal("pip_freeze_packages: Unrecognized output line: %s" % line)
+                self.fatal(
+                    "pip_freeze_packages: Unrecognized output line: %s" % line)
             package, version = line.split('==', 1)
             packages[package] = version
 
         if log_output:
             self.info("Current package versions:")
             for package in sorted(packages):
                 self.info("  %s == %s" % (package, packages[package]))
 
@@ -236,17 +239,18 @@ class VirtualenvMixin(object):
             for requirement in requirements:
                 command += ["-r", requirement]
             if c.get('find_links') and not c["pip_index"]:
                 command += ['--no-index']
             for opt in global_options:
                 command += ["--global-option", opt]
         elif install_method == 'easy_install':
             if not module:
-                self.fatal("module parameter required with install_method='easy_install'")
+                self.fatal(
+                    "module parameter required with install_method='easy_install'")
             if requirements:
                 # Install pip requirements files separately, since they're
                 # not understood by easy_install.
                 self.install_module(requirements=requirements,
                                     install_method='pip')
             command = [self.query_python_path(), '-m', 'easy_install']
         else:
             self.fatal(
@@ -441,29 +445,31 @@ class VirtualenvMixin(object):
 # This is (sadly) a mixin for logging methods.
 class PerfherderResourceOptionsMixin(ScriptMixin):
     def perfherder_resource_options(self):
         """Obtain a list of extraOptions values to identify the env."""
         opts = []
 
         if 'TASKCLUSTER_INSTANCE_TYPE' in os.environ:
             # Include the instance type so results can be grouped.
-            opts.append('taskcluster-%s' % os.environ['TASKCLUSTER_INSTANCE_TYPE'])
+            opts.append('taskcluster-%s' %
+                        os.environ['TASKCLUSTER_INSTANCE_TYPE'])
         else:
             # We assume !taskcluster => buildbot.
             instance = 'unknown'
 
             # Try to load EC2 instance type from metadata file. This file
             # may not exist in many scenarios (including when inside a chroot).
             # So treat it as optional.
             try:
                 # This file should exist on Linux in EC2.
                 with open('/etc/instance_metadata.json', 'rb') as fh:
                     im = json.load(fh)
-                    instance = im.get('aws_instance_type', u'unknown').encode('ascii')
+                    instance = im.get('aws_instance_type',
+                                      u'unknown').encode('ascii')
             except IOError as e:
                 if e.errno != errno.ENOENT:
                     raise
                 self.info('instance_metadata.json not found; unable to '
                           'determine instance type')
             except Exception:
                 self.warning('error reading instance_metadata: %s' %
                              traceback.format_exc())
@@ -482,16 +488,17 @@ class ResourceMonitoringMixin(Perfherder
     This class requires the VirtualenvMixin in order to install a package used
     for recording resource usage.
 
     While we would like to record resource usage for the entirety of a script,
     since we require an external package, we can only record resource usage
     after that package is installed (as part of creating the virtualenv).
     That's just the way things have to be.
     """
+
     def __init__(self, *args, **kwargs):
         super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)
 
         self.register_virtualenv_module('psutil>=5.6.3', method='pip',
                                         optional=True)
         self.register_virtualenv_module('mozsystemmonitor==0.4',
                                         method='pip', optional=True)
         self.register_virtualenv_module('jsonschema==2.5.1',
@@ -505,17 +512,18 @@ class ResourceMonitoringMixin(Perfherder
 
     @PostScriptAction('create-virtualenv')
     def _start_resource_monitoring(self, action, success=None):
         self.activate_virtualenv()
 
         # Resource Monitor requires Python 2.7, however it's currently optional.
         # Remove when all machines have had their Python version updated (bug 711299).
         if sys.version_info[:2] < (2, 7):
-            self.warning('Resource monitoring will not be enabled! Python 2.7+ required.')
+            self.warning(
+                'Resource monitoring will not be enabled! Python 2.7+ required.')
             return
 
         try:
             from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
 
             self.info("Starting resource monitoring.")
             self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
             self._resource_monitor.start()
@@ -743,17 +751,17 @@ class Python3Virtualenv(object):
         '''We don't use __init__ to allow integrating with other mixins.
 
         python_path - Path to Python 3 binary.
         venv_path - Path to virtual environment to be created.
         '''
         self.py3_initialized_venv = True
         self.py3_python_path = os.path.abspath(python_path)
         version = self.get_output_from_command(
-                    [self.py3_python_path, '--version'], env=self.query_env()).split()[-1]
+            [self.py3_python_path, '--version'], env=self.query_env()).split()[-1]
         # Using -m venv is only used on 3.5+ versions
         assert version > '3.5.0'
         self.py3_venv_path = os.path.abspath(venv_path)
         self.py3_pip_path = os.path.join(self.py3_path_to_executables(), 'pip')
 
     def py3_path_to_executables(self):
         platform = self.platform_name()
         if platform.startswith('win'):