Bug 1595982 - make mozharness::base::vcs python3 compatible r=aki
authorEdwin Takahashi <egao@mozilla.com>
Tue, 19 Nov 2019 00:30:19 +0000
changeset 502518 4f123ce2cc07772d932aae792a39a38f3ce765fa
parent 502517 09e3e82fb4396ae250cc5e92c7af8447ff5a5683
child 502519 59d5dcf22862ff998d5f08fcf42309bdb0fc066d
push id114172
push userdluca@mozilla.com
push dateTue, 19 Nov 2019 11:31:10 +0000
treeherdermozilla-inbound@b5c5ba07d3db [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1595982
milestone72.0a1
Bug 1595982 - make mozharness::base::vcs python3 compatible r=aki Changes: Run `isort` and `autopep8` to fix formatting mistakes. Replace deprecated imports with `try/except` block to make it compatible with Python2 and 3. Differential Revision: https://phabricator.services.mozilla.com/D53727
testing/mozharness/mozharness/base/vcs/gittool.py
testing/mozharness/mozharness/base/vcs/mercurial.py
testing/mozharness/mozharness/base/vcs/vcsbase.py
--- a/testing/mozharness/mozharness/base/vcs/gittool.py
+++ b/testing/mozharness/mozharness/base/vcs/gittool.py
@@ -1,15 +1,18 @@
 import os
 import re
-import urlparse
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
 
+from mozharness.base.errors import GitErrorList, VCSException
+from mozharness.base.log import LogMixin, OutputParser
 from mozharness.base.script import ScriptMixin
-from mozharness.base.log import LogMixin, OutputParser
-from mozharness.base.errors import GitErrorList, VCSException
 
 
 class GittoolParser(OutputParser):
     """
     A class that extends OutputParser such that it can find the "Got revision"
     string from gittool.py output
     """
 
@@ -54,17 +57,19 @@ class GittoolVCS(ScriptMixin, LogMixin):
         c = self.vcs_config
         for conf_item in ('dest', 'repo'):
             assert self.vcs_config[conf_item]
         dest = os.path.abspath(c['dest'])
         repo = c['repo']
         revision = c.get('revision')
         branch = c.get('branch')
         clean = c.get('clean')
-        share_base = c.get('vcs_share_base', os.environ.get("GIT_SHARE_BASE_DIR", None))
+        share_base = c.get(
+            'vcs_share_base', os.environ.get(
+                "GIT_SHARE_BASE_DIR", None))
         env = {'PATH': os.environ.get('PATH')}
         env.update(c.get('env', {}))
         if self._is_windows():
             # git.exe is not in the PATH by default
             env['PATH'] = '%s;C:/mozilla-build/Git/bin' % env['PATH']
             # SYSTEMROOT is needed for 'import random'
             if 'SYSTEMROOT' not in env:
                 env['SYSTEMROOT'] = os.environ.get('SYSTEMROOT')
@@ -83,14 +88,18 @@ class GittoolVCS(ScriptMixin, LogMixin):
                 'gittool_base_mirror_urls', self.config.get('vcs_base_mirror_urls', [])):
             bits = urlparse.urlparse(repo)
             mirror_url = urlparse.urljoin(base_mirror_url, bits.path)
             cmd.extend(['--mirror', mirror_url])
 
         cmd.extend([repo, dest])
         parser = GittoolParser(config=self.config, log_obj=self.log_obj,
                                error_list=GitErrorList)
-        retval = self.run_command(cmd, error_list=GitErrorList, env=env, output_parser=parser)
+        retval = self.run_command(
+            cmd,
+            error_list=GitErrorList,
+            env=env,
+            output_parser=parser)
 
         if retval != 0:
             raise VCSException("Unable to checkout")
 
         return parser.got_revision
--- a/testing/mozharness/mozharness/base/vcs/mercurial.py
+++ b/testing/mozharness/mozharness/base/vcs/mercurial.py
@@ -2,32 +2,37 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """Mercurial VCS support.
 """
 
+import hashlib
 import os
 import re
 import subprocess
+import sys
 from collections import namedtuple
-from urlparse import urlsplit
-import hashlib
-
-import sys
-sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0]))))
+try:
+    from urlparse import urlsplit
+except ImportError:
+    from urllib.parse import urlsplit
 
 import mozharness
 from mozharness.base.errors import HgErrorList, VCSException
 from mozharness.base.log import LogMixin, OutputParser
 from mozharness.base.script import ScriptMixin
 from mozharness.base.transfer import TransferMixin
 
+sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(
+                sys.path[0]))))
+
+
 external_tools_path = os.path.join(
     os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
     'external_tools',
 )
 
 
 HG_OPTIONS = ['--config', 'ui.merge=internal:merge']
 
@@ -43,17 +48,18 @@ class RepositoryUpdateRevisionParser(Out
     revision = None
     RE_UPDATED = re.compile('^updated to ([a-f0-9]{40})$')
 
     def parse_single_line(self, line):
         m = self.RE_UPDATED.match(line)
         if m:
             self.revision = m.group(1)
 
-        return super(RepositoryUpdateRevisionParser, self).parse_single_line(line)
+        return super(RepositoryUpdateRevisionParser,
+                     self).parse_single_line(line)
 
 
 def make_hg_url(hg_host, repo_path, protocol='http', revision=None,
                 filename=None):
     """Helper function.
 
     Construct a valid hg url from a base hg url (hg.mozilla.org),
     repo_path, revision and possible filename
@@ -62,17 +68,18 @@ def make_hg_url(hg_host, repo_path, prot
     repo = '/'.join(p.strip('/') for p in [base, repo_path])
     if not filename:
         if not revision:
             return repo
         else:
             return '/'.join([p.strip('/') for p in [repo, 'rev', revision]])
     else:
         assert revision
-        return '/'.join([p.strip('/') for p in [repo, 'raw-file', revision, filename]])
+        return '/'.join([p.strip('/')
+                         for p in [repo, 'raw-file', revision, filename]])
 
 
 class MercurialVCS(ScriptMixin, LogMixin, TransferMixin):
     # For the most part, scripts import mercurial, update
     # tag-release.py imports
     #  apply_and_push, update, get_revision, out, BRANCH, REVISION,
     #  get_branches, cleanOutgoingRevs
 
@@ -131,17 +138,17 @@ class MercurialVCS(ScriptMixin, LogMixin
                                                  cwd=path).splitlines():
             branches.append(line.split()[0])
         return branches
 
     def hg_ver(self):
         """Returns the current version of hg, as a tuple of
         (major, minor, build)"""
         ver_string = self.get_output_from_command(self.hg + ['-q', 'version'])
-        match = re.search("\(version ([0-9.]+)\)", ver_string)
+        match = re.search(r"\(version ([0-9.]+)\)", ver_string)
         if match:
             bits = match.group(1).split(".")
             if len(bits) < 3:
                 bits += (0,)
             ver = tuple(int(b) for b in bits)
         else:
             ver = (0, 0, 0)
         self.debug("Running hg version %s" % str(ver))
@@ -159,17 +166,19 @@ class MercurialVCS(ScriptMixin, LogMixin
         if branch:
             msg += " to branch %s" % branch
         if revision:
             msg += " revision %s" % revision
         self.info("%s." % msg)
         if revision is not None:
             cmd = self.hg + ['update', '-C', '-r', revision]
             if self.run_command(cmd, cwd=dest, error_list=HgErrorList):
-                raise VCSException("Unable to update %s to %s!" % (dest, revision))
+                raise VCSException(
+                    "Unable to update %s to %s!" %
+                    (dest, revision))
         else:
             # Check & switch branch
             local_branch = self.get_branch_from_path(dest)
 
             cmd = self.hg + ['update', '-C']
 
             # If this is different, checkout the other branch
             if branch and branch != local_branch:
@@ -277,17 +286,19 @@ class MercurialVCS(ScriptMixin, LogMixin
             branch = self.vcs_config.get('branch')
             revision = self.vcs_config.get('revision')
             return self.update(dest, branch=branch, revision=revision)
 
     # Defines the places of attributes in the tuples returned by `out'
 
     def out(self, src, remote, **kwargs):
         """Check for outgoing changesets present in a repo"""
-        self.info("Checking for outgoing changesets from %s to %s." % (src, remote))
+        self.info(
+            "Checking for outgoing changesets from %s to %s." %
+            (src, remote))
         cmd = self.hg + ['-q', 'out', '--template', '{node} {branches}\n']
         cmd.extend(self.common_args(**kwargs))
         cmd.append(remote)
         if os.path.exists(src):
             try:
                 revs = []
                 for line in self.get_output_from_command(
                         cmd, cwd=src, throw_exception=True).rstrip().split("\n"):
@@ -346,37 +357,42 @@ class MercurialVCS(ScriptMixin, LogMixin
         upstream = c.get('clone_upstream_url')
 
         # The API here is kind of bad because we're relying on state in
         # self.vcs_config instead of passing arguments. This confuses
         # scripts that have multiple repos. This includes the clone_tools()
         # step :(
 
         if not rev and not branch:
-            self.warning('did not specify revision or branch; assuming "default"')
+            self.warning(
+                'did not specify revision or branch; assuming "default"')
             branch = 'default'
 
-        share_base = c.get('vcs_share_base') or os.environ.get('HG_SHARE_BASE_DIR')
+        share_base = c.get('vcs_share_base') or os.environ.get(
+            'HG_SHARE_BASE_DIR')
         if share_base and c.get('use_vcs_unique_share'):
             # Bug 1277041 - update migration scripts to support robustcheckout
             # fake a share but don't really share
-            share_base = os.path.join(share_base, hashlib.md5(dest).hexdigest())
+            share_base = os.path.join(
+                share_base, hashlib.md5(dest).hexdigest())
 
         # We require shared storage is configured because it guarantees we
         # only have 1 local copy of logical repo stores.
         if not share_base:
             raise VCSException('vcs share base not defined; '
                                'refusing to operate sub-optimally')
 
         if not self.robustcheckout_path:
-            raise VCSException('could not find the robustcheckout Mercurial extension')
+            raise VCSException(
+                'could not find the robustcheckout Mercurial extension')
 
         # Log HG version and install info to aid debugging.
         self.run_command(self.hg + ['--version'])
-        self.run_command(self.hg + ['debuginstall', '--config=ui.username=worker'])
+        self.run_command(self.hg +
+                         ['debuginstall', '--config=ui.username=worker'])
 
         args = self.hg + [
             '--config', 'extensions.robustcheckout=%s' % self.robustcheckout_path,
             'robustcheckout', repo_url, dest, '--sharebase', share_base,
         ]
         if purge:
             args.append('--purge')
         if upstream:
@@ -394,17 +410,19 @@ class MercurialVCS(ScriptMixin, LogMixin
 
         if not parser.revision:
             raise VCSException('could not identify revision updated to')
 
         return parser.revision
 
     def cleanOutgoingRevs(self, reponame, remote, username, sshKey):
         # TODO retry
-        self.info("Wiping outgoing local changes from %s to %s." % (reponame, remote))
+        self.info(
+            "Wiping outgoing local changes from %s to %s." %
+            (reponame, remote))
         outgoingRevs = self.out(src=reponame, remote=remote,
                                 ssh_username=username, ssh_key=sshKey)
         for r in reversed(outgoingRevs):
             self.run_command(self.hg + ['strip', '-n', r[REVISION]],
                              cwd=reponame, error_list=HgErrorList)
 
     def query_pushinfo(self, repository, revision):
         """Query the pushdate and pushid of a repository/revision.
@@ -426,17 +444,17 @@ class MercurialVCS(ScriptMixin, LogMixin
             #    ],
             #    "date": 1428072488,
             #    "user": "user@mozilla.com"
             #   }
             # }
             #
             # So we grab the first element ("28537" in this case) and then pull
             # out the 'date' field.
-            pushid = contents.iterkeys().next()
+            pushid = next(contents.keys())
             self.info('Pushid is: %s' % pushid)
             pushdate = contents[pushid]['date']
             self.info('Pushdate is: %s' % pushdate)
             return PushInfo(pushid, pushdate)
 
         except Exception:
             self.exception("Failed to get push info from hg.mozilla.org")
             raise
--- a/testing/mozharness/mozharness/base/vcs/vcsbase.py
+++ b/testing/mozharness/mozharness/base/vcs/vcsbase.py
@@ -2,40 +2,46 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """Generic VCS support.
 """
 
-from copy import deepcopy
 import os
 import sys
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0]))))
+from copy import deepcopy
 
 from mozharness.base.errors import VCSException
 from mozharness.base.log import FATAL
 from mozharness.base.script import BaseScript
+from mozharness.base.vcs.gittool import GittoolVCS
 from mozharness.base.vcs.mercurial import MercurialVCS
-from mozharness.base.vcs.gittool import GittoolVCS
+
+sys.path.insert(
+    1, os.path.dirname(
+        os.path.dirname(
+            os.path.dirname(
+                sys.path[0]))))
+
 
 # Update this with supported VCS name : VCS object
 VCS_DICT = {
     'hg': MercurialVCS,
     'gittool': GittoolVCS,
 }
 
 
 # VCSMixin {{{1
 class VCSMixin(object):
     """Basic VCS methods that are vcs-agnostic.
     The vcs_class handles all the vcs-specific tasks.
     """
+
     def query_dest(self, kwargs):
         if 'dest' in kwargs:
             return kwargs['dest']
         dest = os.path.basename(kwargs['repo'])
         # Git fun
         if dest.endswith('.git'):
             dest = dest.replace('.git', '')
         return dest
@@ -45,33 +51,37 @@ class VCSMixin(object):
             got_revision = vcs_obj.ensure_repo_and_revision()
             if got_revision:
                 return got_revision
         except VCSException:
             self.rmtree(dest)
             raise
 
     def _get_vcs_class(self, vcs):
-        vcs = vcs or self.config.get('default_vcs', getattr(self, 'default_vcs', None))
+        vcs = vcs or self.config.get(
+            'default_vcs', getattr(
+                self, 'default_vcs', None))
         vcs_class = VCS_DICT.get(vcs)
         return vcs_class
 
     def vcs_checkout(self, vcs=None, error_level=FATAL, **kwargs):
         """ Check out a single repo.
         """
         c = self.config
         vcs_class = self._get_vcs_class(vcs)
         if not vcs_class:
             self.error("Running vcs_checkout with kwargs %s" % str(kwargs))
             raise VCSException("No VCS set!")
         # need a better way to do this.
         if 'dest' not in kwargs:
             kwargs['dest'] = self.query_dest(kwargs)
         if 'vcs_share_base' not in kwargs:
-            kwargs['vcs_share_base'] = c.get('%s_share_base' % vcs, c.get('vcs_share_base'))
+            kwargs['vcs_share_base'] = c.get(
+                '%s_share_base' %
+                vcs, c.get('vcs_share_base'))
         vcs_obj = vcs_class(
             log_obj=self.log_obj,
             config=self.config,
             vcs_config=kwargs,
             script_obj=self,
         )
         return self.retry(
             self._get_revision,