Bug 1595982 - make mozharness::base::vcs python3 compatible r=aki
authorEdwin Takahashi <egao@mozilla.com>
Tue, 19 Nov 2019 00:30:19 +0000
changeset 502535 4f123ce2cc07772d932aae792a39a38f3ce765fa
parent 502534 09e3e82fb4396ae250cc5e92c7af8447ff5a5683
child 502536 59d5dcf22862ff998d5f08fcf42309bdb0fc066d
push id100893
push useregao@mozilla.com
push dateTue, 19 Nov 2019 00:34:05 +0000
treeherderautoland@4f123ce2cc07 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1595982
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1595982 - make mozharness::base::vcs python3 compatible r=aki Changes: Run `isort` and `autopep8` to fix formatting mistakes. Replace deprecated imports with `try/except` block to make it compatible with Python2 and 3. Differential Revision: https://phabricator.services.mozilla.com/D53727
testing/mozharness/mozharness/base/vcs/gittool.py
testing/mozharness/mozharness/base/vcs/mercurial.py
testing/mozharness/mozharness/base/vcs/vcsbase.py
--- a/testing/mozharness/mozharness/base/vcs/gittool.py
+++ b/testing/mozharness/mozharness/base/vcs/gittool.py
@@ -1,15 +1,18 @@
 import os
 import re
-import urlparse
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
 
+from mozharness.base.errors import GitErrorList, VCSException
+from mozharness.base.log import LogMixin, OutputParser
 from mozharness.base.script import ScriptMixin
-from mozharness.base.log import LogMixin, OutputParser
-from mozharness.base.errors import GitErrorList, VCSException
 
 
 class GittoolParser(OutputParser):
     """
     A class that extends OutputParser such that it can find the "Got revision"
     string from gittool.py output
     """
 
@@ -54,17 +57,19 @@ class GittoolVCS(ScriptMixin, LogMixin):
         c = self.vcs_config
         for conf_item in ('dest', 'repo'):
             assert self.vcs_config[conf_item]
         dest = os.path.abspath(c['dest'])
         repo = c['repo']
         revision = c.get('revision')
         branch = c.get('branch')
         clean = c.get('clean')
-        share_base = c.get('vcs_share_base', os.environ.get("GIT_SHARE_BASE_DIR", None))
+        share_base = c.get(
+            'vcs_share_base', os.environ.get(
+                "GIT_SHARE_BASE_DIR", None))
         env = {'PATH': os.environ.get('PATH')}
         env.update(c.get('env', {}))
         if self._is_windows():
             # git.exe is not in the PATH by default
             env['PATH'] = '%s;C:/mozilla-build/Git/bin' % env['PATH']
             # SYSTEMROOT is needed for 'import random'
             if 'SYSTEMROOT' not in env:
                 env['SYSTEMROOT'] = os.environ.get('SYSTEMROOT')
@@ -83,14 +88,18 @@ class GittoolVCS(ScriptMixin, LogMixin):
                 'gittool_base_mirror_urls', self.config.get('vcs_base_mirror_urls', [])):
             bits = urlparse.urlparse(repo)
             mirror_url = urlparse.urljoin(base_mirror_url, bits.path)
             cmd.extend(['--mirror', mirror_url])
 
         cmd.extend([repo, dest])
         parser = GittoolParser(config=self.config, log_obj=self.log_obj,
                                error_list=GitErrorList)
-        retval = self.run_command(cmd, error_list=GitErrorList, env=env, output_parser=parser)
+        retval = self.run_command(
+            cmd,
+            error_list=GitErrorList,
+            env=env,
+            output_parser=parser)
 
         if retval != 0:
             raise VCSException("Unable to checkout")
 
         return parser.got_revision
--- a/testing/mozharness/mozharness/base/vcs/mercurial.py
+++ b/testing/mozharness/mozharness/base/vcs/mercurial.py
@@ -2,32 +2,37 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """Mercurial VCS support.
 """
 
+import hashlib
 import os
 import re
 import subprocess
+import sys
 from collections import namedtuple
-from urlparse import urlsplit
-import hashlib
-
-import sys
-sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0]))))
+try:
+    from urlparse import urlsplit
+except ImportError:
+    from urllib.parse import urlsplit
 
 import mozharness
 from mozharness.base.errors import HgErrorList, VCSException
 from mozharness.base.log import LogMixin, OutputParser
 from mozharness.base.script import ScriptMixin
 from mozharness.base.transfer import TransferMixin
 
+sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(
+                sys.path[0]))))
+
+
 external_tools_path = os.path.join(
     os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
     'external_tools',
 )
 
 
 HG_OPTIONS = ['--config', 'ui.merge=internal:merge']
 
@@ -43,17 +48,18 @@ class RepositoryUpdateRevisionParser(Out
     revision = None
     RE_UPDATED = re.compile('^updated to ([a-f0-9]{40})$')
 
     def parse_single_line(self, line):
         m = self.RE_UPDATED.match(line)
         if m:
             self.revision = m.group(1)
 
-        return super(RepositoryUpdateRevisionParser, self).parse_single_line(line)
+        return super(RepositoryUpdateRevisionParser,
+                     self).parse_single_line(line)
 
 
 def make_hg_url(hg_host, repo_path, protocol='http', revision=None,
                 filename=None):
     """Helper function.
 
     Construct a valid hg url from a base hg url (hg.mozilla.org),
     repo_path, revision and possible filename
@@ -62,17 +68,18 @@ def make_hg_url(hg_host, repo_path, prot
     repo = '/'.join(p.strip('/') for p in [base, repo_path])
     if not filename:
         if not revision:
             return repo
         else:
             return '/'.join([p.strip('/') for p in [repo, 'rev', revision]])
     else:
         assert revision
-        return '/'.join([p.strip('/') for p in [repo, 'raw-file', revision, filename]])
+        return '/'.join([p.strip('/')
+                         for p in [repo, 'raw-file', revision, filename]])
 
 
 class MercurialVCS(ScriptMixin, LogMixin, TransferMixin):
     # For the most part, scripts import mercurial, update
     # tag-release.py imports
     #  apply_and_push, update, get_revision, out, BRANCH, REVISION,
     #  get_branches, cleanOutgoingRevs
 
@@ -131,17 +138,17 @@ class MercurialVCS(ScriptMixin, LogMixin
                                                  cwd=path).splitlines():
             branches.append(line.split()[0])
         return branches
 
     def hg_ver(self):
         """Returns the current version of hg, as a tuple of
         (major, minor, build)"""
         ver_string = self.get_output_from_command(self.hg + ['-q', 'version'])
-        match = re.search("\(version ([0-9.]+)\)", ver_string)
+        match = re.search(r"\(version ([0-9.]+)\)", ver_string)
         if match:
             bits = match.group(1).split(".")
             if len(bits) < 3:
                 bits += (0,)
             ver = tuple(int(b) for b in bits)
         else:
             ver = (0, 0, 0)
         self.debug("Running hg version %s" % str(ver))
@@ -159,17 +166,19 @@ class MercurialVCS(ScriptMixin, LogMixin
         if branch:
             msg += " to branch %s" % branch
         if revision:
             msg += " revision %s" % revision
         self.info("%s." % msg)
         if revision is not None:
             cmd = self.hg + ['update', '-C', '-r', revision]
             if self.run_command(cmd, cwd=dest, error_list=HgErrorList):
-                raise VCSException("Unable to update %s to %s!" % (dest, revision))
+                raise VCSException(
+                    "Unable to update %s to %s!" %
+                    (dest, revision))
         else:
             # Check & switch branch
             local_branch = self.get_branch_from_path(dest)
 
             cmd = self.hg + ['update', '-C']
 
             # If this is different, checkout the other branch
             if branch and branch != local_branch:
@@ -277,17 +286,19 @@ class MercurialVCS(ScriptMixin, LogMixin
             branch = self.vcs_config.get('branch')
             revision = self.vcs_config.get('revision')
             return self.update(dest, branch=branch, revision=revision)
 
     # Defines the places of attributes in the tuples returned by `out'
 
     def out(self, src, remote, **kwargs):
         """Check for outgoing changesets present in a repo"""
-        self.info("Checking for outgoing changesets from %s to %s." % (src, remote))
+        self.info(
+            "Checking for outgoing changesets from %s to %s." %
+            (src, remote))
         cmd = self.hg + ['-q', 'out', '--template', '{node} {branches}\n']
         cmd.extend(self.common_args(**kwargs))
         cmd.append(remote)
         if os.path.exists(src):
             try:
                 revs = []
                 for line in self.get_output_from_command(
                         cmd, cwd=src, throw_exception=True).rstrip().split("\n"):
@@ -346,37 +357,42 @@ class MercurialVCS(ScriptMixin, LogMixin
         upstream = c.get('clone_upstream_url')
 
         # The API here is kind of bad because we're relying on state in
         # self.vcs_config instead of passing arguments. This confuses
         # scripts that have multiple repos. This includes the clone_tools()
         # step :(
 
         if not rev and not branch:
-            self.warning('did not specify revision or branch; assuming "default"')
+            self.warning(
+                'did not specify revision or branch; assuming "default"')
             branch = 'default'
 
-        share_base = c.get('vcs_share_base') or os.environ.get('HG_SHARE_BASE_DIR')
+        share_base = c.get('vcs_share_base') or os.environ.get(
+            'HG_SHARE_BASE_DIR')
         if share_base and c.get('use_vcs_unique_share'):
             # Bug 1277041 - update migration scripts to support robustcheckout
             # fake a share but don't really share
-            share_base = os.path.join(share_base, hashlib.md5(dest).hexdigest())
+            share_base = os.path.join(
+                share_base, hashlib.md5(dest).hexdigest())
 
         # We require shared storage is configured because it guarantees we
         # only have 1 local copy of logical repo stores.
         if not share_base:
             raise VCSException('vcs share base not defined; '
                                'refusing to operate sub-optimally')
 
         if not self.robustcheckout_path:
-            raise VCSException('could not find the robustcheckout Mercurial extension')
+            raise VCSException(
+                'could not find the robustcheckout Mercurial extension')
 
         # Log HG version and install info to aid debugging.
         self.run_command(self.hg + ['--version'])
-        self.run_command(self.hg + ['debuginstall', '--config=ui.username=worker'])
+        self.run_command(self.hg +
+                         ['debuginstall', '--config=ui.username=worker'])
 
         args = self.hg + [
             '--config', 'extensions.robustcheckout=%s' % self.robustcheckout_path,
             'robustcheckout', repo_url, dest, '--sharebase', share_base,
         ]
         if purge:
             args.append('--purge')
         if upstream:
@@ -394,17 +410,19 @@ class MercurialVCS(ScriptMixin, LogMixin
 
         if not parser.revision:
             raise VCSException('could not identify revision updated to')
 
         return parser.revision
 
     def cleanOutgoingRevs(self, reponame, remote, username, sshKey):
         # TODO retry
-        self.info("Wiping outgoing local changes from %s to %s." % (reponame, remote))
+        self.info(
+            "Wiping outgoing local changes from %s to %s." %
+            (reponame, remote))
         outgoingRevs = self.out(src=reponame, remote=remote,
                                 ssh_username=username, ssh_key=sshKey)
         for r in reversed(outgoingRevs):
             self.run_command(self.hg + ['strip', '-n', r[REVISION]],
                              cwd=reponame, error_list=HgErrorList)
 
     def query_pushinfo(self, repository, revision):
         """Query the pushdate and pushid of a repository/revision.
@@ -426,17 +444,17 @@ class MercurialVCS(ScriptMixin, LogMixin
             #    ],
             #    "date": 1428072488,
             #    "user": "user@mozilla.com"
             #   }
             # }
             #
             # So we grab the first element ("28537" in this case) and then pull
             # out the 'date' field.
-            pushid = contents.iterkeys().next()
+            pushid = next(contents.keys())
             self.info('Pushid is: %s' % pushid)
             pushdate = contents[pushid]['date']
             self.info('Pushdate is: %s' % pushdate)
             return PushInfo(pushid, pushdate)
 
         except Exception:
             self.exception("Failed to get push info from hg.mozilla.org")
             raise
--- a/testing/mozharness/mozharness/base/vcs/vcsbase.py
+++ b/testing/mozharness/mozharness/base/vcs/vcsbase.py
@@ -2,40 +2,46 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 """Generic VCS support.
 """
 
-from copy import deepcopy
 import os
 import sys
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0]))))
+from copy import deepcopy
 
 from mozharness.base.errors import VCSException
 from mozharness.base.log import FATAL
 from mozharness.base.script import BaseScript
+from mozharness.base.vcs.gittool import GittoolVCS
 from mozharness.base.vcs.mercurial import MercurialVCS
-from mozharness.base.vcs.gittool import GittoolVCS
+
+sys.path.insert(
+    1, os.path.dirname(
+        os.path.dirname(
+            os.path.dirname(
+                sys.path[0]))))
+
 
 # Update this with supported VCS name : VCS object
 VCS_DICT = {
     'hg': MercurialVCS,
     'gittool': GittoolVCS,
 }
 
 
 # VCSMixin {{{1
 class VCSMixin(object):
     """Basic VCS methods that are vcs-agnostic.
     The vcs_class handles all the vcs-specific tasks.
     """
+
     def query_dest(self, kwargs):
         if 'dest' in kwargs:
             return kwargs['dest']
         dest = os.path.basename(kwargs['repo'])
         # Git fun
         if dest.endswith('.git'):
             dest = dest.replace('.git', '')
         return dest
@@ -45,33 +51,37 @@ class VCSMixin(object):
             got_revision = vcs_obj.ensure_repo_and_revision()
             if got_revision:
                 return got_revision
         except VCSException:
             self.rmtree(dest)
             raise
 
     def _get_vcs_class(self, vcs):
-        vcs = vcs or self.config.get('default_vcs', getattr(self, 'default_vcs', None))
+        vcs = vcs or self.config.get(
+            'default_vcs', getattr(
+                self, 'default_vcs', None))
         vcs_class = VCS_DICT.get(vcs)
         return vcs_class
 
     def vcs_checkout(self, vcs=None, error_level=FATAL, **kwargs):
         """ Check out a single repo.
         """
         c = self.config
         vcs_class = self._get_vcs_class(vcs)
         if not vcs_class:
             self.error("Running vcs_checkout with kwargs %s" % str(kwargs))
             raise VCSException("No VCS set!")
         # need a better way to do this.
         if 'dest' not in kwargs:
             kwargs['dest'] = self.query_dest(kwargs)
         if 'vcs_share_base' not in kwargs:
-            kwargs['vcs_share_base'] = c.get('%s_share_base' % vcs, c.get('vcs_share_base'))
+            kwargs['vcs_share_base'] = c.get(
+                '%s_share_base' %
+                vcs, c.get('vcs_share_base'))
         vcs_obj = vcs_class(
             log_obj=self.log_obj,
             config=self.config,
             vcs_config=kwargs,
             script_obj=self,
         )
         return self.retry(
             self._get_revision,