Bug 1632348 - Convert mach artifact to python 3. r=rstewart
☠☠ backed out by e9e35474d57a ☠ ☠
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 24 Apr 2020 20:51:24 +0000
changeset 526090 90bd5342c58f92923a4e26ddd3b00c37810ace97
parent 526089 59f59e806221dfe16fd3df6a4ba4bb78705c4131
child 526091 646b2f07e39803242f0cb0b04c17bec6f0282b10
push id37350
push usernbeleuzu@mozilla.com
push dateSun, 26 Apr 2020 09:43:12 +0000
treeherdermozilla-central@21659f178a12 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrstewart
bugs1632348
milestone77.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1632348 - Convert mach artifact to python 3. r=rstewart Differential Revision: https://phabricator.services.mozilla.com/D72114
mach
python/mozbuild/mozbuild/action/tooltool.py
python/mozbuild/mozbuild/artifact_cache.py
python/mozbuild/mozbuild/artifact_commands.py
python/mozbuild/mozbuild/artifacts.py
--- a/mach
+++ b/mach
@@ -9,17 +9,16 @@
 
 # Embeds a shell script inside a Python triple quote. This pattern is valid
 # shell because `''':'`, `':'` and `:` are all equivalent, and `:` is a no-op.
 ''':'
 py2commands="
     analyze
     android
     android-emulator
-    artifact
     awsy-test
     browsertime
     cargo
     check-spidermonkey
     clang-format
     cppunittest
     cramtest
     crashtest
--- a/python/mozbuild/mozbuild/action/tooltool.py
+++ b/python/mozbuild/mozbuild/action/tooltool.py
@@ -700,17 +700,17 @@ def execute(cmd):
         log.info(line.replace('\n', ' '))
     return process.wait() == 0
 
 
 def open_manifest(manifest_file):
     """I know how to take a filename and load it into a Manifest object"""
     if os.path.exists(manifest_file):
         manifest = Manifest()
-        with open(manifest_file, "rb") as f:
+        with open(manifest_file, "r" if PY3 else "rb") as f:
             manifest.load(f)
             log.debug("loaded manifest from file '%s'" % manifest_file)
         return manifest
     else:
         log.debug("tried to load absent file '%s' as manifest" % manifest_file)
         raise InvalidManifest(
             "manifest file '%s' does not exist" % manifest_file)
 
--- a/python/mozbuild/mozbuild/artifact_cache.py
+++ b/python/mozbuild/mozbuild/artifact_cache.py
@@ -157,17 +157,17 @@ class ArtifactCache(object):
 
     def fetch(self, url, force=False):
         fname = os.path.basename(url)
         try:
             # Use the file name from the url if it looks like a hash digest.
             if len(fname) not in (32, 40, 56, 64, 96, 128):
                 raise TypeError()
             binascii.unhexlify(fname)
-        except TypeError:
+        except (TypeError, binascii.Error):
             # We download to a temporary name like HASH[:16]-basename to
             # differentiate among URLs with the same basenames.  We used to then
             # extract the build ID from the downloaded artifact and use it to make a
             # human readable unique name, but extracting build IDs is time consuming
             # (especially on Mac OS X, where we must mount a large DMG file).
             hash = hashlib.sha256(six.ensure_binary(url)).hexdigest()[:16]
             # Strip query string and fragments.
             basename = os.path.basename(urlparse.urlparse(url).path)
--- a/python/mozbuild/mozbuild/artifact_commands.py
+++ b/python/mozbuild/mozbuild/artifact_commands.py
@@ -407,17 +407,17 @@ class PackageFrontend(MachCommandBase):
             # when possible.
             try:
                 os.link(record.filename, local)
             except Exception:
                 shutil.copy(record.filename, local)
             # Keep a sha256 of each downloaded file, for the chain-of-trust
             # validation.
             if artifact_manifest is not None:
-                with open(local) as fh:
+                with open(local, 'rb') as fh:
                     h = hashlib.sha256()
                     while True:
                         data = fh.read(1024 * 1024)
                         if not data:
                             break
                         h.update(data)
                 artifacts[record.url] = {
                     'sha256': h.hexdigest(),
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -892,16 +892,17 @@ class Artifacts(object):
     def log(self, *args, **kwargs):
         if self._log:
             self._log(*args, **kwargs)
 
     def run_hg(self, *args, **kwargs):
         env = kwargs.get('env', {})
         env['HGPLAIN'] = '1'
         kwargs['env'] = ensure_subprocess_env(env)
+        kwargs['universal_newlines'] = True
         return subprocess.check_output([self._hg] + list(args),
                                        **kwargs)
 
     def _guess_artifact_job(self):
         # Add the "-debug" suffix to the guessed artifact job name
         # if MOZ_DEBUG is enabled.
         if self._substs.get('MOZ_DEBUG'):
             target_suffix = '-debug'
@@ -974,21 +975,21 @@ class Artifacts(object):
 
         return candidate_pushheads
 
     def _get_hg_revisions_from_git(self):
         rev_list = subprocess.check_output([
             self._git, 'rev-list', '--topo-order',
             '--max-count={num}'.format(num=NUM_REVISIONS_TO_QUERY),
             'HEAD',
-        ], cwd=self._topsrcdir)
+        ], universal_newlines=True, cwd=self._topsrcdir)
 
         hg_hash_list = subprocess.check_output([
             self._git, 'cinnabar', 'git2hg'
-        ] + rev_list.splitlines(), cwd=self._topsrcdir)
+        ] + rev_list.splitlines(), universal_newlines=True, cwd=self._topsrcdir)
 
         zeroes = "0" * 40
 
         hashes = []
         for hg_hash in hg_hash_list.splitlines():
             hg_hash = hg_hash.strip()
             if not hg_hash or hg_hash == zeroes:
                 continue
@@ -1196,31 +1197,33 @@ see https://developer.mozilla.org/en-US/
         revision = None
         try:
             if self._hg:
                 revision = self.run_hg('log', '--template', '{node}\n', '-r', revset,
                                        cwd=self._topsrcdir).strip()
             elif self._git:
                 revset = subprocess.check_output([
                     self._git, 'rev-parse', '%s^{commit}' % revset],
-                    stderr=open(os.devnull, 'w'), cwd=self._topsrcdir).strip()
+                    stderr=open(os.devnull, 'w'), universal_newlines=True,
+                    cwd=self._topsrcdir).strip()
             else:
                 # Fallback to the exception handling case from both hg and git
                 raise subprocess.CalledProcessError()
         except subprocess.CalledProcessError:
             # If the mercurial of git commands above failed, it means the given
             # revset is not known locally to the VCS. But if the revset looks
             # like a complete sha1, assume it is a mercurial sha1 that hasn't
             # been pulled, and use that.
             if re.match(r'^[A-Fa-f0-9]{40}$', revset):
                 revision = revset
 
         if revision is None and self._git:
             revision = subprocess.check_output(
-                [self._git, 'cinnabar', 'git2hg', revset], cwd=self._topsrcdir).strip()
+                [self._git, 'cinnabar', 'git2hg', revset], universal_newlines=True,
+                cwd=self._topsrcdir).strip()
 
         if revision == "0" * 40 or revision is None:
             raise ValueError('revision specification must resolve to a commit known to hg')
         if len(revision.split('\n')) != 1:
             raise ValueError('revision specification must resolve to exactly one commit')
 
         self.log(logging.INFO, 'artifact',
                  {'revset': revset,