Bug 1314678 - Use hfs and dmg tools rather than mozinstall to unpack downloaded builds during cross artifact builds. r=mshal draft
authorChris Manchester <cmanchester@mozilla.com>
Fri, 03 Mar 2017 14:18:33 -0800
changeset 493407 88cf978106d15426d0b44ac607502a2d5a474c5b
parent 493406 0414463110a8b23e3d0b48f460177e99f4352115
child 547852 042d7d5a6c4f7cdce65942fe2a2a9f866ca0076b
push id47752
push usercmanchester@mozilla.com
push dateFri, 03 Mar 2017 22:19:04 +0000
reviewersmshal
bugs1314678
milestone54.0a1
Bug 1314678 - Use hfs and dmg tools rather than mozinstall to unpack downloaded builds during cross artifact builds. r=mshal Mozinstall uses hdiutil, which doesn't exist on cross osx-linux builders. MozReview-Commit-ID: A4KAIFUp4Gj
python/mozbuild/mozbuild/artifacts.py
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -134,22 +134,23 @@ class ArtifactJob(object):
         ('bin/plugins/*', ('bin/plugins', 'plugins')),
         ('bin/components/*', ('bin/components', 'bin/components')),
     }
 
     # We can tell our input is a test archive by this suffix, which happens to
     # be the same across platforms.
     _test_archive_suffix = '.common.tests.zip'
 
-    def __init__(self, package_re, tests_re, log=None, download_symbols=False):
+    def __init__(self, package_re, tests_re, log=None, download_symbols=False, substs=None):
         self._package_re = re.compile(package_re)
         self._tests_re = None
         if tests_re:
             self._tests_re = re.compile(tests_re)
         self._log = log
+        self._substs = substs
         self._symbols_archive_suffix = None
         if download_symbols:
             self._symbols_archive_suffix = 'crashreporter-symbols.zip'
 
     def log(self, *args, **kwargs):
         if self._log:
             self._log(*args, **kwargs)
 
@@ -303,38 +304,38 @@ class LinuxArtifactJob(ArtifactJob):
 
 
 class MacArtifactJob(ArtifactJob):
 
     product = 'firefox'
 
     def process_package_artifact(self, filename, processed_filename):
         tempdir = tempfile.mkdtemp()
+        oldcwd = os.getcwd()
         try:
             self.log(logging.INFO, 'artifact',
                 {'tempdir': tempdir},
                 'Unpacking DMG into {tempdir}')
-            mozinstall.install(filename, tempdir) # Doesn't handle already mounted DMG files nicely:
-
-            # InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)"
-
-            #   File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install
-            #     return artifacts.install_from(source, self.distdir)
-            #   File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from
-            #     return self.install_from_hg(source, distdir)
-            #   File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg
-            #     return self.install_from_url(url, distdir)
-            #   File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url
-            #     return self.install_from_file(filename, distdir)
-            #   File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file
-            #     mozinstall.install(filename, tempdir)
-            #   File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install
-            #     install_dir = _install_dmg(src, dest)
-            #   File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg
-            #     subprocess.call('hdiutil detach %s -quiet' % appDir,
+            if self._substs['HOST_OS_ARCH'] == 'Linux':
+                # This is a cross build, use hfsplus and dmg tools to extract the dmg.
+                os.chdir(tempdir)
+                with open(os.devnull, 'wb') as devnull:
+                    subprocess.check_call([
+                        self._substs['DMG_TOOL'],
+                        'extract',
+                        filename,
+                        'extracted_img',
+                    ], stdout=devnull, stderr=devnull)
+                    subprocess.check_call([
+                        self._substs['HFS_TOOL'],
+                        'extracted_img',
+                        'extractall'
+                    ], stdout=devnull, stderr=devnull)
+            else:
+                mozinstall.install(filename, tempdir)
 
             bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app'))
             if len(bundle_dirs) != 1:
                 raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs))
             [source] = bundle_dirs
 
             # These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
             paths_no_keep_path = ('Contents/MacOS', [
@@ -390,16 +391,17 @@ class MacArtifactJob(ArtifactJob):
                     for p, f in finder.find(path):
                         self.log(logging.INFO, 'artifact',
                             {'path': p},
                             'Adding {path} to processed archive')
                         destpath = mozpath.join('bin', p)
                         writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode)
 
         finally:
+            os.chdir(oldcwd)
             try:
                 shutil.rmtree(tempdir)
             except (OSError, IOError):
                 self.log(logging.WARN, 'artifact',
                     {'tempdir': tempdir},
                     'Unable to delete {tempdir}')
                 pass
 
@@ -482,19 +484,20 @@ JOB_DETAILS = {
     'win64-opt': (WinArtifactJob, (r'public/build/firefox-(.*)\.win64.zip',
                                    r'public/build/firefox-(.*)\.common\.tests\.zip')),
     'win64-debug': (WinArtifactJob, (r'public/build/firefox-(.*)\.win64.zip',
                                      r'public/build/firefox-(.*)\.common\.tests\.zip')),
 }
 
 
 
-def get_job_details(job, log=None, download_symbols=False):
+def get_job_details(job, log=None, download_symbols=False, substs=None):
     cls, (package_re, tests_re) = JOB_DETAILS[job]
-    return cls(package_re, tests_re, log=log, download_symbols=download_symbols)
+    return cls(package_re, tests_re, log=log, download_symbols=download_symbols,
+               substs=substs)
 
 def cachedmethod(cachefunc):
     '''Decorator to wrap a class or instance method with a memoizing callable that
     saves results in a (possibly shared) cache.
     '''
     def decorator(method):
         def wrapper(self, *args, **kwargs):
             mapping = cachefunc(self)
@@ -795,17 +798,19 @@ class Artifacts(object):
         self._log = log
         self._hg = hg
         self._git = git
         self._cache_dir = cache_dir
         self._skip_cache = skip_cache
         self._topsrcdir = topsrcdir
 
         try:
-            self._artifact_job = get_job_details(self._job, log=self._log, download_symbols=self._download_symbols)
+            self._artifact_job = get_job_details(self._job, log=self._log,
+                                                 download_symbols=self._download_symbols,
+                                                 substs=self._substs)
         except KeyError:
             self.log(logging.INFO, 'artifact',
                 {'job': self._job},
                 'Unknown job {job}')
             raise KeyError("Unknown job")
 
         self._task_cache = TaskCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache)
         self._artifact_cache = ArtifactCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache)