Bug 1330421 - Change the structure of test archives include generated test support files in common.tests.zip for the benefit of artifact builds. r=gps
☠☠ backed out by d434f5251333 ☠ ☠
authorChris Manchester <cmanchester@mozilla.com>
Thu, 12 Jan 2017 15:29:48 -0800
changeset 380652 9778c00980aa18fb37e6f0e3cb2387cdccad43fc
parent 380651 deb7b3426d0a43253c0e76f24a1de310f8cab103
child 380653 89e2853bf0e8f520ac17d566bca20386c67fb339
push id1468
push userasasaki@mozilla.com
push dateMon, 05 Jun 2017 19:31:07 +0000
treeherdermozilla-release@0641fc6ee9d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1330421
milestone54.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1330421 - Change the structure of test archives include generated test support files in common.tests.zip for the benefit of artifact builds. r=gps Generated support files for specific tests generally end up in a harness specific test archive, but artifact builds, which might not want to generate these files themselves, get all of their test related files from the common.tests.zip. This effectively moves a small number of these support files from harness specific archives to the common test archive, making it possible to run tests that depend on these files against artifact builds. MozReview-Commit-ID: BvEj1ot1OTw
python/mozbuild/mozbuild/action/test_archive.py
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -15,17 +15,19 @@ import itertools
 import os
 import sys
 import time
 
 from manifestparser import TestManifest
 from reftest import ReftestManifest
 
 from mozbuild.util import ensureParentDir
-from mozpack.files import FileFinder
+from mozpack.copier import FileRegistry
+from mozpack.files import ExistingFile, FileFinder
+from mozpack.manifests import InstallManifest
 from mozpack.mozjar import JarWriter
 import mozpack.path as mozpath
 
 import buildconfig
 
 STAGE = mozpath.join(buildconfig.topobjdir, 'dist', 'test-stage')
 
 TEST_HARNESS_BINS = [
@@ -67,16 +69,32 @@ TEST_PLUGIN_DIRS = [
 ]
 
 GMP_TEST_PLUGIN_DIRS = [
     'gmp-clearkey/**',
     'gmp-fake/**',
     'gmp-fakeopenh264/**',
 ]
 
+# These entries will be used by artifact builds to re-construct an
+# objdir with the appropriate generated support files.
+OBJDIR_TEST_FILES = {
+    'xpcshell': {
+        'source': buildconfig.topobjdir,
+        'base': '_tests/xpcshell',
+        'pattern': '**',
+        'dest': 'xpcshell/tests',
+    },
+    'mochitest': {
+        'source': buildconfig.topobjdir,
+        'base': '_tests/testing',
+        'pattern': 'mochitest/**',
+    },
+}
+
 
 ARCHIVE_FILES = {
     'common': [
         {
             'source': STAGE,
             'base': '',
             'pattern': '**',
             'ignore': [
@@ -297,16 +315,17 @@ ARCHIVE_FILES = {
     'gtest': [
         {
             'source': STAGE,
             'base': '',
             'pattern': 'gtest/**',
         },
     ],
     'mochitest': [
+        OBJDIR_TEST_FILES['mochitest'],
         {
             'source': buildconfig.topobjdir,
             'base': '_tests/testing',
             'pattern': 'mochitest/**',
         },
         {
             'source': STAGE,
             'base': '',
@@ -379,22 +398,17 @@ ARCHIVE_FILES = {
         {
             'source': buildconfig.topobjdir,
             'base': '',
             'pattern': 'mozinfo.json',
             'dest': 'web-platform',
         },
     ],
     'xpcshell': [
-        {
-            'source': buildconfig.topobjdir,
-            'base': '_tests/xpcshell',
-            'pattern': '**',
-            'dest': 'xpcshell/tests',
-        },
+        OBJDIR_TEST_FILES['xpcshell'],
         {
             'source': buildconfig.topsrcdir,
             'base': 'testing/xpcshell',
             'patterns': [
                 'head.js',
                 'mach_test_package_commands.py',
                 'moz-http2/**',
                 'moz-spdy/**',
@@ -438,18 +452,61 @@ for k, v in ARCHIVE_FILES.items():
 
     ignores = set(itertools.chain(*(e.get('ignore', [])
                                   for e in ARCHIVE_FILES['common'])))
 
     if not any(p.startswith('%s/' % k) for p in ignores):
         raise Exception('"common" ignore list probably should contain %s' % k)
 
 
+def find_generated_harness_files():
+    # TEST_HARNESS_FILES end up in an install manifest at
+    # $topsrcdir/_build_manifests/install/_tests.
+    manifest = InstallManifest(mozpath.join(buildconfig.topobjdir,
+                                            '_build_manifests',
+                                            'install',
+                                            '_tests'))
+    registry = FileRegistry()
+    manifest.populate_registry(registry)
+    # Conveniently, the generated files we care about will already
+    # exist in the objdir, so we can identify relevant files if
+    # they're an `ExistingFile` instance.
+    return [mozpath.join('_tests', p) for p in registry.paths()
+            if isinstance(registry[p], ExistingFile)]
+
+
 def find_files(archive):
-    for entry in ARCHIVE_FILES[archive]:
+    extra_entries = []
+    generated_harness_files = find_generated_harness_files()
+
+    if archive == 'common':
+        # Construct entries ensuring all our generated harness files are
+        # packaged in the common tests zip.
+        packaged_paths = set()
+        for entry in OBJDIR_TEST_FILES.values():
+            pat = mozpath.join(entry['base'], entry['pattern'])
+            del entry['pattern']
+            patterns = []
+            for path in generated_harness_files:
+                if mozpath.match(path, pat):
+                    patterns.append(path[len(entry['base']) + 1:])
+                    packaged_paths.add(path)
+            if patterns:
+                entry['patterns'] = patterns
+                extra_entries.append(entry)
+        entry = {
+            'source': buildconfig.topobjdir,
+            'base': '_tests',
+            'patterns': [],
+        }
+        for path in set(generated_harness_files) - packaged_paths:
+            entry['patterns'].append(path[len('_tests') + 1:])
+        extra_entries.append(entry)
+
+    for entry in ARCHIVE_FILES[archive] + extra_entries:
         source = entry['source']
         dest = entry.get('dest')
         base = entry.get('base', '')
 
         pattern = entry.get('pattern')
         patterns = entry.get('patterns', [])
         if pattern:
             patterns.append(pattern)
@@ -464,16 +521,22 @@ def find_files(archive):
 
         ignore = list(entry.get('ignore', []))
         ignore.extend([
             '**/.flake8',
             '**/.mkdir.done',
             '**/*.pyc',
         ])
 
+        if archive != 'common' and base.startswith('_tests'):
+            # We may have generated_harness_files to exclude from this entry.
+            for path in generated_harness_files:
+                if path.startswith(base):
+                    ignore.append(path[len(base) + 1:])
+
         common_kwargs = {
             'find_executables': False,
             'find_dotfiles': True,
             'ignore': ignore,
         }
 
         finder = FileFinder(os.path.join(source, base), **common_kwargs)