Backed out 2 changesets (bug 1283919) for being the wrong way to fix this a=backout
authorWes Kocher <wkocher@mozilla.com>
Wed, 07 Sep 2016 14:22:21 -0700
changeset 313072 ab70808cd4b6c6ad9a57a9f71cfa495fcea0aecd
parent 312973 33e7ae9b3104e90ce56bbde1906efa97fb116449
child 313073 7c655e03eef77b961ad44f62aaa0221b7cc51a43
child 313122 4b0cc8a6ddd5fac8be71477ddb71811bc2a2e33d
push id20479
push userkwierso@gmail.com
push dateThu, 08 Sep 2016 01:08:46 +0000
treeherderfx-team@fb7c6b034329 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1283919
milestone51.0a1
backs outa328778db08dbbe7111f0c7948a05b0bca719373
7e1f25f59298f7dbed0862f0e24cdf6cca8d9234
Backed out 2 changesets (bug 1283919) for being the wrong way to fix this a=backout Backed out changeset a328778db08d (bug 1283919) Backed out changeset 7e1f25f59298 (bug 1283919)
browser/components/migration/moz.build
layout/base/moz.build
python/mozbuild/mozbuild/action/test_archive.py
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozpack/mozjar.py
testing/marionette/harness/marionette/tests/print-manifest-dirs.py
testing/marionette/moz.build
testing/testsuite-targets.mk
--- a/browser/components/migration/moz.build
+++ b/browser/components/migration/moz.build
@@ -1,16 +1,18 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 XPCSHELL_TESTS_MANIFESTS += ['tests/unit/xpcshell.ini']
 
+MARIONETTE_UNIT_MANIFESTS += ['tests/marionette/manifest.ini']
+
 JAR_MANIFESTS += ['jar.mn']
 
 XPIDL_SOURCES += [
     'nsIBrowserProfileMigrator.idl',
 ]
 
 XPIDL_MODULE = 'migration'
 
--- a/layout/base/moz.build
+++ b/layout/base/moz.build
@@ -199,16 +199,17 @@ LOCAL_INCLUDES += [
 if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'android':
     LOCAL_INCLUDES += [
         '/widget/android',
     ]
 
 FINAL_LIBRARY = 'xul'
 
 BROWSER_CHROME_MANIFESTS += ['tests/browser.ini']
+MARIONETTE_LAYOUT_MANIFESTS += ['tests/marionette/manifest.ini']
 MOCHITEST_MANIFESTS += ['tests/mochitest.ini']
 MOCHITEST_CHROME_MANIFESTS += ['tests/chrome/chrome.ini']
 
 CXXFLAGS += CONFIG['MOZ_CAIRO_CFLAGS']
 
 if CONFIG['_MSC_VER']:
     # This is intended as a temporary hack to support building with VS2015.
     # 'type cast': conversion from 'unsigned int' to 'void *' of greater size
--- a/python/mozbuild/mozbuild/action/test_archive.py
+++ b/python/mozbuild/mozbuild/action/test_archive.py
@@ -11,19 +11,16 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import itertools
 import os
 import sys
 import time
 
-from manifestparser import TestManifest
-from reftest import ReftestManifest
-
 from mozbuild.util import ensureParentDir
 from mozpack.files import FileFinder
 from mozpack.mozjar import JarWriter
 import mozpack.path as mozpath
 
 import buildconfig
 
 STAGE = mozpath.join(buildconfig.topobjdir, 'dist', 'test-stage')
@@ -92,45 +89,16 @@ ARCHIVE_FILES = {
             ],
         },
         {
             'source': buildconfig.topobjdir,
             'base': '_tests',
             'pattern': 'modules/**',
         },
         {
-            'source': buildconfig.topsrcdir,
-            'base': 'testing/marionette',
-            'patterns': [
-                'client/**',
-                'mach_test_package_commands.py',
-            ],
-            'dest': 'marionette',
-        },
-        {
-            'source': buildconfig.topsrcdir,
-            'base': 'testing/marionette/harness',
-            'pattern': '**',
-            'dest': 'marionette',
-            'ignore': [
-                'marionette/tests'
-            ]
-        },
-        {
-            'source': buildconfig.topsrcdir,
-            'base': '',
-            'manifests': [
-                'testing/marionette/harness/marionette/tests/unit-tests.ini',
-                'testing/marionette/harness/marionette/tests/webapi-tests.ini',
-            ],
-            # We also need the manifests and harness_unit tests
-            'pattern': 'testing/marionette/harness/marionette/tests/**',
-            'dest': 'marionette/tests',
-        },
-        {
             'source': buildconfig.topobjdir,
             'base': '_tests',
             'pattern': 'mozbase/**',
         },
         {
             'source': buildconfig.topsrcdir,
             'base': 'testing',
             'pattern': 'firefox-ui/**',
@@ -342,25 +310,16 @@ ARCHIVE_FILES = {
             'pattern': 'reftest/**',
         },
         {
             'source': buildconfig.topobjdir,
             'base': '',
             'pattern': 'mozinfo.json',
             'dest': 'reftest',
         },
-        {
-            'source': buildconfig.topsrcdir,
-            'base': '',
-            'manifests': [
-                'layout/reftests/reftest.list',
-                'testing/crashtest/crashtests.list',
-            ],
-            'dest': 'reftest/tests',
-        }
     ],
     'talos': [
         {
             'source': buildconfig.topsrcdir,
             'base': 'testing',
             'pattern': 'talos/**',
         },
     ],
@@ -450,77 +409,49 @@ for k, v in ARCHIVE_FILES.items():
 
     if not any(p.startswith('%s/' % k) for p in ignores):
         raise Exception('"common" ignore list probably should contain %s' % k)
 
 
 def find_files(archive):
     for entry in ARCHIVE_FILES[archive]:
         source = entry['source']
-        dest = entry.get('dest')
         base = entry.get('base', '')
-
         pattern = entry.get('pattern')
         patterns = entry.get('patterns', [])
         if pattern:
             patterns.append(pattern)
-
-        manifest = entry.get('manifest')
-        manifests = entry.get('manifests', [])
-        if manifest:
-            manifests.append(manifest)
-        if manifests:
-            dirs = find_manifest_dirs(buildconfig.topsrcdir, manifests)
-            patterns.extend({'{}/**'.format(d) for d in dirs})
-
+        dest = entry.get('dest')
         ignore = list(entry.get('ignore', []))
-        ignore.extend([
-            '**/.flake8',
-            '**/.mkdir.done',
-            '**/*.pyc',
-        ])
+        ignore.append('**/.mkdir.done')
+        ignore.append('**/*.pyc')
 
         common_kwargs = {
             'find_executables': False,
             'find_dotfiles': True,
             'ignore': ignore,
         }
 
         finder = FileFinder(os.path.join(source, base), **common_kwargs)
 
         for pattern in patterns:
             for p, f in finder.find(pattern):
                 if dest:
                     p = mozpath.join(dest, p)
                 yield p, f
 
 
-def find_manifest_dirs(topsrcdir, manifests):
-    """Routine to retrieve directories specified in a manifest, relative to topsrcdir.
-
-    It does not recurse into manifests, as we currently have no need for that.
-    """
-    dirs = set()
-
-    for p in manifests:
-        p = os.path.join(topsrcdir, p)
+def find_reftest_dirs(topsrcdir, manifests):
+    from reftest import ReftestManifest
 
-        if p.endswith('.ini'):
-            test_manifest = TestManifest()
-            test_manifest.read(p)
-            dirs |= set([os.path.dirname(m) for m in test_manifest.manifests()])
-
-        elif p.endswith('.list'):
-            m = ReftestManifest()
-            m.load(p)
-            dirs |= m.dirs
-
-        else:
-            raise Exception('"{}" is not a supported manifest format.'.format(
-                os.path.splitext(p)[1]))
+    dirs = set()
+    for p in manifests:
+        m = ReftestManifest()
+        m.load(os.path.join(topsrcdir, p))
+        dirs |= m.dirs
 
     dirs = {mozpath.normpath(d[len(topsrcdir):]).lstrip('/') for d in dirs}
 
     # Filter out children captured by parent directories because duplicates
     # will confuse things later on.
     def parents(p):
         while True:
             p = mozpath.dirname(p)
@@ -531,40 +462,66 @@ def find_manifest_dirs(topsrcdir, manife
     seen = set()
     for d in sorted(dirs, key=len):
         if not any(p in seen for p in parents(d)):
             seen.add(d)
 
     return sorted(seen)
 
 
+def insert_reftest_entries(entries):
+    """Reftests have their own mechanism for defining tests and locations.
+
+    This function is called when processing the reftest archive to process
+    reftest test manifests and insert the results into the existing list of
+    archive entries.
+    """
+    manifests = (
+        'layout/reftests/reftest.list',
+        'testing/crashtest/crashtests.list',
+    )
+
+    for base in find_reftest_dirs(buildconfig.topsrcdir, manifests):
+        entries.append({
+            'source': buildconfig.topsrcdir,
+            'base': '',
+            'pattern': '%s/**' % base,
+            'dest': 'reftest/tests',
+        })
+
+
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Produce test archives')
     parser.add_argument('archive', help='Which archive to generate')
     parser.add_argument('outputfile', help='File to write output to')
 
     args = parser.parse_args(argv)
 
     if not args.outputfile.endswith('.zip'):
         raise Exception('expected zip output file')
 
+    # Adjust reftest entries only if processing reftests (because it is
+    # unnecessary overhead otherwise).
+    if args.archive == 'reftest':
+        insert_reftest_entries(ARCHIVE_FILES['reftest'])
+
     file_count = 0
     t_start = time.time()
     ensureParentDir(args.outputfile)
     with open(args.outputfile, 'wb') as fh:
         # Experimentation revealed that level 5 is significantly faster and has
         # marginally larger sizes than higher values and is the sweet spot
         # for optimal compression. Read the detailed commit message that
         # introduced this for raw numbers.
         with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
             res = find_files(args.archive)
             for p, f in res:
-                writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
                 file_count += 1
+                writer.add(p.encode('utf-8'), f.read(), mode=f.mode)
 
     duration = time.time() - t_start
     zip_size = os.path.getsize(args.outputfile)
     basename = os.path.basename(args.outputfile)
     print('Wrote %d files in %d bytes to %s in %.2fs' % (
           file_count, zip_size, basename, duration))
 
 
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -1510,16 +1510,36 @@ VARIABLES = {
     'JETPACK_ADDON_MANIFESTS': (ManifestparserManifestList, list,
         """List of manifest files defining jetpack addon tests.
         """),
 
     'ANDROID_INSTRUMENTATION_MANIFESTS': (ManifestparserManifestList, list,
         """List of manifest files defining Android instrumentation tests.
         """),
 
+    'MARIONETTE_LAYOUT_MANIFESTS': (ManifestparserManifestList, list,
+        """List of manifest files defining marionette-layout tests.
+        """),
+
+    'MARIONETTE_LOOP_MANIFESTS': (ManifestparserManifestList, list,
+        """List of manifest files defining marionette-loop tests.
+        """),
+
+    'MARIONETTE_UNIT_MANIFESTS': (ManifestparserManifestList, list,
+        """List of manifest files defining marionette-unit tests.
+        """),
+
+    'MARIONETTE_UPDATE_MANIFESTS': (ManifestparserManifestList, list,
+        """List of manifest files defining marionette-update tests.
+        """),
+
+    'MARIONETTE_WEBAPI_MANIFESTS': (ManifestparserManifestList, list,
+        """List of manifest files defining marionette-webapi tests.
+        """),
+
     'METRO_CHROME_MANIFESTS': (ManifestparserManifestList, list,
         """List of manifest files defining metro browser chrome tests.
         """),
 
     'MOCHITEST_CHROME_MANIFESTS': (ManifestparserManifestList, list,
         """List of manifest files defining mochitest chrome tests.
         """),
 
--- a/python/mozbuild/mozpack/mozjar.py
+++ b/python/mozbuild/mozpack/mozjar.py
@@ -565,37 +565,35 @@ class JarWriter(object):
         if not self._optimize:
             end['cdir_offset'] = offset
             for entry, _ in self._contents.itervalues():
                 self._data.write(entry.serialize())
         # Store the end of central directory.
         self._data.write(end.serialize())
         self._data.close()
 
-    def add(self, name, data, compress=None, mode=None, skip_duplicates=False):
+    def add(self, name, data, compress=None, mode=None):
         '''
         Add a new member to the jar archive, with the given name and the given
         data.
         The compress option indicates if the given data should be compressed
         (True), not compressed (False), or compressed according to the default
         defined when creating the JarWriter (None).
         When the data should be compressed (True or None with self.compress ==
         True), it is only really compressed if the compressed size is smaller
         than the uncompressed size.
         The mode option gives the unix permissions that should be stored
         for the jar entry.
-        If a duplicated member is found skip_duplicates will prevent raising
-        an exception if set to True.
         The given data may be a buffer, a file-like instance, a Deflater or a
         JarFileReader instance. The latter two allow to avoid uncompressing
         data to recompress it.
         '''
         name = mozpath.normsep(name)
 
-        if name in self._contents and not skip_duplicates:
+        if name in self._contents:
             raise JarWriterError("File %s already in JarWriter" % name)
         if compress is None:
             compress = self._compress
         if (isinstance(data, JarFileReader) and data.compressed == compress) \
                 or (isinstance(data, Deflater) and data.compress == compress):
             deflater = data
         else:
             deflater = Deflater(compress, compress_level=self._compress_level)
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette/tests/print-manifest-dirs.py
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from manifestparser import TestManifest
+import os.path
+import sys
+
+def print_test_dirs(topsrcdir, manifest_file):
+    """
+    Simple routine which prints the paths of directories specified
+    in a Marionette manifest, relative to topsrcdir.  This does not recurse 
+    into manifests, as we currently have no need for that.
+    """
+
+    dirs = set()
+    # output the directory of this (parent) manifest
+    topsrcdir = os.path.abspath(topsrcdir)
+    scriptdir = os.path.abspath(os.path.dirname(__file__))
+    dirs.add(scriptdir[len(topsrcdir) + 1:])
+
+    # output the directories of all the other manifests
+    manifest = TestManifest()
+    manifest.read(manifest_file)
+    for i in manifest.get():
+        d = os.path.dirname(i['manifest'])[len(topsrcdir) + 1:]
+        dirs.add(d)
+    for path in dirs:
+        path = path.replace('\\', '/')
+        print path
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        print >>sys.stderr, "Usage: %s topsrcdir manifest.ini" % sys.argv[0]
+        sys.exit(1)
+
+    print_test_dirs(sys.argv[1], sys.argv[2])
+
--- a/testing/marionette/moz.build
+++ b/testing/marionette/moz.build
@@ -1,12 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 DIRS += ["components"]
 
 JAR_MANIFESTS += ["jar.mn"]
-
+MARIONETTE_UNIT_MANIFESTS += ["harness/marionette/tests/unit/unit-tests.ini"]
+MARIONETTE_UPDATE_MANIFESTS += ["harness/marionette/tests/update-tests.ini"]
+MARIONETTE_WEBAPI_MANIFESTS += ["harness/marionette/tests/webapi-tests.ini"]
 XPCSHELL_TESTS_MANIFESTS += ["unit.ini"]
 
 with Files("**"):
     BUG_COMPONENT = ("Testing", "Marionette")
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -177,16 +177,17 @@ PKG_STAGE = $(DIST)/test-stage
 
 stage-all: \
   stage-config \
   stage-mach \
   stage-extensions \
   stage-mochitest \
   stage-jstests \
   stage-jetpack \
+  stage-marionette \
   stage-luciddream \
   test-packages-manifest \
   $(NULL)
 ifdef MOZ_WEBRTC
 stage-all: stage-steeplechase
 endif
 
 ifdef COMPILE_ENVIRONMENT
@@ -325,16 +326,34 @@ stage-steeplechase: make-stage-dir
 	cp -RL $(DIST)/xpi-stage/specialpowers $(PKG_STAGE)/steeplechase
 	cp -RL $(topsrcdir)/testing/profiles/prefs_general.js $(PKG_STAGE)/steeplechase
 
 LUCIDDREAM_DIR=$(PKG_STAGE)/luciddream
 stage-luciddream: make-stage-dir
 	$(NSINSTALL) -D $(LUCIDDREAM_DIR)
 	@(cd $(topsrcdir)/testing/luciddream && tar $(TAR_CREATE_FLAGS) - *) | (cd $(LUCIDDREAM_DIR)/ && tar -xf -)
 
+MARIONETTE_DIR=$(PKG_STAGE)/marionette
+stage-marionette: make-stage-dir
+	$(NSINSTALL) -D $(MARIONETTE_DIR)/tests
+	$(NSINSTALL) -D $(MARIONETTE_DIR)/client
+	@(cd $(topsrcdir)/testing/marionette/harness && tar --exclude marionette/tests $(TAR_CREATE_FLAGS) - *) | (cd $(MARIONETTE_DIR)/ && tar -xf -)
+	@(cd $(topsrcdir)/testing/marionette/client && tar $(TAR_CREATE_FLAGS) - *) | (cd $(MARIONETTE_DIR)/client && tar -xf -)
+	cp $(topsrcdir)/testing/marionette/mach_test_package_commands.py $(MARIONETTE_DIR)
+	$(PYTHON) $(topsrcdir)/testing/marionette/harness/marionette/tests/print-manifest-dirs.py \
+          $(topsrcdir) \
+          $(topsrcdir)/testing/marionette/harness/marionette/tests/unit-tests.ini \
+          | (cd $(topsrcdir) && xargs tar $(TAR_CREATE_FLAGS) -) \
+          | (cd $(MARIONETTE_DIR)/tests && tar -xf -)
+	$(PYTHON) $(topsrcdir)/testing/marionette/harness/marionette/tests/print-manifest-dirs.py \
+          $(topsrcdir) \
+          $(topsrcdir)/testing/marionette/harness/marionette/tests/webapi-tests.ini \
+          | (cd $(topsrcdir) && xargs tar $(TAR_CREATE_FLAGS) -) \
+          | (cd $(MARIONETTE_DIR)/tests && tar -xf -)
+
 stage-instrumentation-tests: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/instrumentation stage-package
 
 TEST_EXTENSIONS := \
     specialpowers@mozilla.org.xpi \
 	$(NULL)
 
 stage-extensions: make-stage-dir
@@ -352,13 +371,14 @@ stage-extensions: make-stage-dir
   make-stage-dir \
   stage-all \
   stage-b2g \
   stage-config \
   stage-mochitest \
   stage-jstests \
   stage-android \
   stage-jetpack \
+  stage-marionette \
   stage-steeplechase \
   stage-instrumentation-tests \
   stage-luciddream \
   test-packages-manifest \
   $(NULL)