Bug 1302796 - Add --check-clean flag to mach manifest update and mozlint integration, r=ahal,Ms2ger
authorJames Graham <james@hoppipolla.co.uk>
Tue, 13 Sep 2016 14:20:20 +0100
changeset 314218 f06b81832d6fd8507a2591cc45771d3aadb4553e
parent 314217 d3fc2851c3fc76920cd6759e8d50b6e698518ac5
child 314219 efd117281a0291362b8450a5ccc08e88c886417b
push id30711
push userkwierso@gmail.com
push dateFri, 16 Sep 2016 20:38:50 +0000
treeherdermozilla-central@85647c98a790 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal, Ms2ger
bugs1302796
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1302796 - Add --check-clean flag to mach manifest update and mozlint integration, r=ahal,Ms2ger MozReview-Commit-ID: 8Z4ywNEbF8G
testing/web-platform/mach_commands.py
testing/web-platform/manifestupdate.py
tools/lint/wpt_manifest.lint
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -232,34 +232,23 @@ testing/web-platform/tests for tests tha
             wpt_kwargs = vars(p.parse_args(["--manifest-update", path]))
             context.commands.dispatch("web-platform-tests", context, **wpt_kwargs)
 
         if proc:
             proc.wait()
 
 
 class WPTManifestUpdater(MozbuildObject):
-    def run_update(self):
-        import imp
+    def run_update(self, check_clean=False, **kwargs):
+        import manifestupdate
         from wptrunner import wptlogging
-        from wptrunner.wptcommandline import get_test_paths, set_from_config
-        from wptrunner.testloader import ManifestLoader
-
-        wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
 
-        localpaths = imp.load_source("localpaths",
-                                     os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
-        kwargs = {"config": os.path.join(wpt_dir, "wptrunner.ini"),
-                  "tests_root": None,
-                  "metadata_root": None}
-
-        wptlogging.setup({}, {"mach": sys.stdout})
-        set_from_config(kwargs)
-        test_paths = get_test_paths(kwargs["config"])
-        ManifestLoader(test_paths, force_manifest_update=True).load()
+        logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
+        wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
+        manifestupdate.update(logger, wpt_dir, check_clean)
 
 
 def create_parser_wpt():
     from wptrunner import wptcommandline
     return wptcommandline.create_parser(["firefox"])
 
 def create_parser_update():
     from update import updatecommandline
@@ -287,18 +276,26 @@ def create_parser_create():
     p.add_argument("--mismatch", action="store_true",
                    help="Create a mismatch reftest")
     p.add_argument("--wait", action="store_true",
                    help="Create a reftest that waits until takeScreenshot() is called")
     p.add_argument("path", action="store", help="Path to the test file")
     return p
 
 
+def create_parser_manifest_update():
+    import manifestupdate
+    return manifestupdate.create_parser()
+
+
 @CommandProvider
 class MachCommands(MachCommandBase):
+    def setup(self):
+        self._activate_virtualenv()
+
     @Command("web-platform-tests",
              category="testing",
              conditions=[conditions.is_firefox],
              parser=create_parser_wpt)
     def run_web_platform_tests(self, **params):
         self.setup()
 
         if "test_objects" in params:
@@ -331,19 +328,16 @@ class MachCommands(MachCommandBase):
         return wpt_updater.run_update(**params)
 
     @Command("wpt-update",
              category="testing",
              parser=create_parser_update)
     def update_wpt(self, **params):
         return self.update_web_platform_tests(**params)
 
-    def setup(self):
-        self._activate_virtualenv()
-
     @Command("web-platform-tests-reduce",
              category="testing",
              conditions=[conditions.is_firefox],
              parser=create_parser_reduce)
     def unstable_web_platform_tests(self, **params):
         self.setup()
         wpt_reduce = self._spawn(WebPlatformTestsReduce)
         return wpt_reduce.run_reduce(**params)
@@ -367,13 +361,14 @@ class MachCommands(MachCommandBase):
     @Command("wpt-create",
              category="testing",
              conditions=[conditions.is_firefox],
              parser=create_parser_create)
     def create_wpt(self, **params):
         return self.create_web_platform_test(**params)
 
     @Command("wpt-manifest-update",
-             category="testing")
-    def wpt_manifest_update(self, **parms):
+             category="testing",
+             parser=create_parser_manifest_update)
+    def wpt_manifest_update(self, **params):
         self.setup()
         wpt_manifest_updater = self._spawn(WPTManifestUpdater)
-        wpt_manifest_updater.run_update()
+        return wpt_manifest_updater.run_update(**params)
new file mode 100644
--- /dev/null
+++ b/testing/web-platform/manifestupdate.py
@@ -0,0 +1,89 @@
+import argparse
+import imp
+import os
+import sys
+
+from mozlog.structured import commandline
+from wptrunner.wptcommandline import get_test_paths, set_from_config
+from wptrunner.testloader import ManifestLoader
+
+def create_parser():
+    p = argparse.ArgumentParser()
+    p.add_argument("--check-clean", action="store_true",
+                   help="Check that updating the manifest doesn't lead to any changes")
+    commandline.add_logging_group(p)
+
+    return p
+
+
+def update(logger, wpt_dir, check_clean=True):
+    localpaths = imp.load_source("localpaths",
+                                 os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
+    kwargs = {"config": os.path.join(wpt_dir, "wptrunner.ini"),
+              "tests_root": None,
+              "metadata_root": None}
+
+    set_from_config(kwargs)
+    config = kwargs["config"]
+    test_paths = get_test_paths(config)
+
+    if check_clean:
+        old_manifests = {}
+        for data in test_paths.itervalues():
+            path = os.path.join(data["metadata_path"], "MANIFEST.json")
+            with open(path) as f:
+                old_manifests[path] = f.readlines()
+
+    try:
+        ManifestLoader(test_paths, force_manifest_update=True).load()
+
+        rv = 0
+
+        if check_clean:
+            clean = diff_manifests(logger, old_manifests)
+            if not clean:
+                rv = 1
+    finally:
+        if check_clean:
+            for path, data in old_manifests.iteritems():
+                logger.info("Restoring manifest %s" % path)
+                with open(path, "w") as f:
+                    f.writelines(data)
+
+    return rv
+
+def diff_manifests(logger, old_manifests):
+    logger.info("Diffing old and new manifests")
+    import difflib
+
+    clean = True
+    for path, old in old_manifests.iteritems():
+        with open(path) as f:
+            new = f.readlines()
+
+        if old != new:
+            clean = False
+            sm = difflib.SequenceMatcher(a=old, b=new)
+            for group in sm.get_grouped_opcodes():
+                logged = False
+                message = []
+                for op, old_0, old_1, new_0, new_1 in group:
+                    if op != "equal" and not logged:
+                        logged = True
+                        logger.lint_error(path=path,
+                                          message="Manifest changed",
+                                          lineno=(old_0 + 1),
+                                          source="\n".join(old[old_0:old_1]),
+                                          linter="wpt-manifest")
+                    if op == "equal":
+                        message.extend(' ' + line for line in old[old_0:old_1])
+                    if op in ('replace', 'delete'):
+                        message.extend('-' + line for line in old[old_0:old_1])
+                    if op in ('replace', 'insert'):
+                        message.extend('+' + line for line in new[new_0:new_1])
+                logger.info("".join(message))
+    if clean:
+        logger.info("No differences found")
+
+    return clean
+
new file mode 100644
--- /dev/null
+++ b/tools/lint/wpt_manifest.lint
@@ -0,0 +1,34 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import imp
+import json
+import os
+import sys
+
+from mozprocess import ProcessHandler
+
+from mozlint import result
+
+
+def lint(files, logger, **kwargs):
+    wpt_dir = os.path.join(kwargs["root"], "testing", "web-platform")
+    manifestupdate = imp.load_source("manifestupdate",
+                                     os.path.join(wpt_dir, "manifestupdate.py"))
+    manifestupdate.update(logger, wpt_dir, True)
+
+
+LINTER = {
+    'name': "wpt_manifest",
+    'description': "web-platform-tests manifest lint",
+    'include': [
+        'testing/web-platform/tests',
+        'testing/web-platform/mozilla/tests',
+    ],
+    'exclude': [],
+    'type': 'structured_log',
+    'payload': lint,
+}