Bug 1488692 - Remove old ini files when updating wpt metadata r=jdm
authorJames Graham <james@hoppipolla.co.uk>
Wed, 05 Sep 2018 17:21:12 +0000
changeset 434998 403376728e2a6f023afe6ff2db8e576b062557fa
parent 434997 5020b4bda0efe420691c844a416272f98feea7aa
child 434999 0adb1474cc23ca5dc12363a7755e0bdb2b833709
push id107530
push userapavel@mozilla.com
push dateThu, 06 Sep 2018 04:44:27 +0000
treeherdermozilla-inbound@5f5d7a3ce332 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1488692 - Remove old ini files when updating wpt metadata r=jdm The easiest way to ensure that old metadata files are removed is simply to check that all the existing metadata files still correspond to a test file, and remove those that do not. This happens before metadata update so that if the metadata update causes a patch to be created, these changes are included in the patch. Differential Revision: https://phabricator.services.mozilla.com/D5015
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py
@@ -93,21 +93,48 @@ class UpdateMetadata(Step):
             state.suite_name = kwargs["suite_name"]
             state.product = kwargs["product"]
             state.config = kwargs["config"]
             state.extra_properties = kwargs["extra_property"]
             runner = MetadataUpdateRunner(self.logger, state)
+class RemoveObsolete(Step):
+    """Remove metadata files that don't corespond to an existing test file"""
+    def create(self, state):
+        if not state.kwargs["remove_obsolete"]:
+            return
+        paths = state.kwargs["test_paths"]
+        state.tests_path = state.paths["/"]["tests_path"]
+        state.metadata_path = state.paths["/"]["metadata_path"]
+        for url_paths in paths.itervalues():
+            tests_path = url_paths["tests_path"]
+            metadata_path = url_paths["metadata_path"]
+            for dirpath, dirnames, filenames in os.walk(metadata_path):
+                for filename in filenames:
+                    if filename == "__dir__.ini":
+                        continue
+                    if filename.endswith(".ini"):
+                        full_path = os.path.join(dirpath, filename)
+                        rel_path = os.path.relpath(full_path, metadata_path)
+                        test_path = os.path.join(tests_path, rel_path[:-4])
+                        if not os.path.exists(test_path):
+                            os.unlink(full_path)
 class UpdateRunner(StepRunner):
     """Runner for doing an overall update."""
     steps = [LoadConfig,
+             RemoveObsolete,
 class WPTUpdate(object):
     def __init__(self, logger, runner_cls=UpdateRunner, **kwargs):
         """Object that controls the running of a whole wptupdate.
         :param runner_cls: Runner subclass holding the overall list of
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -549,16 +549,18 @@ def create_parser_update(product_choices
                         help="Don't create a VCS commit containing the changes.")
     parser.add_argument("--sync", dest="sync", action="store_true", default=False,
                         help="Sync the tests with the latest from upstream (implies --patch)")
     parser.add_argument("--ignore-existing", action="store_true",
                         help="When updating test results only consider results from the logfiles provided, not existing expectations.")
     parser.add_argument("--stability", nargs="?", action="store", const="unstable", default=None,
         help=("Reason for disabling tests. When updating test results, disable tests that have "
               "inconsistent results across many runs with the given reason."))
+    parser.add_argument("--no-remove-obsolete", action="store_false", dest="remove_obsolete", default=True,
+                        help=("Don't remove metadata files that no longer correspond to a test file"))
     parser.add_argument("--no-store-state", action="store_false", dest="store_state",
                         help="Store state so that steps can be resumed after failure")
     parser.add_argument("--continue", action="store_true",
                         help="Continue a previously started run of the update script")
     parser.add_argument("--abort", action="store_true",
                         help="Clear state from a previous incomplete run of the update script")
     parser.add_argument("--exclude", action="store", nargs="*",
                         help="List of glob-style paths to exclude when syncing tests")
--- a/testing/web-platform/update/update.py
+++ b/testing/web-platform/update/update.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os
 from wptrunner.update.base import Step, StepRunner
-from wptrunner.update.update import LoadConfig, SyncFromUpstream, UpdateMetadata
+from wptrunner.update.update import LoadConfig, SyncFromUpstream, UpdateMetadata, RemoveObsolete
 from wptrunner.update.tree import NoVCSTree
 from .tree import GitTree, HgTree, GeckoCommit
 from .upstream import SyncToUpstream
 class LoadTrees(Step):
     """Load gecko tree and sync tree containing web-platform-tests"""
@@ -34,9 +34,10 @@ class LoadTrees(Step):
 class UpdateRunner(StepRunner):
     """Overall runner for updating web-platform-tests in Gecko."""
     steps = [LoadConfig,
+             RemoveObsolete,