Bug 1473915 - Set up infra so we can move the wpt-manifest out of tree r=gps
authorAhilya Sinha <asinha@mozilla.com>
Thu, 20 Sep 2018 12:07:58 +0000
changeset 437435 fcb3abf6410154560a362dbe1aad16f2d11fd42e
parent 437434 7fb27b858fda61306e1de21604438f39e98c5fd0
child 437436 75a963db26819ad193180efcd003b6e696718edd
push id34683
push userapavel@mozilla.com
push dateThu, 20 Sep 2018 21:54:05 +0000
treeherdermozilla-central@4d3cd0ab7277 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1473915
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1473915 - Set up infra so we can move the wpt-manifest out of tree r=gps Changes the wpt manifest path to the topobjdir instead so it can be moved out of tree. Other changes so that the manifest download and update, and |mach wpt| and |mach test <wpt-test>| work with the new path. The manifest is also downloaded and updated when creating the tests-archive to ensure that it exists when we run tests on TC. MozReview-Commit-ID: Fp6UsKJjhTU Differential Revision: https://phabricator.services.mozilla.com/D5312
python/mozbuild/mozbuild/action/download_wpt_manifest.py
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/emitter.py
taskcluster/ci/source-test/mozlint.yml
testing/mozbase/moztest/moztest/resolve.py
testing/mozbase/moztest/tests/test_resolve.py
testing/testsuite-targets.mk
testing/web-platform/mach_commands.py
testing/web-platform/manifestdownload.py
testing/web-platform/manifestupdate.py
testing/web-platform/moz.build
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/download_wpt_manifest.py
@@ -0,0 +1,22 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This action is used to generate the wpt manifest
+
+import os
+import sys
+
+import buildconfig
+
+
+def main():
+    print("Downloading wpt manifest")
+    man_path = os.path.join(buildconfig.topobjdir, '_tests', 'web-platform')
+    sys.path.insert(0, buildconfig.topsrcdir)
+    import manifestdownload
+    manifestdownload.run(man_path, buildconfig.topsrcdir, force=True)
+
+
+if __name__ == '__main__':
+    sys.exit(main())
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -39,17 +39,16 @@ from mozbuild.util import (
 )
 
 from .. import schedules
 
 from ..testing import (
     all_test_flavors,
     read_manifestparser_manifest,
     read_reftest_manifest,
-    read_wpt_manifest,
 )
 
 import mozpack.path as mozpath
 from types import FunctionType
 
 import itertools
 
 
@@ -890,22 +889,18 @@ def TypedListWithAction(typ, action):
     """
     class _TypedListWithAction(ContextDerivedValue, TypedList(typ), ListWithAction):
         def __init__(self, context, *args):
             def _action(item):
                 return item, action(context, item)
             super(_TypedListWithAction, self).__init__(action=_action, *args)
     return _TypedListWithAction
 
-WebPlatformTestManifest = TypedNamedTuple("WebPlatformTestManifest",
-                                          [("manifest_path", unicode),
-                                           ("test_root", unicode)])
 ManifestparserManifestList = OrderedPathListWithAction(read_manifestparser_manifest)
 ReftestManifestList = OrderedPathListWithAction(read_reftest_manifest)
-WptManifestList = TypedListWithAction(WebPlatformTestManifest, read_wpt_manifest)
 
 OrderedSourceList = ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList)
 OrderedTestFlavorList = TypedList(Enum(*all_test_flavors()),
                                   StrictOrderingOnAppendList)
 OrderedStringList = TypedList(unicode, StrictOrderingOnAppendList)
 DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList),
                                                 ('tags', OrderedStringList),
                                                 ('flavors', OrderedTestFlavorList))
@@ -1877,20 +1872,16 @@ VARIABLES = {
         """),
 
     'CRASHTEST_MANIFESTS': (ReftestManifestList, list,
         """List of manifest files defining crashtests.
 
         These are commonly named crashtests.list.
         """),
 
-    'WEB_PLATFORM_TESTS_MANIFESTS': (WptManifestList, list,
-        """List of (manifest_path, test_path) defining web-platform-tests.
-        """),
-
     'WEBRTC_SIGNALLING_TEST_MANIFESTS': (ManifestparserManifestList, list,
         """List of manifest files defining WebRTC signalling tests.
         """),
 
     'XPCSHELL_TESTS_MANIFESTS': (ManifestparserManifestList, list,
         """List of manifest files defining xpcshell tests.
         """),
 
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -78,17 +78,16 @@ from mozpack.chrome.manifest import (
     Manifest,
 )
 
 from .reader import SandboxValidationError
 
 from ..testing import (
     TEST_MANIFESTS,
     REFTEST_FLAVORS,
-    WEB_PLATFORM_TESTS_FLAVORS,
     SupportFilesConverter,
 )
 
 from .context import (
     Context,
     SourcePath,
     ObjDirPath,
     Path,
@@ -1453,21 +1452,16 @@ class TreeMetadataEmitter(LoggingMixin):
                 for obj in self._process_test_manifest(context, info, path, manifest):
                     yield obj
 
         for flavor in REFTEST_FLAVORS:
             for path, manifest in context.get('%s_MANIFESTS' % flavor.upper(), []):
                 for obj in self._process_reftest_manifest(context, flavor, path, manifest):
                     yield obj
 
-        for flavor in WEB_PLATFORM_TESTS_FLAVORS:
-            for path, manifest in context.get("%s_MANIFESTS" % flavor.upper().replace('-', '_'), []):
-                for obj in self._process_web_platform_tests_manifest(context, path, manifest):
-                    yield obj
-
     def _process_test_manifest(self, context, info, manifest_path, mpmanifest):
         flavor, install_root, install_subdir, package_tests = info
 
         path = manifest_path.full_path
         manifest_dir = mozpath.dirname(path)
         manifest_reldir = mozpath.dirname(mozpath.relpath(path,
             context.config.topsrcdir))
         manifest_sources = [mozpath.relpath(pth, context.config.topsrcdir)
@@ -1586,50 +1580,16 @@ class TreeMetadataEmitter(LoggingMixin):
                 'name': mozpath.basename(test),
                 'head': '',
                 'support-files': '',
                 'subsuite': '',
             })
 
         yield obj
 
-    def _process_web_platform_tests_manifest(self, context, paths, manifest):
-        manifest_path, tests_root = paths
-        manifest_full_path = mozpath.normpath(mozpath.join(
-            context.srcdir, manifest_path))
-        manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path,
-            context.config.topsrcdir))
-        tests_root = mozpath.normpath(mozpath.join(context.srcdir, tests_root))
-
-        # Create a equivalent TestManifest object
-        obj = TestManifest(context, manifest_full_path, manifest,
-                           flavor="web-platform-tests",
-                           relpath=mozpath.join(manifest_reldir,
-                                                mozpath.basename(manifest_path)),
-                           install_prefix="web-platform/")
-
-
-        for test_type, path, tests in manifest:
-            path = mozpath.join(tests_root, path)
-            if test_type not in ["testharness", "reftest", "wdspec"]:
-                continue
-
-            for test in tests:
-                obj.tests.append({
-                    'path': path,
-                    'here': mozpath.dirname(path),
-                    'manifest': manifest_path,
-                    'name': test.id,
-                    'head': '',
-                    'support-files': '',
-                    'subsuite': '',
-                })
-
-        yield obj
-
     def _process_jar_manifests(self, context):
         jar_manifests = context.get('JAR_MANIFESTS', [])
         if len(jar_manifests) > 1:
             raise SandboxValidationError('While JAR_MANIFESTS is a list, '
                 'it is currently limited to one value.', context)
 
         for path in jar_manifests:
             yield JARManifest(context, path)
--- a/taskcluster/ci/source-test/mozlint.yml
+++ b/taskcluster/ci/source-test/mozlint.yml
@@ -134,23 +134,21 @@ test-manifest:
             - 'tools/lint/**'
 
 wptlint-gecko:
     description: web-platform-tests linter
     platform: lint/opt
     treeherder:
         symbol: W
     run:
-        mach: lint -l wpt -l wpt_manifest -f treeherder
+        mach: lint -l wpt -f treeherder
     when:
         files-changed:
             - 'testing/web-platform/tests/**'
             - 'testing/web-platform/mozilla/tests/**'
-            - 'testing/web-platform/meta/MANIFEST.json'
-            - 'testing/web-platform/mozilla/meta/MANIFEST.json'
 
 yaml:
     description: yamllint run over the gecko codebase
     platform: lint/opt
     treeherder:
         symbol: yaml
     run:
         mach: lint -l yaml -f treeherder
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -277,20 +277,23 @@ def rewrite_test_base(test, new_base, ho
 
 class TestMetadata(object):
     """Holds information about tests.
 
     This class provides an API to query tests active in the build
     configuration.
     """
 
-    def __init__(self, all_tests, test_defaults=None):
+    def __init__(self, all_tests, srcdir, test_defaults=None):
         self._tests_by_path = OrderedDefaultDict(list)
         self._tests_by_flavor = defaultdict(set)
         self._test_dirs = set()
+        self._objdir = os.path.abspath(os.path.join(all_tests, os.pardir))
+        self._wpt_loaded = False
+        self._srcdir = srcdir
 
         with open(all_tests, 'rb') as fh:
             test_data = pickle.load(fh)
         defaults = None
         if test_defaults:
             with open(test_defaults, 'rb') as fh:
                 defaults = pickle.load(fh)
         for path, tests in test_data.items():
@@ -373,16 +376,19 @@ class TestMetadata(object):
 
         paths = paths or []
         paths = [mozpath.normpath(p) for p in paths]
         if not paths:
             paths = [None]
 
         candidate_paths = set()
 
+        if any(self.is_wpt_path(path) for path in paths):
+            self.add_wpt_manifest_data()
+
         for path in sorted(paths):
             if path is None:
                 candidate_paths |= set(self._tests_by_path.keys())
                 continue
 
             if '*' in path:
                 candidate_paths |= {p for p in self._tests_by_path
                                     if mozpath.match(p, path)}
@@ -400,16 +406,71 @@ class TestMetadata(object):
             candidate_paths |= {p for p in self._tests_by_path if path in p}
 
         for p in sorted(candidate_paths):
             tests = self._tests_by_path[p]
 
             for test in fltr(tests):
                 yield test
 
+    def is_wpt_path(self, path):
+        if path is None:
+            return True
+        if mozpath.match(path, "testing/web-platform/tests/**"):
+            return True
+        if mozpath.match(path, "testing/web-platform/mozilla/tests/**"):
+            return True
+        return False
+
+    def add_wpt_manifest_data(self):
+        if self._wpt_loaded:
+            return
+
+        wpt_path = os.path.join(self._srcdir, "testing", "web-platform")
+        wptrunner_path = os.path.join(wpt_path, "tests", "tools", "wptrunner")
+        manifest_path = os.path.join(self._objdir, "_tests", "web-platform")
+
+        sys.path = [wpt_path, wptrunner_path] + sys.path
+
+        import manifestdownload
+        import wptrunner
+        from wptrunner.wptrunner import testloader
+
+        manifestdownload.run(manifest_path, self._srcdir)
+
+        kwargs = {"config": os.path.join(self._objdir, "_tests", "web-platform",
+                                         "wptrunner.local.ini"),
+                  "tests_root": None,
+                  "metadata_root": None}
+
+        wptrunner.wptcommandline.set_from_config(kwargs)
+        manifests = testloader.ManifestLoader(kwargs["test_paths"]).load()
+
+        for manifest, data in manifests.iteritems():
+            tests_root = data["tests_path"]
+            for test_type, path, tests in manifest:
+                path = os.path.join(tests_root, path)
+                if test_type not in ["testharness", "reftest", "wdspec"]:
+                    continue
+                for test in tests:
+                    self._tests_by_path[path].append({
+                        "path": path,
+                        "flavor": "web-platform-tests",
+                        "here": os.path.dirname(path),
+                        "manifest": data["manifest_path"],
+                        "name": test.id,
+                        "file_relpath": path,
+                        "head": "",
+                        "support-files": "",
+                        "subsuite": test_type,
+                        "dir_relpath": os.path.relpath(os.path.dirname(path), wpt_path)
+                        })
+
+        self._wpt_loaded = True
+
 
 class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
 
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
 
         # If installing tests is going to result in re-generating the build
@@ -423,16 +484,17 @@ class TestResolver(MozbuildObject):
                 b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend',
                 b'BACKEND_GENERATION_SCRIPT': mozpath.join(
                     self.topsrcdir, 'build', 'gen_test_backend.py'),
             },
         )
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
                                                 'all-tests.pkl'),
+                                   self.topsrcdir,
                                    test_defaults=os.path.join(self.topobjdir,
                                                               'test-defaults.pkl'))
 
         self._test_rewrites = {
             'a11y': os.path.join(self.topobjdir, '_tests', 'testing',
                                  'mochitest', 'a11y'),
             'browser-chrome': os.path.join(self.topobjdir, '_tests', 'testing',
                                            'mochitest', 'browser'),
--- a/testing/mozbase/moztest/tests/test_resolve.py
+++ b/testing/mozbase/moztest/tests/test_resolve.py
@@ -173,17 +173,19 @@ class Base(unittest.TestCase):
         all_tests.flush()
         self._temp_files.append(all_tests)
 
         test_defaults = NamedTemporaryFile(mode='wb')
         pickle.dump(TEST_DEFAULTS, test_defaults)
         test_defaults.flush()
         self._temp_files.append(test_defaults)
 
-        return TestMetadata(all_tests.name, test_defaults=test_defaults.name)
+        rv = TestMetadata(all_tests.name, "/firefox/", test_defaults=test_defaults.name)
+        rv._wpt_loaded = True  # Don't try to load the wpt manifest
+        return rv
 
 
 class TestTestMetadata(Base):
     def test_load(self):
         t = self._get_test_metadata()
         self.assertEqual(len(t._tests_by_path), 8)
 
         self.assertEqual(len(list(t.tests_with_flavor('xpcshell'))), 3)
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -148,18 +148,21 @@ ifdef UPLOAD_PATH
 test_archive_dir = $(UPLOAD_PATH)
 else
 test_archive_dir = $(DIST)/$(PKG_PATH)
 endif
 
 package-tests-prepare-dest:
 	$(NSINSTALL) -D $(test_archive_dir)
 
+download-wpt-manifest:
+	$(call py_action,download_wpt_manifest)
+
 define package_archive
-package-tests-$(1): stage-all package-tests-prepare-dest
+package-tests-$(1): stage-all package-tests-prepare-dest download-wpt-manifest
 	$$(call py_action,test_archive, \
 		$(1) \
 		'$$(abspath $$(test_archive_dir))/$$(PKG_BASENAME).$(1).tests.$(2)')
 package-tests: package-tests-$(1)
 endef
 
 $(foreach name,$(TEST_PKGS_ZIP),$(eval $(call package_archive,$(name),zip)))
 $(foreach name,$(TEST_PKGS_TARGZ),$(eval $(call package_archive,$(name),tar.gz)))
@@ -263,16 +266,17 @@ check::
 
 
 .PHONY: \
   reftest \
   crashtest \
   xpcshell-tests \
   jstestbrowser \
   package-tests \
+  download-wpt-manifest \
   package-tests-prepare-dest \
   package-tests-common \
   make-stage-dir \
   stage-all \
   stage-config \
   stage-mochitest \
   stage-jstests \
   stage-android \
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -34,17 +34,17 @@ class WebPlatformTestsRunnerSetup(Mozbui
     def kwargs_common(self, kwargs):
         build_path = os.path.join(self.topobjdir, 'build')
         here = os.path.split(__file__)[0]
         tests_src_path = os.path.join(here, "tests")
         if build_path not in sys.path:
             sys.path.append(build_path)
 
         if kwargs["config"] is None:
-            kwargs["config"] = os.path.join(here, 'wptrunner.ini')
+            kwargs["config"] = os.path.join(self.topobjdir, '_tests', 'web-platform', 'wptrunner.local.ini')
 
         if kwargs["prefs_root"] is None:
             kwargs["prefs_root"] = os.path.join(self.topsrcdir, 'testing', 'profiles')
 
         if kwargs["stackfix_dir"] is None:
             kwargs["stackfix_dir"] = self.bindir
 
         if kwargs["exclude"] is None and kwargs["include"] is None and not sys.platform.startswith("linux"):
@@ -131,17 +131,17 @@ class WebPlatformTestsRunnerSetup(Mozbui
 
 class WebPlatformTestsUpdater(MozbuildObject):
     """Update web platform tests."""
     def run_update(self, **kwargs):
         import update
         from update import updatecommandline
 
         if kwargs["config"] is None:
-            kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
+            kwargs["config"] = os.path.join(self.topobjdir, '_tests', 'web-platform', 'wptrunner.local.ini')
         if kwargs["product"] is None:
             kwargs["product"] = "firefox"
 
         kwargs["store_state"] = False
 
         kwargs = updatecommandline.check_args(kwargs)
         logger = update.setup_logging(kwargs, {"mach": sys.stdout})
 
@@ -288,31 +288,32 @@ testing/web-platform/tests for tests tha
 
         if proc:
             proc.wait()
 
         context.commands.dispatch("wpt-manifest-update", context)
 
 
 class WPTManifestUpdater(MozbuildObject):
-    def run_update(self, check_clean=False, rebuild=False, **kwargs):
+    def run_update(self, rebuild=False, **kwargs):
         import manifestupdate
         from wptrunner import wptlogging
         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
         wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
-        manifestupdate.update(logger, wpt_dir, check_clean, rebuild)
+        config_dir = os.path.abspath(os.path.join(self.topobjdir, '_tests', 'web-platform'))
+        manifestupdate.update(logger, wpt_dir, rebuild, config_dir)
 
 
 class WPTManifestDownloader(MozbuildObject):
-    def run_download(self, path=None, tests_root=None, force=False, **kwargs):
+    def run_download(self, manifest_update=True, force=False, **kwargs):
         import manifestdownload
         from wptrunner import wptlogging
         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
-        wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
-        manifestdownload.run(logger, wpt_dir, self.topsrcdir, force)
+        wpt_dir = os.path.abspath(os.path.join(self.topobjdir, '_tests', 'web-platform'))
+        manifestdownload.run(wpt_dir, self.topsrcdir, logger, force, manifest_update)
 
 
 def create_parser_update():
     from update import updatecommandline
     return updatecommandline.create_parser()
 
 
 def create_parser_reduce():
@@ -366,16 +367,18 @@ class MachCommands(MachCommandBase):
                 params["product"] = "fennec"
             else:
                 raise ValueError("Must specify --product=fennec in Android environment.")
         if "test_objects" in params:
             for item in params["test_objects"]:
                 params["include"].append(item["name"])
             del params["test_objects"]
 
+        self.wpt_manifest_download(**params)
+        params["manifest_update"] = False
         wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
         wpt_runner = WebPlatformTestsRunner(wpt_setup)
         return wpt_runner.run(**params)
 
     @Command("wpt",
              category="testing",
              conditions=[is_firefox_or_android],
              parser=create_parser_wpt)
@@ -432,16 +435,15 @@ class MachCommands(MachCommandBase):
     @Command("wpt-manifest-update",
              category="testing",
              parser=create_parser_manifest_update)
     def wpt_manifest_update(self, **params):
         self.setup()
         wpt_manifest_updater = self._spawn(WPTManifestUpdater)
         return wpt_manifest_updater.run_update(**params)
 
-
     @Command("wpt-manifest-download",
              category="testing",
              parser=create_parser_manifest_download)
     def wpt_manifest_download(self, **params):
         self.setup()
         wpt_manifest_downloader = self._spawn(WPTManifestDownloader)
         return wpt_manifest_downloader.run_download(**params)
--- a/testing/web-platform/manifestdownload.py
+++ b/testing/web-platform/manifestdownload.py
@@ -1,135 +1,223 @@
 from __future__ import absolute_import
 
 import argparse
-import json
 import os
 from datetime import datetime, timedelta
 import tarfile
+import requests
 import vcs
-import requests
 from cStringIO import StringIO
+import logging
+
+HEADERS = {'User-Agent': "wpt manifest download"}
+
 
 def abs_path(path):
     return os.path.abspath(os.path.expanduser(path))
 
 
 def hg_commits(repo_root):
     hg = vcs.Mercurial.get_func(repo_root)
     for item in hg("log", "-fl50", "--template={node}\n", "testing/web-platform/tests",
                    "testing/web-platform/mozilla/tests").splitlines():
         yield item
 
 
 def git_commits(repo_root):
     git = vcs.Git.get_func(repo_root)
     for item in git("log", "--format=%H", "-n50", "testing/web-platform/tests",
                     "testing/web-platform/mozilla/tests").splitlines():
-        yield git("cinnabar", "git2hg", item)
+        yield git("cinnabar", "git2hg", item).strip()
 
 
 def get_commits(logger, repo_root):
     if vcs.Mercurial.is_hg_repo(repo_root):
         return hg_commits(repo_root)
 
     elif vcs.Git.is_git_repo(repo_root):
         return git_commits(repo_root)
 
     logger.warning("No VCS found")
     return False
 
 
-def should_download(logger, manifest_path, rebuild_time=timedelta(days=5)):
+def should_download(logger, manifest_paths, rebuild_time=timedelta(days=5)):
     # TODO: Improve logic for when to download. Maybe if x revisions behind?
-    if not os.path.exists(manifest_path):
-        return True
-    mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
-    if mtime < datetime.now() - rebuild_time:
-        return True
+    for manifest_path in manifest_paths:
+        if not os.path.exists(manifest_path):
+            return True
+        mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
+        if mtime < datetime.now() - rebuild_time:
+            return True
+
     logger.info("Skipping manifest download because existing file is recent")
     return False
 
 
 def taskcluster_url(logger, commits):
     cset_url = ('https://hg.mozilla.org/mozilla-central/json-pushes?'
                 'changeset={changeset}&version=2&tipsonly=1')
 
     tc_url = ('https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central.'
               'revision.{changeset}.source.manifest-upload')
 
     for revision in commits:
-        req = requests.get(cset_url.format(changeset=revision),
-                           headers={'Accept': 'application/json'})
-
-        req.raise_for_status()
+        if revision == 40 * "0":
+            continue
+        try:
+            req_headers = HEADERS.copy()
+            req_headers.update({'Accept': 'application/json'})
+            req = requests.get(cset_url.format(changeset=revision),
+                               headers=req_headers)
+            req.raise_for_status()
+        except requests.exceptions.RequestException:
+            if req.status_code == 404:
+                # The API returns a 404 if it can't find a changeset for the revision.
+                continue
+            else:
+                return False
 
         result = req.json()
-        [cset] = result['pushes'].values()[0]['changesets']
-        req = requests.get(tc_url.format(changeset=cset))
+
+        pushes = result['pushes']
+        if not pushes:
+            continue
+        [cset] = pushes.values()[0]['changesets']
+
+        try:
+            req = requests.get(tc_url.format(changeset=cset),
+                               headers=HEADERS)
+        except requests.exceptions.RequestException:
+            return False
 
         if req.status_code == 200:
             return tc_url.format(changeset=cset)
 
     logger.info("Can't find a commit-specific manifest so just using the most"
                 "recent one")
 
     return ("https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central."
             "latest.source.manifest-upload")
 
 
 def download_manifest(logger, wpt_dir, commits_func, url_func, force=False):
-    if not force and not should_download(logger, os.path.join(wpt_dir, "meta", "MANIFEST.json")):
-        return False
+    manifest_path = os.path.join(wpt_dir, "meta", "MANIFEST.json")
+    mozilla_manifest_path = os.path.join(wpt_dir, "mozilla", "meta", "MANIFEST.json")
+
+    if not force and not should_download(logger, [manifest_path, mozilla_manifest_path]):
+        return True
 
     commits = commits_func()
     if not commits:
         return False
-    url = url_func(logger, commits) + "/artifacts/public/manifests.tar.gz"
 
+    url = url_func(logger, commits)
     if not url:
         logger.warning("No generated manifest found")
         return False
+    url+= "/artifacts/public/manifests.tar.gz"
 
     logger.info("Downloading manifest from %s" % url)
     try:
-        req = requests.get(url)
+        req = requests.get(url, headers=HEADERS)
     except Exception:
         logger.warning("Downloading pregenerated manifest failed")
         return False
 
     if req.status_code != 200:
         logger.warning("Downloading pregenerated manifest failed; got"
                         "HTTP status %d" % req.status_code)
         return False
 
     tar = tarfile.open(mode="r:gz", fileobj=StringIO(req.content))
     try:
         tar.extractall(path=wpt_dir)
     except IOError:
         logger.warning("Failed to decompress downloaded file")
         return False
 
-    os.utime(os.path.join(wpt_dir, "meta", "MANIFEST.json"), None)
-    os.utime(os.path.join(wpt_dir, "mozilla", "meta", "MANIFEST.json"), None)
+    os.utime(manifest_path, None)
+    os.utime(mozilla_manifest_path, None)
 
     logger.info("Manifest downloaded")
     return True
 
 
 def create_parser():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "-p", "--path", type=abs_path, help="Path to manifest file.")
     parser.add_argument(
         "--force", action="store_true",
         help="Always download, even if the existing manifest is recent")
+    parser.add_argument(
+        "--no-manifest-update", action="store_false", dest="manifest_update",
+        default=True, help="Don't update the downloaded manifest")
     return parser
 
 
 def download_from_taskcluster(logger, wpt_dir, repo_root, force=False):
     return download_manifest(logger, wpt_dir, lambda: get_commits(logger, repo_root),
                              taskcluster_url, force)
 
 
-def run(logger, wpt_dir, repo_root, force=False):
+def generate_config(path):
+    """Generate the local wptrunner.ini file to use locally"""
+    import ConfigParser
+    here = os.path.split(os.path.abspath(__file__))[0]
+    config_path = os.path.join(here, 'wptrunner.ini')
+    path = os.path.join(path, 'wptrunner.local.ini')
+
+    if os.path.exists(path):
+        return True
+
+    parser = ConfigParser.SafeConfigParser()
+    success = parser.read(config_path)
+    assert config_path in success, success
+
+    parser.set('manifest:upstream', 'tests', os.path.join(here, 'tests'))
+    parser.set('manifest:mozilla', 'tests', os.path.join(here, 'mozilla', 'tests'))
+    parser.set('paths', 'prefs', os.path.join(os.getcwd(), 'testing', 'profiles'))
+
+    with open(path, 'wb') as config_file:
+        parser.write(config_file)
+    return True
+
+
+def update_manifest(logger, config_dir, manifest_update=True):
+    if manifest_update:
+        logger.info("Updating manifests")
+        import manifestupdate
+        here = os.path.split(os.path.abspath(__file__))[0]
+        return manifestupdate.update(logger, here, config_dir=config_dir) is 0
+    else:
+        logger.info("Skipping manifest update")
+        return True
+
+def check_dirs(logger, success, wpt_dir):
+    if success:
+        return
+    else:
+        logger.info("Could not download manifests.")
+        logger.info("Generating from scratch instead.")
+        try:
+            os.mkdir(os.path.join(wpt_dir, "meta"))
+        except OSError:
+            pass
+        try:
+            os.makedirs(os.path.join(wpt_dir, "mozilla", "meta"))
+        except OSError:
+            pass
+
+
+def run(wpt_dir, repo_root, logger=None, force=False, manifest_update=True):
+    if not logger:
+        logger = logging.getLogger(__name__)
+        handler = logging.FileHandler(os.devnull)
+        logger.addHandler(handler)
+
     success = download_from_taskcluster(logger, wpt_dir, repo_root, force)
+    check_dirs(logger, success, wpt_dir)
+    generate_config(wpt_dir)
+    success |= update_manifest(logger, wpt_dir, manifest_update)
     return 0 if success else 1
--- a/testing/web-platform/manifestupdate.py
+++ b/testing/web-platform/manifestupdate.py
@@ -11,144 +11,62 @@ manifest = None
 
 
 def do_delayed_imports(wpt_dir):
     global manifest
     sys.path.insert(0, os.path.join(wpt_dir, "tools", "manifest"))
     import manifest
 
 
+
 def create_parser():
     p = argparse.ArgumentParser()
-    p.add_argument("--check-clean", action="store_true",
-                   help="Check that updating the manifest doesn't lead to any changes")
     p.add_argument("--rebuild", action="store_true",
                    help="Rebuild the manifest from scratch")
     commandline.add_logging_group(p)
 
     return p
 
 
-def update(logger, wpt_dir, check_clean=True, rebuild=False):
+def update(logger, wpt_dir, rebuild=False, config_dir=None,):
     localpaths = imp.load_source("localpaths",
                                  os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
-    kwargs = {"config": os.path.join(wpt_dir, "wptrunner.ini"),
+
+    if not config_dir or not os.path.exists(os.path.join(config_dir, 'wptrunner.local.ini')):
+        config_dir = wpt_dir
+        config_name = "wptrunner.ini"
+    else:
+        config_name = "wptrunner.local.ini"
+
+    kwargs = {"config": os.path.join(config_dir, config_name),
               "tests_root": None,
               "metadata_root": None}
 
     set_from_config(kwargs)
     config = kwargs["config"]
     test_paths = get_test_paths(config)
-
     do_delayed_imports(wpt_dir)
 
-    if check_clean:
-        return _check_clean(logger, test_paths)
-
     return _update(logger, test_paths, rebuild)
 
 
 def _update(logger, test_paths, rebuild):
     for url_base, paths in test_paths.iteritems():
         manifest_path = os.path.join(paths["metadata_path"], "MANIFEST.json")
         m = None
-        if not rebuild:
+        if not rebuild and os.path.exists(manifest_path):
             try:
                 m = manifest.manifest.load(paths["tests_path"], manifest_path)
             except manifest.manifest.ManifestVersionMismatch:
                 logger.info("Manifest format changed, rebuilding")
         if m is None:
             m = manifest.manifest.Manifest(url_base)
         manifest.update.update(paths["tests_path"], m, working_copy=True)
         manifest.manifest.write(m, manifest_path)
     return 0
 
 
-def _check_clean(logger, test_paths):
-    manifests_by_path = {}
-    rv = 0
-    for url_base, paths in test_paths.iteritems():
-        tests_path = paths["tests_path"]
-        manifest_path = os.path.join(paths["metadata_path"], "MANIFEST.json")
-        old_manifest = manifest.manifest.load(tests_path, manifest_path)
-        new_manifest = manifest.manifest.Manifest.from_json(tests_path,
-                                                            old_manifest.to_json())
-        manifest.update.update(tests_path, new_manifest, working_copy=True)
-        manifests_by_path[manifest_path] = (old_manifest, new_manifest)
-
-    for manifest_path, (old_manifest, new_manifest) in manifests_by_path.iteritems():
-        if not diff_manifests(logger, manifest_path, old_manifest, new_manifest):
-            rv = 1
-    if rv:
-        logger.error("Manifest %s is outdated, use |mach wpt-manifest-update| to fix." % manifest_path)
-
-    return rv
-
-
-def diff_manifests(logger, manifest_path, old_manifest, new_manifest):
-    """Lint the differences between old and new versions of a
-    manifest. Differences are considered significant (and so produce
-    lint errors) if they produce a meaningful difference in the actual
-    tests run.
-
-    :param logger: mozlog logger to use for output
-    :param manifest_path: Path to the manifest being linted
-    :param old_manifest: Manifest object representing the initial manifest
-    :param new_manifest: Manifest object representing the updated manifest
-    """
-    logger.info("Diffing old and new manifests %s" % manifest_path)
-    old_items, new_items = defaultdict(set), defaultdict(set)
-    for manifest, items in [(old_manifest, old_items),
-                            (new_manifest, new_items)]:
-        for test_type, path, tests in manifest:
-            for test in tests:
-                test_id = [test.id]
-                test_id.extend(tuple(item) if isinstance(item, list) else item
-                               for item in test.meta_key())
-                if hasattr(test, "references"):
-                    test_id.extend(tuple(item) for item in test.references)
-                test_id = tuple(test_id)
-                items[path].add((test_type, test_id))
-
-    old_paths = set(old_items.iterkeys())
-    new_paths = set(new_items.iterkeys())
-
-    added_paths = new_paths - old_paths
-    deleted_paths = old_paths - new_paths
-
-    common_paths = new_paths & old_paths
-
-    clean = True
-
-    for path in added_paths:
-        clean = False
-        log_error(logger, manifest_path, "%s in source but not in manifest." % path)
-    for path in deleted_paths:
-        clean = False
-        log_error(logger, manifest_path, "%s in manifest but removed from source." % path)
-
-    for path in common_paths:
-        old_tests = old_items[path]
-        new_tests = new_items[path]
-        added_tests = new_tests - old_tests
-        removed_tests = old_tests - new_tests
-        if added_tests or removed_tests:
-            clean = False
-            log_error(logger, manifest_path, "%s changed test types or metadata" % path)
-
-    if clean:
-        # Manifest currently has some list vs tuple inconsistencies that break
-        # a simple equality comparison.
-        new_paths = {(key, value[0], value[1])
-                     for (key, value) in new_manifest.to_json()["paths"].iteritems()}
-        old_paths = {(key, value[0], value[1])
-                     for (key, value) in old_manifest.to_json()["paths"].iteritems()}
-        if old_paths != new_paths:
-            logger.warning("Manifest %s contains correct tests but file hashes changed; please update" % manifest_path)
-
-    return clean
-
 def log_error(logger, manifest_path, msg):
     logger.lint_error(path=manifest_path,
                       message=msg,
                       lineno=0,
                       source="",
                       linter="wpt-manifest")
--- a/testing/web-platform/moz.build
+++ b/testing/web-platform/moz.build
@@ -1,19 +1,14 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-WEB_PLATFORM_TESTS_MANIFESTS += [
-    ('meta/MANIFEST.json', 'tests/'),
-    ('mozilla/meta/MANIFEST.json', 'mozilla/tests/')
-]
-
 TEST_HARNESS_FILES['web-platform'] += [
     'mach_commands_base.py',
     'mach_test_package_commands.py',
     'outbound/**',
     'runtests.py',
     'wptrunner.ini',
 ]
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -42,17 +42,17 @@ def create_parser(product_choices=None):
         config_data = config.load()
         product_choices = products.products_enabled(config_data)
 
     parser = argparse.ArgumentParser(description="""Runner for web-platform-tests tests.""",
                                      usage="""%(prog)s [OPTION]... [TEST]...
 
 TEST is either the full path to a test file to run, or the URL of a test excluding
 scheme host and port.""")
-    parser.add_argument("--manifest-update", action="store_true", default=None,
+    parser.add_argument("--manifest-update", action="store_true", default=True,
                         help="Regenerate the test manifest.")
     parser.add_argument("--no-manifest-update", action="store_false", dest="manifest_update",
                         help="Prevent regeneration of the test manifest.")
     parser.add_argument("--manifest-download", action="store_true", default=None,
                         help="Attempt to download a preexisting manifest when updating.")
 
     parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
                         help="Multiplier relative to standard test timeout to use")