Bug 1495372 - Unify wpt manifest download and update, r=ato
authorJames Graham <james@hoppipolla.co.uk>
Tue, 02 Oct 2018 16:46:42 +0100
changeset 495405 2adff75db60c9f2f8bfaa7c72093757ed20adcc5
parent 495404 a06f75df15b516db05c8b98fc0e2c0b507d92b81
child 495406 a5a84cd8b79e56414b7ff6a66385ae1a13fa37d5
push id9984
push userffxbld-merge
push dateMon, 15 Oct 2018 21:07:35 +0000
treeherdermozilla-beta@183d27ea8570 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato
bugs1495372
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1495372 - Unify wpt manifest download and update, r=ato The previous code split a lot of logic between the update and download parts, and exposed two different mach commands. In order to simplify things it's better to have a single entry point for manifest download and update, and ensure that's called consistently. Differential Revision: https://phabricator.services.mozilla.com/D7497
python/mozbuild/mozbuild/action/download_wpt_manifest.py
taskcluster/ci/source-test/wpt-manifest.yml
testing/mozbase/moztest/moztest/resolve.py
testing/web-platform/mach_commands.py
testing/web-platform/mach_commands_base.py
testing/web-platform/manifestdownload.py
testing/web-platform/manifestupdate.py
testing/web-platform/tests/tools/wptrunner/wptrunner/config.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
--- a/python/mozbuild/mozbuild/action/download_wpt_manifest.py
+++ b/python/mozbuild/mozbuild/action/download_wpt_manifest.py
@@ -7,16 +7,15 @@
 import os
 import sys
 
 import buildconfig
 
 
 def main():
     print("Downloading wpt manifest")
-    man_path = os.path.join(buildconfig.topobjdir, '_tests', 'web-platform')
     sys.path.insert(0, buildconfig.topsrcdir)
-    import manifestdownload
-    manifestdownload.run(man_path, buildconfig.topsrcdir, force=True)
+    import manifestupdate
+    return 0 if manifestupdate.run(buildconfig.topsrcdir, buildconfig.topobjdir) else 1
 
 
 if __name__ == '__main__':
     sys.exit(main())
--- a/taskcluster/ci/source-test/wpt-manifest.yml
+++ b/taskcluster/ci/source-test/wpt-manifest.yml
@@ -15,17 +15,17 @@ upload:
     index:
         product: source
         job-name: manifest-upload
         rank: build_date
     run:
         using: run-task
         command: >
             cd /builds/worker/checkouts/gecko
-            && ./mach wpt-manifest-update
+            && ./mach wpt-manifest-update --config testing/web-platform/wptrunner.ini
             && tar -cvzf manifests.tar.gz -C testing/web-platform/ meta/MANIFEST.json mozilla/meta/MANIFEST.json
     worker:
         artifacts:
             - type: file
               path: /builds/worker/checkouts/gecko/manifests.tar.gz
               name: public/manifests.tar.gz
 
         max-run-time: 3600
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -445,34 +445,24 @@ class TestMetadata(object):
             return True
         return False
 
     def add_wpt_manifest_data(self):
         if self._wpt_loaded:
             return
 
         wpt_path = os.path.join(self._srcdir, "testing", "web-platform")
-        wptrunner_path = os.path.join(wpt_path, "tests", "tools", "wptrunner")
-        manifest_path = os.path.join(self._objdir, "_tests", "web-platform")
-
-        sys.path = [wpt_path, wptrunner_path] + sys.path
-
-        import manifestdownload
-        import wptrunner
-        from wptrunner.wptrunner import testloader
+        sys.path = [wpt_path] + sys.path
 
-        manifestdownload.run(manifest_path, self._srcdir)
-
-        kwargs = {"config": os.path.join(self._objdir, "_tests", "web-platform",
-                                         "wptrunner.local.ini"),
-                  "tests_root": None,
-                  "metadata_root": None}
-
-        wptrunner.wptcommandline.set_from_config(kwargs)
-        manifests = testloader.ManifestLoader(kwargs["test_paths"]).load()
+        import manifestupdate
+        manifests = manifestupdate.run(self._srcdir, self._objdir, rebuild=False, download=True,
+                                       config_path=None, rewrite_config=True, update=True)
+        if not manifests:
+            print("Loading wpt manifest failed")
+            return
 
         for manifest, data in manifests.iteritems():
             tests_root = data["tests_path"]
             for test_type, path, tests in manifest:
                 full_path = os.path.join(tests_root, path)
                 src_path = os.path.relpath(full_path, self._srcdir)
                 if test_type not in ["testharness", "reftest", "wdspec"]:
                     continue
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -268,39 +268,16 @@ testing/web-platform/tests for tests tha
             if ref_path:
                 path = "%s %s" % (path, ref_path)
             proc = subprocess.Popen("%s %s" % (editor, path), shell=True)
 
         if proc:
             proc.wait()
 
 
-class WPTManifestUpdater(MozbuildObject):
-    def setup_logging(self, **kwargs):
-        from wptrunner import wptlogging
-        logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
-
-    def run_update(self, logger, rebuild=False, **kwargs):
-        import manifestupdate
-        wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
-        config_dir = os.path.abspath(os.path.join(self.topobjdir, '_tests', 'web-platform'))
-        manifestupdate.update(logger, wpt_dir, rebuild, config_dir)
-
-
-class WPTManifestDownloader(MozbuildObject):
-    def setup_logging(self, **kwargs):
-        from wptrunner import wptlogging
-        logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
-
-    def run_download(self, logger, manifest_update=True, force=False, **kwargs):
-        import manifestdownload
-        wpt_dir = os.path.abspath(os.path.join(self.topobjdir, '_tests', 'web-platform'))
-        manifestdownload.run(wpt_dir, self.topsrcdir, logger, force, manifest_update)
-
-
 def create_parser_update():
     from update import updatecommandline
     return updatecommandline.create_parser()
 
 
 def create_parser_create():
     import argparse
     p = argparse.ArgumentParser()
@@ -321,20 +298,16 @@ def create_parser_create():
     p.add_argument("path", action="store", help="Path to the test file")
     return p
 
 
 def create_parser_manifest_update():
     import manifestupdate
     return manifestupdate.create_parser()
 
-def create_parser_manifest_download():
-    import manifestdownload
-    return manifestdownload.create_parser()
-
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     def setup(self):
         self._activate_virtualenv()
 
     @Command("web-platform-tests",
              category="testing",
@@ -352,18 +325,16 @@ class MachCommands(MachCommandBase):
                 params["include"].append(item["name"])
             del params["test_objects"]
 
         wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
         wpt_runner = WebPlatformTestsRunner(wpt_setup)
 
         logger = wpt_runner.setup_logging(**params)
 
-        self.wpt_manifest_download(logger, **params)
-        params["manifest_update"] = False
         return wpt_runner.run(logger, **params)
 
     @Command("wpt",
              category="testing",
              conditions=[conditions.is_firefox_or_android],
              parser=create_parser_wpt)
     def run_wpt(self, **params):
         return self.run_web_platform_tests(**params)
@@ -401,24 +372,12 @@ class MachCommands(MachCommandBase):
     def create_wpt(self, **params):
         return self.create_web_platform_test(**params)
 
     @Command("wpt-manifest-update",
              category="testing",
              parser=create_parser_manifest_update)
     def wpt_manifest_update(self, **params):
         self.setup()
-        self.wpt_manifest_download(**params)
-        wpt_manifest_updater = self._spawn(WPTManifestUpdater)
-        logger = wpt_manifest_updater.setup_logging(**params)
-        self.wpt_manifest_download(logger, **params)
-        return wpt_manifest_updater.run_update(logger, **params)
-
-    @Command("wpt-manifest-download",
-             category="testing",
-             parser=create_parser_manifest_download)
-    def wpt_manifest_download(self, logger=None, **params):
-        self.setup()
-        if logger is None:
-            from wptrunner import wptlogging
-            logger = wptlogging.setup(params, {"mach": sys.stdout})
-        wpt_manifest_downloader = self._spawn(WPTManifestDownloader)
-        return wpt_manifest_downloader.run_download(logger, **params)
+        wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
+        wpt_runner = WebPlatformTestsRunner(wpt_setup)
+        logger = wpt_runner.setup_logging(**params)
+        return 0 if wpt_runner.update_manifest(logger, **params) else 1
--- a/testing/web-platform/mach_commands_base.py
+++ b/testing/web-platform/mach_commands_base.py
@@ -17,19 +17,31 @@ class WebPlatformTestsRunner(object):
         self.setup = setup
 
     def setup_logging(self, **kwargs):
         from wptrunner import wptrunner
         return wptrunner.setup_logging(kwargs, {self.setup.default_log_type: sys.stdout})
 
     def run(self, logger, **kwargs):
         from wptrunner import wptrunner
+
+        if kwargs["manifest_update"] is not False:
+            self.update_manifest(logger)
+        kwargs["manifest_update"] = False
+
         if kwargs["product"] in ["firefox", None]:
             kwargs = self.setup.kwargs_firefox(kwargs)
         elif kwargs["product"] == "fennec":
             from wptrunner import wptcommandline
             kwargs = wptcommandline.check_args(self.setup.kwargs_common(kwargs))
         elif kwargs["product"] in ("chrome", "edge", "servo"):
             kwargs = self.setup.kwargs_wptrun(kwargs)
         else:
             raise ValueError("Unknown product %s" % kwargs["product"])
         result = wptrunner.start(**kwargs)
         return int(not result)
+
+    def update_manifest(self, logger, **kwargs):
+        import manifestupdate
+        return manifestupdate.run(logger=logger,
+                                  src_root=self.setup.topsrcdir,
+                                  obj_root=self.setup.topobjdir,
+                                  **kwargs)
--- a/testing/web-platform/manifestdownload.py
+++ b/testing/web-platform/manifestdownload.py
@@ -50,16 +50,18 @@ def should_download(logger, manifest_pat
         if mtime < datetime.now() - rebuild_time:
             return True
 
     logger.info("Skipping manifest download because existing file is recent")
     return False
 
 
 def taskcluster_url(logger, commits):
+    artifact_path = '/artifacts/public/manifests.tar.gz'
+
     cset_url = ('https://hg.mozilla.org/mozilla-central/json-pushes?'
                 'changeset={changeset}&version=2&tipsonly=1')
 
     tc_url = ('https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central.'
               'revision.{changeset}.source.manifest-upload')
 
     for revision in commits:
         req = None
@@ -88,151 +90,74 @@ def taskcluster_url(logger, commits):
 
         try:
             req = requests.get(tc_url.format(changeset=cset),
                                headers=HEADERS)
         except requests.exceptions.RequestException:
             return False
 
         if req.status_code == 200:
-            return tc_url.format(changeset=cset)
+            return tc_url.format(changeset=cset) + artifact_path
 
     logger.info("Can't find a commit-specific manifest so just using the most"
                 "recent one")
 
     return ("https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central."
-            "latest.source.manifest-upload")
+            "latest.source.manifest-upload" +
+            artifact_path)
 
 
-def download_manifest(logger, wpt_dir, commits_func, url_func, force=False):
-    manifest_path = os.path.join(wpt_dir, "meta", "MANIFEST.json")
-    mozilla_manifest_path = os.path.join(wpt_dir, "mozilla", "meta", "MANIFEST.json")
+def download_manifest(logger, test_paths, commits_func, url_func, force=False):
+    manifest_paths = [item["manifest_path"] for item in test_paths.itervalues()]
 
-    if not force and not should_download(logger, [manifest_path, mozilla_manifest_path]):
+    if not force and not should_download(logger, manifest_paths):
         return True
 
     commits = commits_func()
     if not commits:
         return False
 
     url = url_func(logger, commits)
     if not url:
         logger.warning("No generated manifest found")
         return False
-    url+= "/artifacts/public/manifests.tar.gz"
 
     logger.info("Downloading manifest from %s" % url)
     try:
         req = requests.get(url, headers=HEADERS)
     except Exception:
         logger.warning("Downloading pregenerated manifest failed")
         return False
 
     if req.status_code != 200:
         logger.warning("Downloading pregenerated manifest failed; got"
                         "HTTP status %d" % req.status_code)
         return False
 
     tar = tarfile.open(mode="r:gz", fileobj=StringIO(req.content))
-    try:
-        tar.extractall(path=wpt_dir)
-    except IOError:
-        logger.warning("Failed to decompress downloaded file")
-        return False
-
-    os.utime(manifest_path, None)
-    os.utime(mozilla_manifest_path, None)
-
-    logger.info("Manifest downloaded")
-    return True
-
-
-def create_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-p", "--path", type=abs_path, help="Path to manifest file.")
-    parser.add_argument(
-        "--force", action="store_true",
-        help="Always download, even if the existing manifest is recent")
-    parser.add_argument(
-        "--no-manifest-update", action="store_false", dest="manifest_update",
-        default=True, help="Don't update the downloaded manifest")
-    return parser
-
-
-def download_from_taskcluster(logger, wpt_dir, repo_root, force=False):
-    return download_manifest(logger, wpt_dir, lambda: get_commits(logger, repo_root),
-                             taskcluster_url, force)
-
+    for paths in test_paths.itervalues():
+        try:
+            member = tar.getmember(paths["manifest_rel_path"].replace(os.path.sep, "/"))
+        except KeyError:
+            logger.warning("Failed to find downloaded manifest %s" % paths["manifest_rel_path"])
+        else:
+            try:
+                logger.debug("Unpacking %s to %s" % (member.name, paths["manifest_path"]))
+                src = tar.extractfile(member)
+                with open(paths["manifest_path"], "w") as dest:
+                    dest.write(src.read())
+                src.close()
+            except IOError:
+                import traceback
+                logger.warning("Failed to decompress %s:\n%s" % (paths["manifest_rel_path"], traceback.format_exc()))
+                return False
 
-def generate_config(obj_base_path):
-    """Generate the local wptrunner.ini file to use locally"""
-    import ConfigParser
-    here = os.path.split(os.path.abspath(__file__))[0]
-    config_path = os.path.join(here, 'wptrunner.ini')
-
-    if not os.path.exists(obj_base_path):
-        os.makedirs(obj_base_path)
-
-    path = os.path.join(obj_base_path, 'wptrunner.local.ini')
-
-    if os.path.exists(path):
-        return True
-
-    parser = ConfigParser.SafeConfigParser()
-    success = parser.read(config_path)
-    assert config_path in success, success
-
-    for name, path_prefix in [("upstream", ""),
-                              ("mozilla", "mozilla")]:
-        obj_path = os.path.join(obj_base_path, path_prefix)
-        src_path = os.path.join(here, path_prefix)
-        parser.set('manifest:%s' % name, 'manifest',
-                   os.path.join(obj_path, 'meta', 'MANIFEST.json'))
-
-        for key, dir_path in [("tests", "tests"), ("metadata", "meta")]:
-            parser.set("manifest:%s" % name, key, os.path.join(src_path, dir_path))
-
-    parser.set('paths', 'prefs', os.path.abspath(os.path.join(here, "..", 'profiles')))
-
-    with open(path, 'wb') as config_file:
-        parser.write(config_file)
+        os.utime(paths["manifest_path"], None)
 
     return True
 
 
-def update_manifest(logger, config_dir, manifest_update=True):
-    if manifest_update:
-        logger.info("Updating manifests")
-        import manifestupdate
-        here = os.path.split(os.path.abspath(__file__))[0]
-        return manifestupdate.update(logger, here, config_dir=config_dir) is 0
-    else:
-        logger.info("Skipping manifest update")
-        return True
-
-def check_dirs(logger, success, wpt_dir):
-    if success:
-        return
-    else:
-        logger.info("Could not download manifests.")
-        logger.info("Generating from scratch instead.")
-        try:
-            os.mkdir(os.path.join(wpt_dir, "meta"))
-        except OSError:
-            pass
-        try:
-            os.makedirs(os.path.join(wpt_dir, "mozilla", "meta"))
-        except OSError:
-            pass
-
-
-def run(wpt_dir, repo_root, logger=None, force=False, manifest_update=True):
-    if not logger:
-        logger = logging.getLogger(__name__)
-        handler = logging.FileHandler(os.devnull)
-        logger.addHandler(handler)
-
-    success = download_from_taskcluster(logger, wpt_dir, repo_root, force)
-    check_dirs(logger, success, wpt_dir)
-    generate_config(wpt_dir)
-    success |= update_manifest(logger, wpt_dir, manifest_update)
-    return 0 if success else 1
+def download_from_taskcluster(logger, repo_root, test_paths, force=False):
+    return download_manifest(logger,
+                             test_paths,
+                             lambda: get_commits(logger, repo_root),
+                             taskcluster_url,
+                             force)
--- a/testing/web-platform/manifestupdate.py
+++ b/testing/web-platform/manifestupdate.py
@@ -1,76 +1,207 @@
+import ConfigParser
 import argparse
 import imp
 import os
 import sys
-from collections import defaultdict
 
 from mozlog.structured import commandline
-from wptrunner.wptcommandline import get_test_paths, set_from_config
+from wptrunner.wptcommandline import set_from_config
+
+import manifestdownload
+from wptrunner import wptcommandline
 
 manifest = None
 
 
 def do_delayed_imports(wpt_dir):
     global manifest
+    imp.load_source("localpaths",
+                    os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
     sys.path.insert(0, os.path.join(wpt_dir, "tools", "manifest"))
     import manifest
 
 
-
 def create_parser():
     p = argparse.ArgumentParser()
     p.add_argument("--rebuild", action="store_true",
-                   help="Rebuild the manifest from scratch")
+                   help="Rebuild manifest from scratch")
+    download_group = p.add_mutually_exclusive_group()
+    download_group.add_argument(
+        "--download", dest="download", action="store_true", default=None,
+        help="Always download even if the local manifest is recent")
+    download_group.add_argument(
+        "--no-download", dest="download", action="store_false",
+        help="Don't try to download the manifest")
+    p.add_argument(
+        "--no-update", action="store_false", dest="update",
+        default=True, help="Just download the manifest, don't update")
+    p.add_argument(
+        "--config", action="store", dest="config_path", default=None,
+        help="Path to wptrunner config file")
+    p.add_argument(
+        "--rewrite-config", action="store_true", default=False,
+        help="Force the local configuration to be regenerated")
     commandline.add_logging_group(p)
 
     return p
 
 
-def update(logger, wpt_dir, rebuild=False, config_dir=None):
-    localpaths = imp.load_source("localpaths",
-                                 os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
-
-    if not config_dir:
-        config_dir = wpt_dir
-        config_name = "wptrunner.ini"
-    else:
-        if not os.path.exists(os.path.join(config_dir, 'wptrunner.local.ini')):
-            from manifestdownload import generate_config
-            generate_config(config_dir)
-        config_name = "wptrunner.local.ini"
-
-    kwargs = {"config": os.path.join(config_dir, config_name),
-              "tests_root": None,
-              "metadata_root": None}
-
-    set_from_config(kwargs)
-    do_delayed_imports(wpt_dir)
-
-    return _update(logger, kwargs["test_paths"], rebuild)
+def ensure_kwargs(kwargs):
+    _kwargs = vars(create_parser().parse_args([]))
+    _kwargs.update(kwargs)
+    return _kwargs
 
 
-def _update(logger, test_paths, rebuild):
+def run(src_root, obj_root, logger=None, **kwargs):
+    kwargs = ensure_kwargs(kwargs)
+
+    if logger is None:
+        from wptrunner import wptlogging
+        logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
+
+    src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
+
+    do_delayed_imports(src_wpt_dir)
+
+    if not kwargs["config_path"]:
+        config_path = generate_config(logger,
+                                      src_root,
+                                      src_wpt_dir,
+                                      os.path.join(obj_root, "_tests", "web-platform"),
+                                      kwargs["rewrite_config"])
+    else:
+        config_path = kwargs["config_path"]
+
+    if not os.path.exists(config_path):
+        logger.critical("Config file %s does not exist" % config_path)
+        return None
+
+    logger.debug("Using config path %s" % config_path)
+
+    test_paths = wptcommandline.get_test_paths(
+        wptcommandline.config.read(config_path))
+
+    for paths in test_paths.itervalues():
+        if "manifest_path" not in paths:
+            paths["manifest_path"] = os.path.join(paths["metadata_path"],
+                                                  "MANIFEST.json")
+
+    ensure_manifest_directories(logger, test_paths)
+
+    local_config = read_local_config(src_wpt_dir)
+    for section in ["manifest:upstream", "manifest:mozilla"]:
+        url_base = local_config.get(section, "url_base")
+        manifest_rel_path = os.path.join(local_config.get(section, "metadata"),
+                                         "MANIFEST.json")
+        test_paths[url_base]["manifest_rel_path"] = manifest_rel_path
+
+    if not kwargs["rebuild"] and kwargs["download"] is not False:
+        force_download = False if kwargs["download"] is None else True
+        manifestdownload.download_from_taskcluster(logger,
+                                                   src_root,
+                                                   test_paths,
+                                                   force=force_download)
+    else:
+        logger.debug("Skipping manifest download")
+
+    if kwargs["update"] or kwargs["rebuild"]:
+        manifests = update(logger, src_wpt_dir, test_paths, rebuild=kwargs["rebuild"])
+    else:
+        logger.debug("Skipping manifest update")
+        manifests = load_manifests(test_paths)
+
+    return manifests
+
+
+def ensure_manifest_directories(logger, test_paths):
+    for paths in test_paths.itervalues():
+        manifest_dir = os.path.dirname(paths["manifest_path"])
+        if not os.path.exists(manifest_dir):
+            logger.info("Creating directory %s" % manifest_dir)
+            os.makedirs(manifest_dir)
+        elif not os.path.isdir(manifest_dir):
+            raise IOError("Manifest directory is a file")
+
+
+def read_local_config(wpt_dir):
+    src_config_path = os.path.join(wpt_dir, "wptrunner.ini")
+
+    parser = ConfigParser.SafeConfigParser()
+    success = parser.read(src_config_path)
+    assert src_config_path in success
+    return parser
+
+
+def generate_config(logger, repo_root, wpt_dir, dest_path, force_rewrite=False):
+    """Generate the local wptrunner.ini file to use locally"""
+    if not os.path.exists(dest_path):
+        os.makedirs(dest_path)
+
+    dest_config_path = os.path.join(dest_path, 'wptrunner.local.ini')
+
+    if not force_rewrite and os.path.exists(dest_config_path):
+        logger.debug("Config is up to date, not regenerating")
+        return dest_config_path
+
+    logger.info("Creating config file %s" % dest_config_path)
+
+    parser = read_local_config(wpt_dir)
+
+    for section in ["manifest:upstream", "manifest:mozilla"]:
+        meta_rel_path = parser.get(section, "metadata")
+        tests_rel_path = parser.get(section, "tests")
+
+        parser.set(section, "manifest",
+                   os.path.join(dest_path, meta_rel_path, 'MANIFEST.json'))
+        parser.set(section, "metadata", os.path.join(wpt_dir, meta_rel_path))
+        parser.set(section, "tests", os.path.join(wpt_dir, tests_rel_path))
+
+    parser.set('paths', 'prefs', os.path.abspath(os.path.join(wpt_dir, parser.get("paths", "prefs"))))
+
+    with open(dest_config_path, 'wb') as config_file:
+        parser.write(config_file)
+
+    return dest_config_path
+
+
+def update(logger, wpt_dir, test_paths, rebuild=False, config_dir=None):
+    rv = {}
+
     for url_base, paths in test_paths.iteritems():
-        if "manifest_path" in paths:
-            manifest_path = paths["manifest_path"]
-        else:
-            manifest_path = os.path.join(paths["metadata_path"], "MANIFEST.json")
         m = None
+        manifest_path = paths["manifest_path"]
         if not rebuild and os.path.exists(manifest_path):
+            logger.info("Updating manifest %s" % manifest_path)
             try:
                 m = manifest.manifest.load(paths["tests_path"], manifest_path)
             except manifest.manifest.ManifestVersionMismatch:
                 logger.info("Manifest format changed, rebuilding")
         if m is None:
+            logger.info("Recreating manifest %s" % manifest_path)
             m = manifest.manifest.Manifest(url_base)
         manifest.update.update(paths["tests_path"], m, working_copy=True)
         manifest.manifest.write(m, manifest_path)
-    return 0
+
+        path_data = {"url_base": url_base}
+        path_data.update(paths)
+        rv[m] = path_data
+
+    return rv
+
+
+def load_manifests(test_paths):
+    rv = {}
+    for url_base, paths in test_paths.iteritems():
+        m = manifest.manifest.load(paths["tests_path"], manifest_path)
+        path_data = {"url_base": url_base}
+        path_data.update(paths)
+        rv[m] = path_data
+    return rv
 
 
 def log_error(logger, manifest_path, msg):
     logger.lint_error(path=manifest_path,
                       message=msg,
                       lineno=0,
                       source="",
                       linter="wpt-manifest")
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/config.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/config.py
@@ -1,44 +1,47 @@
 import ConfigParser
 import os
 import sys
 from collections import OrderedDict
 
 here = os.path.split(__file__)[0]
 
+
 class ConfigDict(dict):
     def __init__(self, base_path, *args, **kwargs):
         self.base_path = base_path
         dict.__init__(self, *args, **kwargs)
 
     def get_path(self, key, default=None):
         if key not in self:
             return default
         path = self[key]
         os.path.expanduser(path)
         return os.path.abspath(os.path.join(self.base_path, path))
 
+
 def read(config_path):
     config_path = os.path.abspath(config_path)
-    config_root = os.path.split(config_path)[0]
+    config_root = os.path.dirname(config_path)
     parser = ConfigParser.SafeConfigParser()
     success = parser.read(config_path)
     assert config_path in success, success
 
     subns = {"pwd": os.path.abspath(os.path.curdir)}
 
     rv = OrderedDict()
     for section in parser.sections():
         rv[section] = ConfigDict(config_root)
         for key in parser.options(section):
             rv[section][key] = parser.get(section, key, False, subns)
 
     return rv
 
+
 def path(argv=None):
     if argv is None:
         argv = []
     path = None
 
     for i, arg in enumerate(argv):
         if arg == "--config":
             if i + 1 < len(argv):
@@ -51,10 +54,11 @@ def path(argv=None):
     if path is None:
         if os.path.exists("wptrunner.ini"):
             path = os.path.abspath("wptrunner.ini")
         else:
             path = os.path.join(here, "..", "wptrunner.default.ini")
 
     return os.path.abspath(path)
 
+
 def load():
     return read(path(sys.argv))
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -361,17 +361,16 @@ def set_from_config(kwargs):
 
     if kwargs.get("manifest_path"):
         if "/" not in kwargs["test_paths"]:
             kwargs["test_paths"]["/"] = {}
         kwargs["test_paths"]["/"]["manifest_path"] = kwargs["manifest_path"]
 
     kwargs["suite_name"] = kwargs["config"].get("web-platform-tests", {}).get("name", "web-platform-tests")
 
-
     check_paths(kwargs)
 
 
 def get_test_paths(config):
     # Set up test_paths
     test_paths = OrderedDict()
 
     for section in config.iterkeys():