Bug 1459598 - Use profiles.json file to map test suites to the base profiles they use, r=jgraham
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 07 May 2018 09:43:12 -0400
changeset 417363 e4b9cb3abd0724e50ced4d0248649f27ad8be9ac
parent 417362 f5cb209a849dd8e2a8dc8a55ee2d435ed578df72
child 417364 0fd5996b2d80ca73b08adf55b5e5261c6d334b1b
push id63722
push userahalberstadt@mozilla.com
push dateTue, 08 May 2018 18:48:50 +0000
treeherderautoland@e4b9cb3abd07 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1459598
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1459598 - Use profiles.json file to map test suites to the base profiles they use, r=jgraham This serves two purposes: 1) It makes web-platform-tests pref downloading/handling a little more robust. When run externally, it now downloads the entire testing/profiles directory. When loading prefs it will look for both prefs_general.js (to support older versions of Firefox) and profiles.json (for support moving forward). This way we can add/remove/rename pref files under these directories without needing to worry about breaking upstream wpt. 2) It provides developers an overview of which harnesses are using which base profiles. Instead of hunting through test harness code to find this information, they can glance at profiles.json. MozReview-Commit-ID: AMzdnD8aGA2
build/pgo/profileserver.py
build/valgrind/mach_commands.py
testing/mochitest/runtests.py
testing/mochitest/tests/python/test_build_profile.py
testing/profiles/moz.build
testing/profiles/profiles.json
testing/web-platform/tests/tools/wpt/browser.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
--- a/build/pgo/profileserver.py
+++ b/build/pgo/profileserver.py
@@ -1,14 +1,15 @@
 #!/usr/bin/python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import json
 import os
 
 from buildconfig import substs
 from mozbuild.base import MozbuildObject
 from mozfile import TemporaryDirectory
 from mozhttpd import MozHttpd
 from mozprofile import FirefoxProfile, Profile, Preferences
 from mozprofile.permissions import ServerLocations
@@ -38,24 +39,27 @@ if __name__ == '__main__':
 
     locations = ServerLocations()
     locations.add_host(host='127.0.0.1',
                        port=PORT,
                        options='primary,privileged')
 
     with TemporaryDirectory() as profilePath:
         # TODO: refactor this into mozprofile
-        prefpath = os.path.join(
-            build.topsrcdir, "testing", "profiles", "common", "user.js")
-        overridepath = os.path.join(
-            build.topsrcdir, "build", "pgo", "prefs_override.js")
+        profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles')
+        with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
+            base_profiles = json.load(fh)['profileserver']
+
+        prefpaths = [os.path.join(profile_data_dir, profile, 'user.js')
+                     for profile in base_profiles]
+        prefpaths.append(os.path.join(build.topsrcdir, "build", "pgo", "prefs_override.js"))
 
         prefs = {}
-        prefs.update(Preferences.read_prefs(prefpath))
-        prefs.update(Preferences.read_prefs(overridepath))
+        for path in prefpaths:
+            prefs.update(Preferences.read_prefs(path))
 
         interpolation = {"server": "%s:%d" % httpd.httpd.server_address,
                          "OOP": "false"}
         for k, v in prefs.items():
             if isinstance(v, string_types):
                 v = v.format(**interpolation)
             prefs[k] = Preferences.cast(v)
 
--- a/build/valgrind/mach_commands.py
+++ b/build/valgrind/mach_commands.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
+import json
 import logging
 import mozinfo
 import os
 import subprocess
 
 from mach.decorators import (
     Command,
     CommandArgument,
@@ -60,19 +61,26 @@ class MachCommands(MachCommandBase):
 
         # XXX: currently we just use the PGO inputs for Valgrind runs.  This may
         # change in the future.
         httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
         httpd.start(block=False)
 
         with TemporaryDirectory() as profilePath:
             #TODO: refactor this into mozprofile
-            prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'common', 'user.js')
+            profile_data_dir = os.path.join(self.topsrcdir, 'testing', 'profiles')
+            with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
+                base_profiles = json.load(fh)['valgrind']
+
+            prefpaths = [os.path.join(profile_data_dir, profile, 'user.js')
+                         for profile in base_profiles]
             prefs = {}
-            prefs.update(Preferences.read_prefs(prefpath))
+            for path in prefpaths:
+                prefs.update(Preferences.read_prefs(path))
+
             interpolation = {
                 'server': '%s:%d' % httpd.httpd.server_address,
             }
             for k, v in prefs.items():
                 if isinstance(v, string_types):
                     v = v.format(**interpolation)
                 prefs[k] = Preferences.cast(v)
 
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -846,17 +846,16 @@ class MochitestDesktop(object):
     CHROME_PATH = "redirect.html"
 
     certdbNew = False
     sslTunnel = None
     DEFAULT_TIMEOUT = 60.0
     mediaDevices = None
 
     patternFiles = {}
-    base_profiles = ('common',)
 
     # XXX use automation.py for test name to avoid breaking legacy
     # TODO: replace this with 'runtests.py' or 'mochitest' or the like
     test_name = 'automation.py'
 
     def __init__(self, flavor, logger_options, quiet=False):
         update_mozinfo()
         self.flavor = flavor
@@ -1846,22 +1845,25 @@ toolbar#nav-bar {
         # If possible, read profile data from topsrcdir. This prevents us from
         # requiring a re-build to pick up newly added extensions in the
         # <profile>/extensions directory.
         if build_obj:
             path = os.path.join(build_obj.topsrcdir, 'testing', 'profiles')
             if os.path.isdir(path):
                 profile_data_dir = path
 
+        with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
+            base_profiles = json.load(fh)['mochitest']
+
         # values to use when interpolating preferences
         interpolation = {
             "server": "%s:%s" % (options.webServer, options.httpPort),
         }
 
-        for profile in self.base_profiles:
+        for profile in base_profiles:
             path = os.path.join(profile_data_dir, profile)
             self.profile.merge(path, interpolation=interpolation)
 
     def buildProfile(self, options):
         """ create the profile and add optional chrome bits and files if requested """
         # get extensions to install
         extensions = self.getExtensionsToInstall(options)
 
--- a/testing/mochitest/tests/python/test_build_profile.py
+++ b/testing/mochitest/tests/python/test_build_profile.py
@@ -1,25 +1,29 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
 
+import json
 import os
 from argparse import Namespace
 
+from mozbuild.base import MozbuildObject
 from mozprofile.prefs import Preferences
 from mozprofile import Profile
 from six import string_types
 
 import mozunit
 import pytest
 from conftest import setup_args
 
+here = os.path.abspath(os.path.dirname(__file__))
+
 
 @pytest.fixture
 def build_profile(monkeypatch, setup_test_harness, parser):
     setup_test_harness(*setup_args)
     runtests = pytest.importorskip('runtests')
     md = runtests.MochitestDesktop('plain', {'log_tbpl': '-'})
     monkeypatch.setattr(md, 'fillCertificateDB', lambda *args, **kwargs: None)
 
@@ -31,29 +35,33 @@ def build_profile(monkeypatch, setup_tes
         opts.update(kwargs)
 
         return md, md.buildProfile(Namespace(**opts))
 
     return inner
 
 
 @pytest.fixture
-def profile_data_dir(build_obj):
-    return os.path.join(build_obj.topsrcdir, 'testing', 'profiles')
+def profile_data_dir():
+    build = MozbuildObject.from_environment(cwd=here)
+    return os.path.join(build.topsrcdir, 'testing', 'profiles')
 
 
 def test_common_prefs_are_all_set(build_profile, profile_data_dir):
     # We set e10s=False here because MochitestDesktop.buildProfile overwrites
     # the value defined in the base profile.
     # TODO stop setting browser.tabs.remote.autostart in the base profile
     md, result = build_profile(e10s=False)
 
+    with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
+        base_profiles = json.load(fh)['mochitest']
+
     # build the expected prefs
     expected_prefs = {}
-    for profile in md.base_profiles:
+    for profile in base_profiles:
         for name in Profile.preference_file_names:
             path = os.path.join(profile_data_dir, profile, name)
             if os.path.isfile(path):
                 expected_prefs.update(Preferences.read_prefs(path))
 
     # read the actual prefs
     actual_prefs = {}
     for name in Profile.preference_file_names:
--- a/testing/profiles/moz.build
+++ b/testing/profiles/moz.build
@@ -1,15 +1,16 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-profiles = [
+profile_files = [
     'common/*',
+    'profiles.json',
 ]
 
-TEST_HARNESS_FILES.testing.mochitest.profile_data += profiles
-TEST_HARNESS_FILES['web-platform'].prefs += ['common/user.js']
+TEST_HARNESS_FILES.testing.mochitest.profile_data += profile_files
+TEST_HARNESS_FILES['web-platform'].prefs += profile_files
 
 with Files("**"):
-    BUG_COMPONENT = ("Testing", "Mochitest")
+    BUG_COMPONENT = ("Testing", "General")
new file mode 100644
--- /dev/null
+++ b/testing/profiles/profiles.json
@@ -0,0 +1,6 @@
+{
+    "mochitest": ["common"],
+    "profileserver": ["common"],
+    "web-platform-tests": ["common"],
+    "valgrind": ["common"]
+}
--- a/testing/web-platform/tests/tools/wpt/browser.py
+++ b/testing/web-platform/tests/tools/wpt/browser.py
@@ -1,16 +1,17 @@
 import logging
 import os
 import platform
 import re
 import shutil
 import stat
 import subprocess
 import sys
+import tempfile
 from abc import ABCMeta, abstractmethod
 from ConfigParser import RawConfigParser
 from datetime import datetime, timedelta
 from distutils.spawn import find_executable
 from io import BytesIO
 
 from utils import call, get, untar, unzip
 
@@ -186,68 +187,69 @@ class Firefox(Browser):
         stdout.strip()
         m = version_re.match(stdout)
         if not m:
             return None, "nightly"
         version, status = m.groups()
         channel = {"a": "nightly", "b": "beta"}
         return version, channel.get(status, "stable")
 
-    def get_prefs_url(self, version, channel):
+    def get_profile_bundle_url(self, version, channel):
         if channel == "stable":
             repo = "https://hg.mozilla.org/releases/mozilla-release"
             tag = "FIREFOX_%s_RELEASE" % version.replace(".", "_")
         else:
             repo = "https://hg.mozilla.org/mozilla-central"
             if channel == "beta":
                 tag = "FIREFOX_%s_BETA" % version.split(".", 1)[0]
             else:
                 # Always use tip as the tag for nightly; this isn't quite right
                 # but to do better we need the actual build revision, which we
                 # can get if we have an application.ini file
                 tag = "tip"
 
-        return "%s/raw-file/%s/testing/profiles/common/user.js" % (repo, tag)
+        return "%s/archive/%s.zip/testing/profiles/" % (repo, tag)
 
     def install_prefs(self, binary, dest=None):
         version, channel = self.get_version_number(binary)
 
         if dest is None:
             dest = os.pwd
 
-        dest = os.path.join(dest, "profiles", "common")
-        if not os.path.exists(dest):
-            os.makedirs(dest)
-        prefs_file = os.path.join(dest, "user.js")
-        cache_file = os.path.join(dest,
-                                  "%s-%s.cache" % (version, channel)
-                                  if channel != "nightly"
-                                  else "nightly.cache")
-
+        dest = os.path.join(dest, "profiles", channel, version)
         have_cache = False
-        if os.path.exists(cache_file):
+        if os.path.exists(dest):
             if channel != "nightly":
                 have_cache = True
             else:
                 now = datetime.now()
-                have_cache = (datetime.fromtimestamp(os.stat(cache_file).st_mtime) >
+                have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) >
                               now - timedelta(days=1))
 
-        # If we don't have a recent download, grab the url
+        # If we don't have a recent download, grab and extract the latest one
         if not have_cache:
-            url = self.get_prefs_url(version, channel)
+            if os.path.exists(dest):
+                shutil.rmtree(dest)
+            os.makedirs(dest)
+
+            url = self.get_profile_bundle_url(version, channel)
 
-            with open(cache_file, "wb") as f:
-                print("Installing test prefs from %s" % url)
-                resp = get(url)
-                f.write(resp.content)
+            print("Installing test prefs from %s" % url)
+            try:
+                extract_dir = tempfile.mkdtemp()
+                unzip(get(url).raw, dest=extract_dir)
+
+                profiles = os.path.join(extract_dir, os.listdir(extract_dir)[0], 'testing', 'profiles')
+                for name in os.listdir(profiles):
+                    path = os.path.join(profiles, name)
+                    shutil.move(path, dest)
+            finally:
+                shutil.rmtree(extract_dir)
         else:
-            print("Using cached test prefs from %s" % cache_file)
-
-        shutil.copyfile(cache_file, prefs_file)
+            print("Using cached test prefs from %s" % dest)
 
         return dest
 
     def _latest_geckodriver_version(self):
         """Get and return latest version number for geckodriver."""
         # This is used rather than an API call to avoid rate limits
         tags = call("git", "ls-remote", "--tags", "--refs",
                     "https://github.com/mozilla/geckodriver.git")
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -1,8 +1,9 @@
+import json
 import os
 import platform
 import signal
 import subprocess
 import sys
 import tempfile
 
 import mozinfo
@@ -240,21 +241,33 @@ class FirefoxBrowser(Browser):
         self.logger.debug("Starting Firefox")
 
         self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
         self.logger.debug("Firefox Started")
 
     def load_prefs(self):
         prefs = Preferences()
 
-        prefs_path = os.path.join(self.prefs_root, "user.js")
-        if os.path.exists(prefs_path):
-            prefs.add(Preferences.read_prefs(prefs_path))
-        else:
-            self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
+        pref_paths = []
+        prefs_general = os.path.join(self.prefs_root, 'prefs_general.js')
+        if os.path.isfile(prefs_general):
+            # Old preference file used in Firefox 60 and earlier (remove when no longer supported)
+            pref_paths.append(prefs_general)
+
+        profiles = os.path.join(self.prefs_root, 'profiles.json')
+        if os.path.isfile(profiles):
+            with open(profiles, 'r') as fh:
+                for name in json.load(fh)['web-platform-tests']:
+                    pref_paths.append(os.path.join(self.prefs_root, name, 'user.js'))
+
+        for path in pref_paths:
+            if os.path.exists(path):
+                prefs.add(Preferences.read_prefs(path))
+            else:
+                self.logger.warning("Failed to find base prefs file in %s" % path)
 
         # Add any custom preferences
         prefs.add(self.extra_prefs, cast=True)
 
         return prefs()
 
     def stop(self, force=False):
         if self.runner is not None and self.runner.is_running():