Bug 1459598 - Use profiles.json file to map test suites to the base profiles they use, r?jgraham draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 07 May 2018 09:43:12 -0400
changeset 792554 0aa91997e57b8d6be5fa6c6819ff19cf674fef1b
parent 792466 dc2ea9f1bf9d7a6b9ff4c111ce8211f6e7fb4142
push id109139
push userahalberstadt@mozilla.com
push dateTue, 08 May 2018 15:24:59 +0000
reviewersjgraham
bugs1459598
milestone62.0a1
Bug 1459598 - Use profiles.json file to map test suites to the base profiles they use, r?jgraham This serves two purposes: 1) It makes web-platform-tests pref downloading/handling a little more robust. When run externally, it now downloads the entire testing/profiles directory. When loading prefs it will look for both prefs_general.js (to support older versions of Firefox) and profiles.json (for support moving forward). This way we can add/remove/rename pref files under these directories without needing to worry about breaking upstream wpt. 2) It provides developers an overview of which harnesses are using which base profiles. Instead of hunting through test harness code to find this information, they can glance at profiles.json. MozReview-Commit-ID: AMzdnD8aGA2
build/pgo/profileserver.py
build/valgrind/mach_commands.py
testing/mochitest/runtests.py
testing/mochitest/tests/python/test_build_profile.py
testing/profiles/moz.build
testing/profiles/profiles.json
testing/web-platform/tests/tools/wpt/browser.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
--- a/build/pgo/profileserver.py
+++ b/build/pgo/profileserver.py
@@ -1,14 +1,15 @@
 #!/usr/bin/python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import json
 import os
 
 from buildconfig import substs
 from mozbuild.base import MozbuildObject
 from mozfile import TemporaryDirectory
 from mozhttpd import MozHttpd
 from mozprofile import FirefoxProfile, Profile, Preferences
 from mozprofile.permissions import ServerLocations
@@ -38,24 +39,27 @@ if __name__ == '__main__':
 
     locations = ServerLocations()
     locations.add_host(host='127.0.0.1',
                        port=PORT,
                        options='primary,privileged')
 
     with TemporaryDirectory() as profilePath:
         # TODO: refactor this into mozprofile
-        prefpath = os.path.join(
-            build.topsrcdir, "testing", "profiles", "common", "user.js")
-        overridepath = os.path.join(
-            build.topsrcdir, "build", "pgo", "prefs_override.js")
+        profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles')
+        with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
+            base_profiles = json.load(fh)['profileserver']
+
+        prefpaths = [os.path.join(profile_data_dir, profile, 'user.js')
+                     for profile in base_profiles]
+        prefpaths.append(os.path.join(build.topsrcdir, "build", "pgo", "prefs_override.js"))
 
         prefs = {}
-        prefs.update(Preferences.read_prefs(prefpath))
-        prefs.update(Preferences.read_prefs(overridepath))
+        for path in prefpaths:
+            prefs.update(Preferences.read_prefs(path))
 
         interpolation = {"server": "%s:%d" % httpd.httpd.server_address,
                          "OOP": "false"}
         for k, v in prefs.items():
             if isinstance(v, string_types):
                 v = v.format(**interpolation)
             prefs[k] = Preferences.cast(v)
 
--- a/build/valgrind/mach_commands.py
+++ b/build/valgrind/mach_commands.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
+import json
 import logging
 import mozinfo
 import os
 import subprocess
 
 from mach.decorators import (
     Command,
     CommandArgument,
@@ -60,19 +61,26 @@ class MachCommands(MachCommandBase):
 
         # XXX: currently we just use the PGO inputs for Valgrind runs.  This may
         # change in the future.
         httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
         httpd.start(block=False)
 
         with TemporaryDirectory() as profilePath:
             #TODO: refactor this into mozprofile
-            prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'common', 'user.js')
+            profile_data_dir = os.path.join(self.topsrcdir, 'testing', 'profiles')
+            with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
+                base_profiles = json.load(fh)['valgrind']
+
+            prefpaths = [os.path.join(profile_data_dir, profile, 'user.js')
+                         for profile in base_profiles]
             prefs = {}
-            prefs.update(Preferences.read_prefs(prefpath))
+            for path in prefpaths:
+                prefs.update(Preferences.read_prefs(path))
+
             interpolation = {
                 'server': '%s:%d' % httpd.httpd.server_address,
             }
             for k, v in prefs.items():
                 if isinstance(v, string_types):
                     v = v.format(**interpolation)
                 prefs[k] = Preferences.cast(v)
 
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -846,17 +846,16 @@ class MochitestDesktop(object):
     CHROME_PATH = "redirect.html"
 
     certdbNew = False
     sslTunnel = None
     DEFAULT_TIMEOUT = 60.0
     mediaDevices = None
 
     patternFiles = {}
-    base_profiles = ('common',)
 
     # XXX use automation.py for test name to avoid breaking legacy
     # TODO: replace this with 'runtests.py' or 'mochitest' or the like
     test_name = 'automation.py'
 
     def __init__(self, flavor, logger_options, quiet=False):
         update_mozinfo()
         self.flavor = flavor
@@ -1846,22 +1845,25 @@ toolbar#nav-bar {
         # If possible, read profile data from topsrcdir. This prevents us from
         # requiring a re-build to pick up newly added extensions in the
         # <profile>/extensions directory.
         if build_obj:
             path = os.path.join(build_obj.topsrcdir, 'testing', 'profiles')
             if os.path.isdir(path):
                 profile_data_dir = path
 
+        with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
+            base_profiles = json.load(fh)['mochitest']
+
         # values to use when interpolating preferences
         interpolation = {
             "server": "%s:%s" % (options.webServer, options.httpPort),
         }
 
-        for profile in self.base_profiles:
+        for profile in base_profiles:
             path = os.path.join(profile_data_dir, profile)
             self.profile.merge(path, interpolation=interpolation)
 
     def buildProfile(self, options):
         """ create the profile and add optional chrome bits and files if requested """
         # get extensions to install
         extensions = self.getExtensionsToInstall(options)
 
--- a/testing/mochitest/tests/python/test_build_profile.py
+++ b/testing/mochitest/tests/python/test_build_profile.py
@@ -1,25 +1,29 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
 
+import json
 import os
 from argparse import Namespace
 
+from mozbuild.base import MozbuildObject
 from mozprofile.prefs import Preferences
 from mozprofile import Profile
 from six import string_types
 
 import mozunit
 import pytest
 from conftest import setup_args
 
+here = os.path.abspath(os.path.dirname(__file__))
+
 
 @pytest.fixture
 def build_profile(monkeypatch, setup_test_harness, parser):
     setup_test_harness(*setup_args)
     runtests = pytest.importorskip('runtests')
     md = runtests.MochitestDesktop('plain', {'log_tbpl': '-'})
     monkeypatch.setattr(md, 'fillCertificateDB', lambda *args, **kwargs: None)
 
@@ -31,29 +35,33 @@ def build_profile(monkeypatch, setup_tes
         opts.update(kwargs)
 
         return md, md.buildProfile(Namespace(**opts))
 
     return inner
 
 
 @pytest.fixture
-def profile_data_dir(build_obj):
-    return os.path.join(build_obj.topsrcdir, 'testing', 'profiles')
+def profile_data_dir():
+    build = MozbuildObject.from_environment(cwd=here)
+    return os.path.join(build.topsrcdir, 'testing', 'profiles')
 
 
 def test_common_prefs_are_all_set(build_profile, profile_data_dir):
     # We set e10s=False here because MochitestDesktop.buildProfile overwrites
     # the value defined in the base profile.
     # TODO stop setting browser.tabs.remote.autostart in the base profile
     md, result = build_profile(e10s=False)
 
+    with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
+        base_profiles = json.load(fh)['mochitest']
+
     # build the expected prefs
     expected_prefs = {}
-    for profile in md.base_profiles:
+    for profile in base_profiles:
         for name in Profile.preference_file_names:
             path = os.path.join(profile_data_dir, profile, name)
             if os.path.isfile(path):
                 expected_prefs.update(Preferences.read_prefs(path))
 
     # read the actual prefs
     actual_prefs = {}
     for name in Profile.preference_file_names:
--- a/testing/profiles/moz.build
+++ b/testing/profiles/moz.build
@@ -1,15 +1,16 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-profiles = [
+profile_files = [
     'common/*',
+    'profiles.json',
 ]
 
-TEST_HARNESS_FILES.testing.mochitest.profile_data += profiles
-TEST_HARNESS_FILES['web-platform'].prefs += ['common/user.js']
+TEST_HARNESS_FILES.testing.mochitest.profile_data += profile_files
+TEST_HARNESS_FILES['web-platform'].prefs += profile_files
 
 with Files("**"):
-    BUG_COMPONENT = ("Testing", "Mochitest")
+    BUG_COMPONENT = ("Testing", "General")
new file mode 100644
--- /dev/null
+++ b/testing/profiles/profiles.json
@@ -0,0 +1,6 @@
+{
+    "mochitest": ["common"],
+    "profileserver": ["common"],
+    "web-platform-tests": ["common"],
+    "valgrind": ["common"]
+}
--- a/testing/web-platform/tests/tools/wpt/browser.py
+++ b/testing/web-platform/tests/tools/wpt/browser.py
@@ -1,16 +1,17 @@
 import logging
 import os
 import platform
 import re
 import shutil
 import stat
 import subprocess
 import sys
+import tempfile
 from abc import ABCMeta, abstractmethod
 from ConfigParser import RawConfigParser
 from datetime import datetime, timedelta
 from distutils.spawn import find_executable
 from io import BytesIO
 
 from utils import call, get, untar, unzip
 
@@ -186,68 +187,69 @@ class Firefox(Browser):
         stdout.strip()
         m = version_re.match(stdout)
         if not m:
             return None, "nightly"
         version, status = m.groups()
         channel = {"a": "nightly", "b": "beta"}
         return version, channel.get(status, "stable")
 
-    def get_prefs_url(self, version, channel):
+    def get_profile_bundle_url(self, version, channel):
         if channel == "stable":
             repo = "https://hg.mozilla.org/releases/mozilla-release"
             tag = "FIREFOX_%s_RELEASE" % version.replace(".", "_")
         else:
             repo = "https://hg.mozilla.org/mozilla-central"
             if channel == "beta":
                 tag = "FIREFOX_%s_BETA" % version.split(".", 1)[0]
             else:
                 # Always use tip as the tag for nightly; this isn't quite right
                 # but to do better we need the actual build revision, which we
                 # can get if we have an application.ini file
                 tag = "tip"
 
-        return "%s/raw-file/%s/testing/profiles/common/user.js" % (repo, tag)
+        return "%s/archive/%s.zip/testing/profiles/" % (repo, tag)
 
     def install_prefs(self, binary, dest=None):
         version, channel = self.get_version_number(binary)
 
         if dest is None:
             dest = os.pwd
 
-        dest = os.path.join(dest, "profiles", "common")
-        if not os.path.exists(dest):
-            os.makedirs(dest)
-        prefs_file = os.path.join(dest, "user.js")
-        cache_file = os.path.join(dest,
-                                  "%s-%s.cache" % (version, channel)
-                                  if channel != "nightly"
-                                  else "nightly.cache")
-
+        dest = os.path.join(dest, "profiles", channel, version)
         have_cache = False
-        if os.path.exists(cache_file):
+        if os.path.exists(dest):
             if channel != "nightly":
                 have_cache = True
             else:
                 now = datetime.now()
-                have_cache = (datetime.fromtimestamp(os.stat(cache_file).st_mtime) >
+                have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) >
                               now - timedelta(days=1))
 
-        # If we don't have a recent download, grab the url
+        # If we don't have a recent download, grab and extract the latest one
         if not have_cache:
-            url = self.get_prefs_url(version, channel)
+            if os.path.exists(dest):
+                shutil.rmtree(dest)
+            os.makedirs(dest)
+
+            url = self.get_profile_bundle_url(version, channel)
 
-            with open(cache_file, "wb") as f:
-                print("Installing test prefs from %s" % url)
-                resp = get(url)
-                f.write(resp.content)
+            print("Installing test prefs from %s" % url)
+            try:
+                extract_dir = tempfile.mkdtemp()
+                unzip(get(url).raw, dest=extract_dir)
+
+                profiles = os.path.join(extract_dir, os.listdir(extract_dir)[0], 'testing', 'profiles')
+                for name in os.listdir(profiles):
+                    path = os.path.join(profiles, name)
+                    shutil.move(path, dest)
+            finally:
+                shutil.rmtree(extract_dir)
         else:
-            print("Using cached test prefs from %s" % cache_file)
-
-        shutil.copyfile(cache_file, prefs_file)
+            print("Using cached test prefs from %s" % dest)
 
         return dest
 
     def _latest_geckodriver_version(self):
         """Get and return latest version number for geckodriver."""
         # This is used rather than an API call to avoid rate limits
         tags = call("git", "ls-remote", "--tags", "--refs",
                     "https://github.com/mozilla/geckodriver.git")
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -1,8 +1,9 @@
+import json
 import os
 import platform
 import signal
 import subprocess
 import sys
 import tempfile
 
 import mozinfo
@@ -240,21 +241,33 @@ class FirefoxBrowser(Browser):
         self.logger.debug("Starting Firefox")
 
         self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
         self.logger.debug("Firefox Started")
 
     def load_prefs(self):
         prefs = Preferences()
 
-        prefs_path = os.path.join(self.prefs_root, "user.js")
-        if os.path.exists(prefs_path):
-            prefs.add(Preferences.read_prefs(prefs_path))
-        else:
-            self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
+        pref_paths = []
+        prefs_general = os.path.join(self.prefs_root, 'prefs_general.js')
+        if os.path.isfile(prefs_general):
+            # Old preference file used in Firefox 60 and earlier (remove when no longer supported)
+            pref_paths.append(prefs_general)
+
+        profiles = os.path.join(self.prefs_root, 'profiles.json')
+        if os.path.isfile(profiles):
+            with open(profiles, 'r') as fh:
+                for name in json.load(fh)['web-platform-tests']:
+                    pref_paths.append(os.path.join(self.prefs_root, name, 'user.js'))
+
+        for path in pref_paths:
+            if os.path.exists(path):
+                prefs.add(Preferences.read_prefs(path))
+            else:
+                self.logger.warning("Failed to find base prefs file in %s" % path)
 
         # Add any custom preferences
         prefs.add(self.extra_prefs, cast=True)
 
         return prefs()
 
     def stop(self, force=False):
         if self.runner is not None and self.runner.is_running():