author | Andrew Halberstadt <ahalberstadt@mozilla.com> |
Mon, 07 May 2018 09:43:12 -0400 | |
changeset 417363 | e4b9cb3abd0724e50ced4d0248649f27ad8be9ac |
parent 417362 | f5cb209a849dd8e2a8dc8a55ee2d435ed578df72 |
child 417364 | 0fd5996b2d80ca73b08adf55b5e5261c6d334b1b |
push id | 63722 |
push user | ahalberstadt@mozilla.com |
push date | Tue, 08 May 2018 18:48:50 +0000 |
treeherder | autoland@e4b9cb3abd07 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | jgraham |
bugs | 1459598 |
milestone | 62.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/build/pgo/profileserver.py +++ b/build/pgo/profileserver.py @@ -1,14 +1,15 @@ #!/usr/bin/python # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json import os from buildconfig import substs from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Profile, Preferences from mozprofile.permissions import ServerLocations @@ -38,24 +39,27 @@ if __name__ == '__main__': locations = ServerLocations() locations.add_host(host='127.0.0.1', port=PORT, options='primary,privileged') with TemporaryDirectory() as profilePath: # TODO: refactor this into mozprofile - prefpath = os.path.join( - build.topsrcdir, "testing", "profiles", "common", "user.js") - overridepath = os.path.join( - build.topsrcdir, "build", "pgo", "prefs_override.js") + profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles') + with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: + base_profiles = json.load(fh)['profileserver'] + + prefpaths = [os.path.join(profile_data_dir, profile, 'user.js') + for profile in base_profiles] + prefpaths.append(os.path.join(build.topsrcdir, "build", "pgo", "prefs_override.js")) prefs = {} - prefs.update(Preferences.read_prefs(prefpath)) - prefs.update(Preferences.read_prefs(overridepath)) + for path in prefpaths: + prefs.update(Preferences.read_prefs(path)) interpolation = {"server": "%s:%d" % httpd.httpd.server_address, "OOP": "false"} for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v)
--- a/build/valgrind/mach_commands.py +++ b/build/valgrind/mach_commands.py @@ -1,14 +1,15 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, unicode_literals +import json import logging import mozinfo import os import subprocess from mach.decorators import ( Command, CommandArgument, @@ -60,19 +61,26 @@ class MachCommands(MachCommandBase): # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: #TODO: refactor this into mozprofile - prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'common', 'user.js') + profile_data_dir = os.path.join(self.topsrcdir, 'testing', 'profiles') + with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: + base_profiles = json.load(fh)['valgrind'] + + prefpaths = [os.path.join(profile_data_dir, profile, 'user.js') + for profile in base_profiles] prefs = {} - prefs.update(Preferences.read_prefs(prefpath)) + for path in prefpaths: + prefs.update(Preferences.read_prefs(path)) + interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v)
--- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -846,17 +846,16 @@ class MochitestDesktop(object): CHROME_PATH = "redirect.html" certdbNew = False sslTunnel = None DEFAULT_TIMEOUT = 60.0 mediaDevices = None patternFiles = {} - base_profiles = ('common',) # XXX use automation.py for test name to avoid breaking legacy # TODO: replace this with 'runtests.py' or 'mochitest' or the like test_name = 'automation.py' def __init__(self, flavor, logger_options, quiet=False): update_mozinfo() self.flavor = flavor @@ -1846,22 +1845,25 @@ toolbar#nav-bar { # If possible, read profile data from topsrcdir. This prevents us from # requiring a re-build to pick up newly added extensions in the # <profile>/extensions directory. if build_obj: path = os.path.join(build_obj.topsrcdir, 'testing', 'profiles') if os.path.isdir(path): profile_data_dir = path + with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: + base_profiles = json.load(fh)['mochitest'] + # values to use when interpolating preferences interpolation = { "server": "%s:%s" % (options.webServer, options.httpPort), } - for profile in self.base_profiles: + for profile in base_profiles: path = os.path.join(profile_data_dir, profile) self.profile.merge(path, interpolation=interpolation) def buildProfile(self, options): """ create the profile and add optional chrome bits and files if requested """ # get extensions to install extensions = self.getExtensionsToInstall(options)
--- a/testing/mochitest/tests/python/test_build_profile.py +++ b/testing/mochitest/tests/python/test_build_profile.py @@ -1,25 +1,29 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import print_function, unicode_literals +import json import os from argparse import Namespace +from mozbuild.base import MozbuildObject from mozprofile.prefs import Preferences from mozprofile import Profile from six import string_types import mozunit import pytest from conftest import setup_args +here = os.path.abspath(os.path.dirname(__file__)) + @pytest.fixture def build_profile(monkeypatch, setup_test_harness, parser): setup_test_harness(*setup_args) runtests = pytest.importorskip('runtests') md = runtests.MochitestDesktop('plain', {'log_tbpl': '-'}) monkeypatch.setattr(md, 'fillCertificateDB', lambda *args, **kwargs: None) @@ -31,29 +35,33 @@ def build_profile(monkeypatch, setup_tes opts.update(kwargs) return md, md.buildProfile(Namespace(**opts)) return inner @pytest.fixture -def profile_data_dir(build_obj): - return os.path.join(build_obj.topsrcdir, 'testing', 'profiles') +def profile_data_dir(): + build = MozbuildObject.from_environment(cwd=here) + return os.path.join(build.topsrcdir, 'testing', 'profiles') def test_common_prefs_are_all_set(build_profile, profile_data_dir): # We set e10s=False here because MochitestDesktop.buildProfile overwrites # the value defined in the base profile. # TODO stop setting browser.tabs.remote.autostart in the base profile md, result = build_profile(e10s=False) + with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: + base_profiles = json.load(fh)['mochitest'] + # build the expected prefs expected_prefs = {} - for profile in md.base_profiles: + for profile in base_profiles: for name in Profile.preference_file_names: path = os.path.join(profile_data_dir, profile, name) if os.path.isfile(path): expected_prefs.update(Preferences.read_prefs(path)) # read the actual prefs actual_prefs = {} for name in Profile.preference_file_names:
--- a/testing/profiles/moz.build +++ b/testing/profiles/moz.build @@ -1,15 +1,16 @@ # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- # vim: set filetype=python: # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -profiles = [ +profile_files = [ 'common/*', + 'profiles.json', ] -TEST_HARNESS_FILES.testing.mochitest.profile_data += profiles -TEST_HARNESS_FILES['web-platform'].prefs += ['common/user.js'] +TEST_HARNESS_FILES.testing.mochitest.profile_data += profile_files +TEST_HARNESS_FILES['web-platform'].prefs += profile_files with Files("**"): - BUG_COMPONENT = ("Testing", "Mochitest") + BUG_COMPONENT = ("Testing", "General")
new file mode 100644 --- /dev/null +++ b/testing/profiles/profiles.json @@ -0,0 +1,6 @@ +{ + "mochitest": ["common"], + "profileserver": ["common"], + "web-platform-tests": ["common"], + "valgrind": ["common"] +}
--- a/testing/web-platform/tests/tools/wpt/browser.py +++ b/testing/web-platform/tests/tools/wpt/browser.py @@ -1,16 +1,17 @@ import logging import os import platform import re import shutil import stat import subprocess import sys +import tempfile from abc import ABCMeta, abstractmethod from ConfigParser import RawConfigParser from datetime import datetime, timedelta from distutils.spawn import find_executable from io import BytesIO from utils import call, get, untar, unzip @@ -186,68 +187,69 @@ class Firefox(Browser): stdout.strip() m = version_re.match(stdout) if not m: return None, "nightly" version, status = m.groups() channel = {"a": "nightly", "b": "beta"} return version, channel.get(status, "stable") - def get_prefs_url(self, version, channel): + def get_profile_bundle_url(self, version, channel): if channel == "stable": repo = "https://hg.mozilla.org/releases/mozilla-release" tag = "FIREFOX_%s_RELEASE" % version.replace(".", "_") else: repo = "https://hg.mozilla.org/mozilla-central" if channel == "beta": tag = "FIREFOX_%s_BETA" % version.split(".", 1)[0] else: # Always use tip as the tag for nightly; this isn't quite right # but to do better we need the actual build revision, which we # can get if we have an application.ini file tag = "tip" - return "%s/raw-file/%s/testing/profiles/common/user.js" % (repo, tag) + return "%s/archive/%s.zip/testing/profiles/" % (repo, tag) def install_prefs(self, binary, dest=None): version, channel = self.get_version_number(binary) if dest is None: dest = os.pwd - dest = os.path.join(dest, "profiles", "common") - if not os.path.exists(dest): - os.makedirs(dest) - prefs_file = os.path.join(dest, "user.js") - cache_file = os.path.join(dest, - "%s-%s.cache" % (version, channel) - if channel != "nightly" - else "nightly.cache") - + dest = os.path.join(dest, "profiles", channel, version) have_cache = False - if os.path.exists(cache_file): + if os.path.exists(dest): if channel != "nightly": have_cache = True else: now = datetime.now() - have_cache = (datetime.fromtimestamp(os.stat(cache_file).st_mtime) > + have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) > now - timedelta(days=1)) - # If we don't have a recent download, grab the url + # If we don't have a recent download, grab and extract the latest one if not have_cache: - url = self.get_prefs_url(version, channel) + if os.path.exists(dest): + shutil.rmtree(dest) + os.makedirs(dest) + + url = self.get_profile_bundle_url(version, channel) - with open(cache_file, "wb") as f: - print("Installing test prefs from %s" % url) - resp = get(url) - f.write(resp.content) + print("Installing test prefs from %s" % url) + try: + extract_dir = tempfile.mkdtemp() + unzip(get(url).raw, dest=extract_dir) + + profiles = os.path.join(extract_dir, os.listdir(extract_dir)[0], 'testing', 'profiles') + for name in os.listdir(profiles): + path = os.path.join(profiles, name) + shutil.move(path, dest) + finally: + shutil.rmtree(extract_dir) else: - print("Using cached test prefs from %s" % cache_file) - - shutil.copyfile(cache_file, prefs_file) + print("Using cached test prefs from %s" % dest) return dest def _latest_geckodriver_version(self): """Get and return latest version number for geckodriver.""" # This is used rather than an API call to avoid rate limits tags = call("git", "ls-remote", "--tags", "--refs", "https://github.com/mozilla/geckodriver.git")
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py @@ -1,8 +1,9 @@ +import json import os import platform import signal import subprocess import sys import tempfile import mozinfo @@ -240,21 +241,33 @@ class FirefoxBrowser(Browser): self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started") def load_prefs(self): prefs = Preferences() - prefs_path = os.path.join(self.prefs_root, "user.js") - if os.path.exists(prefs_path): - prefs.add(Preferences.read_prefs(prefs_path)) - else: - self.logger.warning("Failed to find base prefs file in %s" % prefs_path) + pref_paths = [] + prefs_general = os.path.join(self.prefs_root, 'prefs_general.js') + if os.path.isfile(prefs_general): + # Old preference file used in Firefox 60 and earlier (remove when no longer supported) + pref_paths.append(prefs_general) + + profiles = os.path.join(self.prefs_root, 'profiles.json') + if os.path.isfile(profiles): + with open(profiles, 'r') as fh: + for name in json.load(fh)['web-platform-tests']: + pref_paths.append(os.path.join(self.prefs_root, name, 'user.js')) + + for path in pref_paths: + if os.path.exists(path): + prefs.add(Preferences.read_prefs(path)) + else: + self.logger.warning("Failed to find base prefs file in %s" % path) # Add any custom preferences prefs.add(self.extra_prefs, cast=True) return prefs() def stop(self, force=False): if self.runner is not None and self.runner.is_running():