--- a/configs/merge_day/release_to_esr.py
+++ b/configs/merge_day/release_to_esr.py
@@ -1,11 +1,11 @@
-NEW_ESR_REPO = "ssh://hg.mozilla.org/releases/mozilla-esr38"
-OLD_ESR_REPO = "https://hg.mozilla.org/releases/mozilla-esr31"
-OLD_ESR_CHANGESET = "450086c0ded0"
+NEW_ESR_REPO = "ssh://hg.mozilla.org/releases/mozilla-esr45"
+OLD_ESR_REPO = "https://hg.mozilla.org/releases/mozilla-esr38"
+OLD_ESR_CHANGESET = "16351963d75c"
config = {
"log_name": "relese_to_esr",
# Disallow sharing, since we want pristine .hg directories.
# "vcs_share_base": None,
# "hg_share_base": None,
"tools_repo_url": "https://hg.mozilla.org/build/tools",
"tools_repo_revision": "default",
--- a/mozharness/base/python.py
+++ b/mozharness/base/python.py
@@ -557,16 +557,20 @@ class InfluxRecordingMixin(object):
mozharness are recorded in the 'mozharness' table.
"""
@PreScriptRun
def influxdb_recording_init(self):
self.recording = False
self.post = None
self.posturl = None
+ self.res_props = os.path.join(
+ self.query_abs_dirs()['abs_obj_dir'], '.mozbuild', 'build_resources.json'
+ )
+ self.rmtree(self.res_props)
try:
site_packages_path = self.query_python_site_packages_path()
if site_packages_path not in sys.path:
sys.path.append(site_packages_path)
import requests
self.post = requests.post
@@ -626,16 +630,100 @@ class InfluxRecordingMixin(object):
"branch",
"slavename",
"gecko_revision",
"gaia_revision",
"buildid",
],
}])
+ def _get_resource_usage(self, res, name, iolen, cpulen):
+ c = {}
+ p = {}
+ if self.buildbot_config:
+ c = self.buildbot_config.get('properties', {})
+ if self.buildbot_properties:
+ p = self.buildbot_properties
+
+ data = [
+ # Build properties
+ c.get('buildername'),
+ c.get('product'),
+ c.get('platform'),
+ c.get('branch'),
+ c.get('slavename'),
+ c.get('revision'),
+ p.get('gaia_revision'),
+ c.get('buildid'),
+
+ # Mach step properties
+ name,
+ res.get('start'),
+ res.get('end'),
+ res.get('duration'),
+ res.get('cpu_percent'),
+ ]
+ # The io and cpu_times fields are arrays, though they aren't always
+ # present if a step completes before resource utilization is measured.
+ # We add the arrays if they exist, otherwise we just do an array of None
+ # to fill up the stat point.
+ data.extend(res.get('io', [None] * iolen))
+ data.extend(res.get('cpu_times', [None] * cpulen))
+ return data
+
+ @PostScriptAction('build')
+ def record_mach_stats(self, action, success=None):
+ if not self.recording:
+ return
+ if not os.path.exists(self.res_props):
+ self.info('No build_resources.json found, not logging stats')
+ return
+ with open(self.res_props) as fh:
+ resources = json.load(fh)
+ data = {
+ "points": [
+ ],
+ "name": "mach",
+ "columns": [
+ # Build properties
+ "buildername",
+ "product",
+ "platform",
+ "branch",
+ "slavename",
+ "gecko_revision",
+ "gaia_revision",
+ "buildid",
+
+ # Mach step properties
+ "name",
+ "start",
+ "end",
+ "duration",
+ "cpu_percent",
+ ],
+ }
+ # The io and cpu_times fields aren't static - they may vary based
+ # on the specific platform being measured. Mach records the field
+ # names, which we use as the column names here.
+ data['columns'].extend(resources['io_fields'])
+ data['columns'].extend(resources['cpu_times_fields'])
+ iolen = len(resources['io_fields'])
+ cpulen = len(resources['cpu_times_fields'])
+
+ # The top-level data has the overall resource usage, which we record
+ # under the name 'TOTAL' to separate it from the individual tiers.
+ data['points'].append(self._get_resource_usage(resources, 'TOTAL', iolen, cpulen))
+
+ # Each tier also has the same resource stats as the top-level.
+ for tier in resources['tiers']:
+ data['points'].append(self._get_resource_usage(tier, tier['name'], iolen, cpulen))
+
+ self.record_influx_stat([data])
+
def record_influx_stat(self, json_data):
try:
r = self.post(self.posturl, data=json.dumps(json_data), timeout=5)
if r.status_code != 200:
self.warning("Failed to log stats. Return code = %i, stats = %s" % (r.status_code, json_data))
# Disable recording for the rest of this job. Even if it's just
# intermittent, we don't want to keep the build from progressing.
--- a/mozharness/mozilla/building/buildbase.py
+++ b/mozharness/mozilla/building/buildbase.py
@@ -808,22 +808,25 @@ or run without that action (ie: --no-{ac
env["MOZ_UPDATE_CHANNEL"] = c['update_channel']
else: # let's just give the generic channel based on branch
env["MOZ_UPDATE_CHANNEL"] = "nightly-%s" % (self.branch,)
if self.config.get('pgo_build') or self._compile_against_pgo():
env['MOZ_PGO'] = '1'
if c.get('enable_signing'):
- moz_sign_cmd = subprocess.list2cmdline(
- self.query_moz_sign_cmd(formats=None)
- )
- # windows fix. This is passed to mach build env and we call that
- # with python, not with bash so we need to fix the slashes here
- env['MOZ_SIGN_CMD'] = moz_sign_cmd.replace('\\', '\\\\\\\\')
+ if os.environ.get('MOZ_SIGNING_SERVERS'):
+ moz_sign_cmd = subprocess.list2cmdline(
+ self.query_moz_sign_cmd(formats=None)
+ )
+ # windows fix. This is passed to mach build env and we call that
+ # with python, not with bash so we need to fix the slashes here
+ env['MOZ_SIGN_CMD'] = moz_sign_cmd.replace('\\', '\\\\\\\\')
+ else:
+ self.warning("signing disabled because MOZ_SIGNING_SERVERS is not set")
# to activate the right behaviour in mozonfigs while we transition
if c.get('enable_release_promotion'):
env['ENABLE_RELEASE_PROMOTION'] = "1"
# we can't make env an attribute of self because env can change on
# every call for reasons like MOZ_SIGN_CMD
return env
--- a/mozharness/mozilla/tooltool.py
+++ b/mozharness/mozilla/tooltool.py
@@ -41,21 +41,22 @@ class TooltoolMixin(object):
return '/builds/relengapi.tok'
def tooltool_fetch(self, manifest, bootstrap_cmd=None,
output_dir=None, privileged=False, cache=None):
"""docstring for tooltool_fetch"""
tooltool = self.query_exe('tooltool.py', return_type='list')
if self.config.get("developer_mode"):
- if not os.path.exists(str(tooltool)):
- tooltool = self._fetch_tooltool_py()
- cmd = [tooltool,
- "--authentication-file",
- get_credentials_path()]
+ tooltool = [bin for bin in tooltool if os.path.exists(bin)]
+ if tooltool:
+ cmd = [tooltool[0]]
+ else:
+ cmd = [self._fetch_tooltool_py()]
+ cmd.extend(["--authentication-file", get_credentials_path()])
else:
cmd = tooltool
# get the tooltool servers from configuration
default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
# add slashes (bug 1155630)
def add_slash(url):
--- a/scripts/b2g_build.py
+++ b/scripts/b2g_build.py
@@ -152,16 +152,17 @@ class B2GBuild(LocalesMixin, PurgeMixin,
require_config_file=require_config_file,
config=default_config,
all_actions=all_actions,
default_actions=default_actions,
)
dirs = self.query_abs_dirs()
self.objdir = os.path.join(dirs['work_dir'], 'objdir-gecko')
+ self.abs_dirs['abs_obj_dir'] = self.objdir
if self.config.get("update_type", "ota") == "fota":
self.make_updates_cmd = ['./build.sh', 'gecko-update-fota']
self.extra_update_attrs = 'isOsUpdate="true"'
self.isOSUpdate = True
else:
self.make_updates_cmd = ['./build.sh', 'gecko-update-full']
self.extra_update_attrs = None
self.isOSUpdate = False