Merge default -> production production
authorMike Shal <mshal@mozilla.com>
Wed, 06 May 2015 13:10:42 -0400
branchproduction
changeset 3976 4d260376921f226f7bc13281cb10cdad074745e0
parent 3973 3b64e3e737fd574531ded7623fb14f287fe92d91 (current diff)
parent 3975 fe52c16631e48964da7323b86b172f4513fcf212 (diff)
child 3979 28b10345448dafe13f79780b6a775fe4b9d8f55d
push id3148
push usermshal@mozilla.com
push dateWed, 06 May 2015 17:11:13 +0000
Merge default -> production
--- a/configs/merge_day/release_to_esr.py
+++ b/configs/merge_day/release_to_esr.py
@@ -1,11 +1,11 @@
-NEW_ESR_REPO = "ssh://hg.mozilla.org/releases/mozilla-esr38"
-OLD_ESR_REPO = "https://hg.mozilla.org/releases/mozilla-esr31"
-OLD_ESR_CHANGESET = "450086c0ded0"
+NEW_ESR_REPO = "ssh://hg.mozilla.org/releases/mozilla-esr45"
+OLD_ESR_REPO = "https://hg.mozilla.org/releases/mozilla-esr38"
+OLD_ESR_CHANGESET = "16351963d75c"
 
 config = {
     "log_name": "relese_to_esr",
     # Disallow sharing, since we want pristine .hg directories.
     # "vcs_share_base": None,
     # "hg_share_base": None,
     "tools_repo_url": "https://hg.mozilla.org/build/tools",
     "tools_repo_revision": "default",
--- a/mozharness/base/python.py
+++ b/mozharness/base/python.py
@@ -557,16 +557,20 @@ class InfluxRecordingMixin(object):
     mozharness are recorded in the 'mozharness' table.
     """
 
     @PreScriptRun
     def influxdb_recording_init(self):
         self.recording = False
         self.post = None
         self.posturl = None
+        self.res_props = os.path.join(
+            self.query_abs_dirs()['abs_obj_dir'], '.mozbuild', 'build_resources.json'
+        )
+        self.rmtree(self.res_props)
 
         try:
             site_packages_path = self.query_python_site_packages_path()
             if site_packages_path not in sys.path:
                 sys.path.append(site_packages_path)
 
             import requests
             self.post = requests.post
@@ -626,16 +630,100 @@ class InfluxRecordingMixin(object):
                 "branch",
                 "slavename",
                 "gecko_revision",
                 "gaia_revision",
                 "buildid",
             ],
         }])
 
+    def _get_resource_usage(self, res, name, iolen, cpulen):
+        c = {}
+        p = {}
+        if self.buildbot_config:
+            c = self.buildbot_config.get('properties', {})
+        if self.buildbot_properties:
+            p = self.buildbot_properties
+
+        data = [
+            # Build properties
+            c.get('buildername'),
+            c.get('product'),
+            c.get('platform'),
+            c.get('branch'),
+            c.get('slavename'),
+            c.get('revision'),
+            p.get('gaia_revision'),
+            c.get('buildid'),
+
+            # Mach step properties
+            name,
+            res.get('start'),
+            res.get('end'),
+            res.get('duration'),
+            res.get('cpu_percent'),
+        ]
+        # The io and cpu_times fields are arrays, though they aren't always
+        # present if a step completes before resource utilization is measured.
+        # We add the arrays if they exist, otherwise we just do an array of None
+        # to fill up the stat point.
+        data.extend(res.get('io', [None] * iolen))
+        data.extend(res.get('cpu_times', [None] * cpulen))
+        return data
+
+    @PostScriptAction('build')
+    def record_mach_stats(self, action, success=None):
+        if not self.recording:
+            return
+        if not os.path.exists(self.res_props):
+            self.info('No build_resources.json found, not logging stats')
+            return
+        with open(self.res_props) as fh:
+            resources = json.load(fh)
+            data = {
+                "points": [
+                ],
+                "name": "mach",
+                "columns": [
+                    # Build properties
+                    "buildername",
+                    "product",
+                    "platform",
+                    "branch",
+                    "slavename",
+                    "gecko_revision",
+                    "gaia_revision",
+                    "buildid",
+
+                    # Mach step properties
+                    "name",
+                    "start",
+                    "end",
+                    "duration",
+                    "cpu_percent",
+                ],
+            }
+            # The io and cpu_times fields aren't static - they may vary based
+            # on the specific platform being measured. Mach records the field
+            # names, which we use as the column names here.
+            data['columns'].extend(resources['io_fields'])
+            data['columns'].extend(resources['cpu_times_fields'])
+            iolen = len(resources['io_fields'])
+            cpulen = len(resources['cpu_times_fields'])
+
+            # The top-level data has the overall resource usage, which we record
+            # under the name 'TOTAL' to separate it from the individual tiers.
+            data['points'].append(self._get_resource_usage(resources, 'TOTAL', iolen, cpulen))
+
+            # Each tier also has the same resource stats as the top-level.
+            for tier in resources['tiers']:
+                data['points'].append(self._get_resource_usage(tier, tier['name'], iolen, cpulen))
+
+            self.record_influx_stat([data])
+
     def record_influx_stat(self, json_data):
         try:
             r = self.post(self.posturl, data=json.dumps(json_data), timeout=5)
             if r.status_code != 200:
                 self.warning("Failed to log stats. Return code = %i, stats = %s" % (r.status_code, json_data))
 
                 # Disable recording for the rest of this job. Even if it's just
                 # intermittent, we don't want to keep the build from progressing.
--- a/mozharness/mozilla/building/buildbase.py
+++ b/mozharness/mozilla/building/buildbase.py
@@ -808,22 +808,25 @@ or run without that action (ie: --no-{ac
                 env["MOZ_UPDATE_CHANNEL"] = c['update_channel']
             else:  # let's just give the generic channel based on branch
                 env["MOZ_UPDATE_CHANNEL"] = "nightly-%s" % (self.branch,)
 
         if self.config.get('pgo_build') or self._compile_against_pgo():
             env['MOZ_PGO'] = '1'
 
         if c.get('enable_signing'):
-            moz_sign_cmd = subprocess.list2cmdline(
-                self.query_moz_sign_cmd(formats=None)
-            )
-            # windows fix. This is passed to mach build env and we call that
-            # with python, not with bash so we need to fix the slashes here
-            env['MOZ_SIGN_CMD'] = moz_sign_cmd.replace('\\', '\\\\\\\\')
+            if os.environ.get('MOZ_SIGNING_SERVERS'):
+                moz_sign_cmd = subprocess.list2cmdline(
+                    self.query_moz_sign_cmd(formats=None)
+                )
+                # windows fix. This is passed to mach build env and we call that
+                # with python, not with bash so we need to fix the slashes here
+                env['MOZ_SIGN_CMD'] = moz_sign_cmd.replace('\\', '\\\\\\\\')
+            else:
+                self.warning("signing disabled because MOZ_SIGNING_SERVERS is not set")
 
         # to activate the right behaviour in mozonfigs while we transition
         if c.get('enable_release_promotion'):
             env['ENABLE_RELEASE_PROMOTION'] = "1"
 
         # we can't make env an attribute of self because env can change on
         # every call for reasons like MOZ_SIGN_CMD
         return env
--- a/mozharness/mozilla/tooltool.py
+++ b/mozharness/mozilla/tooltool.py
@@ -41,21 +41,22 @@ class TooltoolMixin(object):
             return '/builds/relengapi.tok'
 
     def tooltool_fetch(self, manifest, bootstrap_cmd=None,
                        output_dir=None, privileged=False, cache=None):
         """docstring for tooltool_fetch"""
         tooltool = self.query_exe('tooltool.py', return_type='list')
 
         if self.config.get("developer_mode"):
-            if not os.path.exists(str(tooltool)):
-                tooltool = self._fetch_tooltool_py()
-            cmd = [tooltool,
-                   "--authentication-file",
-                   get_credentials_path()]
+            tooltool = [bin for bin in tooltool if os.path.exists(bin)]
+            if tooltool:
+                cmd = [tooltool[0]]
+            else:
+                cmd = [self._fetch_tooltool_py()]
+            cmd.extend(["--authentication-file", get_credentials_path()])
         else:
             cmd = tooltool
 
         # get the tooltool servers from configuration
         default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
 
         # add slashes (bug 1155630)
         def add_slash(url):
--- a/scripts/b2g_build.py
+++ b/scripts/b2g_build.py
@@ -152,16 +152,17 @@ class B2GBuild(LocalesMixin, PurgeMixin,
             require_config_file=require_config_file,
             config=default_config,
             all_actions=all_actions,
             default_actions=default_actions,
         )
 
         dirs = self.query_abs_dirs()
         self.objdir = os.path.join(dirs['work_dir'], 'objdir-gecko')
+        self.abs_dirs['abs_obj_dir'] = self.objdir
         if self.config.get("update_type", "ota") == "fota":
             self.make_updates_cmd = ['./build.sh', 'gecko-update-fota']
             self.extra_update_attrs = 'isOsUpdate="true"'
             self.isOSUpdate = True
         else:
             self.make_updates_cmd = ['./build.sh', 'gecko-update-full']
             self.extra_update_attrs = None
             self.isOSUpdate = False