bug 1237610 - move build system telemetry collection code to a common place. r=gps
authorTed Mielczarek <ted@mielczarek.org>
Tue, 07 Aug 2018 11:27:26 -0400
changeset 437684 a413b8fa5c7f3b12b7161782d2b774e96120a78e
parent 437683 cb1c0195ac65b7739cf68c94b953e2e783542fe6
child 437685 61de9d8b0bb22a0e95dac87b297f727f15e4f346
push id34690
push usercbrindusan@mozilla.com
push dateFri, 21 Sep 2018 17:30:01 +0000
treeherdermozilla-central@ce4e883f7642 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1237610
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1237610 - move build system telemetry collection code to a common place. r=gps The telemetry gathering code is currently split in two places, so move it all to a common place. Followup patches will rewrite most of this code.
build/mach_bootstrap.py
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/telemetry.py
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -201,38 +201,16 @@ def bootstrap(topsrcdir, mozilla_dir=Non
                 raise
         outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
         try:
             os.mkdir(outgoing_dir)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
-        # Add common metadata to help submit sorted data later on.
-        data['argv'] = sys.argv
-        data.setdefault('system', {}).update(dict(
-            architecture=list(platform.architecture()),
-            machine=platform.machine(),
-            python_version=platform.python_version(),
-            release=platform.release(),
-            system=platform.system(),
-            version=platform.version(),
-        ))
-
-        if platform.system() == 'Linux':
-            dist = list(platform.linux_distribution())
-            data['system']['linux_distribution'] = dist
-        elif platform.system() == 'Windows':
-            win32_ver = list((platform.win32_ver())),
-            data['system']['win32_ver'] = win32_ver
-        elif platform.system() == 'Darwin':
-            # mac version is a special Cupertino snowflake
-            r, v, m = platform.mac_ver()
-            data['system']['mac_ver'] = [r, list(v), m]
-
         with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'),
                   'w') as f:
             json.dump(data, f, sort_keys=True)
 
     def should_skip_dispatch(context, handler):
         # The user is performing a maintenance command.
         if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'vcs-setup'):
             return True
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -36,32 +36,34 @@ from mozsystemmonitor.resourcemonitor im
 from mozterm.widgets import Footer
 
 import mozpack.path as mozpath
 
 from .clobber import (
     Clobberer,
 )
 from ..base import (
-    BuildEnvironmentNotFoundException,
     MozbuildObject,
 )
 from ..backend import (
     get_backend_class,
 )
 from ..testing import (
     install_test_files,
 )
 from ..compilation.warnings import (
     WarningsCollector,
     WarningsDatabase,
 )
 from ..shellutil import (
     quote as shell_quote,
 )
+from ..telemetry import (
+    gather_telemetry,
+)
 from ..util import (
     FileAvoidWrite,
     mkdir,
     resolve_target_to_make,
 )
 
 
 FINDER_SLOW_MESSAGE = '''
@@ -1280,62 +1282,35 @@ class BuildDriver(MozbuildObject):
         except ValueError:
             # Just stick with the default
             pass
 
         if monitor.elapsed > notify_minimum_time:
             # Display a notification when the build completes.
             self.notify('Build complete' if not status else 'Build failed')
 
+        gather_telemetry(monitor, mach_context, self.substs, ccache_diff)
+
         if status:
             return status
 
-        long_build = monitor.elapsed > 600
-
-        if long_build:
-            output.on_line('We know it took a while, but your build finally finished successfully!')
-        else:
-            output.on_line('Your build was successful!')
-
         if monitor.have_resource_usage:
             excessive, swap_in, swap_out = monitor.have_excessive_swapping()
             # if excessive:
             #    print(EXCESSIVE_SWAP_MESSAGE)
 
             print('To view resource usage of the build, run |mach '
-                'resource-usage|.')
+                  'resource-usage|.')
 
-            telemetry_handler = getattr(mach_context,
-                                        'telemetry_handler', None)
-            telemetry_data = monitor.get_resource_usage()
+        long_build = monitor.elapsed > 600
 
-            # Record build configuration data. For now, we cherry pick
-            # items we need rather than grabbing everything, in order
-            # to avoid accidentally disclosing PII.
-            telemetry_data['substs'] = {}
-            try:
-                for key in ['MOZ_ARTIFACT_BUILDS', 'MOZ_USING_CCACHE', 'MOZ_USING_SCCACHE']:
-                    value = self.substs.get(key, False)
-                    telemetry_data['substs'][key] = value
-            except BuildEnvironmentNotFoundException:
-                pass
-
-            # Grab ccache stats if available. We need to be careful not
-            # to capture information that can potentially identify the
-            # user (such as the cache location)
-            if ccache_diff:
-                telemetry_data['ccache'] = {}
-                for key in [key[0] for key in ccache_diff.STATS_KEYS]:
-                    try:
-                        telemetry_data['ccache'][key] = ccache_diff._values[key]
-                    except KeyError:
-                        pass
-
-            if telemetry_handler:
-                telemetry_handler(mach_context, telemetry_data)
+        if long_build:
+            output.on_line('We know it took a while, but your build finally finished successfully!')
+        else:
+            output.on_line('Your build was successful!')
 
         # Only for full builds because incremental builders likely don't
         # need to be burdened with this.
         if not what:
             try:
                 # Fennec doesn't have useful output from just building. We should
                 # arguably make the build action useful for Fennec. Another day...
                 if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
--- a/python/mozbuild/mozbuild/telemetry.py
+++ b/python/mozbuild/mozbuild/telemetry.py
@@ -3,25 +3,31 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 '''
 This file contains a voluptuous schema definition for build system telemetry.
 '''
 
-from mozbuild.configure.constants import CompilerType
+import platform
+import sys
 from voluptuous import (
     Any,
     Optional,
     Required,
     Schema,
 )
 from voluptuous.validators import Datetime
 
+from .base import (
+    BuildEnvironmentNotFoundException,
+)
+from .configure.constants import CompilerType
+
 schema = Schema({
     Required('client_id', description='A UUID to uniquely identify a client'): basestring,
     Required('time', description='Time at which this event happened'): Datetime(),
     Required('command', description='The mach command that was invoked'): basestring,
     Required('argv', description=(
         'Full mach commandline. ' +
         'If the commandline contains absolute paths they will be sanitized.')): [basestring],
     Required('success', description='true if the command succeeded'): bool,
@@ -55,8 +61,62 @@ schema = Schema({
         Optional('physical_cores', description='Number of physical CPU cores present'): int,
         Optional('memory_gb', description='System memory in GB'): int,
         Optional('drive_is_ssd',
                  description='true if the source directory is on a solid-state disk'): bool,
         Optional('virtual_machine',
                  description='true if the OS appears to be running in a virtual machine'): bool,
     },
 })
+
+
+def gather_telemetry(monitor, mach_context, substs, ccache_diff):
+    if monitor.have_resource_usage:
+        telemetry_handler = getattr(mach_context,
+                                    'telemetry_handler', None)
+        data = monitor.get_resource_usage()
+
+        # Record build configuration data. For now, we cherry pick
+        # items we need rather than grabbing everything, in order
+        # to avoid accidentally disclosing PII.
+        data['substs'] = {}
+        try:
+            for key in ['MOZ_ARTIFACT_BUILDS', 'MOZ_USING_CCACHE', 'MOZ_USING_SCCACHE']:
+                value = substs.get(key, False)
+                data['substs'][key] = value
+        except BuildEnvironmentNotFoundException:
+            pass
+
+        # Grab ccache stats if available. We need to be careful not
+        # to capture information that can potentially identify the
+        # user (such as the cache location)
+        if ccache_diff:
+            data['ccache'] = {}
+            for key in [key[0] for key in ccache_diff.STATS_KEYS]:
+                try:
+                    data['ccache'][key] = ccache_diff._values[key]
+                except KeyError:
+                    pass
+
+        # Add common metadata to help submit sorted data later on.
+        data['argv'] = sys.argv
+        data.setdefault('system', {}).update(dict(
+            architecture=list(platform.architecture()),
+            machine=platform.machine(),
+            python_version=platform.python_version(),
+            release=platform.release(),
+            system=platform.system(),
+            version=platform.version(),
+        ))
+
+        if platform.system() == 'Linux':
+            dist = list(platform.linux_distribution())
+            data['system']['linux_distribution'] = dist
+        elif platform.system() == 'Windows':
+            win32_ver = list((platform.win32_ver())),
+            data['system']['win32_ver'] = win32_ver
+        elif platform.system() == 'Darwin':
+            # mac version is a special Cupertino snowflake
+            r, v, m = platform.mac_ver()
+            data['system']['mac_ver'] = [r, list(v), m]
+
+        if telemetry_handler:
+            telemetry_handler(mach_context, data)