Bug 1244160 - Create json-schema for build telemetry data r=gps
authorDan Minor <dminor@mozilla.com>
Tue, 02 Feb 2016 09:32:49 -0500
changeset 323391 930c12a120abdefcfc6093164112063cb477b4ce
parent 323390 44348b12ec6ac4ee58dc4fc3fcc034c389e23f8a
child 323392 6ae0e2d591f61d8d248ce7d404a63bbe52843cbc
push id1128
push userjlund@mozilla.com
push dateWed, 01 Jun 2016 01:31:59 +0000
treeherdermozilla-release@fe0d30de989d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1244160
milestone47.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1244160 - Create json-schema for build telemetry data r=gps This adds a simple schema for build telemetry data. We can make it more restrictive once we have a better feeling for what kind of data we want to submit. This also moves more common data about the system to the telemetry handler. We leave psutil derivied information in the resource usage data as not every system will have psutil installed. MozReview-Commit-ID: CFRq1Ow6AOf
build/mach_bootstrap.py
build/telemetry-schema.json
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/resources/html-build-viewer/index.html
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -249,18 +249,36 @@ def bootstrap(topsrcdir, mozilla_dir=Non
         outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
         try:
             os.mkdir(outgoing_dir)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         # Add common metadata to help submit sorted data later on.
-        # For now, we'll just record the mach command that was invoked.
         data['argv'] = sys.argv
+        data.setdefault('system', {}).update(dict(
+            architecture=list(platform.architecture()),
+            machine=platform.machine(),
+            python_version=platform.python_version(),
+            release=platform.release(),
+            system=platform.system(),
+            version=platform.version(),
+        ))
+
+        if platform.system() == 'Linux':
+            dist = list(platform.linux_distribution())
+            data['system']['linux_distribution'] = dist
+        elif platform.system() == 'Windows':
+            win32_ver=list((platform.win32_ver())),
+            data['system']['win32_ver'] = win32_ver
+        elif platform.system() == 'Darwin':
+            # mac version is a special Cupertino snowflake
+            r, v, m = platform.mac_ver()
+            data['system']['mac_ver'] = [r, list(v), m]
 
         with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'),
                   'w') as f:
             json.dump(data, f, sort_keys=True)
 
     def should_skip_dispatch(context, handler):
         # The user is performing a maintenance command.
         if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'mercurial-setup'):
new file mode 100644
--- /dev/null
+++ b/build/telemetry-schema.json
@@ -0,0 +1,24 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "argv": {"type": "array"},
+    "system": {
+      "type": "object",
+      "properties": {
+        "architecture": {"type": "array"},
+        "linux_distribution": {"type": "array"},
+        "mac_ver": {"type": "array"},
+        "machine": {"type": "string"},
+        "python_version": {"type": "string"},
+        "release": {"type": "string"},
+        "system": {"type": "string"},
+        "version": {"type": "string"},
+        "win_ver": {"type": "array"}
+      },
+      "required": ["architecture", "machine", "python_version",
+                   "release", "system", "version"]
+    }
+  },
+  "required": ["argv", "system"]
+}
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -372,17 +372,17 @@ class BuildMonitor(MozbuildObject):
         if excessive is not None and (sin or sout):
             sin /= 1048576
             sout /= 1048576
             self.log(logging.WARNING, 'swap_activity',
                 {'sin': sin, 'sout': sout},
                 'Swap in/out (MB): {sin}/{sout}')
 
         o = dict(
-            version=2,
+            version=3,
             argv=sys.argv,
             start=self.start_time,
             end=self.end_time,
             duration=self.end_time - self.start_time,
             resources=[],
             cpu_percent=cpu_percent,
             cpu_times=cpu_times,
             io=io,
@@ -406,51 +406,33 @@ class BuildMonitor(MozbuildObject):
                 swap=list(usage.swap),
             )
 
             self.tiers.add_resources_to_dict(entry, start=usage.start,
                     end=usage.end)
 
             o['resources'].append(entry)
 
-        # TODO: it would be nice to collect data on the storage device as well
-        o['system'] = dict(
-            architecture=list(platform.architecture()),
-            machine=platform.machine(),
-            python_version=platform.python_version(),
-            release=platform.release(),
-            system=platform.system(),
-            version=platform.version(),
-        )
 
         # If the imports for this file ran before the in-tree virtualenv
         # was bootstrapped (for instance, for a clobber build in automation),
         # psutil might not be available.
         #
         # Treat psutil as optional to avoid an outright failure to log resources
+        # TODO: it would be nice to collect data on the storage device as well
         # in this case.
+        o['system'] = {}
         if psutil:
             o['system'].update(dict(
                 logical_cpu_count=psutil.cpu_count(),
                 physical_cpu_count=psutil.cpu_count(logical=False),
                 swap_total=psutil.swap_memory()[0],
                 vmem_total=psutil.virtual_memory()[0],
             ))
 
-        if platform.system() == 'Linux':
-            dist = list(platform.linux_distribution())
-            o['system']['linux_distribution'] = dist
-        elif platform.system() == 'Windows':
-            win32_ver=list((platform.win32_ver())),
-            o['system']['win32_ver'] = win32_ver
-        elif platform.system() == 'Darwin':
-            # mac version is a special Cupertino snowflake
-            r, v, m = platform.mac_ver()
-            o['system']['mac_ver'] = [r, list(v), m]
-
         return o
 
     def _log_resource_usage(self, prefix, m_type, duration, cpu_percent,
         cpu_times, io, extra_params={}):
 
         params = dict(
             duration=duration,
             cpu_percent=cpu_percent,
--- a/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
+++ b/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
@@ -51,17 +51,17 @@ svg {
   <body>
     <script>
 var currentResources;
 
 /**
  * Interface for a build resources JSON file.
  */
 function BuildResources(data) {
-  if (data.version != 1 && data.version != 2) {
+  if (data.version < 1 || data.version > 3) {
     throw new Error("Unsupported version of the JSON format: " + data.version);
   }
 
   this.resources = [];
 
   var cpu_fields = data.cpu_times_fields;
   var io_fields = data.io_fields;
   var virt_fields = data.virt_fields;