Bug 1361661 - Part 1: Generate headers with process data from Processes.yaml. r=dexter
☠☠ backed out by ff997cf7f594 ☠ ☠
authorGeorg Fritzsche <georg.fritzsche@googlemail.com>
Mon, 22 May 2017 15:33:29 +0700
changeset 359954 af4327fd49cd93c717328e61caf1af692a0e52a0
parent 359953 ec6edd4fec8f62db57c1f075d1d3b536c74af5f8
child 359955 8fd3861205d1647a0d644206eab5cea160d15201
push id90549
push userryanvm@gmail.com
push dateMon, 22 May 2017 18:28:46 +0000
treeherdermozilla-inbound@6caffcb2589c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdexter
bugs1361661
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1361661 - Part 1: Generate headers with process data from Processes.yaml. r=dexter Adding the Gecko enums to Processes.yaml allows us to generate mappings from ProcessID to GeckoProcessType. We generate string tables with the Telemetry process names, so we can use these names consistently throughout Telemetry.
CLOBBER
toolkit/components/telemetry/Processes.yaml
toolkit/components/telemetry/gen-process-data.py
toolkit/components/telemetry/gen-process-enum.py
toolkit/components/telemetry/moz.build
toolkit/components/telemetry/shared_telemetry_utils.py
--- a/CLOBBER
+++ b/CLOBBER
@@ -17,9 +17,10 @@
 #
 # Modifying this file will now automatically clobber the buildbot machines \o/
 #
 
 # Are you updating CLOBBER because you think it's needed for your WebIDL
 # changes to stick? As of bug 928195, this shouldn't be necessary! Please
 # don't change CLOBBER for WebIDL changes any more.
 
-Bug 1340627 - clobber for Skia update
+Bug 1361661 - Update Telemetry build and headers.
+
--- a/toolkit/components/telemetry/Processes.yaml
+++ b/toolkit/components/telemetry/Processes.yaml
@@ -1,17 +1,21 @@
 # This lists the known child processes we collect Telemetry for.
 # The entries are keyed with the names used in Telemetry internally, the same name that is used
 # in the main pings payload, i.e. "payload/processes/<process name>". See:
 # https://gecko.readthedocs.io/en/latest/toolkit/components/telemetry/telemetry/data/main-ping.html#processes
 #
 # For now this is only used to inform the data pipeline about new processes, but will be used to
 # generate headers with C++ data later (enums, strings, ...).
 parent:
+  gecko_enum: GeckoProcessType_Default
   description: This is the main process. It is also known as the parent or chrome process.
 content:
+  gecko_enum: GeckoProcessType_Content
   description: This is for processes web content is rendered in.
 extension:
+  gecko_enum: GeckoProcessType_Content
   description: >
     This is the WebExtension process. It is a re-used content process, with the data submitted
     separately to avoid skewing other content process Telemetry.
 gpu:
+  gecko_enum: GeckoProcessType_GPU
   description: This is the compositor or GPU process.
new file mode 100644
--- /dev/null
+++ b/toolkit/components/telemetry/gen-process-data.py
@@ -0,0 +1,72 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Write out processes data for C++. The processes are defined
+# in a file provided as a command-line argument.
+
+from __future__ import print_function
+from shared_telemetry_utils import ParserError, load_processes
+
+import sys
+import collections
+
+# The banner/text at the top of the generated file.
+banner = """/* This file is auto-generated from Telemetry build scripts,
+   see gen-processes-data.py. */
+"""
+
+file_header = """\
+#ifndef mozilla_TelemetryProcessData_h
+#define mozilla_TelemetryProcessData_h
+
+#include "mozilla/TelemetryProcessEnums.h"
+
+namespace mozilla {
+namespace Telemetry {
+"""
+
+file_footer = """
+} // namespace Telemetry
+} // namespace mozilla
+#endif // mozilla_TelemetryProcessData_h"""
+
+
+def to_enum_label(name):
+    return name.title().replace('_', '')
+
+
+def write_processes_data(processes, output):
+    def p(line):
+        print(line, file=output)
+    processes = collections.OrderedDict(processes)
+
+    p("static GeckoProcessType ProcessIDToGeckoProcessType[%d] = {" % len(processes))
+    for i, (name, value) in enumerate(processes.iteritems()):
+        p("  /* %d: ProcessID::%s = */ %s," % (i, to_enum_label(name), value['gecko_enum']))
+    p("};")
+    p("")
+    p("static const char* const ProcessIDToString[%d] = {" % len(processes))
+    for i, (name, value) in enumerate(processes.iteritems()):
+        p("  /* %d: ProcessID::%s = */ \"%s\"," % (i, to_enum_label(name), name))
+    p("};")
+
+
+def main(output, *filenames):
+    if len(filenames) > 1:
+        raise Exception('We don\'t support loading from more than one file.')
+
+    try:
+        processes = load_yaml_file(filenames[0])
+
+        # Write the process data file.
+        print(banner, file=output)
+        print(file_header, file=output)
+        write_processes_data(processes, output)
+        print(file_footer, file=output)
+    except ParserError as ex:
+        print("\nError generating processes data:\n" + str(ex) + "\n")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main(sys.stdout, *sys.argv[1:])
new file mode 100644
--- /dev/null
+++ b/toolkit/components/telemetry/gen-process-enum.py
@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Write out processes data for C++. The processes are defined
+# in a file provided as a command-line argument.
+
+from __future__ import print_function
+from shared_telemetry_utils import ParserError, load_processes
+
+import sys
+import collections
+
+# The banner/text at the top of the generated file.
+banner = """/* This file is auto-generated from Telemetry build scripts,
+   see gen-processes-enum.py. */
+"""
+
+file_header = """\
+#ifndef mozilla_TelemetryProcessEnums_h
+#define mozilla_TelemetryProcessEnums_h
+
+namespace mozilla {
+namespace Telemetry {
+"""
+
+file_footer = """
+} // namespace Telemetry
+} // namespace mozilla
+#endif // mozilla_TelemetryProcessEnums_h"""
+
+
+def to_enum_label(name):
+    return name.title().replace('_', '')
+
+
+def write_processes_enum(processes, output):
+    def p(line):
+        print(line, file=output)
+    processes = collections.OrderedDict(processes)
+
+    p("enum class ProcessID : uint32_t {")
+    for i, (name, _) in enumerate(processes.iteritems()):
+        p("  %s = %d," % (to_enum_label(name), i))
+    p("  Count = %d" % len(processes))
+    p("};")
+
+
+def main(output, *filenames):
+    if len(filenames) > 1:
+        raise Exception('We don\'t support loading from more than one file.')
+
+    try:
+        processes = load_yaml_file(filenames[0])
+
+        # Write the process data file.
+        print(banner, file=output)
+        print(file_header, file=output)
+        write_processes_enum(processes, output)
+        print(file_footer, file=output)
+    except ParserError as ex:
+        print("\nError generating processes enums:\n" + str(ex) + "\n")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main(sys.stdout, *sys.argv[1:])
--- a/toolkit/components/telemetry/moz.build
+++ b/toolkit/components/telemetry/moz.build
@@ -38,16 +38,17 @@ XPIDL_SOURCES += [
     'nsITelemetry.idl',
 ]
 
 XPIDL_MODULE = 'telemetry'
 
 EXPORTS.mozilla += [
     '!TelemetryEventEnums.h',
     '!TelemetryHistogramEnums.h',
+    '!TelemetryProcessEnums.h',
     '!TelemetryScalarEnums.h',
     'ipc/TelemetryComms.h',
     'ipc/TelemetryIPC.h',
     'ProcessedStack.h',
     'Telemetry.h',
     'ThreadHangStats.h',
 ]
 
@@ -89,16 +90,18 @@ TESTING_JS_MODULES += [
   'tests/unit/TelemetryArchiveTesting.jsm',
 ]
 
 GENERATED_FILES = [
     'TelemetryEventData.h',
     'TelemetryEventEnums.h',
     'TelemetryHistogramData.inc',
     'TelemetryHistogramEnums.h',
+    'TelemetryProcessData.h',
+    'TelemetryProcessEnums.h',
     'TelemetryScalarData.h',
     'TelemetryScalarEnums.h',
 ]
 
 # Generate histogram files.
 histogram_files = [
     'Histograms.json',
     '/dom/base/UseCounters.conf',
@@ -134,10 +137,23 @@ event_files = [
 event_data = GENERATED_FILES['TelemetryEventData.h']
 event_data.script = 'gen-event-data.py'
 event_data.inputs = event_files
 
 event_enums = GENERATED_FILES['TelemetryEventEnums.h']
 event_enums.script = 'gen-event-enum.py'
 event_enums.inputs = event_files
 
+# Generate data from Processes.yaml
+processes_files = [
+    'Processes.yaml',
+]
+
+processes_enum = GENERATED_FILES['TelemetryProcessEnums.h']
+processes_enum.script = 'gen-process-enum.py'
+processes_enum.inputs = processes_files
+
+processes_data = GENERATED_FILES['TelemetryProcessData.h']
+processes_data.script = 'gen-process-data.py'
+processes_data.inputs = processes_files
+
 with Files('**'):
     BUG_COMPONENT = ('Toolkit', 'Telemetry')
--- a/toolkit/components/telemetry/shared_telemetry_utils.py
+++ b/toolkit/components/telemetry/shared_telemetry_utils.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # This file contains utility functions shared by the scalars and the histogram generation
 # scripts.
 
 from __future__ import print_function
 
 import re
+import yaml
 
 # This is a list of flags that determine which process a measurement is allowed
 # to record from.
 KNOWN_PROCESS_FLAGS = {
     'all': 'All',
     'all_childs': 'AllChilds',
     'main': 'Main',
     'content': 'Content',
@@ -125,8 +126,20 @@ def add_expiration_postfix(expiration):
     """
     if re.match(r'^[1-9][0-9]*$', expiration):
         return expiration + ".0a1"
 
     if re.match(r'^[1-9][0-9]*\.0$', expiration):
         return expiration + "a1"
 
     return expiration
+
+
+def load_yaml_file(filename):
+    """ Load a YAML file from disk, throw a ParserError on failure."""
+    try:
+        with open(filename, 'r') as f:
+            return yaml.safe_load(f)
+    except IOError, e:
+        raise ParserError('Error opening ' + filename + ': ' + e.message)
+    except ValueError, e:
+        raise ParserError('Error parsing processes in {}: {}'
+                          .format(filename, e.message))