Bug 968923 - part 1 - add infrastructure for defining use counters from UseCounters.conf; original-author=heycam; r=heycam,gfritzsche,mshal
authorNathan Froyd <froydnj@mozilla.com>
Wed, 04 Feb 2015 17:00:00 -0500
changeset 245184 c729634e55d1529886f03ffaa919e41d7ce1816f
parent 245183 a3206897842a96ac1b7cfff364db17913165d4b4
child 245185 1a67af1cbf1bf26a4122d2f4e153d7d4d92a7943
push id28799
push userphilringnalda@gmail.com
push dateSat, 23 May 2015 20:31:44 +0000
treeherdermozilla-central@c3c0928c3cde [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersheycam, gfritzsche, mshal
bugs968923
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 968923 - part 1 - add infrastructure for defining use counters from UseCounters.conf; original-author=heycam; r=heycam,gfritzsche,mshal This patch adds UseCounter.conf for defining use counters along with infrastructure to generate enums and telemetry identifiers from UseCounter.conf.
dom/base/Makefile.in
dom/base/UseCounter.h
dom/base/UseCounters.conf
dom/base/gen-usecounters.py
dom/base/moz.build
dom/base/usecounters.py
toolkit/components/telemetry/Makefile.in
toolkit/components/telemetry/gen-histogram-bucket-ranges.py
toolkit/components/telemetry/gen-histogram-data.py
toolkit/components/telemetry/gen-histogram-enum.py
toolkit/components/telemetry/histogram_tools.py
new file mode 100644
--- /dev/null
+++ b/dom/base/Makefile.in
@@ -0,0 +1,8 @@
+ # This Source Code Form is subject to the terms of the Mozilla Public
+ # License, v. 2.0. If a copy of the MPL was not distributed with this
+ # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+INSTALL_TARGETS += usecounterlist
+usecounterlist_FILES := UseCounterList.h
+usecounterlist_DEST = $(DIST)/include/mozilla/dom
+usecounterlist_TARGET := export
new file mode 100644
--- /dev/null
+++ b/dom/base/UseCounter.h
@@ -0,0 +1,27 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#ifndef UseCounter_h_
+#define UseCounter_h_
+
+namespace mozilla {
+
+enum UseCounter {
+  eUseCounter_UNKNOWN = -1,
+#define USE_COUNTER_DOM_METHOD(interface_, name_) \
+    eUseCounter_##interface_##_##name_,
+#define USE_COUNTER_DOM_ATTRIBUTE(interface_, name_) \
+    eUseCounter_##interface_##_##name_,
+#define USE_COUNTER_CSS_PROPERTY(name_, id_) \
+    eUseCounter_property_##id_,
+#include "mozilla/dom/UseCounterList.h"
+#undef USE_COUNTER_DOM_METHOD
+#undef USE_COUNTER_DOM_ATTRIBUTE
+#undef USE_COUNTER_CSS_PROPERTY
+  eUseCounter_Count
+};
+
+}
+
+#endif
new file mode 100644
--- /dev/null
+++ b/dom/base/UseCounters.conf
@@ -0,0 +1,33 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// This file defines a list of use counters, which are things that can
+// record usage of Web platform features and then report this information
+// through Telemetry.
+//
+// The format of this file is very strict.  Each line can be:
+//
+//   (a) a blank line
+//
+//   (b) a comment, which is a line that begins with "//"
+//
+//   (c) an #if/ifdef/else/endif preprocessor directive
+//
+//   (d) one of three possible use counter declarations:
+//
+//         method <IDL interface name>.<IDL operation name>
+//         attribute <IDL interface name>.<IDL attribute name>
+//         property <CSS property name>
+//
+// To actually cause use counters to be incremented, DOM methods
+// and attributes must have a [UseCounter] extended attribute in
+// the Web IDL file, and CSS properties must be declared with
+// the CSS_PROPERTY_HAS_USE_COUNTER flag in nsCSSPropList.h.
+//
+// You might reasonably ask why we have this file and we require
+// annotating things with [UseCounter] in the relevant WebIDL file as
+// well.  Generating things from bindings codegen and ensuring all the
+// dependencies were correct would have been rather difficult, and
+// annotating the WebIDL files does nothing for identifying CSS
+// property usage, which we would also like to track.
new file mode 100755
--- /dev/null
+++ b/dom/base/gen-usecounters.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function
+
+import json
+import os
+import sys
+sys.path.append(os.path.dirname(__file__))
+
+import usecounters
+
+AUTOGENERATED_WARNING_COMMENT = "/* THIS FILE IS AUTOGENERATED BY gen-usercounters.py - DO NOT EDIT */"
+
+def generate_list(f, counters):
+    def print_optional_macro_declare(name):
+        print('''
+#ifndef %(name)s
+#define %(name)s(interface_, name_) // nothing
+#define DEFINED_%(name)s
+#endif
+''' % { 'name': name }, file=f)
+
+    def print_optional_macro_undeclare(name):
+        print('''
+#ifdef DEFINED_%(name)s
+#undef DEFINED_%(name)s
+#undef %(name)s
+#endif
+''' % { 'name': name }, file=f)
+
+    print(AUTOGENERATED_WARNING_COMMENT, file=f)
+
+    print_optional_macro_declare('USE_COUNTER_DOM_METHOD')
+    print_optional_macro_declare('USE_COUNTER_DOM_ATTRIBUTE')
+    print_optional_macro_declare('USE_COUNTER_CSS_PROPERTY')
+
+    for counter in counters:
+        if counter['type'] == 'method':
+            print('USE_COUNTER_DOM_METHOD(%s, %s)' % (counter['interface_name'], counter['method_name']), file=f)
+        elif counter['type'] == 'attribute':
+            print('USE_COUNTER_DOM_ATTRIBUTE(%s, %s)' % (counter['interface_name'], counter['attribute_name']), file=f)
+        elif counter['type'] == 'property':
+            prop = counter['property_name']
+            print('USE_COUNTER_CSS_PROPERTY(%s, %s)' % (prop, prop.replace('-', '_')), file=f)
+
+    print_optional_macro_undeclare('USE_COUNTER_DOM_METHOD')
+    print_optional_macro_undeclare('USE_COUNTER_DOM_ATTRIBUTE')
+    print_optional_macro_undeclare('USE_COUNTER_CSS_PROPERTY')
+
+def generate_property_map(f, counters):
+    print(AUTOGENERATED_WARNING_COMMENT, file=f)
+    print('''
+enum {
+  // XXX is this the right define?
+  #define CSS_PROP_LIST_INCLUDE_LOGICAL
+  #define CSS_PROP(name_, id_, method_, flags_, pref_, parsevariant_,     \\
+                   kwtable_, stylestruct_, stylestructoffset_, animtype_) \\
+    USE_COUNTER_FOR_CSS_PROPERTY_##id_ = eUseCounter_UNKNOWN,
+  #include "nsCSSPropList.h"
+  #undef CSS_PROP
+  #undef CSS_PROP_LIST_INCLUDE_LOGICAL
+};
+''', file=f)
+    for counter in counters:
+        if counter['type'] == 'property':
+            prop = counter['property_name'].replace('-', '_')
+            print('#define USE_COUNTER_FOR_CSS_PROPERTY_%s eUseCounter_property_%s' % (prop, prop), file=f)
+
+def use_counter_list(output_header, conf_filename):
+    counters = usecounters.read_conf(conf_filename)
+    generate_list(output_header, counters)
+
+def property_map(output_map, conf_filename):
+    counters = usecounters.read_conf(conf_filename)
+    generate_property_map(output_map, counters)
--- a/dom/base/moz.build
+++ b/dom/base/moz.build
@@ -139,16 +139,17 @@ if CONFIG['MOZ_WEBRTC']:
         'nsDOMDataChannel.h',
         'nsDOMDataChannelDeclarations.h',
     ]
 
 EXPORTS.mozilla += [
     'CORSMode.h',
     'FeedWriterEnabled.h',
     'TextInputProcessor.h',
+    'UseCounter.h',
 ]
 
 EXPORTS.mozilla.dom += [
     'AnonymousContent.h',
     'Attr.h',
     'BarProps.h',
     'BlobSet.h',
     'ChildIterator.h',
@@ -462,8 +463,21 @@ for var in ('MOZ_B2G_RIL', 'MOZ_B2G_FM')
     if CONFIG[var]:
         DEFINES[var] = True
 
 if CONFIG['MOZ_BUILD_APP'] in ['browser', 'mobile/android', 'xulrunner']:
     DEFINES['HAVE_SIDEBAR'] = True
 
 if CONFIG['MOZ_X11']:
     CXXFLAGS += CONFIG['TK_CFLAGS']
+
+GENERATED_FILES += [
+    'PropertyUseCounterMap.inc',
+    'UseCounterList.h',
+]
+
+countermap = GENERATED_FILES['PropertyUseCounterMap.inc']
+countermap.script = 'gen-usecounters.py:property_map'
+countermap.inputs = ['UseCounters.conf']
+
+counterlist = GENERATED_FILES['UseCounterList.h']
+counterlist.script = 'gen-usecounters.py:use_counter_list'
+counterlist.inputs = ['UseCounters.conf']
new file mode 100644
--- /dev/null
+++ b/dom/base/usecounters.py
@@ -0,0 +1,78 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import buildconfig
+import collections
+import re
+import StringIO
+import sys
+
+from mozbuild.preprocessor import preprocess
+
+def read_conf(conf_filename):
+    # Invoking the preprocessor ourselves is easier than writing build rules
+    # to do it for us.
+    processed = StringIO.StringIO()
+    preprocess(includes=[conf_filename],
+               defines=buildconfig.defines,
+               output=processed)
+
+    # Can't read/write from a single StringIO, so make a new one for reading.
+    stream = StringIO.StringIO(processed.getvalue())
+
+    def parse_counters(stream):
+        for line_num, line in enumerate(stream):
+            line = line.rstrip('\n')
+            if not line or line.startswith('//'):
+                # empty line or comment
+                continue
+            m = re.match(r'method ([A-Za-z0-9]+)\.([A-Za-z0-9]+)$', line)
+            if m:
+                interface_name, method_name = m.groups()
+                yield { 'type': 'method',
+                        'interface_name': interface_name,
+                        'method_name': method_name }
+                continue
+            m = re.match(r'attribute ([A-Za-z0-9]+)\.([A-Za-z0-9]+)$', line)
+            if m:
+                interface_name, attribute_name = m.groups()
+                yield { 'type': 'attribute',
+                        'interface_name': interface_name,
+                        'attribute_name': attribute_name }
+                continue
+            m = re.match(r'property ([a-z0-9-]+)$', line)
+            if m:
+                property_name = m.group(1)
+                yield { 'type': 'property',
+                        'property_name': property_name }
+                continue
+            raise ValueError('error parsing %s at line %d' % (conf_filename, line_num))
+
+    return parse_counters(stream)
+
+def generate_histograms(filename):
+    # The mapping for use counters to telemetry histograms depends on the
+    # ordering of items in the dictionary.
+    items = collections.OrderedDict()
+    for counter in read_conf(filename):
+        def append_counter(name, desc):
+            items[name] = { 'expires_in_version': 'never',
+                            'kind' : 'boolean',
+                            'description': desc }
+
+        def append_counters(name, desc):
+            append_counter('USE_COUNTER_%s_DOCUMENT' % name, 'Whether a document %s' % desc)
+            append_counter('USE_COUNTER_%s_PAGE' % name, 'Whether a page %s' % desc)
+
+        if counter['type'] == 'method':
+            method = '%s.%s' % (counter['interface_name'], counter['method_name'])
+            append_counters(method.replace('.', '_').upper(), 'called %s' % method)
+        elif counter['type'] == 'attribute':
+            attr = '%s.%s' % (counter['interface_name'], counter['attribute_name'])
+            append_counters(attr.replace('.', '_').upper(), 'got or set %s' % attr)
+        elif counter['type'] == 'property':
+            prop = counter['property_name']
+            append_counters('PROPERTY_%s' % prop.replace('-', '_').upper(), "used the '%s' property" % prop)
+
+    return items
--- a/toolkit/components/telemetry/Makefile.in
+++ b/toolkit/components/telemetry/Makefile.in
@@ -17,34 +17,34 @@ endif
 
 INSTALL_TARGETS += histoenums
 histoenums_FILES := TelemetryHistogramEnums.h
 histoenums_DEST = $(DIST)/include/mozilla
 histoenums_TARGET := export
 
 include $(topsrcdir)/config/rules.mk
 
-histograms_file := $(srcdir)/Histograms.json
+histogram_files := $(srcdir)/Histograms.json $(topsrcdir)/dom/base/UseCounters.conf
 histogram_enum_file := TelemetryHistogramEnums.h
 histogram_data_file := TelemetryHistogramData.inc
 
 enum_python_deps := \
   $(srcdir)/gen-histogram-enum.py \
   $(srcdir)/histogram_tools.py \
   $(NULL)
 
 data_python_deps := \
   $(srcdir)/gen-histogram-data.py \
   $(srcdir)/histogram_tools.py \
   $(NULL)
 
-$(histogram_enum_file): $(histograms_file) $(enum_python_deps)
-	$(PYTHON) $(srcdir)/gen-histogram-enum.py $< > $@
-$(histogram_data_file): $(histograms_file) $(data_python_deps)
-	$(PYTHON) $(srcdir)/gen-histogram-data.py $< > $@
+$(histogram_enum_file): $(histogram_files) $(enum_python_deps)
+	$(PYTHON) $(srcdir)/gen-histogram-enum.py $(histogram_files) > $@
+$(histogram_data_file): $(histogram_files) $(data_python_deps)
+	$(PYTHON) $(srcdir)/gen-histogram-data.py $(histogram_files) > $@
 
 GARBAGE += $(histogram_enum_file)
 
 # This is so hacky. Waiting on bug 988938.
 addondir = $(srcdir)/tests/addons
 testdir = $(abspath $(DEPTH)/_tests/xpcshell/toolkit/components/telemetry/tests/unit)
 
 misc:: $(call mkdir_deps,$(testdir))
--- a/toolkit/components/telemetry/gen-histogram-bucket-ranges.py
+++ b/toolkit/components/telemetry/gen-histogram-bucket-ranges.py
@@ -12,20 +12,21 @@ import histogram_tools
 import json
 
 from collections import OrderedDict
 
 # Keep this in sync with TelemetryController.
 startup_histogram_re = re.compile("SQLITE|HTTP|SPDY|CACHE|DNS")
 
 def main(argv):
-    filename = argv[0]
+    filenames = argv
+
     all_histograms = OrderedDict()
 
-    for histogram in histogram_tools.from_file(filename):
+    for histogram in histogram_tools.from_files(filenames):
         name = histogram.name()
         parameters = OrderedDict()
         table = {
             'boolean': '2',
             'flag': '3',
             'enumerated': '1',
             'linear': '1',
             'exponential': '0',
--- a/toolkit/components/telemetry/gen-histogram-data.py
+++ b/toolkit/components/telemetry/gen-histogram-data.py
@@ -182,18 +182,18 @@ def write_debug_histogram_ranges(histogr
         print "{ %d, %d }," % (offset, range_length)
         if cpp_guard:
             print "#endif"
         offset += range_length
     print "};"
     print "#endif"
 
 def main(argv):
-    filename = argv[0]
+    filenames = argv
 
-    histograms = list(histogram_tools.from_file(filename))
+    histograms = list(histogram_tools.from_files(filenames))
 
     print banner
     write_histogram_table(histograms)
     write_histogram_static_asserts(histograms)
     write_debug_histogram_ranges(histograms)
 
 main(sys.argv[1:])
--- a/toolkit/components/telemetry/gen-histogram-enum.py
+++ b/toolkit/components/telemetry/gen-histogram-enum.py
@@ -1,30 +1,65 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # Write out a C++ enum definition whose members are the names of
-# histograms.  The histograms are defined in a file provided as a
-# command-line argument.
+# histograms as well as the following other members:
+#
+#   - HistogramCount
+#   - HistogramFirstUseCounter
+#   - HistogramLastUseCounter
+#   - HistogramUseCounterCount
+#
+# The histograms are defined in files provided as command-line arguments.
 
+import histogram_tools
+import itertools
 import sys
-import histogram_tools
 
 banner = """/* This file is auto-generated, see gen-histogram-enum.py.  */
 """
 
 def main(argv):
-    filename = argv[0]
+    filenames = argv
 
     print banner
     print "enum ID : uint32_t {"
-    for histogram in histogram_tools.from_file(filename):
-        cpp_guard = histogram.cpp_guard()
-        if cpp_guard:
-            print "#if defined(%s)" % cpp_guard
-        print "  %s," % histogram.name()
-        if cpp_guard:
-            print "#endif"
-    print "  HistogramCount"
+
+    groups = itertools.groupby(histogram_tools.from_files(filenames),
+                               lambda h: h.name().startswith("USE_COUNTER_"))
+    seen_use_counters = False
+
+    # Note that histogram_tools.py guarantees that all of the USE_COUNTER_*
+    # histograms are defined in a contiguous block.  We therefore assume
+    # that there's at most one group for which use_counter_group is true.
+    for (use_counter_group, histograms) in groups:
+        if use_counter_group:
+            seen_use_counters = True
+
+        # The HistogramDUMMY* enum variables are used to make the computation
+        # of Histogram{First,Last}UseCounter easier.  Otherwise, we'd have to
+        # special case the first and last histogram in the group.
+        if use_counter_group:
+            print "  HistogramFirstUseCounter,"
+            print "  HistogramDUMMY1 = HistogramFirstUseCounter - 1,"
+
+        for histogram in histograms:
+            cpp_guard = histogram.cpp_guard()
+            if cpp_guard:
+                print "#if defined(%s)" % cpp_guard
+            print "  %s," % histogram.name()
+            if cpp_guard:
+                print "#endif"
+
+        if use_counter_group:
+            print "  HistogramDUMMY2,"
+            print "  HistogramLastUseCounter = HistogramDUMMY2 - 1,"
+
+    print "  HistogramCount,"
+    if seen_use_counters:
+        print "  HistogramUseCounterCount = HistogramLastUseCounter - HistogramFirstUseCounter + 1"
+    else:
+        print "  HistogramUseCounterCount = 0"
     print "};"
 
 main(sys.argv[1:])
--- a/toolkit/components/telemetry/histogram_tools.py
+++ b/toolkit/components/telemetry/histogram_tools.py
@@ -1,15 +1,22 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import buildconfig
 import json
 import math
+import os
 import re
+import sys
+
+# Need to update sys.path to be able to find usecounters.
+sys.path.append(os.path.join(buildconfig.topsrcdir, 'dom/base/'))
+import usecounters
 
 from collections import OrderedDict
 
 def table_dispatch(kind, table, body):
     """Call body with table[kind] if it exists.  Raise an error otherwise."""
     if kind in table:
         return body(table[kind])
     else:
@@ -241,16 +248,62 @@ is enabled."""
         return (1, n_values, "%s+1" % n_values)
 
     @staticmethod
     def exponential_bucket_parameters(definition):
         return (definition.get('low', 1),
                 definition['high'],
                 definition['n_buckets'])
 
-def from_file(filename):
+# We support generating histograms from multiple different input files, not
+# just Histograms.json.  For each file's basename, we have a specific
+# routine to parse that file, and return a dictionary mapping histogram
+# names to histogram parameters.
+def from_Histograms_json(filename):
+    with open(filename, 'r') as f:
+        try:
+            histograms = json.load(f, object_pairs_hook=OrderedDict)
+        except ValueError, e:
+            raise BaseException, "error parsing histograms in %s: %s" % (filename, e.message)
+    return histograms
+
+def from_UseCounters_conf(filename):
+    return usecounters.generate_histograms(filename)
+
+FILENAME_PARSERS = {
+    'Histograms.json': from_Histograms_json,
+    'UseCounters.conf': from_UseCounters_conf,
+}
+
+def from_files(filenames):
     """Return an iterator that provides a sequence of Histograms for
-the histograms defined in filename.
+the histograms defined in filenames.
     """
-    with open(filename, 'r') as f:
-        histograms = json.load(f, object_pairs_hook=OrderedDict)
+    all_histograms = OrderedDict()
+    for filename in filenames:
+        parser = FILENAME_PARSERS[os.path.basename(filename)]
+        histograms = parser(filename)
+
+        # OrderedDicts are important, because then the iteration order over
+        # the parsed histograms is stable, which makes the insertion into
+        # all_histograms stable, which makes ordering in generated files
+        # stable, which makes builds more deterministic.
+        if not isinstance(histograms, OrderedDict):
+            raise BaseException, "histogram parser didn't provide an OrderedDict"
+
         for (name, definition) in histograms.iteritems():
-            yield Histogram(name, definition)
+            if all_histograms.has_key(name):
+                raise DefinitionException, "duplicate histogram name %s" % name
+            all_histograms[name] = definition
+
+    # We require that all USE_COUNTER_* histograms be defined in a contiguous
+    # block.
+    use_counter_indices = filter(lambda x: x[1].startswith("USE_COUNTER_"),
+                                 enumerate(all_histograms.iterkeys()));
+    if use_counter_indices:
+        lower_bound = use_counter_indices[0][0]
+        upper_bound = use_counter_indices[-1][0]
+        n_counters = upper_bound - lower_bound + 1
+        if n_counters != len(use_counter_indices):
+            raise DefinitionException, "use counter histograms must be defined in a contiguous block"
+
+    for (name, definition) in all_histograms.iteritems():
+        yield Histogram(name, definition)