Backed out 4 changesets (bug 1526072) for causing build bustages in test_clobber.py CLOSED TREE
authorNoemi Erli <nerli@mozilla.com>
Wed, 27 Mar 2019 22:51:22 +0200
changeset 466451 bdbde35b573df50c66a8e973fceb8772ab261cce
parent 466450 dcfe21ab348eda2fdd073fc24299b375963937f7
child 466452 8d043fdfac71edb9ccda79987f95dae1ed02b434
push id35768
push useropoprus@mozilla.com
push dateThu, 28 Mar 2019 09:55:54 +0000
treeherdermozilla-central@c045dd97faf2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1526072
milestone68.0a1
backs outd3d56eca307f057c6a691cc4b79f4b7adaa28088
37942b0f911b0532a2b5c8acfdee9444fda3f006
a0eb0f43c928a95a7af543ab9339cd03a88bd27b
c1d1576431d7d3a6e1a394ca52f2802343038258
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 4 changesets (bug 1526072) for causing build bustages in test_clobber.py CLOSED TREE Backed out changeset d3d56eca307f (bug 1526072) Backed out changeset 37942b0f911b (bug 1526072) Backed out changeset a0eb0f43c928 (bug 1526072) Backed out changeset c1d1576431d7 (bug 1526072)
build/docs/telemetry.rst
build/mach_bootstrap.py
python/mach/mach/base.py
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/export_telemetry_schema.py
python/mozbuild/mozbuild/telemetry.py
--- a/build/docs/telemetry.rst
+++ b/build/docs/telemetry.rst
@@ -129,46 +129,16 @@ true if build is optimized (--enable-opt
 sccache
 #######
 
 true if ccache in use is sccache
 
 :type: ``boolean``
 
 
-.. _telemetry.json#/properties/build_attrs:
-
-build_attrs
-+++++++++++
-
-Selected runtime attributes of the build
-
-:type: ``object``
-
-**Properties:** :ref:`telemetry.json#/properties/build_attrs/properties/cpu_percent`, :ref:`telemetry.json#/properties/build_attrs/properties/clobber`
-
-.. _telemetry.json#/properties/build_attrs/properties/cpu_percent:
-
-cpu_percent
-###########
-
-cpu utilization observed during the build
-
-:type: ``number``
-
-.. _telemetry.json#/properties/build_attrs/properties/clobber:
-
-clobber
-#######
-
-true if the build was a clobber/full build
-
-:type: ``boolean``
-
-
 .. _telemetry.json#/properties/client_id:
 
 client_id
 +++++++++
 
 A UUID to uniquely identify a client
 
 :type: ``string``
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -245,32 +245,30 @@ def bootstrap(topsrcdir, mozilla_dir=Non
         if not isinstance(instance, MozbuildObject):
             instance = MozbuildObject.from_environment()
 
         try:
             substs = instance.substs
         except Exception:
             substs = {}
 
-        command_attrs = getattr(context, 'command_attrs', {})
-
         # We gather telemetry for every operation.
         paths = {
             instance.topsrcdir: '$topsrcdir/',
             instance.topobjdir: '$topobjdir/',
             mozpath.normpath(os.path.expanduser('~')): '$HOME/',
         }
         # This might override one of the existing entries, that's OK.
         # We don't use a sigil here because we treat all arguments as potentially relative
         # paths, so we'd like to get them back as they were specified.
         paths[mozpath.normpath(os.getcwd())] = ''
         data = gather_telemetry(command=handler.name, success=(result == 0),
                                 start_time=start_time, end_time=end_time,
                                 mach_context=context, substs=substs,
-                                command_attrs=command_attrs, paths=paths)
+                                paths=paths)
         if data:
             telemetry_dir = os.path.join(get_state_dir(), 'telemetry')
             try:
                 os.mkdir(telemetry_dir)
             except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
             outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
--- a/python/mach/mach/base.py
+++ b/python/mach/mach/base.py
@@ -7,17 +7,16 @@ from __future__ import absolute_import, 
 
 class CommandContext(object):
     """Holds run-time state so it can easily be passed to command providers."""
     def __init__(self, cwd=None, settings=None, log_manager=None, commands=None, **kwargs):
         self.cwd = cwd
         self.settings = settings
         self.log_manager = log_manager
         self.commands = commands
-        self.command_attrs = {}
 
         for k, v in kwargs.items():
             setattr(self, k, v)
 
 
 class MachError(Exception):
     """Base class for all errors raised by mach itself."""
 
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -973,28 +973,23 @@ class CCacheStats(object):
             return '%.1f Mbytes' % (float(v) / CCacheStats.MiB)
         else:
             return '%.1f Kbytes' % (float(v) / CCacheStats.KiB)
 
 
 class BuildDriver(MozbuildObject):
     """Provides a high-level API for build actions."""
 
-    def __init__(self, *args, **kwargs):
-        MozbuildObject.__init__(self, *args, **kwargs)
-        self.mach_context = None
-
     def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
               directory=None, verbose=False, keep_going=False, mach_context=None):
         """Invoke the build backend.
 
         ``what`` defines the thing to build. If not defined, the default
         target is used.
         """
-        self.mach_context = mach_context
         warnings_path = self._get_state_filename('warnings.json')
         monitor = self._spawn(BuildMonitor)
         monitor.init(warnings_path)
         ccache_start = monitor.ccache_stats()
         footer = BuildProgressFooter(self.log_manager.terminal, monitor)
 
         # Disable indexing in objdir because it is not necessary and can slow
         # down builds.
@@ -1054,25 +1049,21 @@ class BuildDriver(MozbuildObject):
                     if not os.path.isfile(mozpath.join(self.topobjdir, output)):
                         return True
 
                 dep_file = '%s.in' % backend_file
                 return build_out_of_date(backend_file, dep_file)
 
             monitor.start_resource_recording()
 
-            self.mach_context.command_attrs['clobber'] = False
             config = None
             try:
                 config = self.config_environment
             except Exception:
-                # If we don't already have a config environment this is either
-                # a fresh objdir or $OBJDIR/config.status has been removed for
-                # some reason, which indicates a clobber of sorts.
-                self.mach_context.command_attrs['clobber'] = True
+                pass
 
             # Record whether a clobber was requested so we can print
             # a special message later if the build fails.
             clobber_requested = False
 
             # Write out any changes to the current mozconfig in case
             # they should invalidate configure.
             self._write_mozconfig_json()
@@ -1231,21 +1222,16 @@ class BuildDriver(MozbuildObject):
                          "failing the build due to exception: {ex}.")
                 if not status:
                     # If the underlying build provided a failing status, pass
                     # it through; otherwise, fail.
                     status = 1
 
             monitor.finish(record_usage=status == 0)
 
-        if status == 0:
-            usage = monitor.get_resource_usage()
-            if usage:
-                self.mach_context.command_attrs['usage'] = usage
-
         # Print the collected compiler warnings. This is redundant with
         # inline output from the compiler itself. However, unlike inline
         # output, this list is sorted and grouped by file, making it
         # easier to triage output.
         #
         # Only do this if we had a successful build. If the build failed,
         # there are more important things in the log to look for than
         # whatever code we warned about.
@@ -1544,18 +1530,16 @@ class BuildDriver(MozbuildObject):
         res = clobberer.maybe_do_clobber(os.getcwd(), auto_clobber,
                                          clobber_output)
         clobber_output.seek(0)
         for line in clobber_output.readlines():
             self.log(logging.WARNING, 'clobber',
                      {'msg': line.rstrip()}, '{msg}')
 
         clobber_required, clobber_performed, clobber_message = res
-        if clobber_performed:
-            self.mach_context.command_attrs['clobber'] = True
         if not clobber_required or clobber_performed:
             if clobber_performed and env.get('TINDERBOX_OUTPUT'):
                 self.log(logging.WARNING, 'clobber',
                          {'msg': 'TinderboxPrint: auto clobber'}, '{msg}')
         else:
             for line in clobber_message.splitlines():
                 self.log(logging.WARNING, 'clobber',
                          {'msg': line.rstrip()}, '{msg}')
--- a/python/mozbuild/mozbuild/export_telemetry_schema.py
+++ b/python/mozbuild/mozbuild/export_telemetry_schema.py
@@ -32,13 +32,13 @@ def main():
 
     parser = argparse.ArgumentParser(
         description='Output build system telemetry schema in json-schema format')
     parser.add_argument('output', help='JSON output destination')
     args = parser.parse_args()
 
     schema = luscious.get_jsonschema(mozbuild.telemetry.schema)
     with open(args.output, 'wb') as f:
-        json.dump(schema, f, indent=2, separators=(',', ': '), sort_keys=True)
+        json.dump(schema, f, indent=2, sort_keys=True)
 
 
 if __name__ == '__main__':
     main()
--- a/python/mozbuild/mozbuild/telemetry.py
+++ b/python/mozbuild/mozbuild/telemetry.py
@@ -56,20 +56,16 @@ schema = Schema({
             Any(*CompilerType.POSSIBLE_VALUES),
         Optional('artifact', description='true if --enable-artifact-builds'): bool,
         Optional('debug', description='true if build is debug (--enable-debug)'): bool,
         Optional('opt', description='true if build is optimized (--enable-optimize)'): bool,
         Optional('ccache', description='true if ccache is in use (--with-ccache)'): bool,
         Optional('sccache', description='true if ccache in use is sccache'): bool,
         Optional('icecream', description='true if icecream in use'): bool,
     },
-    Optional('build_attrs', description='Attributes characterizing a build'): {
-        Optional('cpu_percent', description='cpu utilization observed during a build'): int,
-        Optional('clobber', description='true if the build was a clobber/full build'): bool,
-    },
     Required('system'): {
         # We don't need perfect granularity here.
         Required('os', description='Operating system'): Any('windows', 'macos', 'linux', 'other'),
         Optional('cpu_brand', description='CPU brand string from CPUID'): basestring,
         Optional('logical_cores', description='Number of logical CPU cores present'): int,
         Optional('physical_cores', description='Number of physical CPU cores present'): int,
         Optional('memory_gb', description='System memory in GB'): int,
         Optional('drive_is_ssd',
@@ -223,29 +219,16 @@ def get_build_opts(substs):
         prefix = os.path.basename(substs.get('CCACHE_PREFIX', ''))
         if substs.get('CXX_IS_ICECREAM', None) or prefix == 'icecc':
             opts['icecream'] = True
         return opts
     except BuildEnvironmentNotFoundException:
         return {}
 
 
-def get_build_attrs(attrs):
-    '''
-    Extracts clobber and cpu usage info from command attributes.
-    '''
-    res = {}
-    clobber = attrs.get('clobber')
-    if clobber:
-        res['clobber'] = clobber
-    usage = attrs.get('usage')
-    if usage:
-        res['cpu_percent'] = int(round(usage['cpu_percent']))
-    return res
-
 def filter_args(command, argv, paths):
     '''
     Given the full list of command-line arguments, remove anything up to and including `command`,
     and attempt to filter absolute pathnames out of any arguments after that.
 
     `paths` is a dict whose keys are pathnames and values are sigils that should be used to
     replace those pathnames.
     '''
@@ -261,17 +244,17 @@ def filter_args(command, argv, paths):
         if base:
             return paths[base] + mozpath.relpath(p, base)
         # Best-effort.
         return '<path omitted>'
     return [filter_path(arg) for arg in args]
 
 
 def gather_telemetry(command='', success=False, start_time=None, end_time=None,
-                     mach_context=None, substs={}, paths={}, command_attrs=None):
+                     mach_context=None, substs={}, paths={}):
     '''
     Gather telemetry about the build and the user's system and pass it to the telemetry
     handler to be stored for later submission.
 
     `paths` is a dict whose keys are pathnames and values are sigils that should be used to
     replace those pathnames.
 
     Any absolute paths on the command line will be made relative to `paths` or replaced
@@ -282,17 +265,16 @@ def gather_telemetry(command='', success
         # Get an rfc3339 datetime string.
         'time': datetime.utcfromtimestamp(start_time).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
         'command': command,
         'argv': filter_args(command, sys.argv, paths),
         'success': success,
         # TODO: use a monotonic clock: https://bugzilla.mozilla.org/show_bug.cgi?id=1481624
         'duration_ms': int((end_time - start_time) * 1000),
         'build_opts': get_build_opts(substs),
-        'build_attrs': get_build_attrs(command_attrs),
         'system': get_system_info(),
         # TODO: exception: https://bugzilla.mozilla.org/show_bug.cgi?id=1481617
         # TODO: file_types_changed: https://bugzilla.mozilla.org/show_bug.cgi?id=1481774
     }
     try:
         # Validate against the schema.
         schema(data)
         return data