python/mozbuild/mozbuild/mach_commands.py
author Birunthan Mohanathas <birunthan@mohanathas.com>
Fri, 25 Jul 2014 14:15:35 -0700
changeset 196078 bf3d1eb30dcf
parent 195922 95239dc01939
child 196083 3f25f71326ca
permissions -rw-r--r--
Bug 946065 - Part 7: Move content/mathml/ to dom/ and flatten subdirectories. r=karlt
gps@108214
     1
# This Source Code Form is subject to the terms of the Mozilla Public
gps@108214
     2
# License, v. 2.0. If a copy of the MPL was not distributed with this
gps@108214
     3
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
gps@108214
     4
gps@112494
     5
from __future__ import print_function, unicode_literals
gps@108214
     6
mh+mozilla@146817
     7
import itertools
gps@108214
     8
import logging
gps@112494
     9
import operator
gps@108214
    10
import os
gps@132815
    11
import sys
gps@108214
    12
mh+mozilla@191955
    13
import mozpack.path as mozpath
mh+mozilla@191955
    14
gps@112492
    15
from mach.decorators import (
gps@112494
    16
    CommandArgument,
gps@112492
    17
    CommandProvider,
gps@112492
    18
    Command,
gps@112492
    19
)
gps@112492
    20
gps@132815
    21
from mach.mixin.logging import LoggingMixin
gps@132815
    22
gps@141815
    23
from mozbuild.base import (
gps@141815
    24
    MachCommandBase,
gps@141815
    25
    MozbuildObject,
gps@141815
    26
    MozconfigFindException,
gps@141815
    27
    MozconfigLoadException,
gps@150500
    28
    ObjdirMismatchException,
gps@141815
    29
)
gps@108214
    30
gps@108214
    31
gps@120252
    32
BUILD_WHAT_HELP = '''
gps@120252
    33
What to build. Can be a top-level make target or a relative directory. If
nalexander@130542
    34
multiple options are provided, they will be built serially. Takes dependency
nalexander@130542
    35
information from `topsrcdir/build/dumbmake-dependencies` to build additional
nalexander@130542
    36
targets as needed. BUILDING ONLY PARTS OF THE TREE CAN RESULT IN BAD TREE
nalexander@130542
    37
STATE. USE AT YOUR OWN RISK.
gps@120252
    38
'''.strip()
gps@120252
    39
gps@124815
    40
FINDER_SLOW_MESSAGE = '''
gps@124815
    41
===================
gps@124815
    42
PERFORMANCE WARNING
gps@124815
    43
gps@124815
    44
The OS X Finder application (file indexing used by Spotlight) used a lot of CPU
gps@124815
    45
during the build - an average of %f%% (100%% is 1 core). This made your build
gps@124815
    46
slower.
gps@124815
    47
gps@124815
    48
Consider adding ".noindex" to the end of your object directory name to have
gps@124815
    49
Finder ignore it. Or, add an indexing exclusion through the Spotlight System
gps@124815
    50
Preferences.
gps@124815
    51
===================
gps@124815
    52
'''.strip()
gps@124815
    53
gps@159852
    54
EXCESSIVE_SWAP_MESSAGE = '''
gps@159852
    55
===================
gps@159852
    56
PERFORMANCE WARNING
gps@159852
    57
gps@159852
    58
Your machine experienced a lot of swap activity during the build. This is
gps@159852
    59
possibly a sign that your machine doesn't have enough physical memory or
gps@159852
    60
not enough available memory to perform the build. It's also possible some
gps@159852
    61
other system activity during the build is to blame.
gps@159852
    62
gps@159852
    63
If you feel this message is not appropriate for your machine configuration,
gps@159852
    64
please file a Core :: Build Config bug at
gps@159852
    65
https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=Build%20Config
gps@159852
    66
and tell us about your machine and build configuration so we can adjust the
gps@159852
    67
warning heuristic.
gps@159852
    68
===================
gps@159852
    69
'''
gps@159852
    70
gps@120252
    71
gps@132815
    72
class TerminalLoggingHandler(logging.Handler):
gps@132815
    73
    """Custom logging handler that works with terminal window dressing.
gps@132815
    74
gps@132815
    75
    This class should probably live elsewhere, like the mach core. Consider
gps@132815
    76
    this a proving ground for its usefulness.
gps@132815
    77
    """
gps@132815
    78
    def __init__(self):
gps@132815
    79
        logging.Handler.__init__(self)
gps@132815
    80
gps@132815
    81
        self.fh = sys.stdout
gps@132815
    82
        self.footer = None
gps@132815
    83
gps@132815
    84
    def flush(self):
gps@132815
    85
        self.acquire()
gps@132815
    86
gps@132815
    87
        try:
gps@132815
    88
            self.fh.flush()
gps@132815
    89
        finally:
gps@132815
    90
            self.release()
gps@132815
    91
gps@132815
    92
    def emit(self, record):
gps@132815
    93
        msg = self.format(record)
gps@132815
    94
gps@135647
    95
        self.acquire()
gps@132815
    96
gps@135647
    97
        try:
gps@135647
    98
            if self.footer:
gps@137363
    99
                self.footer.clear()
gps@132815
   100
gps@135647
   101
            self.fh.write(msg)
gps@135647
   102
            self.fh.write('\n')
gps@132815
   103
gps@135647
   104
            if self.footer:
gps@135647
   105
                self.footer.draw()
gps@135647
   106
gps@135647
   107
            # If we don't flush, the footer may not get drawn.
gps@135647
   108
            self.fh.flush()
gps@135647
   109
        finally:
gps@135647
   110
            self.release()
gps@132815
   111
gps@132815
   112
gps@132815
   113
class BuildProgressFooter(object):
gps@132815
   114
    """Handles display of a build progress indicator in a terminal.
gps@132815
   115
gps@132815
   116
    When mach builds inside a blessings-supported terminal, it will render
gps@132815
   117
    progress information collected from a BuildMonitor. This class converts the
gps@132815
   118
    state of BuildMonitor into terminal output.
gps@132815
   119
    """
gps@132815
   120
gps@132815
   121
    def __init__(self, terminal, monitor):
gps@132815
   122
        # terminal is a blessings.Terminal.
gps@132815
   123
        self._t = terminal
gps@132815
   124
        self._fh = sys.stdout
gps@132815
   125
        self._monitor = monitor
gps@132815
   126
gps@132815
   127
    def _clear_lines(self, n):
Sahilc@143226
   128
        self._fh.write(self._t.move_x(0))
gps@137720
   129
        self._fh.write(self._t.clear_eos())
gps@132815
   130
gps@132815
   131
    def clear(self):
gps@132815
   132
        """Removes the footer from the current terminal."""
gps@132815
   133
        self._clear_lines(1)
gps@132815
   134
gps@132815
   135
    def draw(self):
gps@132815
   136
        """Draws this footer in the terminal."""
gps@143189
   137
        tiers = self._monitor.tiers
gps@143189
   138
gps@143189
   139
        if not tiers.tiers:
gps@132815
   140
            return
gps@132815
   141
gps@132815
   142
        # The drawn terminal looks something like:
gps@132815
   143
        # TIER: base nspr nss js platform app SUBTIER: static export libs tools DIRECTORIES: 06/09 (memory)
gps@132815
   144
gps@132815
   145
        # This is a list of 2-tuples of (encoding function, input). None means
gps@132815
   146
        # no encoding. For a full reason on why we do things this way, read the
gps@132815
   147
        # big comment below.
gps@132815
   148
        parts = [('bold', 'TIER'), ':', ' ']
gps@132815
   149
gps@143189
   150
        for tier, active, finished in tiers.tier_status():
gps@143189
   151
            if active:
maxli@137664
   152
                parts.extend([('underline_yellow', tier), ' '])
gps@143189
   153
            elif finished:
gps@132815
   154
                parts.extend([('green', tier), ' '])
gps@132815
   155
            else:
gps@132815
   156
                parts.extend([tier, ' '])
gps@132815
   157
gps@132815
   158
        # We don't want to write more characters than the current width of the
gps@132815
   159
        # terminal otherwise wrapping may result in weird behavior. We can't
gps@132815
   160
        # simply truncate the line at terminal width characters because a)
gps@132815
   161
        # non-viewable escape characters count towards the limit and b) we
gps@132815
   162
        # don't want to truncate in the middle of an escape sequence because
gps@132815
   163
        # subsequent output would inherit the escape sequence.
gps@132815
   164
        max_width = self._t.width
gps@132815
   165
        written = 0
gps@132815
   166
        write_pieces = []
gps@132815
   167
        for part in parts:
gps@132815
   168
            if isinstance(part, tuple):
gps@132815
   169
                func, arg = part
gps@132815
   170
gps@132815
   171
                if written + len(arg) > max_width:
gps@132815
   172
                    write_pieces.append(arg[0:max_width - written])
gps@132815
   173
                    written += len(arg)
gps@132815
   174
                    break
gps@132815
   175
gps@132815
   176
                encoded = getattr(self._t, func)(arg)
gps@132815
   177
gps@132815
   178
                write_pieces.append(encoded)
gps@132815
   179
                written += len(arg)
gps@132815
   180
            else:
gps@132815
   181
                if written + len(part) > max_width:
gps@139322
   182
                    write_pieces.append(part[0:max_width - written])
gps@132815
   183
                    written += len(part)
gps@132815
   184
                    break
gps@132815
   185
gps@132815
   186
                write_pieces.append(part)
gps@132815
   187
                written += len(part)
Sahilc@143226
   188
        with self._t.location():
Sahilc@143226
   189
            self._t.move(self._t.height-1,0)
Sahilc@143226
   190
            self._fh.write(''.join(write_pieces))
gps@132815
   191
        self._fh.flush()
gps@132815
   192
gps@132815
   193
gps@132815
   194
class BuildOutputManager(LoggingMixin):
gps@132815
   195
    """Handles writing build output to a terminal, to logs, etc."""
gps@132815
   196
gps@132815
   197
    def __init__(self, log_manager, monitor):
gps@132815
   198
        self.populate_logger()
gps@132815
   199
gps@132815
   200
        self.monitor = monitor
gps@132815
   201
        self.footer = None
gps@132815
   202
gps@132815
   203
        terminal = log_manager.terminal
gps@132815
   204
gps@132815
   205
        # TODO convert terminal footer to config file setting.
gps@132815
   206
        if not terminal or os.environ.get('MACH_NO_TERMINAL_FOOTER', None):
gps@132815
   207
            return
gps@132815
   208
gps@132815
   209
        self.t = terminal
gps@132815
   210
        self.footer = BuildProgressFooter(terminal, monitor)
gps@132815
   211
gps@143249
   212
        self._handler = TerminalLoggingHandler()
gps@143249
   213
        self._handler.setFormatter(log_manager.terminal_formatter)
gps@143249
   214
        self._handler.footer = self.footer
gps@132815
   215
gps@143249
   216
        old = log_manager.replace_terminal_handler(self._handler)
gps@143249
   217
        self._handler.level = old.level
gps@132815
   218
gps@132815
   219
    def __enter__(self):
gps@132815
   220
        return self
gps@132815
   221
gps@132815
   222
    def __exit__(self, exc_type, exc_value, traceback):
gps@132815
   223
        if self.footer:
gps@132815
   224
            self.footer.clear()
gps@143249
   225
            # Prevents the footer from being redrawn if logging occurs.
gps@143249
   226
            self._handler.footer = None
gps@132815
   227
gps@132815
   228
    def write_line(self, line):
gps@132815
   229
        if self.footer:
gps@132815
   230
            self.footer.clear()
gps@132815
   231
gps@132815
   232
        print(line)
gps@132815
   233
gps@132815
   234
        if self.footer:
gps@132815
   235
            self.footer.draw()
gps@132815
   236
gps@132815
   237
    def refresh(self):
gps@132815
   238
        if not self.footer:
gps@132815
   239
            return
gps@132815
   240
gps@132815
   241
        self.footer.clear()
gps@132815
   242
        self.footer.draw()
gps@132815
   243
gps@132815
   244
    def on_line(self, line):
gps@132815
   245
        warning, state_changed, relevant = self.monitor.on_line(line)
gps@132815
   246
gps@132815
   247
        if warning:
gps@132815
   248
            self.log(logging.INFO, 'compiler_warning', warning,
gps@132815
   249
                'Warning: {flag} in {filename}: {message}')
gps@132815
   250
gps@132815
   251
        if relevant:
gps@132815
   252
            self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
gps@132815
   253
        elif state_changed:
gps@137387
   254
            have_handler = hasattr(self, 'handler')
gps@137387
   255
            if have_handler:
gps@137387
   256
                self.handler.acquire()
gps@137363
   257
            try:
gps@137363
   258
                self.refresh()
gps@137363
   259
            finally:
gps@137387
   260
                if have_handler:
gps@137387
   261
                    self.handler.release()
gps@132815
   262
gps@132815
   263
gps@109513
   264
@CommandProvider
gps@112492
   265
class Build(MachCommandBase):
gps@108214
   266
    """Interface to build the tree."""
gps@108214
   267
gps@131318
   268
    @Command('build', category='build', description='Build the tree.')
snorp@130669
   269
    @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,
snorp@130669
   270
        help='Number of concurrent jobs to run. Default is the number of CPUs.')
gps@120252
   271
    @CommandArgument('what', default=None, nargs='*', help=BUILD_WHAT_HELP)
nalexander@130542
   272
    @CommandArgument('-X', '--disable-extra-make-dependencies',
nalexander@130542
   273
                     default=False, action='store_true',
nalexander@130542
   274
                     help='Do not add extra make dependencies.')
kgupta@132515
   275
    @CommandArgument('-v', '--verbose', action='store_true',
kgupta@132515
   276
        help='Verbose output for what commands the build is running.')
mh+mozilla@190847
   277
    def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
mh+mozilla@190847
   278
        verbose=False):
gps@143951
   279
        import which
gps@132813
   280
        from mozbuild.controller.building import BuildMonitor
nalexander@123889
   281
        from mozbuild.util import resolve_target_to_make
gps@108214
   282
gps@143249
   283
        self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
gps@143249
   284
gps@108214
   285
        warnings_path = self._get_state_filename('warnings.json')
gps@143189
   286
        monitor = self._spawn(BuildMonitor)
gps@143189
   287
        monitor.init(warnings_path)
gps@108214
   288
gps@132815
   289
        with BuildOutputManager(self.log_manager, monitor) as output:
gps@132815
   290
            monitor.start()
gps@108214
   291
gps@132815
   292
            if what:
gps@132815
   293
                top_make = os.path.join(self.topobjdir, 'Makefile')
gps@132815
   294
                if not os.path.exists(top_make):
gps@132815
   295
                    print('Your tree has not been configured yet. Please run '
gps@132815
   296
                        '|mach build| with no arguments.')
gps@120252
   297
                    return 1
gps@120252
   298
gps@132815
   299
                # Collect target pairs.
gps@132815
   300
                target_pairs = []
gps@132815
   301
                for target in what:
gps@132815
   302
                    path_arg = self._wrap_path_argument(target)
nalexander@130542
   303
gps@132815
   304
                    make_dir, make_target = resolve_target_to_make(self.topobjdir,
gps@132815
   305
                        path_arg.relpath())
nalexander@130542
   306
gps@132815
   307
                    if make_dir is None and make_target is None:
gps@132815
   308
                        return 1
gps@120252
   309
mhammond@138618
   310
                    # See bug 886162 - we don't want to "accidentally" build
mhammond@138618
   311
                    # the entire tree (if that's really the intent, it's
mhammond@138618
   312
                    # unlikely they would have specified a directory.)
mhammond@138618
   313
                    if not make_dir and not make_target:
mhammond@138618
   314
                        print("The specified directory doesn't contain a "
mhammond@138618
   315
                              "Makefile and the first parent with one is the "
mhammond@138618
   316
                              "root of the tree. Please specify a directory "
mhammond@138618
   317
                              "with a Makefile or run |mach build| if you "
mhammond@138618
   318
                              "want to build the entire tree.")
mhammond@138618
   319
                        return 1
mhammond@138618
   320
gps@132815
   321
                    target_pairs.append((make_dir, make_target))
gps@120252
   322
gps@132815
   323
                # Possibly add extra make depencies using dumbmake.
gps@132815
   324
                if not disable_extra_make_dependencies:
gps@132815
   325
                    from dumbmake.dumbmake import (dependency_map,
gps@132815
   326
                                                   add_extra_dependencies)
gps@132815
   327
                    depfile = os.path.join(self.topsrcdir, 'build',
gps@132815
   328
                                           'dumbmake-dependencies')
gps@132815
   329
                    with open(depfile) as f:
gps@132815
   330
                        dm = dependency_map(f.readlines())
gps@132815
   331
                    new_pairs = list(add_extra_dependencies(target_pairs, dm))
gps@132815
   332
                    self.log(logging.DEBUG, 'dumbmake',
gps@132815
   333
                             {'target_pairs': target_pairs,
gps@132815
   334
                              'new_pairs': new_pairs},
gps@132815
   335
                             'Added extra dependencies: will build {new_pairs} ' +
gps@132815
   336
                             'instead of {target_pairs}.')
gps@132815
   337
                    target_pairs = new_pairs
gps@108214
   338
gps@152698
   339
                # Ensure build backend is up to date. The alternative is to
gps@152698
   340
                # have rules in the invoked Makefile to rebuild the build
gps@152698
   341
                # backend. But that involves make reinvoking itself and there
gps@152698
   342
                # are undesired side-effects of this. See bug 877308 for a
gps@152698
   343
                # comprehensive history lesson.
gps@152698
   344
                self._run_make(directory=self.topobjdir,
gps@152698
   345
                    target='backend.RecursiveMakeBackend',
mh+mozilla@190847
   346
                    line_handler=output.on_line, log=False,
mh+mozilla@190847
   347
                    print_directory=False)
gps@152698
   348
gps@132815
   349
                # Build target pairs.
gps@132815
   350
                for make_dir, make_target in target_pairs:
gps@144518
   351
                    # We don't display build status messages during partial
gps@144518
   352
                    # tree builds because they aren't reliable there. This
gps@144518
   353
                    # could potentially be fixed if the build monitor were more
gps@144518
   354
                    # intelligent about encountering undefined state.
gps@132815
   355
                    status = self._run_make(directory=make_dir, target=make_target,
gps@132815
   356
                        line_handler=output.on_line, log=False, print_directory=False,
gps@144518
   357
                        ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
mh+mozilla@190847
   358
                        append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'})
gps@132815
   359
gps@132815
   360
                    if status != 0:
gps@132815
   361
                        break
gps@132815
   362
            else:
gps@143249
   363
                monitor.start_resource_recording()
gps@132815
   364
                status = self._run_make(srcdir=True, filename='client.mk',
gps@132815
   365
                    line_handler=output.on_line, log=False, print_directory=False,
gps@132815
   366
                    allow_parallel=False, ensure_exit_code=False, num_jobs=jobs,
mh+mozilla@190847
   367
                    silent=not verbose)
gps@132815
   368
mshal@188593
   369
                make_extra = self.mozconfig['make_extra'] or []
mshal@188593
   370
                make_extra = dict(m.split('=', 1) for m in make_extra)
mshal@188593
   371
mshal@195272
   372
                # For universal builds, we need to run the automation steps in
mshal@195272
   373
                # the first architecture from MOZ_BUILD_PROJECTS
mshal@195272
   374
                projects = make_extra.get('MOZ_BUILD_PROJECTS')
mshal@195272
   375
                if projects:
mshal@195272
   376
                    subdir = os.path.join(self.topobjdir, projects.split()[0])
mshal@195272
   377
                else:
mshal@195272
   378
                    subdir = self.topobjdir
mshal@188593
   379
                moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get('export MOZ_AUTOMATION', None)
mshal@188593
   380
                if moz_automation and status == 0:
mshal@195272
   381
                    status = self._run_make(target='automation/build', directory=subdir,
mshal@188593
   382
                        line_handler=output.on_line, log=False, print_directory=False,
mh+mozilla@190847
   383
                        ensure_exit_code=False, num_jobs=jobs, silent=not verbose)
mshal@188593
   384
gps@132815
   385
                self.log(logging.WARNING, 'warning_summary',
gps@132815
   386
                    {'count': len(monitor.warnings_database)},
gps@132815
   387
                    '{count} compiler warnings present.')
gps@132815
   388
gps@143249
   389
            monitor.finish(record_usage=status==0)
gps@132815
   390
gps@132813
   391
        high_finder, finder_percent = monitor.have_high_finder_usage()
gps@132813
   392
        if high_finder:
gps@132813
   393
            print(FINDER_SLOW_MESSAGE % finder_percent)
nnethercote@117369
   394
gps@143953
   395
        if monitor.elapsed > 300:
gps@143951
   396
            # Display a notification when the build completes.
gps@143951
   397
            # This could probably be uplifted into the mach core or at least
gps@143951
   398
            # into a helper API. It is here as an experimentation to see how it
gps@143951
   399
            # is received.
gps@143951
   400
            try:
gps@143951
   401
                if sys.platform.startswith('darwin'):
gps@143951
   402
                    notifier = which.which('terminal-notifier')
gps@143951
   403
                    self.run_process([notifier, '-title',
gps@143951
   404
                        'Mozilla Build System', '-group', 'mozbuild',
gps@143951
   405
                        '-message', 'Build complete'], ensure_exit_code=False)
evilpies@195922
   406
                elif sys.platform.startswith('linux'):
evilpies@195922
   407
                    try:
evilpies@195922
   408
                        import dbus
evilpies@195922
   409
                        bus = dbus.SessionBus()
evilpies@195922
   410
                        notify = bus.get_object('org.freedesktop.Notifications',
evilpies@195922
   411
                                                '/org/freedesktop/Notifications')
evilpies@195922
   412
                        method = notify.get_dbus_method('Notify',
evilpies@195922
   413
                                                        'org.freedesktop.Notifications')
evilpies@195922
   414
                        method('Mozilla Build System', 0, '', 'Build complete', '', [], [], -1)
evilpies@195922
   415
                    except (dbus.exceptions.DBusException, ImportError):
evilpies@195922
   416
                        pass
evilpies@195922
   417
evilpies@195922
   418
            except (which.WhichError, ImportError):
gps@143951
   419
                pass
gps@143951
   420
            except Exception as e:
gps@143951
   421
                self.log(logging.WARNING, 'notifier-failed', {'error':
gps@143951
   422
                    e.message}, 'Notification center failed: {error}')
gps@125956
   423
gps@125956
   424
        if status:
gps@125956
   425
            return status
gps@125956
   426
gps@143951
   427
        long_build = monitor.elapsed > 600
gps@143951
   428
gps@125956
   429
        if long_build:
gps@125956
   430
            print('We know it took a while, but your build finally finished successfully!')
gps@125956
   431
        else:
gps@125956
   432
            print('Your build was successful!')
gps@125956
   433
gps@143250
   434
        if monitor.have_resource_usage:
gps@159852
   435
            excessive, swap_in, swap_out = monitor.have_excessive_swapping()
ehsan@161285
   436
            # if excessive:
ehsan@161285
   437
            #    print(EXCESSIVE_SWAP_MESSAGE)
gps@159852
   438
gps@143250
   439
            print('To view resource usage of the build, run |mach '
gps@143250
   440
                'resource-usage|.')
gps@143250
   441
gps@125956
   442
        # Only for full builds because incremental builders likely don't
gps@125956
   443
        # need to be burdened with this.
gps@125956
   444
        if not what:
mshal@193549
   445
            try:
mshal@193549
   446
                # Fennec doesn't have useful output from just building. We should
mshal@193549
   447
                # arguably make the build action useful for Fennec. Another day...
mshal@193549
   448
                if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
mshal@193549
   449
                    print('To take your build for a test drive, run: |mach run|')
mshal@193549
   450
                app = self.substs['MOZ_BUILD_APP']
mshal@193549
   451
                if app in ('browser', 'mobile/android'):
mshal@193549
   452
                    print('For more information on what to do now, see '
mshal@193549
   453
                        'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox')
mshal@193549
   454
            except Exception:
mshal@193549
   455
                # Ignore Exceptions in case we can't find config.status (such
mshal@193549
   456
                # as when doing OSX Universal builds)
mshal@193549
   457
                pass
gps@112494
   458
gps@112495
   459
        return status
gps@112495
   460
gps@131318
   461
    @Command('configure', category='build',
janx@132173
   462
        description='Configure the tree (run configure and config.status).')
bokeefe@130048
   463
    def configure(self):
bokeefe@130048
   464
        def on_line(line):
bokeefe@130048
   465
            self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
bokeefe@130048
   466
bokeefe@130048
   467
        status = self._run_make(srcdir=True, filename='client.mk',
bokeefe@130048
   468
            target='configure', line_handler=on_line, log=False,
bokeefe@130048
   469
            print_directory=False, allow_parallel=False, ensure_exit_code=False)
bokeefe@130048
   470
bokeefe@130048
   471
        if not status:
bokeefe@130048
   472
            print('Configure complete!')
bokeefe@130048
   473
            print('Be sure to run |mach build| to pick up any changes');
bokeefe@130048
   474
bokeefe@130048
   475
        return status
bokeefe@130048
   476
gps@143250
   477
    @Command('resource-usage', category='post-build',
gps@143250
   478
        description='Show information about system resource usage for a build.')
gps@143250
   479
    @CommandArgument('--address', default='localhost',
gps@143250
   480
        help='Address the HTTP server should listen on.')
gps@143250
   481
    @CommandArgument('--port', type=int, default=0,
gps@143250
   482
        help='Port number the HTTP server should listen on.')
gps@143250
   483
    @CommandArgument('--browser', default='firefox',
gps@143250
   484
        help='Web browser to automatically open. See webbrowser Python module.')
gps@143250
   485
    def resource_usage(self, address=None, port=None, browser=None):
gps@143250
   486
        import webbrowser
gps@143250
   487
        from mozbuild.html_build_viewer import BuildViewerServer
gps@143250
   488
gps@143250
   489
        last = self._get_state_filename('build_resources.json')
gps@143250
   490
        if not os.path.exists(last):
gps@143250
   491
            print('Build resources not available. If you have performed a '
gps@143250
   492
                'build and receive this message, the psutil Python package '
gps@143250
   493
                'likely failed to initialize properly.')
gps@143250
   494
            return 1
gps@143250
   495
gps@143250
   496
        server = BuildViewerServer(address, port)
gps@143250
   497
        server.add_resource_json_file('last', last)
gps@143250
   498
        try:
gps@143250
   499
            webbrowser.get(browser).open_new_tab(server.url)
gps@143250
   500
        except Exception:
gps@143250
   501
            print('Please open %s in a browser.' % server.url)
gps@143250
   502
gps@143250
   503
        print('Hit CTRL+c to stop server.')
gps@143250
   504
        server.run()
bokeefe@130048
   505
gps@131318
   506
    @Command('clobber', category='build',
gps@131318
   507
        description='Clobber the tree (delete the object directory).')
gps@115172
   508
    def clobber(self):
mbrubeck@121937
   509
        try:
mbrubeck@121937
   510
            self.remove_objdir()
mbrubeck@121937
   511
            return 0
gps@146396
   512
        except OSError as e:
gps@146396
   513
            if sys.platform.startswith('win'):
gps@146396
   514
                if isinstance(e, WindowsError) and e.winerror in (5,32):
gps@146396
   515
                    self.log(logging.ERROR, 'file_access_error', {'error': e},
gps@146396
   516
                        "Could not clobber because a file was in use. If the "
gps@146396
   517
                        "application is running, try closing it. {error}")
gps@146396
   518
                    return 1
gps@146396
   519
gps@146396
   520
            raise
gps@115172
   521
gps@152698
   522
    @Command('build-backend', category='build',
gps@152698
   523
        description='Generate a backend used to build the tree.')
gps@160048
   524
    @CommandArgument('-d', '--diff', action='store_true',
gps@160048
   525
        help='Show a diff of changes.')
nalexander@168739
   526
    # It would be nice to filter the choices below based on
nalexander@168739
   527
    # conditions, but that is for another day.
nalexander@168739
   528
    @CommandArgument('-b', '--backend',
b56girard@172177
   529
        choices=['RecursiveMake', 'AndroidEclipse', 'CppEclipse', 'VisualStudio'],
nalexander@168740
   530
        default='RecursiveMake',
nalexander@168739
   531
        help='Which backend to build (default: RecursiveMake).')
nalexander@168739
   532
    def build_backend(self, backend='RecursiveMake', diff=False):
gps@152703
   533
        python = self.virtualenv_manager.python_path
gps@152698
   534
        config_status = os.path.join(self.topobjdir, 'config.status')
gps@160048
   535
nalexander@168739
   536
        args = [python, config_status, '--backend=%s' % backend]
gps@160048
   537
        if diff:
gps@160048
   538
            args.append('--diff')
gps@160048
   539
gps@160048
   540
        return self._run_command_in_objdir(args=args, pass_thru=True,
gps@160048
   541
            ensure_exit_code=False)
gps@152698
   542
gps@112494
   543
gps@112494
   544
@CommandProvider
gps@112494
   545
class Warnings(MachCommandBase):
gps@112494
   546
    """Provide commands for inspecting warnings."""
gps@112494
   547
gps@112494
   548
    @property
gps@112494
   549
    def database_path(self):
gps@112494
   550
        return self._get_state_filename('warnings.json')
gps@112494
   551
gps@112494
   552
    @property
gps@112494
   553
    def database(self):
gps@112494
   554
        from mozbuild.compilation.warnings import WarningsDatabase
gps@112494
   555
gps@112494
   556
        path = self.database_path
gps@112494
   557
gps@112494
   558
        database = WarningsDatabase()
gps@112494
   559
gps@112494
   560
        if os.path.exists(path):
gps@112494
   561
            database.load_from_file(path)
gps@112494
   562
gps@112494
   563
        return database
gps@112494
   564
gps@131318
   565
    @Command('warnings-summary', category='post-build',
gps@131318
   566
        description='Show a summary of compiler warnings.')
gps@112494
   567
    @CommandArgument('report', default=None, nargs='?',
gps@112494
   568
        help='Warnings report to display. If not defined, show the most '
gps@112494
   569
            'recent report.')
gps@112494
   570
    def summary(self, report=None):
gps@112494
   571
        database = self.database
gps@112494
   572
gps@112494
   573
        type_counts = database.type_counts
gps@112494
   574
        sorted_counts = sorted(type_counts.iteritems(),
gps@112494
   575
            key=operator.itemgetter(1))
gps@112494
   576
gps@112494
   577
        total = 0
gps@112494
   578
        for k, v in sorted_counts:
gps@112494
   579
            print('%d\t%s' % (v, k))
gps@112494
   580
            total += v
gps@112494
   581
gps@112494
   582
        print('%d\tTotal' % total)
gps@112494
   583
gps@131318
   584
    @Command('warnings-list', category='post-build',
gps@131318
   585
        description='Show a list of compiler warnings.')
gps@112494
   586
    @CommandArgument('report', default=None, nargs='?',
gps@112494
   587
        help='Warnings report to display. If not defined, show the most '
gps@112494
   588
            'recent report.')
gps@112494
   589
    def list(self, report=None):
gps@112494
   590
        database = self.database
gps@112494
   591
gps@112494
   592
        by_name = sorted(database.warnings)
gps@112494
   593
gps@112494
   594
        for warning in by_name:
gps@112494
   595
            filename = warning['filename']
gps@112494
   596
gps@112494
   597
            if filename.startswith(self.topsrcdir):
gps@112494
   598
                filename = filename[len(self.topsrcdir) + 1:]
gps@112494
   599
gps@112494
   600
            if warning['column'] is not None:
gps@112494
   601
                print('%s:%d:%d [%s] %s' % (filename, warning['line'],
gps@112494
   602
                    warning['column'], warning['flag'], warning['message']))
gps@112494
   603
            else:
gps@112494
   604
                print('%s:%d [%s] %s' % (filename, warning['line'],
gps@112494
   605
                    warning['flag'], warning['message']))
gps@112494
   606
rnewman@115674
   607
@CommandProvider
b56girard@126577
   608
class GTestCommands(MachCommandBase):
gps@131318
   609
    @Command('gtest', category='testing',
gps@131318
   610
        description='Run GTest unit tests.')
b56girard@130533
   611
    @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter',
b56girard@126577
   612
        help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns),"
b56girard@126577
   613
             "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).")
b56girard@126577
   614
    @CommandArgument('--jobs', '-j', default='1', nargs='?', metavar='jobs', type=int,
b56girard@126577
   615
        help='Run the tests in parallel using multiple processes.')
b56girard@126577
   616
    @CommandArgument('--tbpl-parser', '-t', action='store_true',
b56girard@126577
   617
        help='Output test results in a format that can be parsed by TBPL.')
b56girard@126577
   618
    @CommandArgument('--shuffle', '-s', action='store_true',
b56girard@126577
   619
        help='Randomize the execution order of tests.')
b56girard@126577
   620
    def gtest(self, shuffle, jobs, gtest_filter, tbpl_parser):
b56girard@134882
   621
b56girard@134882
   622
        # We lazy build gtest because it's slow to link
b56girard@134882
   623
        self._run_make(directory="testing/gtest", target='gtest', ensure_exit_code=True)
b56girard@134882
   624
b56girard@126577
   625
        app_path = self.get_binary_path('app')
b56girard@126577
   626
b56girard@126577
   627
        # Use GTest environment variable to control test execution
b56girard@126577
   628
        # For details see:
b56girard@126577
   629
        # https://code.google.com/p/googletest/wiki/AdvancedGuide#Running_Test_Programs:_Advanced_Options
b56girard@126577
   630
        gtest_env = {b'GTEST_FILTER': gtest_filter}
b56girard@126577
   631
b56girard@134882
   632
        gtest_env[b"MOZ_RUN_GTEST"] = b"True"
b56girard@134882
   633
b56girard@126577
   634
        if shuffle:
b56girard@126577
   635
            gtest_env[b"GTEST_SHUFFLE"] = b"True"
b56girard@126577
   636
b56girard@126577
   637
        if tbpl_parser:
b56girard@126577
   638
            gtest_env[b"MOZ_TBPL_PARSER"] = b"True"
b56girard@126577
   639
b56girard@126577
   640
        if jobs == 1:
b56girard@126577
   641
            return self.run_process([app_path, "-unittest"],
b56girard@126577
   642
                                    append_env=gtest_env,
b56girard@126577
   643
                                    ensure_exit_code=False,
b56girard@126577
   644
                                    pass_thru=True)
b56girard@126577
   645
b56girard@126577
   646
        from mozprocess import ProcessHandlerMixin
b56girard@126577
   647
        import functools
b56girard@126577
   648
        def handle_line(job_id, line):
b56girard@126577
   649
            # Prepend the jobId
b56girard@126577
   650
            line = '[%d] %s' % (job_id + 1, line.strip())
b56girard@126577
   651
            self.log(logging.INFO, "GTest", {'line': line}, '{line}')
b56girard@126577
   652
b56girard@126577
   653
        gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs)
b56girard@126577
   654
        processes = {}
b56girard@126577
   655
        for i in range(0, jobs):
b56girard@126577
   656
            gtest_env["GTEST_SHARD_INDEX"] = str(i)
b56girard@126577
   657
            processes[i] = ProcessHandlerMixin([app_path, "-unittest"],
b56girard@126577
   658
                             env=gtest_env,
b56girard@126577
   659
                             processOutputLine=[functools.partial(handle_line, i)],
b56girard@126577
   660
                             universal_newlines=True)
b56girard@126577
   661
            processes[i].run()
b56girard@126577
   662
b56girard@126577
   663
        exit_code = 0
b56girard@126577
   664
        for process in processes.values():
b56girard@126577
   665
            status = process.wait()
b56girard@126577
   666
            if status:
b56girard@126577
   667
                exit_code = status
b56girard@126577
   668
b56girard@126577
   669
        # Clamp error code to 255 to prevent overflowing multiple of
b56girard@126577
   670
        # 256 into 0
b56girard@126577
   671
        if exit_code > 255:
b56girard@126577
   672
            exit_code = 255
b56girard@126577
   673
b56girard@126577
   674
        return exit_code
b56girard@126577
   675
b56girard@126577
   676
@CommandProvider
gps@120653
   677
class ClangCommands(MachCommandBase):
gps@131318
   678
    @Command('clang-complete', category='devenv',
gps@131318
   679
        description='Generate a .clang_complete file.')
gps@120653
   680
    def clang_complete(self):
gps@120653
   681
        import shlex
gps@120653
   682
gps@120653
   683
        build_vars = {}
gps@120653
   684
gps@120653
   685
        def on_line(line):
gps@120653
   686
            elements = [s.strip() for s in line.split('=', 1)]
gps@120653
   687
gps@120653
   688
            if len(elements) != 2:
gps@120653
   689
                return
gps@120653
   690
gps@120653
   691
            build_vars[elements[0]] = elements[1]
gps@120653
   692
gps@120653
   693
        try:
gps@120653
   694
            old_logger = self.log_manager.replace_terminal_handler(None)
gps@120653
   695
            self._run_make(target='showbuild', log=False, line_handler=on_line)
gps@120653
   696
        finally:
gps@120653
   697
            self.log_manager.replace_terminal_handler(old_logger)
gps@120653
   698
gps@120653
   699
        def print_from_variable(name):
gps@120653
   700
            if name not in build_vars:
gps@120653
   701
                return
gps@120653
   702
gps@120653
   703
            value = build_vars[name]
gps@120653
   704
gps@120653
   705
            value = value.replace('-I.', '-I%s' % self.topobjdir)
gps@120653
   706
            value = value.replace(' .', ' %s' % self.topobjdir)
gps@120653
   707
            value = value.replace('-I..', '-I%s/..' % self.topobjdir)
gps@120653
   708
            value = value.replace(' ..', ' %s/..' % self.topobjdir)
gps@120653
   709
gps@120653
   710
            args = shlex.split(value)
gps@120653
   711
            for i in range(0, len(args) - 1):
gps@120653
   712
                arg = args[i]
gps@120653
   713
gps@120653
   714
                if arg.startswith(('-I', '-D')):
gps@120653
   715
                    print(arg)
gps@120653
   716
                    continue
gps@120653
   717
gps@120653
   718
                if arg.startswith('-include'):
gps@120653
   719
                    print(arg + ' ' + args[i + 1])
gps@120653
   720
                    continue
gps@120653
   721
gps@120653
   722
        print_from_variable('COMPILE_CXXFLAGS')
gps@120653
   723
gps@120653
   724
        print('-I%s/ipc/chromium/src' % self.topsrcdir)
gps@120653
   725
        print('-I%s/ipc/glue' % self.topsrcdir)
gps@120653
   726
        print('-I%s/ipc/ipdl/_ipdlheaders' % self.topobjdir)
gps@120653
   727
gps@120653
   728
gps@120653
   729
@CommandProvider
rnewman@115674
   730
class Package(MachCommandBase):
rnewman@115674
   731
    """Package the built product for distribution."""
rnewman@115674
   732
gps@131318
   733
    @Command('package', category='post-build',
gps@131318
   734
        description='Package the built product for distribution as an APK, DMG, etc.')
rnewman@115674
   735
    def package(self):
rnewman@115674
   736
        return self._run_make(directory=".", target='package', ensure_exit_code=False)
gps@123595
   737
ted@124653
   738
@CommandProvider
paul@124910
   739
class Install(MachCommandBase):
paul@124910
   740
    """Install a package."""
paul@124910
   741
gps@131318
   742
    @Command('install', category='post-build',
gps@131318
   743
        description='Install the package on the machine, or on a device.')
paul@124910
   744
    def install(self):
paul@124910
   745
        return self._run_make(directory=".", target='install', ensure_exit_code=False)
paul@124910
   746
paul@124910
   747
@CommandProvider
mbrubeck@126275
   748
class RunProgram(MachCommandBase):
mbrubeck@126275
   749
    """Launch the compiled binary"""
mbrubeck@126275
   750
gps@131318
   751
    @Command('run', category='post-build', allow_all_args=True,
gps@131318
   752
        description='Run the compiled program.')
gps@131318
   753
    @CommandArgument('params', default=None, nargs='...',
mbrubeck@126275
   754
        help='Command-line arguments to pass to the program.')
mbrubeck@131821
   755
    @CommandArgument('+remote', '+r', action='store_true',
mbrubeck@131821
   756
        help='Do not pass the -no-remote argument by default.')
ehsan@133550
   757
    @CommandArgument('+background', '+b', action='store_true',
ehsan@133550
   758
        help='Do not pass the -foreground argument by default on Mac')
philippe@191839
   759
    @CommandArgument('+profile', '+P', action='store_true',
glob@195666
   760
        help='Specify the profile to use')
philippe@191839
   761
    def run(self, params, remote, background, profile):
mbrubeck@126275
   762
        try:
mbrubeck@131821
   763
            args = [self.get_binary_path('app')]
mbrubeck@126275
   764
        except Exception as e:
mbrubeck@126275
   765
            print("It looks like your program isn't built.",
mbrubeck@126275
   766
                "You can run |mach build| to build it.")
mbrubeck@126275
   767
            print(e)
mbrubeck@126275
   768
            return 1
mbrubeck@131821
   769
        if not remote:
mbrubeck@131821
   770
            args.append('-no-remote')
ehsan@133550
   771
        if not background and sys.platform == 'darwin':
ehsan@133550
   772
            args.append('-foreground')
philippe@191839
   773
        if '-profile' not in params and '-P' not in params:
philippe@191839
   774
            path = os.path.join(self.topobjdir, 'tmp', 'scratch_user')
philippe@191839
   775
            if not os.path.isdir(path):
philippe@191839
   776
                os.makedirs(path)
philippe@191839
   777
            args.append('-profile')
philippe@191839
   778
            args.append(path)
mbrubeck@126275
   779
        if params:
mbrubeck@126275
   780
            args.extend(params)
mbrubeck@126275
   781
        return self.run_process(args=args, ensure_exit_code=False,
mbrubeck@126275
   782
            pass_thru=True)
mbrubeck@126275
   783
mbrubeck@126275
   784
@CommandProvider
ehsan@128078
   785
class DebugProgram(MachCommandBase):
ehsan@128078
   786
    """Debug the compiled binary"""
ehsan@128078
   787
gps@131318
   788
    @Command('debug', category='post-build', allow_all_args=True,
gps@131318
   789
        description='Debug the compiled program.')
gps@131318
   790
    @CommandArgument('params', default=None, nargs='...',
ehsan@128078
   791
        help='Command-line arguments to pass to the program.')
mbrubeck@131821
   792
    @CommandArgument('+remote', '+r', action='store_true',
mbrubeck@131821
   793
        help='Do not pass the -no-remote argument by default')
ehsan@133550
   794
    @CommandArgument('+background', '+b', action='store_true',
ehsan@133550
   795
        help='Do not pass the -foreground argument by default on Mac')
robert@179918
   796
    @CommandArgument('+debugger', default=None, type=str,
robert@179918
   797
        help='Name of debugger to launch')
jwillcox@159853
   798
    @CommandArgument('+debugparams', default=None, metavar='params', type=str,
jwillcox@159853
   799
        help='Command-line arguments to pass to GDB or LLDB itself; split as the Bourne shell would.')
philippe@191839
   800
    @CommandArgument('+profile', '+P', action='store_true',
philippe@191839
   801
        help='Specifiy thr profile to use')
asutherland@156655
   802
    # Bug 933807 introduced JS_DISABLE_SLOW_SCRIPT_SIGNALS to avoid clever
asutherland@156655
   803
    # segfaults induced by the slow-script-detecting logic for Ion/Odin JITted
asutherland@156655
   804
    # code.  If we don't pass this, the user will need to periodically type
asutherland@156655
   805
    # "continue" to (safely) resume execution.  There are ways to implement
asutherland@156655
   806
    # automatic resuming; see the bug.
asutherland@156655
   807
    @CommandArgument('+slowscript', action='store_true',
asutherland@156655
   808
        help='Do not set the JS_DISABLE_SLOW_SCRIPT_SIGNALS env variable; when not set, recoverable but misleading SIGSEGV instances may occur in Ion/Odin JIT code')
philippe@191839
   809
    def debug(self, params, remote, background, profile, debugger, debugparams, slowscript):
ehsan@128078
   810
        import which
robert@179918
   811
        if debugger:
jwillcox@159853
   812
            try:
robert@179918
   813
                debugger = which.which(debugger)
jwillcox@159853
   814
            except Exception as e:
robert@179918
   815
                print("You don't have %s in your PATH" % (debugger))
jwillcox@159853
   816
                print(e)
jwillcox@159853
   817
                return 1
robert@179918
   818
        else:
robert@179918
   819
            try:
robert@179918
   820
                debugger = which.which('gdb')
robert@179918
   821
            except Exception:
robert@179918
   822
                try:
robert@179918
   823
                    debugger = which.which('lldb')
robert@179918
   824
                except Exception as e:
robert@179918
   825
                    print("You don't have gdb or lldb in your PATH")
robert@179918
   826
                    print(e)
robert@179918
   827
                    return 1
jimb@140988
   828
        args = [debugger]
froydnj@175068
   829
        extra_env = { 'MOZ_CRASHREPORTER_DISABLE' : '1' }
jwillcox@159853
   830
        if debugparams:
jimb@140988
   831
            import pymake.process
jwillcox@159853
   832
            argv, badchar = pymake.process.clinetoargv(debugparams, os.getcwd())
jimb@140988
   833
            if badchar:
jwillcox@159853
   834
                print("The +debugparams you passed require a real shell to parse them.")
jimb@140988
   835
                print("(We can't handle the %r character.)" % (badchar,))
jimb@140988
   836
                return 1
jimb@140988
   837
            args.extend(argv)
jwillcox@159853
   838
jwillcox@159853
   839
        binpath = None
jwillcox@159853
   840
ehsan@128078
   841
        try:
jwillcox@159853
   842
            binpath = self.get_binary_path('app')
ehsan@128078
   843
        except Exception as e:
ehsan@128078
   844
            print("It looks like your program isn't built.",
ehsan@128078
   845
                "You can run |mach build| to build it.")
ehsan@128078
   846
            print(e)
ehsan@128078
   847
            return 1
jwillcox@159853
   848
tlin@183838
   849
        # args added to separate the debugger and process arguments.
tlin@183838
   850
        args_separator = {
tlin@183838
   851
            'gdb': '--args',
tlin@183838
   852
            'ddd': '--args',
tlin@183838
   853
            'cgdb': '--args',
tlin@183838
   854
            'lldb': '--'
tlin@183838
   855
        }
tlin@183838
   856
tlin@183838
   857
        debugger_name = os.path.basename(debugger)
tlin@183838
   858
        if debugger_name in args_separator:
tlin@183838
   859
            args.append(args_separator[debugger_name])
robert@179918
   860
        args.append(binpath)
jwillcox@159853
   861
mbrubeck@131821
   862
        if not remote:
mbrubeck@131821
   863
            args.append('-no-remote')
ehsan@133550
   864
        if not background and sys.platform == 'darwin':
ehsan@133550
   865
            args.append('-foreground')
ehsan@128078
   866
        if params:
ehsan@128078
   867
            args.extend(params)
philippe@191839
   868
        if '-profile' not in params and '-P' not in params:
philippe@191839
   869
            path = os.path.join(self.topobjdir, 'tmp', 'scratch_user')
philippe@191839
   870
            if not os.path.isdir(path):
philippe@191839
   871
                os.makedirs(path)
philippe@191839
   872
            args.append('-profile')
philippe@191839
   873
            args.append(path)
asutherland@156655
   874
        if not slowscript:
asutherland@156655
   875
            extra_env['JS_DISABLE_SLOW_SCRIPT_SIGNALS'] = '1'
asutherland@156655
   876
        return self.run_process(args=args, append_env=extra_env,
asutherland@156655
   877
            ensure_exit_code=False, pass_thru=True)
ehsan@128078
   878
ehsan@128078
   879
@CommandProvider
ted@124653
   880
class Buildsymbols(MachCommandBase):
ted@124653
   881
    """Produce a package of debug symbols suitable for use with Breakpad."""
ted@124653
   882
gps@131318
   883
    @Command('buildsymbols', category='post-build',
gps@131318
   884
        description='Produce a package of Breakpad-format symbols.')
ted@124653
   885
    def buildsymbols(self):
ted@124653
   886
        return self._run_make(directory=".", target='buildsymbols', ensure_exit_code=False)
gps@123595
   887
gps@123595
   888
@CommandProvider
gps@123595
   889
class Makefiles(MachCommandBase):
gps@131318
   890
    @Command('empty-makefiles', category='build-dev',
gps@131318
   891
        description='Find empty Makefile.in in the tree.')
gps@123595
   892
    def empty(self):
gps@123595
   893
        import pymake.parser
gps@123595
   894
        import pymake.parserdata
gps@123595
   895
gps@123595
   896
        IGNORE_VARIABLES = {
gps@123595
   897
            'DEPTH': ('@DEPTH@',),
gps@123595
   898
            'topsrcdir': ('@top_srcdir@',),
gps@123595
   899
            'srcdir': ('@srcdir@',),
gps@123595
   900
            'relativesrcdir': ('@relativesrcdir@',),
gps@123595
   901
            'VPATH': ('@srcdir@',),
gps@123595
   902
        }
gps@123595
   903
gps@123595
   904
        IGNORE_INCLUDES = [
gps@123595
   905
            'include $(DEPTH)/config/autoconf.mk',
gps@123595
   906
            'include $(topsrcdir)/config/config.mk',
gps@123595
   907
            'include $(topsrcdir)/config/rules.mk',
gps@123595
   908
        ]
gps@123595
   909
gps@123595
   910
        def is_statement_relevant(s):
gps@123595
   911
            if isinstance(s, pymake.parserdata.SetVariable):
gps@123595
   912
                exp = s.vnameexp
gps@123595
   913
                if not exp.is_static_string:
gps@123595
   914
                    return True
gps@123595
   915
gps@123595
   916
                if exp.s not in IGNORE_VARIABLES:
gps@123595
   917
                    return True
gps@123595
   918
gps@123595
   919
                return s.value not in IGNORE_VARIABLES[exp.s]
gps@123595
   920
gps@123595
   921
            if isinstance(s, pymake.parserdata.Include):
gps@123595
   922
                if s.to_source() in IGNORE_INCLUDES:
gps@123595
   923
                    return False
gps@123595
   924
gps@123595
   925
            return True
gps@123595
   926
gps@123595
   927
        for path in self._makefile_ins():
gps@172864
   928
            relpath = os.path.relpath(path, self.topsrcdir)
gps@172864
   929
            try:
gps@172864
   930
                statements = [s for s in pymake.parser.parsefile(path)
gps@172864
   931
                    if is_statement_relevant(s)]
gps@123595
   932
gps@172864
   933
                if not statements:
gps@172864
   934
                    print(relpath)
gps@172864
   935
            except pymake.parser.SyntaxError:
gps@172864
   936
                print('Warning: Could not parse %s' % relpath, file=sys.stderr)
gps@123595
   937
gps@123595
   938
    def _makefile_ins(self):
gps@123595
   939
        for root, dirs, files in os.walk(self.topsrcdir):
gps@123595
   940
            for f in files:
gps@123595
   941
                if f == 'Makefile.in':
gps@123595
   942
                    yield os.path.join(root, f)
gps@123595
   943
gps@141815
   944
@CommandProvider
gps@150500
   945
class MachDebug(MachCommandBase):
gps@141815
   946
    @Command('environment', category='build-dev',
gps@141815
   947
        description='Show info about the mach and build environment.')
mh+mozilla@191955
   948
    @CommandArgument('--format', default='pretty',
mh+mozilla@191955
   949
        choices=['pretty', 'client.mk', 'configure', 'json'],
mh+mozilla@191955
   950
        help='Print data in the given format.')
mh+mozilla@191955
   951
    @CommandArgument('--output', '-o', type=str,
mh+mozilla@191955
   952
        help='Output to the given file.')
gps@141815
   953
    @CommandArgument('--verbose', '-v', action='store_true',
gps@141815
   954
        help='Print verbose output.')
mh+mozilla@191955
   955
    def environment(self, format, output=None, verbose=False):
mh+mozilla@191955
   956
        func = getattr(self, '_environment_%s' % format.replace('.', '_'))
mh+mozilla@191955
   957
mh+mozilla@191955
   958
        if output:
mh+mozilla@191955
   959
            # We want to preserve mtimes if the output file already exists
mh+mozilla@191955
   960
            # and the content hasn't changed.
mh+mozilla@191955
   961
            from mozbuild.util import FileAvoidWrite
mh+mozilla@191955
   962
            with FileAvoidWrite(output) as out:
mh+mozilla@191955
   963
                return func(out, verbose)
mh+mozilla@191955
   964
        return func(sys.stdout, verbose)
mh+mozilla@191955
   965
mh+mozilla@191955
   966
    def _environment_pretty(self, out, verbose):
mh+mozilla@191952
   967
        state_dir = self._mach_context.state_dir
gps@141815
   968
        import platform
mh+mozilla@191955
   969
        print('platform:\n\t%s' % platform.platform(), file=out)
mh+mozilla@191955
   970
        print('python version:\n\t%s' % sys.version, file=out)
mh+mozilla@191955
   971
        print('python prefix:\n\t%s' % sys.prefix, file=out)
mh+mozilla@191955
   972
        print('mach cwd:\n\t%s' % self._mach_context.cwd, file=out)
mh+mozilla@191955
   973
        print('os cwd:\n\t%s' % os.getcwd(), file=out)
mh+mozilla@191955
   974
        print('mach directory:\n\t%s' % self._mach_context.topdir, file=out)
mh+mozilla@191955
   975
        print('state directory:\n\t%s' % state_dir, file=out)
gps@141815
   976
mh+mozilla@191955
   977
        print('object directory:\n\t%s' % self.topobjdir, file=out)
gps@141815
   978
mh+mozilla@191953
   979
        if self.mozconfig['path']:
mh+mozilla@191955
   980
            print('mozconfig path:\n\t%s' % self.mozconfig['path'], file=out)
mh+mozilla@191953
   981
            if self.mozconfig['configure_args']:
mh+mozilla@191955
   982
                print('mozconfig configure args:', file=out)
mh+mozilla@191953
   983
                for arg in self.mozconfig['configure_args']:
mh+mozilla@191955
   984
                    print('\t%s' % arg, file=out)
gps@141815
   985
mh+mozilla@191953
   986
            if self.mozconfig['make_extra']:
mh+mozilla@191955
   987
                print('mozconfig extra make args:', file=out)
mh+mozilla@191953
   988
                for arg in self.mozconfig['make_extra']:
mh+mozilla@191955
   989
                    print('\t%s' % arg, file=out)
gps@141815
   990
mh+mozilla@191953
   991
            if self.mozconfig['make_flags']:
mh+mozilla@191955
   992
                print('mozconfig make flags:', file=out)
mh+mozilla@191953
   993
                for arg in self.mozconfig['make_flags']:
mh+mozilla@191955
   994
                    print('\t%s' % arg, file=out)
gps@141815
   995
gps@141815
   996
        config = None
gps@141815
   997
gps@141815
   998
        try:
mh+mozilla@191953
   999
            config = self.config_environment
gps@141815
  1000
gps@141815
  1001
        except Exception:
gps@141815
  1002
            pass
gps@141815
  1003
gps@141815
  1004
        if config:
mh+mozilla@191955
  1005
            print('config topsrcdir:\n\t%s' % config.topsrcdir, file=out)
mh+mozilla@191955
  1006
            print('config topobjdir:\n\t%s' % config.topobjdir, file=out)
gps@141815
  1007
gps@141815
  1008
            if verbose:
mh+mozilla@191955
  1009
                print('config substitutions:', file=out)
gps@141815
  1010
                for k in sorted(config.substs):
mh+mozilla@191955
  1011
                    print('\t%s: %s' % (k, config.substs[k]), file=out)
gps@141815
  1012
mh+mozilla@191955
  1013
                print('config defines:', file=out)
gps@141815
  1014
                for k in sorted(config.defines):
mh+mozilla@191955
  1015
                    print('\t%s' % k, file=out)
mh+mozilla@191955
  1016
mh+mozilla@191955
  1017
    def _environment_client_mk(self, out, verbose):
mh+mozilla@191955
  1018
        if self.mozconfig['make_extra']:
mh+mozilla@191955
  1019
            for arg in self.mozconfig['make_extra']:
mh+mozilla@191955
  1020
                print(arg, file=out)
mh+mozilla@191955
  1021
        objdir = mozpath.normsep(self.topobjdir)
mh+mozilla@191955
  1022
        print('MOZ_OBJDIR=%s' % objdir, file=out)
mh+mozilla@191955
  1023
        if 'MOZ_CURRENT_PROJECT' in os.environ:
mh+mozilla@191955
  1024
            objdir = mozpath.join(objdir, os.environ['MOZ_CURRENT_PROJECT'])
mh+mozilla@191955
  1025
        print('OBJDIR=%s' % objdir, file=out)
mh+mozilla@191955
  1026
        if self.mozconfig['path']:
mh+mozilla@191955
  1027
            print('FOUND_MOZCONFIG=%s' % mozpath.normsep(self.mozconfig['path']),
mh+mozilla@191955
  1028
                file=out)
mh+mozilla@191955
  1029
mh+mozilla@191955
  1030
    def _environment_configure(self, out, verbose):
mh+mozilla@191955
  1031
        if self.mozconfig['path']:
mh+mozilla@191955
  1032
            # Replace ' with '"'"', so that shell quoting e.g.
mh+mozilla@191955
  1033
            # a'b becomes 'a'"'"'b'.
mh+mozilla@191955
  1034
            quote = lambda s: s.replace("'", """'"'"'""")
mh+mozilla@192744
  1035
            if self.mozconfig['configure_args'] and \
mh+mozilla@192744
  1036
                    'COMM_BUILD' not in os.environ:
mh+mozilla@192744
  1037
                print('echo Adding configure options from %s' %
mh+mozilla@192744
  1038
                    mozpath.normsep(self.mozconfig['path']), file=out)
mh+mozilla@191955
  1039
                for arg in self.mozconfig['configure_args']:
mh+mozilla@191955
  1040
                    quoted_arg = quote(arg)
mh+mozilla@191955
  1041
                    print("echo '  %s'" % quoted_arg, file=out)
mh+mozilla@191955
  1042
                    print("""set -- "$@" '%s'""" % quoted_arg, file=out)
mh+mozilla@192744
  1043
            for key, value in self.mozconfig['env']['added'].items():
mh+mozilla@192744
  1044
                print("export %s='%s'" % (key, quote(value)), file=out)
mh+mozilla@192744
  1045
            for key, (old, value) in self.mozconfig['env']['modified'].items():
mh+mozilla@192744
  1046
                print("export %s='%s'" % (key, quote(value)), file=out)
mh+mozilla@192744
  1047
            for key, value in self.mozconfig['vars']['added'].items():
mh+mozilla@192744
  1048
                print("%s='%s'" % (key, quote(value)), file=out)
mh+mozilla@192744
  1049
            for key, (old, value) in self.mozconfig['vars']['modified'].items():
mh+mozilla@192744
  1050
                print("%s='%s'" % (key, quote(value)), file=out)
mh+mozilla@192744
  1051
            for key in self.mozconfig['env']['removed'].keys() + \
mh+mozilla@192744
  1052
                    self.mozconfig['vars']['removed'].keys():
mh+mozilla@192744
  1053
                print("unset %s" % key, file=out)
mh+mozilla@191955
  1054
mh+mozilla@191955
  1055
    def _environment_json(self, out, verbose):
mh+mozilla@191955
  1056
        import json
mh+mozilla@191955
  1057
        class EnvironmentEncoder(json.JSONEncoder):
mh+mozilla@191955
  1058
            def default(self, obj):
mh+mozilla@191955
  1059
                if isinstance(obj, MozbuildObject):
mh+mozilla@191955
  1060
                    result = {
mh+mozilla@191955
  1061
                        'topsrcdir': obj.topsrcdir,
mh+mozilla@191955
  1062
                        'topobjdir': obj.topobjdir,
mh+mozilla@191955
  1063
                        'mozconfig': obj.mozconfig,
mh+mozilla@191955
  1064
                    }
mh+mozilla@191955
  1065
                    if verbose:
mh+mozilla@191955
  1066
                        result['substs'] = obj.substs
mh+mozilla@191955
  1067
                        result['defines'] = obj.defines
mh+mozilla@191955
  1068
                    return result
mh+mozilla@191955
  1069
                elif isinstance(obj, set):
mh+mozilla@191955
  1070
                    return list(obj)
mh+mozilla@191955
  1071
                return json.JSONEncoder.default(self, obj)
mh+mozilla@191955
  1072
        json.dump(self, cls=EnvironmentEncoder, sort_keys=True, fp=out)