Bug 780329 - Part 5: Add base modules to mozbuild; r=glandium, jhammel
authorGregory Szorc <gps@mozilla.com>
Wed, 26 Sep 2012 09:43:53 -0700
changeset 108104 1ae2d42ad234863b5e9e66c0c727586b280d0e19
parent 108103 df69d95f636c7f55af3d600abfa6974af9cb32a5
child 108105 9226912d54eb802b27ff8109286c4f1c44bfaff7
push id23534
push usergszorc@mozilla.com
push dateWed, 26 Sep 2012 16:48:13 +0000
treeherdermozilla-central@2359243ee2b1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium, jhammel
bugs780329
milestone18.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 780329 - Part 5: Add base modules to mozbuild; r=glandium, jhammel
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.mo
python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po
python/mozbuild/mozbuild/test/test_base.py
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/base.py
@@ -0,0 +1,363 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import logging
+import multiprocessing
+import os
+import pymake.parser
+import shlex
+import subprocess
+import which
+
+from mozprocess.processhandler import ProcessHandlerMixin
+from pymake.data import Makefile
+from tempfile import TemporaryFile
+
+from mozbuild.config import ConfigProvider
+from mozbuild.config import PositiveIntegerType
+
+
+# Perform detection of operating system environment. This is used by command
+# execution. We only do this once to save redundancy. Yes, this can fail module
+# loading. That is arguably OK.
+if 'SHELL' in os.environ:
+    _current_shell = os.environ['SHELL']
+elif 'MOZILLABUILD' in os.environ:
+    _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
+elif 'COMSPEC' in os.environ:
+    _current_shell = os.environ['COMSPEC']
+else:
+    raise Exception('Could not detect environment shell!')
+
+_in_msys = False
+
+if os.environ.get('MSYSTEM', None) == 'MINGW32':
+    _in_msys = True
+
+    if not _current_shell.lower().endswith('.exe'):
+        _current_shell += '.exe'
+
+
+class MozbuildObject(object):
+    """Base class providing basic functionality useful to many modules.
+
+    Modules in this package typically require common functionality such as
+    accessing the current config, getting the location of the source directory,
+    running processes, etc. This classes provides that functionality. Other
+    modules can inherit from this class to obtain this functionality easily.
+    """
+    def __init__(self, topsrcdir, settings, log_manager, topobjdir=None):
+        """Create a new Mozbuild object instance.
+
+        Instances are bound to a source directory, a ConfigSettings instance,
+        and a LogManager instance. The topobjdir may be passed in as well. If
+        it isn't, it will be calculated from the active mozconfig.
+        """
+        self.topsrcdir = topsrcdir
+        self.settings = settings
+        self.config = BuildConfig(settings)
+        self.logger = logging.getLogger(__name__)
+        self.log_manager = log_manager
+
+        self._config_guess_output = None
+        self._make = None
+        self._topobjdir = topobjdir
+
+    @property
+    def topobjdir(self):
+        if self._topobjdir is None:
+            self._load_mozconfig()
+
+        if self._topobjdir is None:
+            self._topobjdir = 'obj-%s' % self._config_guess
+
+        return self._topobjdir
+
+    @property
+    def distdir(self):
+        return os.path.join(self.topobjdir, 'dist')
+
+    @property
+    def bindir(self):
+        return os.path.join(self.topobjdir, 'dist', 'bin')
+
+    @property
+    def statedir(self):
+        return os.path.join(self.topobjdir, '.mozbuild')
+
+    def log(self, level, action, params, format_str):
+        self.logger.log(level, format_str,
+            extra={'action': action, 'params': params})
+
+    def _load_mozconfig(self, path=None):
+        # The mozconfig loader outputs a make file. We parse and load this make
+        # file with pymake and evaluate it in a context similar to client.mk.
+
+        loader = os.path.join(self.topsrcdir, 'build', 'autoconf',
+            'mozconfig2client-mk')
+
+        # os.environ from a library function is somewhat evil. But, mozconfig
+        # files are tightly coupled with the environment by definition. In the
+        # future, perhaps we'll have a more sanitized environment for mozconfig
+        # execution.
+        env = dict(os.environ)
+        if path is not None:
+            env['MOZCONFIG'] = path
+
+        env['CONFIG_GUESS'] = self._config_guess
+
+        output = subprocess.check_output([loader, self.topsrcdir],
+            stderr=subprocess.PIPE, cwd=self.topsrcdir, env=env)
+
+        # The output is make syntax. We parse this in a specialized make
+        # context.
+        statements = pymake.parser.parsestring(output, 'mozconfig')
+
+        makefile = Makefile(workdir=self.topsrcdir, env={
+            'TOPSRCDIR': self.topsrcdir,
+            'CONFIG_GUESS': self._config_guess})
+
+        statements.execute(makefile)
+
+        def get_value(name):
+            exp = makefile.variables.get(name)[2]
+
+            return exp.resolvestr(makefile, makefile.variables)
+
+        for name, flavor, source, value in makefile.variables:
+            # We only care about variables that came from the parsed mozconfig.
+            if source != pymake.data.Variables.SOURCE_MAKEFILE:
+                continue
+
+            # Ignore some pymake built-ins.
+            if name in ('.PYMAKE', 'MAKELEVEL', 'MAKEFLAGS'):
+                continue
+
+            if name == 'MOZ_OBJDIR':
+                self._topobjdir = get_value(name)
+
+            # If we want to extract other variables defined by mozconfig, here
+            # is where we'd do it.
+
+    @property
+    def _config_guess(self):
+        if self._config_guess_output is None:
+            p = os.path.join(self.topsrcdir, 'build', 'autoconf',
+                'config.guess')
+            self._config_guess_output = subprocess.check_output([p],
+                cwd=self.topsrcdir).strip()
+
+        return self._config_guess_output
+
+    def _ensure_objdir_exists(self):
+        if os.path.isdir(self.statedir):
+            return
+
+        os.makedirs(self.statedir)
+
+    def _ensure_state_subdir_exists(self, subdir):
+        path = os.path.join(self.statedir, subdir)
+
+        if os.path.isdir(path):
+            return
+
+        os.makedirs(path)
+
+    def _get_state_filename(self, filename, subdir=None):
+        path = self.statedir
+
+        if subdir:
+            path = os.path.join(path, subdir)
+
+        return os.path.join(path, filename)
+
+    def _get_srcdir_path(self, path):
+        """Convert a relative path in the source directory to a full path."""
+        return os.path.join(self.topsrcdir, path)
+
+    def _get_objdir_path(self, path):
+        """Convert a relative path in the object directory to a full path."""
+        return os.path.join(self.topobjdir, path)
+
+    def _run_make(self, directory=None, filename=None, target=None, log=True,
+            srcdir=False, allow_parallel=True, line_handler=None, env=None,
+            ignore_errors=False):
+        """Invoke make.
+
+        directory -- Relative directory to look for Makefile in.
+        filename -- Explicit makefile to run.
+        target -- Makefile target(s) to make. Can be a string or iterable of
+            strings.
+        srcdir -- If True, invoke make from the source directory tree.
+            Otherwise, make will be invoked from the object directory.
+        """
+        self._ensure_objdir_exists()
+
+        args = [self._make_path]
+
+        if directory:
+            args.extend(['-C', directory])
+
+        if filename:
+            args.extend(['-f', filename])
+
+        if allow_parallel:
+            args.append('-j%d' % self.settings.build.threads)
+
+        if ignore_errors:
+            args.append('-k')
+
+        # Silent mode by default.
+        args.append('-s')
+
+        # Print entering/leaving directory messages. Some consumers look at
+        # these to measure progress. Ideally, we'd do everything with pymake
+        # and use hooks in its API. Unfortunately, it doesn't provide that
+        # feature... yet.
+        args.append('-w')
+
+        if isinstance(target, list):
+            args.extend(target)
+        elif target:
+            args.append(target)
+
+        fn = self._run_command_in_objdir
+
+        if srcdir:
+            fn = self._run_command_in_srcdir
+
+        params = {
+            'args': args,
+            'line_handler': line_handler,
+            'explicit_env': env,
+            'log_level': logging.INFO,
+            'require_unix_environment': True,
+            'ignore_errors': ignore_errors,
+        }
+
+        if log:
+            params['log_name'] = 'make'
+
+        fn(**params)
+
+    @property
+    def _make_path(self):
+        if self._make is None:
+            if self._is_windows():
+                self._make = os.path.join(self.topsrcdir, 'build', 'pymake',
+                    'make.py')
+
+            else:
+                for test in ['gmake', 'make']:
+                    try:
+                        self._make = which.which(test)
+                        break
+                    except which.WhichError:
+                        continue
+
+        if self._make is None:
+            raise Exception('Could not find suitable make binary!')
+
+        return self._make
+
+    def _run_command_in_srcdir(self, **args):
+        self._run_command(cwd=self.topsrcdir, **args)
+
+    def _run_command_in_objdir(self, **args):
+        self._run_command(cwd=self.topobjdir, **args)
+
+    def _run_command(self, args=None, cwd=None, append_env=None,
+        explicit_env=None, log_name=None, log_level=logging.INFO,
+        line_handler=None, require_unix_environment=False,
+        ignore_errors=False):
+        """Runs a single command to completion.
+
+        Takes a list of arguments to run where the first item is the
+        executable. Runs the command in the specified directory and
+        with optional environment variables.
+
+        append_env -- Dict of environment variables to append to the current
+            set of environment variables.
+        explicit_env -- Dict of environment variables to set for the new
+            process. Any existing environment variables will be ignored.
+
+        require_unix_environment if True will ensure the command is executed
+        within a UNIX environment. Basically, if we are on Windows, it will
+        execute the command via an appropriate UNIX-like shell.
+        """
+        assert isinstance(args, list) and len(args)
+
+        if require_unix_environment and _in_msys:
+            # Always munge Windows-style into Unix style for the command.
+            prog = args[0].replace('\\', '/')
+
+            # PyMake removes the C: prefix. But, things seem to work here
+            # without it. Not sure what that's about.
+
+            # We run everything through the msys shell. We need to use
+            # '-c' and pass all the arguments as one argument because that is
+            # how sh works.
+            cline = subprocess.list2cmdline([prog] + args[1:])
+            args = [_current_shell, '-c', cline]
+
+        self.log(logging.INFO, 'process', {'args': args}, ' '.join(args))
+
+        def handleLine(line):
+            if line_handler:
+                line_handler(line)
+
+            if not log_name:
+                return
+
+            self.log(log_level, log_name, {'line': line.strip()}, '{line}')
+
+        use_env = {}
+        if explicit_env:
+            use_env = explicit_env
+        else:
+            use_env.update(os.environ)
+
+            if append_env:
+                use_env.update(env)
+
+        p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
+            processOutputLine=[handleLine], universal_newlines=True)
+        p.run()
+        p.processOutput()
+        status = p.wait()
+
+        if status != 0 and not ignore_errors:
+            raise Exception('Process executed with non-0 exit code: %s' % args)
+
+    def _is_windows(self):
+        return os.name in ('nt', 'ce')
+
+    def _spawn(self, cls):
+        """Create a new MozbuildObject-derived class instance from ourselves.
+
+        This is used as a convenience method to create other
+        MozbuildObject-derived class instances. It can only be used on
+        classes that have the same constructor arguments as us.
+        """
+
+        return cls(self.topsrcdir, self.settings, self.log_manager,
+            topobjdir=self.topobjdir)
+
+
+class BuildConfig(ConfigProvider):
+    """The configuration for mozbuild."""
+
+    def __init__(self, settings):
+        self.settings = settings
+
+    @classmethod
+    def _register_settings(cls):
+        def register(section, option, type_cls, **kwargs):
+            cls.register_setting(section, option, type_cls, domain='mozbuild',
+                **kwargs)
+
+        register('build', 'threads', PositiveIntegerType,
+            default=multiprocessing.cpu_count())
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..be7711cb2fcfc927de59407bac54fb613dfab0fe
GIT binary patch
literal 301
zc${6%K~4iP3<b~uwL4a2+r>XXH78)#3v|&QAf54$2v1zu$y9I;?!c)yOC1#nfu*-`
zV*7t@R-c{MquaWjTe}yxaR>M82KVKz?#bby`%`Vcwazv1*q5jjM(d}_{I6wEv~<Tv
z$aOxGLMHsYp$MuGu8}e>q?72J@+<a-6H+deEiHr+?9F-{V=@eqt*Ee%`a-6Kxb3?H
w4a~&pBa|e$eI|@Fg~|m>;7UmMk>3o1D4CzZyHv#-hL|$-9<)WnUTQAg4eLHs%>V!Z
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po
@@ -0,0 +1,8 @@
+msgid "build.threads.short"
+msgstr "Thread Count"
+
+msgid "build.threads.full"
+msgstr "The number of threads to use when performing CPU intensive tasks. "
+       "This constrols the level of parallelization. The default value is "
+       "the number of cores in your machine."
+
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/test_base.py
@@ -0,0 +1,61 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import unittest
+
+from tempfile import NamedTemporaryFile
+
+from mozbuild.base import BuildConfig
+from mozbuild.base import MozbuildObject
+from mozbuild.config import ConfigSettings
+from mozbuild.logger import LoggingManager
+
+
+curdir = os.path.dirname(__file__)
+topsrcdir = os.path.normpath(os.path.join(curdir, '..', '..', '..', '..'))
+log_manager = LoggingManager()
+
+
+class TestBuildConfig(unittest.TestCase):
+    def test_basic(self):
+        c = ConfigSettings()
+        c.register_provider(BuildConfig)
+
+        c.build.threads = 6
+
+
+class TestMozbuildObject(unittest.TestCase):
+    def get_base(self):
+        settings = ConfigSettings()
+        settings.register_provider(BuildConfig)
+
+        return MozbuildObject(topsrcdir, settings, log_manager)
+
+    def test_mozconfig_parsing(self):
+        with NamedTemporaryFile(mode='wt') as mozconfig:
+            mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/some-objdir')
+            mozconfig.flush()
+
+            os.environ['MOZCONFIG'] = mozconfig.name
+
+            base = self.get_base()
+            base._load_mozconfig()
+
+            self.assertEqual(base.topobjdir, '%s/some-objdir' % topsrcdir)
+
+        del os.environ['MOZCONFIG']
+
+    def test_objdir_config_guess(self):
+        base = self.get_base()
+
+        with NamedTemporaryFile() as mozconfig:
+            os.environ['MOZCONFIG'] = mozconfig.name
+
+            self.assertIsNotNone(base.topobjdir)
+            self.assertEqual(len(base.topobjdir.split()), 1)
+
+        del os.environ['MOZCONFIG']