Bug 784841 - Part 8: Capture and save moz.build tree state; r=ted
authorGregory Szorc <gps@mozilla.com>
Tue, 29 Jan 2013 06:24:24 -0800
changeset 130138 115bf99a25db150fe82c6642555949931d8ced66
parent 130137 7c73b5af624735e5042a40086b45602f3357c0ed
child 130139 9584298d2a6ace4036e1484dd09cfe3763692211
push id2323
push userbbajaj@mozilla.com
push dateMon, 01 Apr 2013 19:47:02 +0000
treeherdermozilla-beta@7712be144d91 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs784841
milestone21.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 784841 - Part 8: Capture and save moz.build tree state; r=ted
python/mozbuild/mozbuild/backend/base.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/sandbox.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozbuild/test/frontend/test_emitter.py
python/mozbuild/mozbuild/test/frontend/test_sandbox.py
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -4,18 +4,22 @@
 
 from __future__ import unicode_literals
 
 from abc import (
     ABCMeta,
     abstractmethod,
 )
 
+import os
+import sys
+
 from mach.mixin.logging import LoggingMixin
 
+from ..frontend.data import SandboxDerived
 from .configenvironment import ConfigEnvironment
 
 
 class BuildBackend(LoggingMixin):
     """Abstract base class for build backends.
 
     A build backend is merely a consumer of the build configuration (the output
     of the frontend processing). It does something with said data. What exactly
@@ -48,16 +52,22 @@ class BuildBackend(LoggingMixin):
         Child classes are not expected to implement this method. Instead, the
         base class consumes objects and calls methods (possibly) implemented by
         child classes.
         """
 
         for obj in objs:
             self.consume_object(obj)
 
+        # Write out a file indicating when this backend was last generated.
+        age_file = os.path.join(self.environment.topobjdir,
+            'backend.%s.built' % self.__class__.__name__)
+        with open(age_file, 'a'):
+            os.utime(age_file, None)
+
         self.consume_finished()
 
     @abstractmethod
     def consume_object(self, obj):
         """Consumes an individual TreeMetadata instance.
 
         This is the main method used by child classes to react to build
         metadata.
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -31,17 +31,21 @@ class BackendMakeFile(object):
 
         self.fh = FileAvoidWrite(os.path.join(objdir, 'backend.mk'))
         self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
 
     def write(self, buf):
         self.fh.write(buf)
 
     def close(self):
-        self.fh.write('BACKEND_OUTPUT_FILES += %s\n' % ' '.join(self.outputs))
+        if len(self.inputs):
+            self.fh.write('BACKEND_INPUT_FILES += %s\n' % ' '.join(self.inputs))
+
+        if len(self.outputs):
+            self.fh.write('BACKEND_OUTPUT_FILES += %s\n' % ' '.join(self.outputs))
 
         self.fh.close()
 
 
 class RecursiveMakeBackend(BuildBackend):
     """Backend that integrates with the existing recursive make build system.
 
     This backend facilitates the transition from Makefile.in to moz.build
@@ -60,16 +64,18 @@ class RecursiveMakeBackend(BuildBackend)
         self._backend_files = {}
 
     def consume_object(self, obj):
         """Write out build files necessary to build with recursive make."""
 
         backend_file = self._backend_files.get(obj.srcdir,
             BackendMakeFile(obj.srcdir, obj.objdir))
 
+        backend_file.inputs |= obj.sandbox_all_paths
+
         if isinstance(obj, DirectoryTraversal):
             self._process_directory_traversal(obj, backend_file)
 
         self._backend_files[obj.srcdir] = backend_file
 
     def consume_finished(self):
         for srcdir in sorted(self._backend_files.keys()):
             bf = self._backend_files[srcdir]
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -29,22 +29,29 @@ class SandboxDerived(TreeMetadata):
 
     It holds fields common to all sandboxes. This class is likely never
     instantiated directly but is instead derived from.
     """
 
     __slots__ = (
         'objdir',
         'relativedir',
+        'sandbox_all_paths',
+        'sandbox_path',
         'srcdir',
         'topobjdir',
         'topsrcdir',
     )
 
     def __init__(self, sandbox):
+        # Capture the files that were evaluated to build this sandbox.
+        self.sandbox_main_path = sandbox.main_path
+        self.sandbox_all_paths = sandbox.all_paths
+
+        # Basic directory state.
         self.topsrcdir = sandbox['TOPSRCDIR']
         self.topobjdir = sandbox['TOPOBJDIR']
 
         self.relativedir = sandbox['RELATIVEDIR']
         self.srcdir = sandbox['SRCDIR']
         self.objdir = sandbox['OBJDIR']
 
 
--- a/python/mozbuild/mozbuild/frontend/sandbox.py
+++ b/python/mozbuild/mozbuild/frontend/sandbox.py
@@ -254,16 +254,18 @@ class Sandbox(object):
         """Initialize a Sandbox ready for execution.
 
         The arguments are proxied to GlobalNamespace.__init__.
         """
         self._globals = GlobalNamespace(allowed_variables=allowed_variables,
             builtins=builtins)
         self._locals = LocalNamespace(self._globals)
         self._execution_stack = []
+        self.main_path = None
+        self.all_paths = set()
 
     def exec_file(self, path):
         """Execute code at a path in the sandbox.
 
         The path must be absolute.
         """
         assert os.path.isabs(path)
 
@@ -285,16 +287,21 @@ class Sandbox(object):
         will be compiled and executed.
 
         You should almost always go through exec_file() because exec_source()
         does not perform extra path normalization. This can cause relative
         paths to behave weirdly.
         """
         self._execution_stack.append(path)
 
+        if self.main_path is None:
+            self.main_path = path
+
+        self.all_paths.add(path)
+
         # We don't have to worry about bytecode generation here because we are
         # too low-level for that. However, we could add bytecode generation via
         # the marshall module if parsing performance were ever an issue.
 
         try:
             # compile() inherits the __future__ from the module by default. We
             # do want Unicode literals.
             code = compile(source, path, 'exec')
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -15,16 +15,18 @@ from mozbuild.frontend.reader import Bui
 
 from mozbuild.test.backend.common import BackendTester
 
 
 class TestRecursiveMakeBackend(BackendTester):
     def test_basic(self):
         """Ensure the RecursiveMakeBackend works without error."""
         env = self._consume('stub0', RecursiveMakeBackend)
+        self.assertTrue(os.path.exists(os.path.join(env.topobjdir,
+            'backend.RecursiveMakeBackend.built')))
 
     def test_output_files(self):
         """Ensure proper files are generated."""
         env = self._consume('stub0', RecursiveMakeBackend)
 
         expected = ['', 'dir1', 'dir2']
 
         for d in expected:
@@ -55,17 +57,17 @@ class TestRecursiveMakeBackend(BackendTe
         ])
 
     def test_backend_mk(self):
         """Ensure backend.mk file is written out properly."""
         env = self._consume('stub0', RecursiveMakeBackend)
 
         p = os.path.join(env.topobjdir, 'backend.mk')
 
-        lines = [l.strip() for l in open(p, 'rt').readlines()[1:-1]]
+        lines = [l.strip() for l in open(p, 'rt').readlines()[1:-2]]
         self.assertEqual(lines, [
             'DIRS := dir1',
             'PARALLEL_DIRS := dir2',
             'TEST_DIRS := dir3',
         ])
 
     def test_no_mtime_bump(self):
         """Ensure mtime is not updated if file content does not change."""
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -38,16 +38,18 @@ class TestEmitterBasic(unittest.TestCase
         for o in objs:
             self.assertIsInstance(o, DirectoryTraversal)
             self.assertEqual(o.parallel_dirs, [])
             self.assertEqual(o.tool_dirs, [])
             self.assertEqual(o.test_dirs, [])
             self.assertEqual(o.test_tool_dirs, [])
             self.assertEqual(len(o.tier_dirs), 0)
             self.assertEqual(len(o.tier_static_dirs), 0)
+            self.assertTrue(os.path.isabs(o.sandbox_main_path))
+            self.assertEqual(len(o.sandbox_all_paths), 1)
 
         reldirs = [o.relativedir for o in objs]
         self.assertEqual(reldirs, ['', 'foo', 'foo/biz', 'bar'])
 
         dirs = [o.dirs for o in objs]
         self.assertEqual(dirs, [['foo', 'bar'], ['biz'], [], []])
 
     def test_traversal_all_vars(self):
--- a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py
@@ -112,25 +112,28 @@ class TestSandbox(unittest.TestCase):
             continue
 
     def test_exec_source_success(self):
         sandbox = self.sandbox()
 
         sandbox.exec_source('foo = True', 'foo.py')
 
         self.assertNotIn('foo', sandbox)
+        self.assertEqual(sandbox.main_path, 'foo.py')
+        self.assertEqual(sandbox.all_paths, set(['foo.py']))
 
     def test_exec_compile_error(self):
         sandbox = self.sandbox()
 
         with self.assertRaises(SandboxExecutionError) as se:
             sandbox.exec_source('2f23;k;asfj', 'foo.py')
 
         self.assertEqual(se.exception.file_stack, ['foo.py'])
         self.assertIsInstance(se.exception.exc_value, SyntaxError)
+        self.assertEqual(sandbox.main_path, 'foo.py')
 
     def test_exec_import_denied(self):
         sandbox = self.sandbox()
 
         with self.assertRaises(SandboxExecutionError) as se:
             sandbox.exec_source('import sys', 'import.py')
 
         self.assertIsInstance(se.exception, SandboxExecutionError)
@@ -205,16 +208,19 @@ add_tier_dir('t1', 'bat', static=True)
             sandbox.exec_source('add_tier_dir("t1", "foo")', 'foo.py')
 
     def test_include_basic(self):
         sandbox = self.sandbox(data_path='include-basic')
 
         sandbox.exec_file('moz.build')
 
         self.assertEqual(sandbox['DIRS'], ['foo', 'bar'])
+        self.assertEqual(sandbox.main_path,
+            os.path.join(sandbox['TOPSRCDIR'], 'moz.build'))
+        self.assertEqual(len(sandbox.all_paths), 2)
 
     def test_include_outside_topsrcdir(self):
         sandbox = self.sandbox(data_path='include-outside-topsrcdir')
 
         with self.assertRaises(SandboxLoadError) as se:
             sandbox.exec_file('relative.build')
 
         expected = os.path.join(test_data_path, 'moz.build')