Backed out changeset 95b3298fd2d4 (bug 1537574) for windows task timeouts a=backout
authorAndreea Pavel <apavel@mozilla.com>
Mon, 08 Apr 2019 22:23:03 +0300
changeset 530287 facb0f655f03a8c3a9c02cec2dc7e607ebbc2514
parent 530286 40456af7da1ce0561a28bc88da23427fa56d3282
child 530288 395a65d512e38c304f5228357de7cade57799175
push id2082
push userffxbld-merge
push dateMon, 01 Jul 2019 08:34:18 +0000
treeherdermozilla-release@2fb19d0466d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1537574
milestone68.0a1
backs out95b3298fd2d462ecf0aab134289838317009fb3c
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 95b3298fd2d4 (bug 1537574) for windows task timeouts a=backout
Makefile.in
build/gen_test_backend.py
build/rebuild-backend.mk
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/gen_test_backend.py
testing/mozbase/moztest/moztest/resolve.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -60,16 +60,36 @@ ifdef JS_STANDALONE
 CLOBBER:
 else
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo 'STOP!  The CLOBBER file has changed.'
 	@echo 'Please run the build through "mach build".'
 	@exit 1
 endif
 
+# Regenerate the build backend if it is out of date. We only have this rule in
+# this main make file because having it in rules.mk and applied to partial tree
+# builds resulted in a world of hurt. Gory details are in bug 877308.
+#
+# The mach build driver will ensure the backend is up to date for partial tree
+# builds. This cleanly avoids most of the pain.
+
+ifndef TEST_MOZBUILD
+
+.PHONY: backend
+backend: $(BUILD_BACKEND_FILES)
+
+include $(topsrcdir)/build/rebuild-backend.mk
+
+Makefile: $(BUILD_BACKEND_FILES)
+	@$(TOUCH) $@
+
+default:: $(BUILD_BACKEND_FILES)
+endif
+
 install_manifests := \
   $(addprefix dist/,branding include public private xpi-stage) \
   _tests \
   $(NULL)
 # Skip the dist/bin install manifest when using the hybrid
 # FasterMake/RecursiveMake backend. This is a hack until bug 1241744 moves
 # xpidl handling to FasterMake in that case, mechanically making the dist/bin
 # install manifest non-existent (non-existent manifests being skipped)
rename from python/mozbuild/mozbuild/gen_test_backend.py
rename to build/gen_test_backend.py
new file mode 100644
--- /dev/null
+++ b/build/rebuild-backend.mk
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+BACKEND_GENERATION_SCRIPT ?= config.status
+
+# A traditional rule would look like this:
+#    backend.%:
+#        @echo do stuff
+#
+# But with -j<n>, and multiple items in BUILD_BACKEND_FILES, the command would
+# run multiple times in parallel.
+#
+# "Fortunately", make has some weird semantics for pattern rules: if there are
+# multiple targets in a pattern rule and each of them is matched at most once,
+# the command will only run once. So:
+#     backend%RecursiveMakeBackend backend%FasterMakeBackend:
+#         @echo do stuff
+#     backend: backend.RecursiveMakeBackend backend.FasterMakeBackend
+# would only execute the command once.
+#
+# Credit where due: http://stackoverflow.com/questions/2973445/gnu-makefile-rule-generating-a-few-targets-from-a-single-source-file/3077254#3077254
+$(subst .,%,$(BUILD_BACKEND_FILES)):
+	@echo 'Build configuration changed. Regenerating backend.'
+	$(PYTHON) $(BACKEND_GENERATION_SCRIPT)
+
+define build_backend_rule
+$(1): $$(wildcard $$(shell cat $(1).in))
+
+endef
+$(foreach file,$(BUILD_BACKEND_FILES),$(eval $(call build_backend_rule,$(file))))
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -192,59 +192,16 @@ class MozbuildObject(ProcessExecutionMix
             topobjdir = topobjdir.replace('@CONFIG_GUESS@',
                 self.resolve_config_guess())
 
         if not os.path.isabs(topobjdir):
             topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir))
 
         return mozpath.normsep(os.path.normpath(topobjdir))
 
-    def build_out_of_date(self, output, dep_file):
-        if not os.path.isfile(output):
-            print(" Output reference file not found: %s" % output)
-            return True
-        if not os.path.isfile(dep_file):
-            print(" Dependency file not found: %s" % dep_file)
-            return True
-
-        deps = []
-        with open(dep_file, 'r') as fh:
-            deps = fh.read().splitlines()
-
-        mtime = os.path.getmtime(output)
-        for f in deps:
-            try:
-                dep_mtime = os.path.getmtime(f)
-            except OSError as e:
-                if e.errno == errno.ENOENT:
-                    print(" Input not found: %s" % f)
-                    return True
-                raise
-            if dep_mtime > mtime:
-                print(" %s is out of date with respect to %s" % (output, f))
-                return True
-        return False
-
-    def backend_out_of_date(self, backend_file):
-        if not os.path.isfile(backend_file):
-            return True
-
-        # Check if any of our output files have been removed since
-        # we last built the backend, re-generate the backend if
-        # so.
-        outputs = []
-        with open(backend_file, 'r') as fh:
-            outputs = fh.read().splitlines()
-        for output in outputs:
-            if not os.path.isfile(mozpath.join(self.topobjdir, output)):
-                return True
-
-        dep_file = '%s.in' % backend_file
-        return self.build_out_of_date(backend_file, dep_file)
-
     @property
     def topobjdir(self):
         if self._topobjdir is None:
             self._topobjdir = self.resolve_mozconfig_topobjdir(
                 default='obj-@CONFIG_GUESS@')
 
         return self._topobjdir
 
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1009,16 +1009,59 @@ class BuildDriver(MozbuildObject):
                 return 1
 
             if directory is not None:
                 disable_extra_make_dependencies=True
                 directory = mozpath.normsep(directory)
                 if directory.startswith('/'):
                     directory = directory[1:]
 
+            def build_out_of_date(output, dep_file):
+                if not os.path.isfile(output):
+                    print(" Output reference file not found: %s" % output)
+                    return True
+                if not os.path.isfile(dep_file):
+                    print(" Configure dependency file not found: %s" % dep_file)
+                    return True
+
+                deps = []
+                with open(dep_file, 'r') as fh:
+                    deps = fh.read().splitlines()
+
+                mtime = os.path.getmtime(output)
+                for f in deps:
+                    try:
+                        dep_mtime = os.path.getmtime(f)
+                    except OSError as e:
+                        if e.errno == errno.ENOENT:
+                            print(" Configure input not found: %s" % f)
+                            return True
+                        raise
+                    if dep_mtime > mtime:
+                        print(" %s is out of date with respect to %s" % (output, f))
+                        return True
+                return False
+
+            def backend_out_of_date(backend_file):
+                if not os.path.isfile(backend_file):
+                    return True
+
+                # Check if any of our output files have been removed since
+                # we last built the backend, re-generate the backend if
+                # so.
+                outputs = []
+                with open(backend_file, 'r') as fh:
+                    outputs = fh.read().splitlines()
+                for output in outputs:
+                    if not os.path.isfile(mozpath.join(self.topobjdir, output)):
+                        return True
+
+                dep_file = '%s.in' % backend_file
+                return build_out_of_date(backend_file, dep_file)
+
             monitor.start_resource_recording()
 
             self.mach_context.command_attrs['clobber'] = False
             config = None
             try:
                 config = self.config_environment
             except Exception:
                 # If we don't already have a config environment this is either
@@ -1036,20 +1079,20 @@ class BuildDriver(MozbuildObject):
 
             previous_backend = None
             if config is not None:
                 previous_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
 
             config_rc = None
             # Even if we have a config object, it may be out of date
             # if something that influences its result has changed.
-            if config is None or self.build_out_of_date(mozpath.join(self.topobjdir,
-                                                                     'config.status'),
-                                                        mozpath.join(self.topobjdir,
-                                                                     'config_status_deps.in')):
+            if config is None or build_out_of_date(mozpath.join(self.topobjdir,
+                                                                'config.status'),
+                                                   mozpath.join(self.topobjdir,
+                                                                'config_status_deps.in')):
                 if previous_backend and 'Make' not in previous_backend:
                     clobber_requested = self._clobber_configure()
 
                 if config is None:
                     print(" Config object not found by mach.")
 
                 config_rc = self.configure(buildstatus_messages=True,
                                            line_handler=output.on_line)
@@ -1058,26 +1101,26 @@ class BuildDriver(MozbuildObject):
                     return config_rc
 
                 config = self.reload_config_environment()
 
             active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
 
             status = None
 
-            if (not config_rc and
-                self.backend_out_of_date(mozpath.join(self.topobjdir,
-                                                      'backend.%sBackend' %
-                                                      active_backend))):
-                print('Build configuration changed. Regenerating backend.')
-                args = [config.substs['PYTHON'],
-                        mozpath.join(self.topobjdir, 'config.status')]
-                self.run_process(args, cwd=self.topobjdir, pass_thru=True)
+            if 'Make' not in active_backend:
+                if (not config_rc and
+                    backend_out_of_date(mozpath.join(self.topobjdir,
+                                                     'backend.%sBackend' %
+                                                     active_backend))):
+                    print('Build configuration changed. Regenerating backend.')
+                    args = [config.substs['PYTHON'],
+                            mozpath.join(self.topobjdir, 'config.status')]
+                    self.run_process(args, cwd=self.topobjdir, pass_thru=True)
 
-            if 'Make' not in active_backend:
                 # client.mk has its own handling of MOZ_PARALLEL_BUILD so the
                 # make backend can determine when to run in single-threaded mode
                 # or parallel mode. For other backends, we can pass in the value
                 # of MOZ_PARALLEL_BUILD if -jX was not specified on the
                 # commandline.
                 if jobs == 0 and 'make_extra' in self.mozconfig:
                     for param in self.mozconfig['make_extra']:
                         key, value = param.split('=')
@@ -1133,16 +1176,25 @@ class BuildDriver(MozbuildObject):
                     new_pairs = list(add_extra_dependencies(target_pairs, dm))
                     self.log(logging.DEBUG, 'dumbmake',
                              {'target_pairs': target_pairs,
                               'new_pairs': new_pairs},
                              'Added extra dependencies: will build {new_pairs} ' +
                              'instead of {target_pairs}.')
                     target_pairs = new_pairs
 
+                # Ensure build backend is up to date. The alternative is to
+                # have rules in the invoked Makefile to rebuild the build
+                # backend. But that involves make reinvoking itself and there
+                # are undesired side-effects of this. See bug 877308 for a
+                # comprehensive history lesson.
+                self._run_make(directory=self.topobjdir, target='backend',
+                    line_handler=output.on_line, log=False,
+                    print_directory=False, keep_going=keep_going)
+
                 # Build target pairs.
                 for make_dir, make_target in target_pairs:
                     # We don't display build status messages during partial
                     # tree builds because they aren't reliable there. This
                     # could potentially be fixed if the build monitor were more
                     # intelligent about encountering undefined state.
                     no_build_status = b'1' if make_dir is not None else b''
                     status = self._run_make(directory=make_dir, target=make_target,
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -539,22 +539,26 @@ class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
 
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
 
         # If installing tests is going to result in re-generating the build
         # backend, we need to do this here, so that the updated contents of
         # all-tests.pkl make it to the set of tests to run.
-        if self.backend_out_of_date(mozpath.join(self.topobjdir,
-                                                 'backend.TestManifestBackend'
-                                                 )):
-            print("Test configuration changed. Regenerating backend.")
-            from mozbuild.gen_test_backend import gen_test_backend
-            gen_test_backend()
+        self._run_make(
+            target='backend.TestManifestBackend', pass_thru=True, print_directory=False,
+            filename=mozpath.join(self.topsrcdir, 'build', 'rebuild-backend.mk'),
+            append_env={
+                b'PYTHON': self.virtualenv_manager.python_path,
+                b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend',
+                b'BACKEND_GENERATION_SCRIPT': mozpath.join(
+                    self.topsrcdir, 'build', 'gen_test_backend.py'),
+            },
+        )
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
                                                 'all-tests.pkl'),
                                    self.topsrcdir,
                                    test_defaults=os.path.join(self.topobjdir,
                                                               'test-defaults.pkl'))
 
         self._test_rewrites = {