bug 1409721 - wire up support for LOCALIZED[_PP]_FILES in the recursive make backend. r=nalexander
authorTed Mielczarek <ted@mielczarek.org>
Thu, 09 Nov 2017 15:22:42 -0800
changeset 444498 b7c381c2e7f951e36ba421478c6820319bea8caa
parent 444497 338963fe12d3b84dfd6105efa3982e7e34f8ec94
child 444499 0af8d899db740b9b80f81d51c373ed105ea809d3
push id8527
push userCallek@gmail.com
push dateThu, 11 Jan 2018 21:05:50 +0000
treeherdermozilla-beta@95342d212a7a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1409721
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1409721 - wire up support for LOCALIZED[_PP]_FILES in the recursive make backend. r=nalexander This commit adds support for handling LOCALIZED_FILES and LOCALIZED_PP_FILES in the recursive make backend. They get special handling because they have a few special needs: * They run during the libs tier, so that repacks work. * The filenames cannot be determined at build-backend generation time, since repacks run configure once and then repack multiple locales using the generated backend files, so they are written with to be expanded with MERGE_FILE by make so that the file gets picked up from the proper locale dir. Other build backends that aren't trying to handle localized builds will silently handle these like FINAL_TARGET_FILES, which is fine until we revamp our l10n repack story. MozReview-Commit-ID: 2LZhPZNhQ4S
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/data/localized-files/en-US/bar.ini
python/mozbuild/mozbuild/test/backend/data/localized-files/en-US/foo.js
python/mozbuild/mozbuild/test/backend/data/localized-files/moz.build
python/mozbuild/mozbuild/test/backend/data/localized-pp-files/en-US/bar.ini
python/mozbuild/mozbuild/test/backend/data/localized-pp-files/en-US/foo.js
python/mozbuild/mozbuild/test/backend/data/localized-pp-files/moz.build
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -54,16 +54,18 @@ from ..frontend.data import (
     HostSimpleProgram,
     HostSources,
     InstallationTarget,
     JARManifest,
     JavaJarData,
     Library,
     Linkable,
     LocalInclude,
+    LocalizedFiles,
+    LocalizedPreprocessedFiles,
     ObjdirFiles,
     ObjdirPreprocessedFiles,
     PerSourceFlag,
     Program,
     RustLibrary,
     HostRustLibrary,
     RustProgram,
     RustTest,
@@ -640,16 +642,22 @@ class RecursiveMakeBackend(CommonBackend
             self._process_linked_libraries(obj, backend_file)
 
         elif isinstance(obj, ObjdirFiles):
             self._process_objdir_files(obj, obj.files, backend_file)
 
         elif isinstance(obj, ObjdirPreprocessedFiles):
             self._process_final_target_pp_files(obj, obj.files, backend_file, 'OBJDIR_PP_FILES')
 
+        elif isinstance(obj, LocalizedFiles):
+            self._process_localized_files(obj, obj.files, backend_file)
+
+        elif isinstance(obj, LocalizedPreprocessedFiles):
+            self._process_localized_pp_files(obj, obj.files, backend_file)
+
         elif isinstance(obj, FinalTargetFiles):
             self._process_final_target_files(obj, obj.files, backend_file)
 
         elif isinstance(obj, FinalTargetPreprocessedFiles):
             self._process_final_target_pp_files(obj, obj.files, backend_file, 'DIST_FILES')
 
         elif isinstance(obj, AndroidResDirs):
             # Order matters.
@@ -1448,16 +1456,64 @@ class RecursiveMakeBackend(CommonBackend
             for f in files:
                 backend_file.write('%s += %s\n' % (
                     var, self._pretty_path(f, backend_file)))
             backend_file.write('%s_PATH := $(DEPTH)/%s\n'
                                % (var, mozpath.join(obj.install_target, path)))
             backend_file.write('%s_TARGET := misc\n' % var)
             backend_file.write('PP_TARGETS += %s\n' % var)
 
+    def _write_localized_files_files(self, files, name, backend_file):
+        for f in files:
+            # The emitter asserts that all files start with `en-US/`
+            e, f = f.split('en-US/')
+            assert(not e)
+            if '*' in f:
+                # We can't use MERGE_FILE for wildcards because it takes
+                # only the first match internally. This is only used
+                # in one place in the tree currently so we'll hardcode
+                # that specific behavior for now.
+                backend_file.write('%s += $(wildcard $(LOCALE_SRCDIR)/%s)\n' % (name, f))
+            else:
+                backend_file.write('%s += $(call MERGE_FILE,%s)\n' % (name, f))
+
+    def _process_localized_files(self, obj, files, backend_file):
+        target = obj.install_target
+        path = mozpath.basedir(target, ('dist/bin', ))
+        if not path:
+            raise Exception('Cannot install localized files to ' + target)
+        for i, (path, files) in enumerate(files.walk()):
+            name = 'LOCALIZED_FILES_%d' % i
+            self._no_skip['libs'].add(backend_file.relobjdir)
+            self._write_localized_files_files(files, name + '_FILES', backend_file)
+            # Use FINAL_TARGET here because some l10n repack rules set
+            # XPI_NAME to generate langpacks.
+            backend_file.write('%s_DEST = $(FINAL_TARGET)/%s\n' % (name, path))
+            backend_file.write('%s_TARGET := libs\n' % name)
+            backend_file.write('INSTALL_TARGETS += %s\n' % name)
+
+    def _process_localized_pp_files(self, obj, files, backend_file):
+        target = obj.install_target
+        path = mozpath.basedir(target, ('dist/bin', ))
+        if not path:
+            raise Exception('Cannot install localized files to ' + target)
+        for i, (path, files) in enumerate(files.walk()):
+            name = 'LOCALIZED_PP_FILES_%d' % i
+            self._no_skip['libs'].add(backend_file.relobjdir)
+            self._write_localized_files_files(files, name, backend_file)
+            # Use FINAL_TARGET here because some l10n repack rules set
+            # XPI_NAME to generate langpacks.
+            backend_file.write('%s_PATH = $(FINAL_TARGET)/%s\n' % (name, path))
+            backend_file.write('%s_TARGET := libs\n' % name)
+            # Localized files will have different content in different
+            # localizations, and some preprocessed files may not have
+            # any preprocessor directives.
+            backend_file.write('%s_FLAGS := --silence-missing-directive-warnings\n' % name)
+            backend_file.write('PP_TARGETS += %s\n' % name)
+
     def _process_objdir_files(self, obj, files, backend_file):
         # We can't use an install manifest for the root of the objdir, since it
         # would delete all the other files that get put there by the build
         # system.
         for i, (path, files) in enumerate(files.walk()):
             self._no_skip['misc'].add(backend_file.relobjdir)
             for f in files:
                 backend_file.write('OBJDIR_%d_FILES += %s\n' % (
new file mode 100644
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/localized-files/moz.build
@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+LOCALIZED_FILES += [
+    'en-US/abc/*.abc',
+    'en-US/bar.ini',
+    'en-US/foo.js',
+]
new file mode 100644
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/backend/data/localized-pp-files/moz.build
@@ -0,0 +1,8 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+LOCALIZED_PP_FILES += [
+    'en-US/bar.ini',
+    'en-US/foo.js',
+]
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -839,16 +839,54 @@ class TestRecursiveMakeBackend(BackendTe
             'DIST_FILES_0_PATH := $(DEPTH)/dist/bin/',
             'DIST_FILES_0_TARGET := misc',
             'PP_TARGETS += DIST_FILES_0',
         ]
 
         found = [str for str in lines if 'DIST_FILES' in str]
         self.assertEqual(found, expected)
 
+    def test_localized_files(self):
+        """Test that LOCALIZED_FILES is written to backend.mk correctly."""
+        env = self._consume('localized-files', RecursiveMakeBackend)
+
+        backend_path = mozpath.join(env.topobjdir, 'backend.mk')
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+
+        expected = [
+            'LOCALIZED_FILES_0_FILES += $(wildcard $(LOCALE_SRCDIR)/abc/*.abc)',
+            'LOCALIZED_FILES_0_FILES += $(call MERGE_FILE,bar.ini)',
+            'LOCALIZED_FILES_0_FILES += $(call MERGE_FILE,foo.js)',
+            'LOCALIZED_FILES_0_DEST = $(FINAL_TARGET)/',
+            'LOCALIZED_FILES_0_TARGET := libs',
+            'INSTALL_TARGETS += LOCALIZED_FILES_0',
+        ]
+
+        found = [str for str in lines if 'LOCALIZED_FILES' in str]
+        self.assertEqual(found, expected)
+
+    def test_localized_pp_files(self):
+        """Test that LOCALIZED_PP_FILES is written to backend.mk correctly."""
+        env = self._consume('localized-pp-files', RecursiveMakeBackend)
+
+        backend_path = mozpath.join(env.topobjdir, 'backend.mk')
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+
+        expected = [
+            'LOCALIZED_PP_FILES_0 += $(call MERGE_FILE,bar.ini)',
+            'LOCALIZED_PP_FILES_0 += $(call MERGE_FILE,foo.js)',
+            'LOCALIZED_PP_FILES_0_PATH = $(FINAL_TARGET)/',
+            'LOCALIZED_PP_FILES_0_TARGET := libs',
+            'LOCALIZED_PP_FILES_0_FLAGS := --silence-missing-directive-warnings',
+            'PP_TARGETS += LOCALIZED_PP_FILES_0',
+        ]
+
+        found = [str for str in lines if 'LOCALIZED_PP_FILES' in str]
+        self.assertEqual(found, expected)
+
     def test_config(self):
         """Test that CONFIGURE_SUBST_FILES are properly handled."""
         env = self._consume('test_config', RecursiveMakeBackend)
 
         self.assertEqual(
             open(os.path.join(env.topobjdir, 'file'), 'r').readlines(), [
                 '#ifdef foo\n',
                 'bar baz\n',