Bug 1231349 - L10n repacks broken on with SM 2.42 and newer - add ignorelist argument to l10n repack. r=frg a=frg DONTBUILD CLOSED TREE SEAMONKEY_2_49_ESR_RELBRANCH
authorEdmund Wong <ewong@pw-wspx.org>
Mon, 15 Jul 2019 18:07:58 +0200
branchSEAMONKEY_2_49_ESR_RELBRANCH
changeset 357559 a7d832fb13d102d77690756bf5e26b046ffacf78
parent 357558 3c7797e7a8ddd3b7ddebcfe400828787e041d161
child 357560 cf09bc568b9a54e2d9286b976f2fe7f24a51b776
push id7836
push userfrgrahl@gmx.net
push dateMon, 15 Jul 2019 16:17:02 +0000
treeherdermozilla-esr52@3a4afe7ecac5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfrg, frg
bugs1231349
milestone52.9.1
Bug 1231349 - L10n repacks broken on with SM 2.42 and newer - add ignorelist argument to l10n repack. r=frg a=frg DONTBUILD CLOSED TREE SeaMonkey ESR52 release branch only.
python/mozbuild/mozpack/packager/l10n.py
python/mozbuild/mozpack/packager/unpack.py
toolkit/locales/l10n.mk
toolkit/mozapps/installer/l10n-repack.py
--- a/python/mozbuild/mozpack/packager/l10n.py
+++ b/python/mozbuild/mozpack/packager/l10n.py
@@ -36,80 +36,138 @@ from mozpack.chrome.manifest import (
     ManifestChrome,
     Manifest,
 )
 from mozpack.errors import errors
 from mozpack.packager.unpack import UnpackFinder
 from createprecomplete import generate_precomplete
 
 
+class UnpackFinder(UnpackFinder):
+
+    def __init__(self, *args, **kwargs):
+        self.ignorelist = kwargs.pop('ignorelist', [])
+        super(UnpackFinder, self).__init__(*args, **kwargs)
+
+    def _maybe_zip(self, path, file):
+        def check_ignore(p):
+            dn = False
+            for i in self.ignorelist:
+                if i in p:
+                    dn = True
+            return dn
+
+        if not check_ignore(path):
+            return super(UnpackFinder, self)._maybe_zip(path, file)
+        return False
+
+
 class LocaleManifestFinder(object):
-    def __init__(self, finder):
+    def __init__(self, finder, ignorelist=[]):
         entries = self.entries = []
         bases = self.bases = []
+        self.ignorelist = ignorelist
 
         class MockFormatter(object):
+            def __init__(self, ignorelist=[]):
+                self.ignorelist = ignorelist
+
             def add_interfaces(self, path, content):
                 pass
 
             def add(self, path, content):
                 pass
 
             def add_manifest(self, entry):
+                def check_ig(p):
+                    dn = False
+                    for i in self.ignorelist:
+                        if hasattr(p, 'name'):
+                            if i in p.name:
+                                dn = True
+                    return dn
                 if entry.localized:
-                    entries.append(entry)
+                    if not check_ig(entry):
+                        entries.append(entry)
 
             def add_base(self, base, addon=False):
-                bases.append(base)
+                def check_ignore(p):
+                    dn = False
+                    for i in self.ignorelist:
+                        if i in p:
+                            dn = True
+                    return dn
+                if not check_ignore(base):
+                    bases.append(base)
 
         # SimplePackager rejects "manifest foo.manifest" entries with
         # additional flags (such as "manifest foo.manifest application=bar").
         # Those type of entries are used by language packs to work as addons,
         # but are not necessary for the purpose of l10n repacking. So we wrap
         # the finder in order to remove those entries.
         class WrapFinder(object):
-            def __init__(self, finder):
+            def __init__(self, finder, ignorelist=[]):
                 self._finder = finder
+                self._ignorelist = []
 
             def find(self, pattern):
-                for p, f in self._finder.find(pattern):
-                    if isinstance(f, ManifestFile):
-                        unwanted = [
-                            e for e in f._entries
-                            if isinstance(e, Manifest) and e.flags
-                        ]
-                        if unwanted:
-                            f = ManifestFile(
-                                f._base,
-                                [e for e in f._entries if e not in unwanted])
-                    yield p, f
+                def check_ignore(pf):
+                    dn = False
+                    for i in self._ignorelist:
+                        if i in pf:
+                            dn = True
+                    return dn
+                if not check_ignore(pattern):
+                    for p, f in self._finder.find(pattern):
+                        if isinstance(f, ManifestFile):
+                            unwanted = [
+                                e for e in f._entries
+                                if isinstance(e, Manifest) and e.flags
+                            ]
+                            if unwanted:
+                                f = ManifestFile(
+                                    f._base,
+                                    [e for e in f._entries if e not in unwanted])
+                        yield p, f
 
-        sink = SimpleManifestSink(WrapFinder(finder), MockFormatter())
+        sink = SimpleManifestSink(WrapFinder(finder,
+                                             ignorelist=self.ignorelist),
+                                  MockFormatter(ignorelist=self.ignorelist))
         sink.add(Component(''), '*')
         sink.close(False)
 
         # Find unique locales used in these manifest entries.
         self.locales = list(set(e.id for e in self.entries
                                 if isinstance(e, ManifestLocale)))
 
 
-def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
-    app = LocaleManifestFinder(app_finder)
+def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set(),
+            ignorelist=[]):
+
+    def check_ignore(p):
+        dn = False
+        for i in ignorelist:
+            if i in p:
+                dn = True
+        return dn
+
+    app = LocaleManifestFinder(app_finder, ignorelist=ignorelist)
     l10n = LocaleManifestFinder(l10n_finder)
 
     # The code further below assumes there's only one locale replaced with
     # another one.
-    if len(app.locales) > 1:
-        errors.fatal("Multiple app locales aren't supported: " +
-                     ",".join(app.locales))
     if len(l10n.locales) > 1:
         errors.fatal("Multiple l10n locales aren't supported: " +
                      ",".join(l10n.locales))
+    l10n_locale = l10n.locales[0]
     locale = app.locales[0]
-    l10n_locale = l10n.locales[0]
+
+    if len(app.locales) > 1:
+        if l10n_locale in app.locales:
+            locale = l10n_locale
 
     # For each base directory, store what path a locale chrome package name
     # corresponds to.
     # e.g., for the following entry under app/chrome:
     #     locale foo en-US path/to/files
     # keep track that the locale path for foo in app is
     # app/chrome/path/to/files.
     l10n_paths = {}
@@ -118,17 +176,18 @@ def _repack(app_finder, l10n_finder, cop
             base = mozpath.basedir(e.path, app.bases)
             l10n_paths.setdefault(base, {})
             l10n_paths[base][e.name] = e.path
 
     # For chrome and non chrome files or directories, store what langpack path
     # corresponds to a package path.
     paths = {}
     for e in app.entries:
-        if isinstance(e, ManifestEntryWithRelPath):
+        if isinstance(e, ManifestEntryWithRelPath) and \
+           not check_ignore(e.path):
             base = mozpath.basedir(e.path, app.bases)
             if base not in l10n_paths:
                 errors.fatal("Locale doesn't contain %s/" % base)
                 # Allow errors to accumulate
                 continue
             if e.name not in l10n_paths[base]:
                 errors.fatal("Locale doesn't have a manifest entry for '%s'" %
                     e.name)
@@ -206,17 +265,18 @@ def _repack(app_finder, l10n_finder, cop
                     formatter.add(p, f)
 
     # Transplant jar preloading information.
     for path, log in app_finder.jarlogs.iteritems():
         assert isinstance(copier[path], Jarrer)
         copier[path].preload([l.replace(locale, l10n_locale) for l in log])
 
 
-def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
+def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set(),
+           ignorelist=[]):
     '''
     Replace localized data from the `source` directory with localized data
     from `l10n` and `extra_l10n`.
 
     The `source` argument points to a directory containing a packaged
     application (in omnijar, jar or flat form).
     The `l10n` argument points to a directory containing the main localized
     data (usually in the form of a language pack addon) to use to replace
@@ -226,34 +286,34 @@ def repack(source, l10n, extra_l10n={}, 
     This can be used to point at different language pack addons for different
     parts of the package application.
     The `non_resources` argument gives a list of relative paths in the source
     that should not be added in an omnijar in case the packaged application
     is in that format.
     The `non_chrome` argument gives a list of file/directory patterns for
     localized files that are not listed in a chrome.manifest.
     '''
-    app_finder = UnpackFinder(source)
-    l10n_finder = UnpackFinder(l10n)
+    app_finder = UnpackFinder(source, ignorelist=ignorelist)
+    l10n_finder = UnpackFinder(l10n, ignorelist=ignorelist)
     if extra_l10n:
         finders = {
             '': l10n_finder,
         }
         for base, path in extra_l10n.iteritems():
-            finders[base] = UnpackFinder(path)
+            finders[base] = UnpackFinder(path, ignorelist=ignorelist)
         l10n_finder = ComposedFinder(finders)
     copier = FileCopier()
     if app_finder.kind == 'flat':
         formatter = FlatFormatter(copier)
     elif app_finder.kind == 'jar':
         formatter = JarFormatter(copier,
                                  optimize=app_finder.optimizedjars,
                                  compress=app_finder.compressed)
     elif app_finder.kind == 'omni':
         formatter = OmniJarFormatter(copier, app_finder.omnijar,
                                      optimize=app_finder.optimizedjars,
                                      compress=app_finder.compressed,
                                      non_resources=non_resources)
 
     with errors.accumulate():
-        _repack(app_finder, l10n_finder, copier, formatter, non_chrome)
+        _repack(app_finder, l10n_finder, copier, formatter, non_chrome, ignorelist=ignorelist)
     copier.copy(source, skip_if_older=False)
     generate_precomplete(source)
--- a/python/mozbuild/mozpack/packager/unpack.py
+++ b/python/mozbuild/mozpack/packager/unpack.py
@@ -62,17 +62,17 @@ class UnpackFinder(BaseFinder):
             # Skip the precomplete file, which is generated at packaging time.
             if p == 'precomplete':
                 continue
             base = mozpath.dirname(p)
             # If the file is a zip/jar that is not a .xpi, and contains a
             # chrome.manifest, it is an omnijar. All the files it contains
             # go in the directory containing the omnijar. Manifests are merged
             # if there is a corresponding manifest in the directory.
-            if not p.endswith('.xpi') and self._maybe_zip(f) and \
+            if not p.endswith('.xpi') and self._maybe_zip(p, f) and \
                     (mozpath.basename(p) == self.omnijar or
                      not self.omnijar):
                 jar = self._open_jar(p, f)
                 if 'chrome.manifest' in jar:
                     self.kind = 'omni'
                     self.omnijar = mozpath.basename(p)
                     self._fill_with_jar(base, jar)
                     continue
@@ -84,17 +84,17 @@ class UnpackFinder(BaseFinder):
                     else ManifestFile(base)
                 for e in parse_manifest(self.base, p, f.open()):
                     m.add(self._handle_manifest_entry(e, jars))
                 if self.files.contains(p):
                     continue
                 f = m
             # If the file is a packed addon, unpack it under a directory named
             # after the xpi.
-            if p.endswith('.xpi') and self._maybe_zip(f):
+            if p.endswith('.xpi') and self._maybe_zip(p, f):
                 self._fill_with_jar(p[:-4], self._open_jar(p, f))
                 continue
             if not p in jars:
                 self.files.add(p, f)
 
     def _fill_with_jar(self, base, jar):
         for j in jar:
             path = mozpath.join(base, j.filename)
@@ -152,17 +152,17 @@ class UnpackFinder(BaseFinder):
             jarlog = jar.entries.keys()
             self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1]
         return jar
 
     def find(self, path):
         for p in self.files.match(path):
             yield p, self.files[p]
 
-    def _maybe_zip(self, file):
+    def _maybe_zip(self, path, file):
         '''
         Return whether the given BaseFile looks like a ZIP/Jar.
         '''
         header = file.open().read(8)
         return len(header) == 8 and (header[0:2] == 'PK' or
                                      header[4:6] == 'PK')
 
     def _unjarize(self, entry, relpath):
--- a/toolkit/locales/l10n.mk
+++ b/toolkit/locales/l10n.mk
@@ -113,17 +113,18 @@ repackage-zip:  libs-$(AB_CD)
 # call a hook for apps to put their uninstall helper.exe into the package
 	$(UNINSTALLER_PACKAGE_HOOK)
 # call a hook for apps to build the stub installer
 ifdef MOZ_STUB_INSTALLER
 	$(STUB_HOOK)
 endif
 	$(PYTHON) $(MOZILLA_DIR)/toolkit/mozapps/installer/l10n-repack.py $(STAGEDIST) $(DIST)/xpi-stage/locale-$(AB_CD) \
 		$(MOZ_PKG_EXTRAL10N) \
-		$(if $(filter omni,$(MOZ_PACKAGER_FORMAT)),$(if $(NON_OMNIJAR_FILES),--non-resource $(NON_OMNIJAR_FILES)))
+		$(if $(filter omni,$(MOZ_PACKAGER_FORMAT)),$(if $(NON_OMNIJAR_FILES),--non-resource $(NON_OMNIJAR_FILES))) \
+		--ignorelist inspector@ {59c81df5-4b modern chatzilla
 
 ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
 ifneq (en,$(LPROJ_ROOT))
 	mv $(STAGEDIST)/en.lproj $(STAGEDIST)/$(LPROJ_ROOT).lproj
 endif
 ifdef MOZ_CRASHREPORTER
 # On Mac OS X, the crashreporter.ini file needs to be moved from under the
 # application bundle's Resources directory where all other l10n files are
--- a/toolkit/mozapps/installer/l10n-repack.py
+++ b/toolkit/mozapps/installer/l10n-repack.py
@@ -43,18 +43,21 @@ def main():
     parser.add_argument('extra_l10n', nargs='*', metavar='BASE=PATH',
                         type=valid_extra_l10n,
                         help='Extra directories with staged localized files '
                              'to be considered under the given base in the '
                              'repacked build')
     parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
                         default=[],
                         help='Extra files not to be considered as resources')
+    parser.add_argument('--ignorelist', nargs='+', default=[],
+                        help='List of ignore patterns.')
     args = parser.parse_args()
 
     buildconfig.substs['USE_ELF_HACK'] = False
     buildconfig.substs['PKG_SKIP_STRIP'] = True
     l10n.repack(args.build, args.l10n, extra_l10n=dict(args.extra_l10n),
-                non_resources=args.non_resource, non_chrome=NON_CHROME)
+                non_resources=args.non_resource, non_chrome=NON_CHROME,
+                ignorelist=args.ignorelist)
 
 
 if __name__ == "__main__":
     main()