Bug 1313640 - Add support for allowing duplicate files in multilocale builds. r=gps,mshal
authorChris AtLee <catlee@mozilla.com>
Fri, 28 Oct 2016 09:25:33 -0400
changeset 320371 a352e8d4b5d92451341bc0bd36d840cda5e11c76
parent 320370 c42b82086c262ad124507ef976f388fbdcfc4f77
child 320372 7053f12766bc5b21ef76e495b550b0413ca70a80
push id83367
push userphilringnalda@gmail.com
push dateTue, 01 Nov 2016 01:39:40 +0000
treeherdermozilla-inbound@50e030bdef66 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps, mshal
bugs1313640
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1313640 - Add support for allowing duplicate files in multilocale builds. r=gps,mshal MozReview-Commit-ID: CTmfSRs4JaS
toolkit/mozapps/installer/find-dupes.py
--- a/toolkit/mozapps/installer/find-dupes.py
+++ b/toolkit/mozapps/installer/find-dupes.py
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import sys
 import hashlib
+import re
 from mozpack.packager.unpack import UnpackFinder
 from mozpack.files import DeflatedFile
 from collections import OrderedDict
 
 '''
 Find files duplicated in a given packaged directory, independently of its
 package format.
 '''
@@ -230,32 +231,73 @@ ALLOWED_DUPES = set([
     'modules/commonjs/sdk/ui/state/events.js',
     'plugin-container.app/Contents/PkgInfo',
     'res/table-remove-column-active.gif',
     'res/table-remove-column-hover.gif',
     'res/table-remove-column.gif',
     'res/table-remove-row-active.gif',
     'res/table-remove-row-hover.gif',
     'res/table-remove-row.gif',
+    # For android multilocale
+    'chrome/en-US/locale/branding/brand.dtd',
+    'chrome/en-US/locale/branding/brand.properties',
+    'chrome/en-US/locale/en-US/browser/aboutHealthReport.dtd',
+    'chrome/en-US/locale/en-US/browser/aboutHome.dtd',
+    'chrome/en-US/locale/en-US/browser/checkbox.dtd',
+    'chrome/en-US/locale/en-US/browser/devicePrompt.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/aboutAbout.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/global.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/global/mozilla.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/intl.css',
+    'chrome/en-US/locale/en-US/browser/region.properties',
+    'chrome/en-US/locale/en-US/browser/searchplugins/amazondotcom.xml',
+    'chrome/en-US/locale/en-US/browser/searchplugins/bing.xml',
+    'chrome/en-US/locale/en-US/browser/searchplugins/duckduckgo.xml',
+    'chrome/en-US/locale/en-US/browser/searchplugins/google-nocodes.xml',
+    'chrome/en-US/locale/en-US/browser/searchplugins/google.xml',
+    'chrome/en-US/locale/en-US/browser/searchplugins/qwant.xml',
+    'chrome/en-US/locale/en-US/browser/searchplugins/twitter.xml',
 ])
 
 
 def normalize_osx_path(p):
     '''
     Strips the first 3 elements of an OSX app path
 
     >>> normalize_osx_path('Nightly.app/foo/bar/baz')
     'baz'
     '''
     bits = p.split('/')
     if len(bits) > 3 and bits[0].endswith('.app'):
         return '/'.join(bits[3:])
     return p
 
 
+def normalize_l10n_path(p):
+    '''
+    Normalizes localized paths to en-US
+
+    >>> normalize_l10n_path('chrome/es-ES/locale/branding/brand.properties')
+    'chrome/en-US/locale/branding/brand.properties'
+    >>> normalize_l10n_path('chrome/fr/locale/fr/browser/aboutHome.dtd')
+    'chrome/en-US/locale/en-US/browser/aboutHome.dtd'
+    '''
+    p = re.sub(r'chrome/(\S+)/locale/\1',
+               'chrome/en-US/locale/en-US',
+               p)
+    p = re.sub(r'chrome/(\S+)/locale',
+               'chrome/en-US/locale',
+               p)
+    return p
+
+
+def normalize_path(p):
+    return normalize_osx_path(normalize_l10n_path(p))
+
+
 def find_dupes(source):
     md5s = OrderedDict()
     for p, f in UnpackFinder(source):
         content = f.open().read()
         m = hashlib.md5(content).digest()
         if m not in md5s:
             if isinstance(f, DeflatedFile):
                 compressed = f.file.compressed_size
@@ -273,17 +315,17 @@ def find_dupes(source):
             print 'Duplicates %d bytes%s%s:' % (size,
                   ' (%d compressed)' % compressed if compressed != size else '',
                   ' (%d times)' % (len(paths) - 1) if len(paths) > 2 else '')
             print ''.join('  %s\n' % p for p in paths)
             total += (len(paths) - 1) * size
             total_compressed += (len(paths) - 1) * compressed
             num_dupes += 1
 
-            unexpected_dupes.extend([p for p in paths if normalize_osx_path(p) not in ALLOWED_DUPES])
+            unexpected_dupes.extend([p for p in paths if normalize_path(p) not in ALLOWED_DUPES])
 
     if num_dupes:
         print "WARNING: Found %d duplicated files taking %d bytes (%s)" % \
               (num_dupes, total,
                '%d compressed' % total_compressed if total_compressed != total
                                                   else 'uncompressed')
 
     if unexpected_dupes: