Bug 1303184 - Add exception list to find-dupes, and error out when unexpected dupes are detected; r=gps
authorChris AtLee <catlee@mozilla.com>
Wed, 26 Oct 2016 17:00:07 -0700
changeset 346347 532415dbd6208ff854e65760f5ea9241de4edd7b
parent 346346 082d0ffb918fa9a2d560d5cca3b40f895c9a43ee
child 346348 6f6ae961e35406c7d0384ce9cdb437bf3a4c8c1e
push id10298
push userraliiev@mozilla.com
push dateMon, 14 Nov 2016 12:33:03 +0000
treeherdermozilla-aurora@7e29173b1641 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1303184
milestone52.0a1
Bug 1303184 - Add exception list to find-dupes, and error out when unexpected dupes are detected; r=gps MozReview-Commit-ID: JRRnj6Kw1s4
toolkit/mozapps/installer/find-dupes.py
--- a/toolkit/mozapps/installer/find-dupes.py
+++ b/toolkit/mozapps/installer/find-dupes.py
@@ -8,48 +8,294 @@ from mozpack.packager.unpack import Unpa
 from mozpack.files import DeflatedFile
 from collections import OrderedDict
 
 '''
 Find files duplicated in a given packaged directory, independently of its
 package format.
 '''
 
+# Known duplicate files
+# This list is ideally empty, but some existing files will be grandfathered in
+# See bug 1303184
+#
+# PLEASE DO NOT ADD MORE EXCEPTIONS TO THIS LIST
+#
+ALLOWED_DUPES = set([
+    # updater on osx is bug 1311194
+    'LaunchServices/org.mozilla.updater',
+    'updater.app/Contents/MacOS/org.mozilla.updater',
+    'updater.app/Contents/PkgInfo',
+    'browser/chrome.manifest',
+    # browser branding / themes is bug 1313106
+    'browser/chrome/browser/content/branding/icon128.png',
+    'browser/chrome/browser/content/branding/icon16.png',
+    'browser/chrome/browser/content/branding/icon32.png',
+    'browser/chrome/browser/content/branding/icon48.png',
+    'browser/chrome/browser/content/browser/defaultthemes/5.footer.png',
+    'browser/chrome/browser/content/browser/defaultthemes/5.header.png',
+    'browser/chrome/browser/content/browser/extension.svg',
+    'browser/chrome/browser/content/browser/places/bookmarkProperties.xul',
+    'browser/chrome/browser/content/browser/places/bookmarkProperties2.xul',
+    'browser/chrome/browser/skin/classic/browser/addons/addon-install-confirm.svg',
+    'browser/chrome/browser/skin/classic/browser/connection-secure.svg',
+    'browser/chrome/browser/skin/classic/browser/controlcenter/warning-gray.svg',
+    'browser/chrome/browser/skin/classic/browser/newtab/close.png',
+    'browser/chrome/browser/skin/classic/browser/theme-switcher-icon.png',
+    # devtools reduction is bug 1311178
+    'browser/chrome/devtools/content/dom/content/dom-view.css',
+    'browser/chrome/devtools/content/dom/dom.html',
+    'browser/chrome/devtools/content/dom/main.js',
+    'browser/chrome/devtools/content/framework/toolbox-options.js',
+    'browser/chrome/devtools/content/inspector/fonts/fonts.js',
+    'browser/chrome/devtools/content/inspector/inspector.xhtml',
+    'browser/chrome/devtools/content/memory/initializer.js',
+    'browser/chrome/devtools/content/projecteditor/lib/helpers/readdir.js',
+    'browser/chrome/devtools/content/shared/frame-script-utils.js',
+    'browser/chrome/devtools/content/shared/theme-switching.js',
+    'browser/chrome/devtools/modules/devtools/client/dom/content/dom-view.css',
+    'browser/chrome/devtools/modules/devtools/client/dom/dom.html',
+    'browser/chrome/devtools/modules/devtools/client/dom/main.js',
+    'browser/chrome/devtools/modules/devtools/client/framework/toolbox-options.js',
+    'browser/chrome/devtools/modules/devtools/client/inspector/fonts/fonts.js',
+    'browser/chrome/devtools/modules/devtools/client/inspector/inspector.xhtml',
+    'browser/chrome/devtools/modules/devtools/client/jsonview/css/controls.png',
+    'browser/chrome/devtools/modules/devtools/client/jsonview/css/controls@2x.png',
+    'browser/chrome/devtools/modules/devtools/client/memory/initializer.js',
+    'browser/chrome/devtools/modules/devtools/client/projecteditor/lib/helpers/readdir.js',
+    'browser/chrome/devtools/modules/devtools/client/shared/frame-script-utils.js',
+    'browser/chrome/devtools/modules/devtools/client/shared/theme-switching.js',
+    'browser/chrome/devtools/modules/devtools/client/themes/common.css',
+    'browser/chrome/devtools/modules/devtools/client/themes/variables.css',
+    'browser/chrome/devtools/skin/common.css',
+    'browser/chrome/devtools/skin/images/command-scratchpad.svg',
+    'browser/chrome/devtools/skin/images/controls.png',
+    'browser/chrome/devtools/skin/images/controls@2x.png',
+    'browser/chrome/devtools/skin/images/debugger-blackbox.svg',
+    'browser/chrome/devtools/skin/images/debugger-prettyprint.svg',
+    'browser/chrome/devtools/skin/images/filetypes/store.svg',
+    'browser/chrome/devtools/skin/images/itemToggle.svg',
+    'browser/chrome/devtools/skin/images/security-state-broken.svg',
+    'browser/chrome/devtools/skin/images/security-state-local.svg',
+    'browser/chrome/devtools/skin/images/security-state-secure.svg',
+    'browser/chrome/devtools/skin/images/tabs-icon.svg',
+    'browser/chrome/devtools/skin/images/tool-scratchpad.svg',
+    'browser/chrome/devtools/skin/images/tool-storage.svg',
+    'browser/chrome/devtools/skin/images/tool-styleeditor.svg',
+    'browser/chrome/devtools/skin/promisedebugger.css',
+    'browser/chrome/devtools/skin/variables.css',
+    'modules/devtools/Console.jsm',
+    'modules/devtools/Loader.jsm',
+    'modules/devtools/Simulator.jsm',
+    'modules/devtools/shared/Console.jsm',
+    'modules/devtools/shared/Loader.jsm',
+    'modules/devtools/shared/apps/Simulator.jsm',
+    'browser/modules/devtools/client/framework/gDevTools.jsm',
+    'browser/modules/devtools/gDevTools.jsm',
+    'browser/chrome/icons/default/default16.png',
+    'browser/chrome/icons/default/default32.png',
+    'browser/chrome/icons/default/default48.png',
+    'browser/chrome/pdfjs/content/web/images/findbarButton-next-rtl.png',
+    'browser/chrome/pdfjs/content/web/images/findbarButton-next-rtl@2x.png',
+    'browser/chrome/pdfjs/content/web/images/findbarButton-next.png',
+    'browser/chrome/pdfjs/content/web/images/findbarButton-next@2x.png',
+    'browser/chrome/pdfjs/content/web/images/findbarButton-previous-rtl.png',
+    'browser/chrome/pdfjs/content/web/images/findbarButton-previous-rtl@2x.png',
+    'browser/chrome/pdfjs/content/web/images/findbarButton-previous.png',
+    'browser/chrome/pdfjs/content/web/images/findbarButton-previous@2x.png',
+    'browser/extensions/{972ce4c6-7e08-4474-a285-3208198ce6fd}/icon.png',
+    'browser/features/firefox@getpocket.com/chrome/skin/linux/menuPanel.png',
+    'browser/features/firefox@getpocket.com/chrome/skin/linux/menuPanel@2x.png',
+    'browser/features/firefox@getpocket.com/chrome/skin/windows/menuPanel.png',
+    'browser/features/firefox@getpocket.com/chrome/skin/windows/menuPanel@2x.png',
+    # flyweb reduction is bug 1313107
+    'browser/features/flyweb@mozilla.org/chrome/skin/linux/flyweb.css',
+    'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-16.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-32-anchored.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-32.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-64-anchored.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/linux/icon-64.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/osx/flyweb.css',
+    'browser/features/flyweb@mozilla.org/chrome/skin/osx/icon-32-anchored.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/osx/icon-64-anchored.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/windows/flyweb.css',
+    'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-16.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-32-anchored.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-32.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-64-anchored.png',
+    'browser/features/flyweb@mozilla.org/chrome/skin/windows/icon-64.png',
+    'browser/icons/mozicon128.png',
+    'chrome.manifest',
+    'chrome/en-US/locale/en-US/browser/overrides/AccessFu.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/about.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/aboutAbout.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/aboutReader.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/aboutRights.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/charsetMenu.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/commonDialogs.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/crashreporter/crashes.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/crashreporter/crashes.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/dom/dom.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/global.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/global/aboutSupport.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/global/aboutSupport.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/global/aboutTelemetry.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/global/aboutTelemetry.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/global/aboutWebrtc.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/global/mozilla.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/intl.css',
+    'chrome/en-US/locale/en-US/browser/overrides/intl.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/passwordmgr.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/plugins.properties',
+    'chrome/en-US/locale/en-US/browser/overrides/plugins/pluginproblem.dtd',
+    'chrome/en-US/locale/en-US/browser/overrides/search/search.properties',
+    'chrome/en-US/locale/en-US/global-platform/mac/intl.properties',
+    'chrome/en-US/locale/en-US/global-platform/unix/accessible.properties',
+    'chrome/en-US/locale/en-US/global-platform/unix/intl.properties',
+    'chrome/en-US/locale/en-US/global-platform/unix/platformKeys.properties',
+    'chrome/en-US/locale/en-US/global-platform/win/accessible.properties',
+    'chrome/en-US/locale/en-US/global-platform/win/intl.properties',
+    'chrome/en-US/locale/en-US/global-platform/win/platformKeys.properties',
+    'chrome/en-US/locale/en-US/global/AccessFu.properties',
+    'chrome/en-US/locale/en-US/global/about.dtd',
+    'chrome/en-US/locale/en-US/global/aboutAbout.dtd',
+    'chrome/en-US/locale/en-US/global/aboutReader.properties',
+    'chrome/en-US/locale/en-US/global/aboutRights.dtd',
+    'chrome/en-US/locale/en-US/global/aboutSupport.dtd',
+    'chrome/en-US/locale/en-US/global/aboutSupport.properties',
+    'chrome/en-US/locale/en-US/global/aboutTelemetry.dtd',
+    'chrome/en-US/locale/en-US/global/aboutTelemetry.properties',
+    'chrome/en-US/locale/en-US/global/aboutWebrtc.properties',
+    'chrome/en-US/locale/en-US/global/charsetMenu.properties',
+    'chrome/en-US/locale/en-US/global/commonDialogs.properties',
+    'chrome/en-US/locale/en-US/global/crashes.dtd',
+    'chrome/en-US/locale/en-US/global/crashes.properties',
+    'chrome/en-US/locale/en-US/global/dom/dom.properties',
+    'chrome/en-US/locale/en-US/global/global.dtd',
+    'chrome/en-US/locale/en-US/global/intl.css',
+    'chrome/en-US/locale/en-US/global/intl.properties',
+    'chrome/en-US/locale/en-US/global/mozilla.dtd',
+    'chrome/en-US/locale/en-US/global/plugins.properties',
+    'chrome/en-US/locale/en-US/global/search/search.properties',
+    'chrome/en-US/locale/en-US/passwordmgr/passwordmgr.properties',
+    'chrome/en-US/locale/en-US/pluginproblem/pluginproblem.dtd',
+    'chrome/toolkit/skin/classic/global/autocomplete.css',
+    'chrome/toolkit/skin/classic/global/button.css',
+    'chrome/toolkit/skin/classic/global/checkbox.css',
+    'chrome/toolkit/skin/classic/global/dialog.css',
+    'chrome/toolkit/skin/classic/global/dropmarker.css',
+    'chrome/toolkit/skin/classic/global/global.css',
+    'chrome/toolkit/skin/classic/global/groupbox.css',
+    'chrome/toolkit/skin/classic/global/icons/close-XPVista7.png',
+    'chrome/toolkit/skin/classic/global/icons/tabprompts-bgtexture.png',
+    'chrome/toolkit/skin/classic/global/listbox.css',
+    'chrome/toolkit/skin/classic/global/media/clicktoplay-bgtexture.png',
+    'chrome/toolkit/skin/classic/global/menu.css',
+    'chrome/toolkit/skin/classic/global/menulist.css',
+    'chrome/toolkit/skin/classic/global/numberbox.css',
+    'chrome/toolkit/skin/classic/global/popup.css',
+    'chrome/toolkit/skin/classic/global/preferences.css',
+    'chrome/toolkit/skin/classic/global/progressmeter.css',
+    'chrome/toolkit/skin/classic/global/radio.css',
+    'chrome/toolkit/skin/classic/global/resizer.css',
+    'chrome/toolkit/skin/classic/global/richlistbox.css',
+    'chrome/toolkit/skin/classic/global/scale.css',
+    'chrome/toolkit/skin/classic/global/scrollbars.css',
+    'chrome/toolkit/skin/classic/global/scrollbox.css',
+    'chrome/toolkit/skin/classic/global/spinbuttons.css',
+    'chrome/toolkit/skin/classic/global/splitter.css',
+    'chrome/toolkit/skin/classic/global/tabbox.css',
+    'chrome/toolkit/skin/classic/global/textbox.css',
+    'chrome/toolkit/skin/classic/global/toolbar.css',
+    'chrome/toolkit/skin/classic/global/toolbarbutton.css',
+    'chrome/toolkit/skin/classic/global/tree.css',
+    'chrome/toolkit/skin/classic/global/wizard.css',
+    'chrome/toolkit/skin/classic/mozapps/downloads/buttons.png',
+    'chrome/toolkit/skin/classic/mozapps/downloads/downloadButtons-XP.png',
+    'chrome/toolkit/skin/classic/mozapps/downloads/downloadButtons.png',
+    'chrome/toolkit/skin/classic/mozapps/extensions/category-dictionaries.png',
+    'chrome/toolkit/skin/classic/mozapps/extensions/category-experiments.png',
+    'chrome/toolkit/skin/classic/mozapps/extensions/dictionaryGeneric.png',
+    'chrome/toolkit/skin/classic/mozapps/extensions/experimentGeneric.png',
+    'chrome/toolkit/skin/classic/mozapps/update/buttons.png',
+    'chrome/toolkit/skin/classic/mozapps/update/downloadButtons-XP.png',
+    'chrome/toolkit/skin/classic/mozapps/update/downloadButtons.png',
+    'components/FxAccountsPush.js',
+    'crashreporter.app/Contents/Resources/English.lproj/MainMenu.nib/classes.nib',
+    'crashreporter.app/Contents/Resources/English.lproj/MainMenuRTL.nib/classes.nib',
+    # firefox/firefox-bin is bug 658850
+    'firefox',
+    'firefox-bin',
+    'modules/FxAccountsPush.js',
+    'modules/commonjs/index.js',
+    'modules/commonjs/sdk/ui/button/view/events.js',
+    'modules/commonjs/sdk/ui/state/events.js',
+    'plugin-container.app/Contents/PkgInfo',
+    'res/table-remove-column-active.gif',
+    'res/table-remove-column-hover.gif',
+    'res/table-remove-column.gif',
+    'res/table-remove-row-active.gif',
+    'res/table-remove-row-hover.gif',
+    'res/table-remove-row.gif',
+])
+
+
+def normalize_osx_path(p):
+    '''
+    Strips the first 3 elements of an OSX app path
+
+    >>> normalize_osx_path('Nightly.app/foo/bar/baz')
+    'baz'
+    '''
+    bits = p.split('/')
+    if len(bits) > 3 and bits[0].endswith('.app'):
+        return '/'.join(bits[3:])
+    return p
+
 
 def find_dupes(source):
     md5s = OrderedDict()
     for p, f in UnpackFinder(source):
         content = f.open().read()
         m = hashlib.md5(content).digest()
         if m not in md5s:
             if isinstance(f, DeflatedFile):
                 compressed = f.file.compressed_size
             else:
                 compressed = len(content)
             md5s[m] = (len(content), compressed, [])
         md5s[m][2].append(p)
     total = 0
     total_compressed = 0
     num_dupes = 0
+    unexpected_dupes = []
     for m, (size, compressed, paths) in sorted(md5s.iteritems(),
                                                key=lambda x: x[1][1]):
         if len(paths) > 1:
             print 'Duplicates %d bytes%s%s:' % (size,
                   ' (%d compressed)' % compressed if compressed != size else '',
                   ' (%d times)' % (len(paths) - 1) if len(paths) > 2 else '')
             print ''.join('  %s\n' % p for p in paths)
             total += (len(paths) - 1) * size
             total_compressed += (len(paths) - 1) * compressed
             num_dupes += 1
+
+            unexpected_dupes.extend([p for p in paths if normalize_osx_path(p) not in ALLOWED_DUPES])
+
     if num_dupes:
         print "WARNING: Found %d duplicated files taking %d bytes (%s)" % \
               (num_dupes, total,
                '%d compressed' % total_compressed if total_compressed != total
                                                   else 'uncompressed')
 
+    if unexpected_dupes:
+        print "ERROR: The following duplicated files are not allowed:"
+        print "\n".join(unexpected_dupes)
+        sys.exit(1)
+
 
 def main():
     if len(sys.argv) != 2:
         import os
         print >>sys.stderr, "Usage: %s directory" % \
                             os.path.basename(sys.argv[0])
         sys.exit(1)