Bug 589368 - Locale repacking support for jar reordering; r=ted a=blocking-betaN+
authorTaras Glek <tglek@mozilla.com>
Fri, 10 Sep 2010 12:30:07 -0400
changeset 52385 4cffee2c859b32632e8a91d3cf30556073c10a56
parent 52384 65161587c8b4bc9563a7a000384559b853dc2411
child 52386 fcae8913e4665d706aeb14a9cde21a27c2b1d129
push id15619
push usereakhgari@mozilla.com
push dateFri, 10 Sep 2010 16:32:44 +0000
treeherdermozilla-central@98fb85e9d3b6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted, blocking-betaN
bugs589368
milestone2.0b6pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 589368 - Locale repacking support for jar reordering; r=ted a=blocking-betaN+
config/optimizejars.py
modules/libjar/nsZipArchive.cpp
toolkit/locales/l10n.mk
toolkit/mozapps/installer/packager.mk
--- a/config/optimizejars.py
+++ b/config/optimizejars.py
@@ -178,102 +178,166 @@ class BinaryBlob:
                 member_data = self.readAt(self.offset, retstruct.__getattr__(member_desc))
             retstruct.addMember(name, member_data)
         # sanity check serialization code
         data = self.readAt(offset, self.offset - offset)
         out_data = retstruct.pack()
         assert_true(out_data == data, "Serialization fail %d !=%d"% (len(out_data), len(data)))
         return retstruct
 
-def optimizejar(log, jar, outjar):
-    entries = open(log).read().rstrip().split("\n")
+def optimizejar(jar, outjar, inlog = None):
+    if inlog is not None:
+        inlog = open(inlog).read().rstrip()
+        # in the case of an empty log still move the index forward
+        if len(inlog) == 0:
+            inlog = []
+        else:
+            inlog = inlog.split("\n")
+    outlog = []
     jarblob = BinaryBlob(jar)
     dirend = jarblob.read_struct(cdir_end, jarblob.length - size_of(cdir_end))
     assert_true(dirend.signature == ENDSIG, "no signature in the end");
+    cdir_offset = dirend.cdir_offset
+    readahead = 0
+    if inlog is None and cdir_offset == 4:
+        readahead = struct.unpack("<I", jarblob.readAt(0, 4))[0]
+        print("%s: startup data ends at byte %d" % (outjar, readahead));
 
-    cdir_offset = dirend.cdir_offset
-   
     jarblob.offset = cdir_offset
     central_directory = []
     for i  in range(0, dirend.cdir_entries):
         entry = jarblob.read_struct(cdir_entry)
         central_directory.append(entry)
         
     reordered_count = 0
-    dup_guard = set()
-    for ordered_name in entries:
-        if ordered_name in dup_guard:
-            continue
-        else:
-            dup_guard.add(ordered_name)
-        found = False
-        for i in range(reordered_count, len(central_directory)):
-            if central_directory[i].filename == ordered_name:
-                # swap the cdir entries
-                tmp = central_directory[i]
-                central_directory[i] = central_directory[reordered_count]
-                central_directory[reordered_count] = tmp
-                reordered_count = reordered_count + 1
-                found = True
-                break
-        assert_true(found, "Can't find %s in %s\n" %(ordered_name, jar))
+    if inlog is not None:
+        dup_guard = set()
+        for ordered_name in inlog:
+            if ordered_name in dup_guard:
+                continue
+            else:
+                dup_guard.add(ordered_name)
+            found = False
+            for i in range(reordered_count, len(central_directory)):
+                if central_directory[i].filename == ordered_name:
+                    # swap the cdir entries
+                    tmp = central_directory[i]
+                    central_directory[i] = central_directory[reordered_count]
+                    central_directory[reordered_count] = tmp
+                    reordered_count = reordered_count + 1
+                    found = True
+                    break
+            if not found:
+                print( "Can't find '%s' in %s" % (ordered_name, jar))
 
-    # have to put central directory at offset 1 cos 0 confuses some tools
-    dirend.cdir_offset = 1
-    dirend_data = dirend.pack()
     outfd = open(outjar, "wb")
-    # make room for central dir + end of dir + 1 extra byte at front
-    out_offset = dirend.cdir_offset + dirend.cdir_size + len(dirend_data)
-    outfd.seek(out_offset)
+    out_offset = 0
+    if inlog is not None:
+        # have to put central directory at offset 4 cos 0 confuses some tools.
+        # This also lets us specify how many entries should be preread
+        dirend.cdir_offset = 4
+        # make room for central dir + end of dir + 4 extra bytes at front
+        out_offset = dirend.cdir_offset + dirend.cdir_size + size_of(cdir_end) 
+        outfd.seek(out_offset)
 
     cdir_data = ""
     written_count = 0
+    # store number of bytes suggested for readahead
     for entry in central_directory:
         # read in the header twice..first for comparison, second time for convenience when writing out
         jarfile = jarblob.read_struct(local_file_header, entry.offset)
         assert_true(jarfile.filename == entry.filename, "Directory/Localheader mismatch")
         data = jarfile.pack()
         outfd.write(data)
+        old_entry_offset = entry.offset
         entry.offset = out_offset
         out_offset = out_offset + len(data)
         entry_data = entry.pack()
         cdir_data += entry_data
         expected_len = entry.filename_size + entry.extrafield_size + entry.filecomment_size
         assert_true(len(entry_data) != expected_len,
                     "%s entry size - expected:%d got:%d" % (entry.filename, len(entry_data), expected_len))
         written_count += 1
-        if written_count == reordered_count:
-            print("%s: startup data ends at byte %d"%( outjar, out_offset));
-        elif written_count < reordered_count:
-            print("%s @ %d" % (entry.filename, entry.offset))
+        if inlog is not None:
+            if written_count == reordered_count:
+                readahead = out_offset
+                print("%s: startup data ends at byte %d"%( outjar, readahead));
+            elif written_count < reordered_count:
+                pass
+                #print("%s @ %d" % (entry.filename, out_offset))
+        elif readahead >= old_entry_offset + len(data):
+            outlog.append(entry.filename)
+            reordered_count += 1
 
+    if inlog is None:
+        dirend.cdir_offset = out_offset
+
+    dirend_data = dirend.pack()
     assert_true(size_of(cdir_end) == len(dirend_data), "Failed to serialize directory end correctly. Serialized size;%d, expected:%d"%(len(dirend_data), size_of(cdir_end)));
-    outfd.write(dirend_data)
+
     outfd.seek(dirend.cdir_offset)
     assert_true(len(cdir_data) == dirend.cdir_size, "Failed to serialize central directory correctly. Serialized size;%d, expected:%d expected-size:%d" % (len(cdir_data), dirend.cdir_size, dirend.cdir_size - len(cdir_data)));
     outfd.write(cdir_data)
     outfd.write(dirend_data)
+
+    # for ordered jars the central directory is written in the begining of the file, so a second central-directory
+    # entry has to be written in the end of the file
+    if inlog is not None:
+        outfd.seek(0)
+        outfd.write(struct.pack("<I", readahead));
+        outfd.seek(out_offset)
+        outfd.write(dirend_data)
+
+    print "%s %d/%d in %s" % (("Ordered" if inlog is not None else "Deoptimized"),
+                              reordered_count, len(central_directory), outjar)
     outfd.close()
-    print "Ordered %d/%d in %s" % (reordered_count, len(central_directory), outjar)
-    
-if len(sys.argv) != 4:
-    print "Usage: %s JAR_LOG_DIR IN_JAR_DIR OUT_JAR_DIR" % sys.argv[0]
+    return outlog
+        
+if len(sys.argv) != 5:
+    print "Usage: --optimize|--deoptimize %s JAR_LOG_DIR IN_JAR_DIR OUT_JAR_DIR" % sys.argv[0]
     exit(1)
 
-JAR_LOG_DIR = sys.argv[1]
-IN_JAR_DIR = sys.argv[2]
-OUT_JAR_DIR = sys.argv[3]
+def optimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR):
+    if not os.path.exists(JAR_LOG_DIR):
+        print("No jar logs found in %s. No jars to optimize." % JAR_LOG_DIR)
+        exit(0)
 
-if not os.path.exists(JAR_LOG_DIR):
-    print "No jar logs found. No jars to optimize."
-    exit(0)
+    ls = os.listdir(JAR_LOG_DIR)
+    for logfile in ls:
+        if not logfile.endswith(".jar.log"):
+            continue
+        injarfile = os.path.join(IN_JAR_DIR, logfile[:-4])
+        outjarfile = os.path.join(OUT_JAR_DIR, logfile[:-4]) 
+        if not os.path.exists(injarfile):
+            print "Warning: Skipping %s, %s doesn't exist" % (logfile, injarfile)
+            continue
+        logfile = os.path.join(JAR_LOG_DIR, logfile)
+        optimizejar(injarfile, outjarfile, logfile)
+
+def deoptimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR):
+    if not os.path.exists(JAR_LOG_DIR):
+        os.makedirs(JAR_LOG_DIR)
 
-ls = os.listdir(JAR_LOG_DIR)
-for logfile in ls:
-    if logfile[-8:] != ".jar.log":
-        continue
-    injarfile = os.path.join(IN_JAR_DIR, logfile[:-4])
-    outjarfile = os.path.join(OUT_JAR_DIR, logfile[:-4]) 
-    if not os.path.exists(injarfile):
-        print "Warning: Skipping %s, %s doesn't exist" % (logfile, injarfile)
-        continue
-    logfile = os.path.join(JAR_LOG_DIR, logfile)
-    optimizejar(logfile, injarfile, outjarfile)
+    ls = os.listdir(IN_JAR_DIR)
+    for jarfile in ls:
+        if not jarfile.endswith(".jar"):
+            continue
+        injarfile = os.path.join(IN_JAR_DIR, jarfile)
+        outjarfile = os.path.join(OUT_JAR_DIR, jarfile) 
+        logfile = os.path.join(JAR_LOG_DIR, jarfile + ".log")
+        log = optimizejar(injarfile, outjarfile, None)
+        open(logfile, "wb").write("\n".join(log))
+
+def main():        
+    MODE = sys.argv[1]
+    JAR_LOG_DIR = sys.argv[2]
+    IN_JAR_DIR = sys.argv[3]
+    OUT_JAR_DIR = sys.argv[4]
+    if MODE == "--optimize":
+        optimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR)
+    elif MODE == "--deoptimize":
+        deoptimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR)
+    else:
+        print("Unknown mode %s" % MODE)
+        exit(1)
+
+if __name__ == '__main__':
+    main()
--- a/modules/libjar/nsZipArchive.cpp
+++ b/modules/libjar/nsZipArchive.cpp
@@ -589,17 +589,17 @@ nsresult nsZipArchive::BuildFileList()
 {
   NS_TIME_FUNCTION;
 
   // Get archive size using end pos
   const PRUint8* buf;
   const PRUint8* startp = mFd->mFileData;
   const PRUint8* endp = startp + mFd->mLen;
   
-  PRUint32 centralOffset = 1;
+  PRUint32 centralOffset = 4;
   if (mFd->mLen > ZIPCENTRAL_SIZE && *(PRUint32*)(startp + centralOffset) == CENTRALSIG) {
     // Success means optimized jar layout from bug 559961 is in effect
   } else {
     for (buf = endp - ZIPEND_SIZE; buf > startp; buf--)
       {
         if (xtolong(buf) == ENDSIG) {
           centralOffset = xtolong(((ZipEnd *)buf)->offset_central_dir);
           break;
--- a/toolkit/locales/l10n.mk
+++ b/toolkit/locales/l10n.mk
@@ -127,16 +127,18 @@ unpack: $(STAGEDIST)
 	@echo done unpacking
 
 # The path to the object dir for the mozilla-central build system,
 # may be overridden if necessary.
 MOZDEPTH ?= $(DEPTH)
 
 repackage-zip: UNPACKAGE="$(ZIP_IN)"
 repackage-zip:
+# Adjust jar logs with the new locale (can't use sed -i because of bug 373784)
+	-$(PERL) -pi -e "s/en-US/$(AB_CD)/g" $(_ABS_DIST)/jarlog/*.jar.log
 # call a hook for apps to put their uninstall helper.exe into the package
 	$(UNINSTALLER_PACKAGE_HOOK)
 # copy xpi-stage over, but not install.rdf and chrome.manifest,
 # those are just for language packs
 	cd $(DIST)/xpi-stage/locale-$(AB_CD) && \
 	  tar --exclude=install.rdf --exclude=chrome.manifest $(TAR_CREATE_FLAGS) - * | ( cd $(STAGEDIST) && tar -xf - )
 	mv $(STAGEDIST)/chrome/$(AB_CD).manifest $(STAGEDIST)/chrome/localized.manifest
 ifneq (en,$(AB))
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -82,16 +82,17 @@ SDK_PATH      = $(PKG_PATH)
 ifeq ($(MOZ_APP_NAME),xulrunner)
 SDK_PATH = sdk/
 endif
 SDK_SUFFIX    = $(PKG_SUFFIX)
 SDK           = $(SDK_PATH)$(PKG_BASENAME).sdk$(SDK_SUFFIX)
 
 MAKE_PACKAGE	= $(error What is a $(MOZ_PKG_FORMAT) package format?);
 MAKE_CAB	= $(error Don't know how to make a CAB!);
+_ABS_DIST = $(call core_abspath,$(DIST))
 
 ifdef WINCE
 ifndef WINCE_WINDOWS_MOBILE
 CABARGS += -s
 endif
 ifdef MOZ_FASTSTART
 CABARGS += -faststart
 endif
@@ -179,17 +180,16 @@ ifdef UNIVERSAL_BINARY
 STAGEPATH = universal/
 endif
 ifndef PKG_DMG_SOURCE
 PKG_DMG_SOURCE = $(STAGEPATH)$(MOZ_PKG_DIR)
 endif
 INNER_MAKE_PACKAGE	= $(_ABS_MOZSRCDIR)/build/package/mac_osx/pkg-dmg \
   --source "$(PKG_DMG_SOURCE)" --target "$(PACKAGE)" \
   --volname "$(MOZ_APP_DISPLAYNAME)" $(PKG_DMG_FLAGS)
-_ABS_DIST = $(call core_abspath,$(DIST))
 INNER_UNMAKE_PACKAGE	= \
   set -ex; \
   rm -rf $(_ABS_DIST)/unpack.tmp; \
   mkdir -p $(_ABS_DIST)/unpack.tmp; \
   $(_ABS_MOZSRCDIR)/build/package/mac_osx/unpack-diskimage $(UNPACKAGE) /tmp/$(MOZ_PKG_APPNAME)-unpack $(_ABS_DIST)/unpack.tmp; \
   rsync -a "$(_ABS_DIST)/unpack.tmp/$(_APPNAME)" $(MOZ_PKG_DIR); \
   test -n "$(MOZ_PKG_MAC_DSSTORE)" && \
     rsync -a "$(_ABS_DIST)/unpack.tmp/.DS_Store" "$(MOZ_PKG_MAC_DSSTORE)"; \
@@ -236,20 +236,21 @@ NON_OMNIJAR_FILES = \
 
 PACK_OMNIJAR	= \
   rm -f omni.jar components/binary.manifest && \
   grep -h '^binary-component' components/*.manifest > binary.manifest ; \
   sed -e 's/^binary-component/\#binary-component/' components/components.manifest > components.manifest && \
   mv components.manifest components && \
   find . | xargs touch -t 201001010000 && \
   zip -r9mX omni.jar $(OMNIJAR_FILES) -x $(NON_OMNIJAR_FILES) && \
-  $(OPTIMIZE_JARS_CMD) $(DIST)/jarlog/ ./ ./ && \
+  $(OPTIMIZE_JARS_CMD) --optimize $(_ABS_DIST)/jarlog/ ./ ./ && \
   mv binary.manifest components && \
   printf "manifest components/binary.manifest\n" > chrome.manifest
 UNPACK_OMNIJAR	= \
+  $(OPTIMIZE_JARS_CMD) --deoptimize $(_ABS_DIST)/jarlog/ ./ ./ && \
   unzip -o omni.jar && \
   rm -f components/binary.manifest && \
   sed -e 's/^\#binary-component/binary-component/' components/components.manifest > components.manifest && \
   mv components.manifest components
 
 MAKE_PACKAGE	= (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && $(INNER_MAKE_PACKAGE)
 UNMAKE_PACKAGE	= $(INNER_UNMAKE_PACKAGE) && (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(UNPACK_OMNIJAR))
 else
@@ -451,17 +452,17 @@ else
 	@cd $(DIST)/bin && tar $(TAR_CREATE_FLAGS) - * | (cd ../$(MOZ_PKG_DIR); tar -xf -)
 	@echo "Linking XPT files..."
 	@rm -rf $(DIST)/xpt
 	@$(NSINSTALL) -D $(DIST)/xpt
 	@($(XPIDL_LINK) $(DIST)/xpt/$(MOZ_PKG_APPNAME).xpt $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)/components/*.xpt && rm -f $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)/components/*.xpt && cp $(DIST)/xpt/$(MOZ_PKG_APPNAME).xpt $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)/components && printf "interfaces $(MOZ_PKG_APPNAME).xpt\n" >$(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)/components/interfaces.manifest) || echo No *.xpt files found in: $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)/components/.  Continuing...
 endif # DMG
 endif # MOZ_PKG_MANIFEST
 endif # UNIVERSAL_BINARY
-	$(OPTIMIZE_JARS_CMD) $(DIST)/jarlog/ $(DIST)/bin/chrome $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)/chrome
+	$(OPTIMIZE_JARS_CMD) --optimize $(DIST)/jarlog/ $(DIST)/bin/chrome $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)/chrome
 ifndef PKG_SKIP_STRIP
 	@echo "Stripping package directory..."
 	@cd $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR); find . ! -type d \
 			! -name "*.js" \
 			! -name "*.xpt" \
 			! -name "*.gif" \
 			! -name "*.jpg" \
 			! -name "*.png" \