bug 526668 - add option to 'unify' to allow files to match if their sorted contents match. r=bsmedberg
authorTed Mielczarek <ted.mielczarek@gmail.com>
Thu, 05 Nov 2009 09:04:49 -0500
changeset 34600 7378ea1411d8ffee07de01ff37087084342a7018
parent 34599 2e4ccecc5d550c9ef0cfce3bc34c47dfe0c06474
child 34601 0cc47ba7304b37de4e04c87ef288a62479596156
push idunknown
push userunknown
push dateunknown
reviewersbsmedberg
bugs526668
milestone1.9.3a1pre
bug 526668 - add option to 'unify' to allow files to match if their sorted contents match. r=bsmedberg
build/Makefile.in
build/macosx/universal/flight.mk
build/macosx/universal/unify
--- a/build/Makefile.in
+++ b/build/Makefile.in
@@ -115,16 +115,18 @@ check::
         fi
 	@if ! test -f ./unify-test-universal; then \
           echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to produce a universal binary!"; \
           false; \
         fi
 	@if ! file -b ./unify-test-universal | head -n1 | grep -q "^Mach-O universal binary"; then \
           echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to produce a universal binary!"; \
           false; \
+        else \
+          echo "TEST-PASS | build/ | unify produced a universal binary!"; \
         fi
 # try unifying two identical Java class files
 	rm -f unifytesta.class unifytestb.class unifytestc.class
 	cp $(srcdir)/unifytest.class ./unifytesta.class
 	cp $(srcdir)/unifytest.class ./unifytestb.class
 	@if ! $(srcdir)/macosx/universal/unify ./unifytesta.class ./unifytestb.class \
           ./unifytestc.class; then \
           echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to unify a Java class file!"; \
@@ -132,16 +134,36 @@ check::
         fi
 	@if ! test -f ./unifytestc.class; then \
           echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to unify a Java class file!"; \
           false; \
         fi
 	@if ! diff -q ./unifytesta.class ./unifytestc.class; then \
           echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to unify a Java class file!"; \
           false; \
+        else \
+          echo "TEST-PASS | build/ | unify unified a Java class file!"; \
+        fi
+# try unifying some files that differ only in line ordering
+	rm -rf unify-sort-test
+	mkdir unify-sort-test unify-sort-test/a unify-sort-test/b
+	printf "lmn\nabc\nxyz\n" > unify-sort-test/a/file.foo
+	printf "xyz\nlmn\nabc\n" > unify-sort-test/b/file.foo
+	printf "abc\nlmn\nxyz\n" > unify-sort-test/expected-result
+	@if ! $(srcdir)/macosx/universal/unify --unify-with-sort "\.foo$$" \
+          ./unify-sort-test/a ./unify-sort-test/b \
+          ./unify-sort-test/c; then \
+          echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to unify files with differing line ordering!"; \
+          false; \
+        fi
+	@if ! diff -q ./unify-sort-test/expected-result ./unify-sort-test/c/file.foo; then \
+          echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to unify files with differing line ordering!"; \
+          false; \
+        else \
+          echo "TEST-PASS | build/ | unify unified files with differing line ordering!"; \
         fi
 endif
 
 ifeq ($(OS_ARCH),Linux)
 libs:: $(topsrcdir)/tools/rb/fix-linux-stack.pl
 	$(INSTALL) $< $(DIST)/bin
 endif
 endif # ENABLE_TESTS
--- a/build/macosx/universal/flight.mk
+++ b/build/macosx/universal/flight.mk
@@ -100,16 +100,17 @@ postflight_all:
 	$(TOPSRCDIR)/build/macosx/universal/fix-buildconfig \
 	  $(DIST_PPC)/$(MOZ_PKG_APPNAME)/$(APPNAME)/$(BUILDCONFIG_JAR) \
 	  $(DIST_X86)/$(MOZ_PKG_APPNAME)/$(APPNAME)/$(BUILDCONFIG_JAR)
 	mkdir -p $(DIST_UNI)/$(MOZ_PKG_APPNAME)
 	rm -f $(DIST_X86)/universal
 	ln -s $(DIST_UNI) $(DIST_X86)/universal
 	rm -rf $(DIST_UNI)/$(MOZ_PKG_APPNAME)/$(APPNAME)
 	$(TOPSRCDIR)/build/macosx/universal/unify \
+          --unify-with-sort "\.manifest$$" \
 	  $(DIST_PPC)/$(MOZ_PKG_APPNAME)/$(APPNAME) \
 	  $(DIST_X86)/$(MOZ_PKG_APPNAME)/$(APPNAME) \
 	  $(DIST_UNI)/$(MOZ_PKG_APPNAME)/$(APPNAME)
 # A universal .dmg can now be produced by making in either architecture's
 # INSTALLER_DIR.
 # Now, repeat the process for the test package.
 	$(MAKE) -C $(OBJDIR_PPC) UNIVERSAL_BINARY= package-tests
 	$(MAKE) -C $(OBJDIR_X86) UNIVERSAL_BINARY= package-tests
@@ -118,11 +119,12 @@ postflight_all:
 # dist/bin. It doesn't matter which one we use.
 	if test -d $(DIST_PPC)/test-package-stage -a                 \
                 -d $(DIST_X86)/test-package-stage; then              \
            cp $(DIST_PPC)/test-package-stage/mochitest/automation.py \
              $(DIST_X86)/test-package-stage/mochitest/;              \
            cp $(DIST_PPC)/test-package-stage/reftest/automation.py   \
              $(DIST_X86)/test-package-stage/reftest/;                \
            $(TOPSRCDIR)/build/macosx/universal/unify                 \
+             --unify-with-sort "all-test-dirs\.list$$"               \
              $(DIST_PPC)/test-package-stage                          \
              $(DIST_X86)/test-package-stage                          \
              $(DIST_UNI)/test-package-stage; fi
--- a/build/macosx/universal/unify
+++ b/build/macosx/universal/unify
@@ -48,16 +48,17 @@ B<unify> - Mac OS X universal binary pac
 
 B<unify>
 I<ppc-path>
 I<x86-path>
 I<universal-path>
 [B<--dry-run>]
 [B<--only-one> I<action>]
 [B<--verbosity> I<level>]
+[B<--unify-with-sort> I<regex>]
 
 =head1 DESCRIPTION
 
 I<unify> merges any two architecture-specific files or directory trees
 into a single file or tree suitable for use on either architecture as a
 "fat" or "universal binary."
 
 Architecture-specific Mach-O files will be merged into fat Mach-O files
@@ -129,16 +130,23 @@ I<level> are:
   0 - B<unify> never prints anything.
       (Other programs that B<unify> calls may still print messages.)
   1 - Fatal error messages are printed to stderr.
   2 - Nonfatal warnings are printed to stderr.
   3 - Commands are printed to stdout as they are executed.
 
 The default I<level> is 2.
 
+=item B<--unify-with-sort> I<regex>
+
+Allows merging files matching I<regex> that differ only by the ordering
+of the lines contained within them. The unified file will have its contents
+sorted. This option may be given multiple times to specify multiple
+regexes for matching files.
+
 =back
 
 =head1 EXAMPLES
 
 =over 5
 
 =item Create a universal .app bundle from two architecture-specific .app
 bundles:
@@ -178,23 +186,27 @@ L<cmp(1)>, L<ditto(1)>, L<lipo(1)>
 
 use Archive::Zip(':ERROR_CODES');
 use Errno;
 use Fcntl;
 use File::Compare;
 use File::Copy;
 use Getopt::Long;
 
-my (%gConfig, $gDryRun, $gOnlyOne, $gVerbosity);
+my (%gConfig, $gDryRun, $gOnlyOne, $gVerbosity, @gSortMatches);
 
 sub argumentEscape(@);
 sub command(@);
 sub compareZipArchives($$);
 sub complain($$@);
 sub copyIfIdentical($$$);
+sub slurp($);
+sub compare_sorted($$);
+sub copy_sorted($$);
+sub copyIfIdenticalWhenSorted($$$);
 sub createUniqueFile($$);
 sub makeUniversal($$$);
 sub makeUniversalDirectory($$$);
 sub makeUniversalInternal($$$$);
 sub makeUniversalFile($$$);
 sub usage();
 sub readZipCRCs($);
 
@@ -223,22 +235,24 @@ sub readZipCRCs($);
 %gConfig = (
   'cmd_lipo' => 'lipo',
   'cmd_rm'   => 'rm',
 );
 
 $gDryRun = 0;
 $gOnlyOne = 'copy';
 $gVerbosity = 2;
+@gSortMatches = ();
 
 Getopt::Long::Configure('pass_through');
-GetOptions('dry-run'     => \$gDryRun,
-           'only-one=s'  => \$gOnlyOne,
-           'verbosity=i' => \$gVerbosity,
-           'config=s'    => \%gConfig); # "hidden" option not in usage()
+GetOptions('dry-run'           => \$gDryRun,
+           'only-one=s'        => \$gOnlyOne,
+           'verbosity=i'       => \$gVerbosity,
+           'unify-with-sort=s' => \@gSortMatches,
+           'config=s'          => \%gConfig); # "hidden" option not in usage()
 
 if (scalar(@ARGV) != 3 || $gVerbosity < 0 || $gVerbosity > 3 ||
     ($gOnlyOne ne 'skip' && $gOnlyOne ne 'copy' && $gOnlyOne ne 'fail')) {
   usage();
   exit(1);
 }
 
 if (!makeUniversal($ARGV[0],$ARGV[1],$ARGV[2])) {
@@ -476,16 +490,126 @@ sub copyIfIdentical($$$) {
       unlink($target);
       return 0;
     }
   }
 
   return 1;
 }
 
+# slurp($file)
+#
+# Read the contents of $file into an array and return it.
+# Returns undef on error.
+sub slurp($) {
+  my $file = $_[0];
+  open FILE, $file or return undef;
+  my @lines = <FILE>;
+  close FILE;
+  return @lines;
+}
+
+# compare_sorted($file1, $file2)
+#
+# Read the contents of both files into arrays, sort the arrays,
+# and then compare the two arrays for equality.
+#
+# Returns 0 if the sorted array contents are equal, or 1 if not.
+# Returns undef on error.
+sub compare_sorted($$) {
+  my ($file1, $file2) = @_;
+  my @lines1 = sort(slurp($file1));
+  my @lines2 = sort(slurp($file2));
+
+  return undef if !@lines1 || !@lines2;
+  return 1 unless scalar @lines1 == scalar @lines2;
+
+  for (my $i = 0; $i < scalar @lines1; $i++) {
+    return 1 if $lines1[$i] ne $lines2[$i];
+  }
+  return 0;
+}
+
+# copy_sorted($source, $destination)
+#
+# $source and $destination are filenames. Read the contents of $source
+# into an array, sort it, and then write the sorted contents to $destination.
+# Returns 1 on success, and undef on failure.
+sub copy_sorted($$) {
+  my ($src, $dest) = @_;
+  my @lines = sort(slurp($src));
+  return undef unless @lines;
+  open FILE, "> $dest" or return undef;
+  print FILE @lines;
+  close FILE;
+  return 1;
+}
+
+# copyIfIdenticalWhenSorted($source1, $source2, $target)
+#
+# $source1 and $source2 are FileAttrCache objects that are compared, and if
+# identical, copied to path string $target.  The comparison is done by
+# sorting the individual lines within the two files and comparing the results.
+#
+# Returns true on success, false for files that are not equivalent,
+# and undef if an error occurs.
+sub copyIfIdenticalWhenSorted($$$) {
+  my ($source1, $source2, $target);
+  ($source1, $source2, $target) = @_;
+
+  if ($gVerbosity >= 3 || $gDryRun) {
+    print('cmp -s '.
+          join(' ',argumentEscape($source1->path(), $source2->path()))."\n");
+  }
+  my ($comparison);
+  if (!defined($comparison = compare_sorted($source1->path(),
+                                            $source2->path())) ||
+      $comparison == -1) {
+    return complain(1, 'copyIfIdenticalWhenSorted: compare: '.$!
+                    .' while comparing:',
+                      $source1->path(),
+                      $source2->path());
+  }
+  if ($comparison != 0) {
+    return complain(1, 'copyIfIdenticalWhenSorted: files differ:',
+                    $source1->path(),
+                    $source2->path());
+  }
+
+  if ($gVerbosity >= 3 || $gDryRun) {
+    print('cp '.
+          join(' ',argumentEscape($source1->path(), $target))."\n");
+  }
+
+  if (!$gDryRun) {
+    my ($isExecutable);
+
+    # Set the execute bits (as allowed by the umask) on the new file if any
+    # execute bit is set on either old file.
+    $isExecutable = $source1->lIsExecutable() ||
+                    (defined($source2) && $source2->lIsExecutable());
+
+    if (!createUniqueFile($target, $isExecutable ? 0777 : 0666)) {
+      # createUniqueFile printed an error.
+      return 0;
+    }
+
+    if (!copy_sorted($source1->path(), $target)) {
+      complain(1, 'copyIfIdenticalWhenSorted: copy_sorted: '.$!
+               .' while copying',
+               $source1->path(),
+               $target);
+      unlink($target);
+      return 0;
+    }
+  }
+
+  return 1;
+}
+
 # createUniqueFile($path, $mode)
 #
 # Creates a new plain empty file at pathname $path, provided it does not
 # yet exist.  $mode is used as the file mode.  The actual file's mode will
 # be modified by the effective umask.  Returns false if the file could
 # not be created, setting $! to the error.  An error message is printed
 # in the event of failure.
 sub createUniqueFile($$) {
@@ -963,16 +1087,22 @@ sub makeUniversalInternal($$$$) {
                   $fileX86->path());
     }
 
     if ($machPPC) {
       # makeUniversalFile will print an error if it fails.
       return makeUniversalFile($filePPC, $fileX86, $fileTargetPath);
     }
 
+    if (grep { $filePPC->path() =~ m/$_/; } @gSortMatches) {
+      # Regular files, but should be compared with sorting first.
+      # copyIfIdenticalWhenSorted will print an error if it fails.
+      return copyIfIdenticalWhenSorted($filePPC, $fileX86, $fileTargetPath);
+    }
+
     # Regular file.  copyIfIdentical will print an error if it fails.
     return copyIfIdentical($filePPC, $fileX86, $fileTargetPath);
   }
 
   # Special file, don't know how to handle.
   return complain(1, 'makeUniversal: cannot handle special file:',
                   $filePPC->path(),
                   $fileX86->path());