Bug 1355625 - DO NOT LAND - Add |mach compare-mach-and-gradle| to compare builds. draft
authorNick Alexander <nalexander@mozilla.com>
Mon, 08 May 2017 16:16:29 -0700
changeset 597751 e97a56abdb670e1d83e4e10b81d3820ee2ac5751
parent 597750 25814a3d27e299bf29ca98d5ff6f3d301a040f02
child 597752 90a2b1b50172199f550d357980ef17cf3e8eb792
push id65017
push usernalexander@mozilla.com
push dateTue, 20 Jun 2017 22:34:49 +0000
bugs1355625
milestone56.0a1
Bug 1355625 - DO NOT LAND - Add |mach compare-mach-and-gradle| to compare builds. I developed and used these scripts to compare the Gradle and moz.build build outputs to each other. MozReview-Commit-ID: 5EXaBlLphAS
mobile/android/mach_commands.py
xmldiff.py
--- a/mobile/android/mach_commands.py
+++ b/mobile/android/mach_commands.py
@@ -209,8 +209,326 @@ class AutophoneCommands(MachCommandBase)
             return 2
         if not runner.configure():
             runner.save_config()
             return 3
         runner.save_config()
         runner.launch_autophone()
         runner.command_prompts()
         return 0
+
+import subprocess
+
+# This unorthodox structure is so these can be accessed via multiprocessing.
+javap = [None]
+compiled_mach = {}
+compiled_gradle = {}
+
+def javap_comparator(name):
+    if name not in compiled_mach:
+        return 0
+    if name not in compiled_gradle:
+        return 0
+    # Short circuit if we can.
+    if compiled_mach[name]['sha256'] == compiled_gradle[name]['sha256']:
+        return 0
+
+    if javap[0]:
+        a = subprocess.check_output([javap[0], '-private', '-constants', compiled_mach[name]['path']])
+        b = subprocess.check_output([javap[0], '-private', '-constants', compiled_gradle[name]['path']])
+        if a == b:
+            return 0
+
+    return name
+
+# Modified from http://stackoverflow.com/a/20380514.
+def png_dimensions(fileobj):
+    import imghdr
+    import struct
+
+    head = fileobj.read(24)
+    if len(head) != 24:
+        return None
+    if imghdr.what(None, head) == 'png':
+        check = struct.unpack('>i', head[4:8])[0]
+        if check != 0x0d0a1a0a:
+            return
+        width, height = struct.unpack('>ii', head[16:24])
+        return (width, height)
+
+    return None
+
+@CommandProvider
+class MachCommands(MachCommandBase):
+    # TODO: explain that match is not anchored to the beginning of the given path.
+    @CommandArgument('--filter', default=None,
+        help='Only compare build outputs with relative paths that match the given regular expression.')
+    @CommandArgument('--compare-merged-manifests', action='store_true',
+        help='Compare the merged manifests of Mach and Gradle build outputs.')
+    @CommandArgument('--compare-merged-resources', action='store_true',
+        help='Compare merged resources of Mach and Gradle build outputs.')
+    @CommandArgument('--compare-resource-ids', action='store_true',
+        help='Compare allocated resource IDs of Mach and Gradle build outputs.')
+    @CommandArgument('--compare-resource-contents', action='store_true',
+        help='Compare resource contents of Mach and Gradle build outputs.')
+    @CommandArgument('--compare-classes', action='store_true',
+        help='Compare compiled .class output of Mach and Gradle build outputs.')
+    @CommandArgument('--compare-javap', action='store_true',
+        help='Compare javap output of Mach and Gradle build outputs.')
+    @CommandArgument('--verbose', action='store_true',
+        help='Log informative status messages.')
+    @Command('compare-mach-and-gradle', category='devenv',
+        description='Compare Mach and Gradle build outputs.',
+        conditions=[conditions.is_android])
+    def compare(self,
+                compare_merged_manifests=True,
+                compare_merged_resources=True,
+                compare_resource_ids=True,
+                compare_resource_contents=True,
+                compare_classes=False, filter=None,
+                compare_javap=False,
+                verbose=False):
+        from collections import defaultdict, OrderedDict
+        import hashlib
+        import itertools
+        import multiprocessing
+        import re
+        from mozbuild.util import hash_file 
+        from mozpack.files import FileFinder
+
+        retval = 0
+
+        if compare_merged_manifests:
+            if verbose:
+                print("Comparing manifests included in Mach and Gradle builds")
+
+            mach = mozpath.join(self.topobjdir,
+                'mobile', 'android', 'base',
+                'AndroidManifest.xml')
+
+            gradle = mozpath.join(self.topobjdir,
+                'gradle', 'build', 'mobile', 'android', 'app',
+                'intermediates', 'manifests', 'full', 'officialAustralis', 'release', 'AndroidManifest.xml')
+
+            print("python xmldiff.py 'diff -U2' {0} {1}".format(mach, gradle))
+
+        if compare_merged_resources:
+            if verbose:
+                print("Comparing the set of resource names after merging produced by Mach and Gradle builds")
+
+            mach = defaultdict(set)
+            gradle = defaultdict(set)
+
+            finder = FileFinder(mozpath.join(self.topobjdir,
+                                             'mobile', 'android', 'base', 'merged'),
+                                find_executables=False)
+
+            for path, _ in finder.find('*/*'):
+                dir, name = path.split('/')
+                mach[name].add(dir)
+
+            finder = FileFinder(mozpath.join(self.topobjdir,
+                                             'gradle', 'build', 'mobile', 'android', 'app', 'intermediates', 'res', 'merged', 'officialAustralis', 'release'),
+                                find_executables=False)
+
+            for path, _ in finder.find('*/*'):
+                dir, name = path.split('/')
+                gradle[name].add(dir)
+                
+            mach_only = list(sorted(set(mach.keys()).difference(set(gradle.keys()))))
+            for name in mach_only:
+                retval |= 0b010000000
+                print("mach-only:", name)
+
+            gradle_only = list(sorted(set(gradle.keys()).difference(set(mach.keys()))))
+            for name in gradle_only:
+                retval |= 0b100000000
+                print("Gradle-only:", name)
+
+            both = list(sorted(set(mach.keys()).intersection(set(gradle.keys()))))
+            for name in both:
+                if mach[name] != gradle[name]:
+                    retval |= 0b001000000
+                    print("Different:", name)
+                    print(list(sorted(mach[name])))
+                    print(list(sorted(gradle[name])))
+
+            if verbose and not (retval & 0b111000000):
+                print("The sets of resource names after merging are identical!")
+
+        if compare_resource_contents:
+            if verbose:
+                print("Comparing the resource contents produced by Mach and Gradle builds")
+
+            # The builds produce .ap_ files containing all of the
+            # resources.  These .ap_ files are really ZIP files.  We
+            # Walk the .ap_ files, verifying the contents are the
+            # same.  Since aapt sorts the resources, we should even
+            # have the same order; we won't have the exact same ZIP
+            # files, however, since the embedded timestamps will be
+            # different.
+            import zipfile
+            mach = zipfile.ZipFile(mozpath.join(self.topobjdir,
+                                                'mobile', 'android', 'base',
+                                                'gecko.ap_'))
+
+            gradle = zipfile.ZipFile(mozpath.join(self.topobjdir,
+                                                  'gradle', 'build', 'mobile', 'android',
+                                                  'app', 'intermediates', 'res', 'resources-official-australis-release.ap_'))
+
+            mach_only = list(sorted(set(mach.namelist()).difference(set(gradle.namelist()))))
+            for name in mach_only:
+                retval |= 0b010000000
+                print("mach-only:", name)
+
+            gradle_only = list(sorted(set(gradle.namelist()).difference(set(mach.namelist()))))
+            for name in gradle_only:
+                retval |= 0b100000000
+                print("Gradle-only:", name)
+
+            both = list(sorted(set(mach.namelist()).intersection(set(gradle.namelist()))))
+
+            for name in both:
+                m = mach.getinfo(name)
+                g = gradle.getinfo(name)
+                if m.CRC != g.CRC or m.file_size != g.file_size or m.compress_size != g.compress_size:
+                    retval |= 0b001000000
+                    if name.endswith('.png'):
+                        print("Different:", name, (m.CRC, g.CRC), (m.file_size, g.file_size), (m.compress_size, g.compress_size), (png_dimensions(mach.open(name)), png_dimensions(gradle.open(name))))
+                    else:
+                        print("Different:", name)
+
+
+        if compare_resource_ids:
+            if verbose:
+                print("Comparing resource IDs allocated in Mach and Gradle build outputs")
+
+            fm = open(mozpath.join(self.topobjdir,
+                                   'mobile', 'android', 'base',
+                                   'R.txt'))
+
+            fg = open(mozpath.join(self.topobjdir,
+                                   'gradle', 'build', 'mobile', 'android', 'app', 'intermediates', 'symbols', 'officialAustralis', 'release', 'R.txt'))
+
+            mach = OrderedDict([])
+            for line in fm:
+                (type, dir, name, id) = line.split(None, 3)
+                if not type.endswith("[]"):
+                    mach[name] = (type, dir, int(id, base=16))
+                else:
+                    # Handle arrays later.
+                    pass
+
+            gradle = OrderedDict([])
+            for line in fg:
+                (type, dir, name, id) = line.split(None, 3)
+                if not type.endswith("[]"):
+                    gradle[name] = (type, dir, int(id, base=16))
+                else:
+                    # Handle arrays later.
+                    pass
+
+            mach_only = list(sorted(set(mach.keys()).difference(set(gradle.keys()))))
+            for name in mach_only:
+                retval |= 0b010000
+                print("mach-only:", name)
+
+            gradle_only = list(sorted(set(gradle.keys()).difference(set(mach.keys()))))
+            for name in gradle_only:
+                retval |= 0b100000
+                print("Gradle-only:", name)
+
+            both = list(sorted(set(mach.keys()).intersection(set(gradle.keys()))))
+
+            jumps = []
+            
+            offset = 0
+            for i, (m, g) in enumerate(zip(mach.items(), gradle.items())):
+                (mname, (mtype, mdir, mid)) = m
+                (gname, (gtype, gdir, gid)) = g
+
+                if mname != gname:
+                    retval |= 0b001000
+                    print("Different resource order at index:", hex(i), mname, gname)
+
+                if mid != gid + offset:
+                    retval |= 0b001000
+
+                    print("Different resource ids at index:", hex(i), (mname, hex(mid)), (gname, hex(gid)))
+                    jumps.append((m, g))
+                    offset = mid - gid
+
+            for jump in jumps:
+                print(jump)
+
+            if verbose and not (retval & 0b000111000):
+                print("The resource ID maps produced in R.txt are identical!")
+
+
+        if compare_classes or compare_javap:
+            if verbose:
+                print("Comparing compiled .class output of Mach and Gradle build outputs{0}"
+                      .format("" if not filter else "matching regular expression {0}".format(filter)))
+
+            if filter:
+                filter = re.compile(filter)
+
+            for part in ('base', 'geckoview', 'javaaddons', 'stumbler', 'thirdparty'):
+                finder = FileFinder(mozpath.join(self.topobjdir,
+                                                 'mobile', 'android', part),
+                                    find_executables=False)
+
+                for path, _ in finder.find('**/*-classes/**/*.class'):
+                    _, name = path.split('-classes/', 1)
+
+                    if filter and not filter.search(name):
+                        continue
+
+                    compiled_mach[name] = {}
+                    compiled_mach[name]['path'] = mozpath.join(finder.base, path)
+                    compiled_mach[name]['sha256'] = hash_file(mozpath.join(finder.base, path))
+
+            for (part, product) in (('app', 'officialAustralis'),
+                                    ('geckoview', ''),
+                                    ('thirdparty', '')):
+                finder = FileFinder(mozpath.join(self.topobjdir,
+                                                 'gradle', 'build', 'mobile', 'android', part,
+                                                 'intermediates', 'classes',
+                                                 product,
+                                                 'release'),
+                                    find_executables=False)
+                for path, _ in finder.find('**/*.class'):
+                    name = path
+
+                    if filter and not filter.search(name):
+                        continue
+
+                    compiled_gradle[name] = {}
+                    compiled_gradle[name]['path'] = mozpath.join(finder.base, path)
+                    compiled_gradle[name]['sha256'] = hash_file(mozpath.join(finder.base, path))
+
+            mach_only = list(sorted(set(compiled_mach.keys()).difference(set(compiled_gradle.keys()))))
+            for name in mach_only:
+                retval |= 0b010
+                print("mach-only:", name)
+
+            gradle_only = list(sorted(set(compiled_gradle.keys()).difference(set(compiled_mach.keys()))))
+            for name in gradle_only:
+                retval |= 0b100
+                print("Gradle-only:", name)
+
+            both = list(sorted(set(compiled_mach.keys()).intersection(set(compiled_gradle.keys()))))
+
+            if compare_javap:
+                javap[0] = mozpath.join(os.path.dirname(self.substs['JAVA']), 'javap')
+
+            pool = multiprocessing.Pool(4)
+            for name in pool.imap(javap_comparator, both):
+                if name:
+                    retval |= 0b001
+                    print("Different:", name)
+                    print(compiled_mach[name]['path'])
+                    print(compiled_gradle[name]['path'])
+
+            if verbose and not (retval & 0b000000111):
+                print("Compiled .class files are identical!")
+
+        return retval
new file mode 100644
--- /dev/null
+++ b/xmldiff.py
@@ -0,0 +1,165 @@
+##########################################################################
+#
+#  xmldiff
+#
+#    Simple utility script to enable a diff of two XML files in a way 
+#     that ignores the order or attributes and elements.
+#
+#    Dale Lane (email@dalelane.co.uk)
+#     6 Oct 2014
+#
+##########################################################################
+#
+#  Overview
+#    The approach is to sort both files by attribute and element, and 
+#     then reuse an existing diff implementation on the sorted files.
+#
+#  Arguments
+#    <diffcommand> the command that should be run to diff the sorted files
+#    <filename1>   the first XML file to diff
+#    <filename2>   the second XML file to diff
+#
+#  Background
+#    http://dalelane.co.uk/blog/?p=3225
+#
+##########################################################################
+
+import os, sys, subprocess, platform
+import lxml.etree as le
+from operator import attrgetter
+
+le.register_namespace('android', 'http://schemas.android.com/apk/res/android')
+
+#
+# Check required arguments
+if len(sys.argv) != 4:
+    print ("Usage: python xmldiff.py <diffcommand> <filename1> <filename2>")
+    quit()
+
+
+#
+# Prepares the location of the temporary file that will be created by xmldiff
+def createFileObj(prefix, name):
+    return { 
+        "filename" : os.path.abspath(name),
+        "tmpfilename" : "." + prefix + "." + os.path.basename(name)
+    }
+
+
+#
+# Function to sort XML elements by id 
+#  (where the elements have an 'id' attribute that can be cast to an int)
+def sortbyid(elem):
+    return elem.get('{http://schemas.android.com/apk/res/android}name') or \
+        elem.get('{http://schemas.android.com/apk/res/android}label')
+    # if name:
+    #     return name
+    # return 0
+
+
+#
+# Function to sort XML elements by their text contents
+def sortbytext(elem):
+    text = elem.text
+    if text:
+        return text
+    else:
+        return ''
+
+
+#
+# Function to sort XML attributes alphabetically by key
+#  The original item is left unmodified, and it's attributes are 
+#  copied to the provided sorteditem
+def sortAttrs(item, sorteditem):
+    attrkeys = sorted(item.keys())
+    for key in attrkeys:
+        sorteditem.set(key, item.get(key))
+
+
+# 
+# Function to sort XML elements
+#  The sorted elements will be added as children of the provided newroot
+#  This is a recursive function, and will be called on each of the children
+#  of items.
+def sortElements(items, newroot):
+    # The intended sort order is to sort by XML element name
+    #  If more than one element has the same name, we want to 
+    #   sort by their text contents.
+    #  If more than one element has the same name and they do 
+    #   not contain any text contents, we want to sort by the 
+    #   value of their ID attribute.
+    #  If more than one element has the same name, but has 
+    #   no text contents or ID attribute, their order is left
+    #   unmodified.
+    #
+    # We do this by performing three sorts in the reverse order
+    items = sorted(items, key=sortbytext)
+    items = sorted(items, key=sortbyid)
+    items = sorted(items, key=attrgetter('tag'))
+
+    # Once sorted, we sort each of the items
+    for item in items:
+        if item.tag is le.Comment:
+            continue
+
+        # Create a new item to represent the sorted version 
+        #  of the next item, and copy the tag name and contents
+        newitem = le.Element(item.tag)
+        if item.text and item.text.isspace() == False:
+            newitem.text = item.text
+
+        # Copy the attributes (sorted by key) to the new item
+        sortAttrs(item, newitem)
+
+        # Copy the children of item (sorted) to the new item
+        sortElements(list(item), newitem)
+
+        # Append this sorted item to the sorted root
+        newroot.append(newitem)
+
+
+# 
+# Function to sort the provided XML file
+#  fileobj.filename will be left untouched
+#  A new sorted copy of it will be created at fileobj.tmpfilename 
+def sortFile(fileobj):
+    with open(fileobj['filename'], 'r') as original:
+        # parse the XML file and get a pointer to the top
+        xmldoc = le.parse(original)
+        xmlroot = xmldoc.getroot()
+
+        # create a new XML element that will be the top of 
+        #  the sorted copy of the XML file
+        newxmlroot = le.Element(xmlroot.tag)
+
+        # create the sorted copy of the XML file
+        sortAttrs(xmlroot, newxmlroot)
+        sortElements(list(xmlroot), newxmlroot)
+
+        # write the sorted XML file to the temp file
+        newtree = le.ElementTree(newxmlroot)
+        with open(fileobj['tmpfilename'], 'wb') as newfile:
+            newtree.write(newfile, pretty_print=True)
+
+
+#
+# sort each of the specified files
+filefrom = createFileObj("from", sys.argv[2])
+sortFile(filefrom)
+fileto = createFileObj("to", sys.argv[3])
+sortFile(fileto)
+
+#
+# invoke the requested diff command to compare the two sorted files
+if platform.system() == "Windows":
+    sp = subprocess.Popen([ "cmd", "/c", sys.argv[1] + " " + filefrom['tmpfilename'] + " " + fileto['tmpfilename'] ])
+    sp.communicate()
+else:
+    sp = subprocess.Popen([ "/bin/bash", "-i", "-c", sys.argv[1] + " " + os.path.abspath(filefrom['tmpfilename']) + " " + os.path.abspath(fileto['tmpfilename']) ])
+    sp.communicate()
+
+#
+# cleanup - delete the temporary sorted files after the diff terminates
+os.remove(filefrom['tmpfilename'])
+os.remove(fileto['tmpfilename'])