Bug 811527 - generate_diff.py does not ensure that m-c mirror of mozbase is internally compatible;r=wlach ; DONTBUILD because NPOTB
authorJeff Hammel <jhammel@mozilla.com>
Fri, 04 Jan 2013 13:55:29 -0800
changeset 126782 fe4e809058e04311a4e0b5d22aad1bb28295fa23
parent 126781 c43c4835dafd782be47f10937da8c85a4a88648a
child 126783 68ded29fc68e26d2a0f7a20c6375d03d8e38b4b8
push id2151
push userlsblakk@mozilla.com
push dateTue, 19 Feb 2013 18:06:57 +0000
treeherdermozilla-beta@4952e88741ec [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerswlach
bugs811527
milestone20.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 811527 - generate_diff.py does not ensure that m-c mirror of mozbase is internally compatible;r=wlach ; DONTBUILD because NPOTB
testing/mozbase/generate_diff.py
testing/mozbase/setup_development.py
--- a/testing/mozbase/generate_diff.py
+++ b/testing/mozbase/generate_diff.py
@@ -17,32 +17,36 @@ somewhere that has the `cp` command avai
 
 Your mozilla-central repository must have no outstanding changes before this
 script is run.  The repository must also have no untracked
 files that show up in `hg st`.
 
 See: https://bugzilla.mozilla.org/show_bug.cgi?id=702832
 """
 
-from __future__ import with_statement
-
+import imp
 import optparse
 import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 
+from pkg_resources import parse_version
 from subprocess import check_call as call
 
 # globals
 here = os.path.dirname(os.path.abspath(__file__))
 MOZBASE = 'git://github.com/mozilla/mozbase.git'
 version_regex = r"""PACKAGE_VERSION *= *['"]([0-9.]+)["'].*"""
+setup_development = imp.load_source('setup_development',
+                                    os.path.join(here, 'setup_development.py'))
+current_package = None
+current_package_info = {}
 
 def error(msg):
     """err out with a message"""
     print >> sys.stdout, msg
     sys.exit(1)
 
 def remove(path):
     """remove a file or directory"""
@@ -118,16 +122,61 @@ def parse_versions(*args):
             directory = arg
             version = None
         retval.append((directory, version))
     return retval
 
 def version_tag(directory, version):
     return '%s-%s' % (directory, version)
 
+def setup(**kwargs):
+    """monkey-patch function for setuptools.setup"""
+    assert current_package
+    current_package_info[current_package] = kwargs
+
+def check_consistency(*package_info):
+    """checks consistency between a set of packages"""
+
+    # set versions and dependencies per package
+    versions = {}
+    dependencies = {}
+    for package in package_info:
+        name = package['name']
+        versions[name] = package['version']
+        for dep in package.get('install_requires', []):
+            dependencies.setdefault(name, []).append(dep)
+
+    func_map = {'==': tuple.__eq__,
+                '<=': tuple.__le__,
+                '>=': tuple.__ge__}
+
+    # check dependencies
+    errors = []
+    for package, deps in dependencies.items():
+        for dep in deps:
+            parsed = setup_development.dependency_info(dep)
+            if parsed['Name'] not in versions:
+                # external dependency
+                continue
+            if parsed.get('Version') is None:
+                # no version specified for dependency
+                continue
+
+            # check versions
+            func = func_map[parsed['Type']]
+            comparison = func(parse_version(versions[parsed['Name']]),
+                              parse_version(parsed['Version']))
+
+            if not comparison:
+                # an error
+                errors.append("Dependency for package '%s' failed: %s-%s not %s %s" % (package, parsed['Name'], versions[parsed['Name']], parsed['Type'], parsed['Version']))
+
+    # raise an Exception if errors exist
+    if errors:
+        raise Exception('\n'.join(errors))
 
 ###
 
 def main(args=sys.argv[1:]):
     """command line entry point"""
 
     # parse command line options
     usage = '%prog [options] package1[=version1] <package2=version2> <...>'
@@ -146,16 +195,34 @@ def main(args=sys.argv[1:]):
     options, args = parser.parse_args(args)
     if args:
         versions = parse_versions(*args)
     else:
         parser.print_help()
         parser.exit()
     output = options.output
 
+    # gather info from current mozbase packages
+    global current_package
+    setuptools = sys.modules.get('setuptools')
+    sys.modules['setuptools'] = sys.modules[__name__]
+    try:
+        for package in setup_development.mozbase_packages:
+            current_package = package
+            imp.load_source('setup', os.path.join(here, package, 'setup.py'))
+    finally:
+        current_package = None
+        sys.modules.pop('setuptools')
+        if setuptools:
+            sys.modules['setuptools'] = setuptools
+    assert set(current_package_info.keys()) == set(setup_development.mozbase_packages)
+
+    # check consistency of current set of packages
+    check_consistency(*current_package_info.values())
+
     # calculate hg root
     hg_root = os.path.dirname(os.path.dirname(here))  # testing/mozbase
     hg_dir = os.path.join(hg_root, '.hg')
     assert os.path.exists(hg_dir) and os.path.isdir(hg_dir)
 
     # ensure there are no outstanding changes to m-c
     process = subprocess.Popen(['hg', 'diff'], cwd=here, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = process.communicate()
@@ -198,16 +265,40 @@ def main(args=sys.argv[1:]):
                             break
                     else:
                         error("Cannot find PACKAGE_VERSION in %s" % setup_py)
 
             tag = version_tag(directory, version)
             if tag not in _tags:
                 error("Tag for '%s' -- %s -- not in tags")
 
+        # ensure that the versions to mirror are compatible with what is in m-c
+        old_package_info = current_package_info.copy()
+        setuptools = sys.modules.get('setuptools')
+        sys.modules['setuptools'] = sys.modules[__name__]
+        try:
+            for directory, version in versions:
+
+                # checkout appropriate revision of mozbase
+                tag = version_tag(directory, version)
+                checkout(src, tag)
+
+                # update the package information
+                setup_py = os.path.join(src, directory, 'setup.py')
+                current_package = directory
+                imp.load_source('setup', setup_py)
+        finally:
+            current_package = None
+            sys.modules.pop('setuptools')
+            if setuptools:
+                sys.modules['setuptools'] = setuptools
+        checkout(src, 'HEAD')
+        current_package_info['mozprocess']['version'] = '0.9'
+        check_consistency(*current_package_info.values())
+
         # copy mozbase directories to m-c
         for directory, version in versions:
 
             # checkout appropriate revision of mozbase
             tag = version_tag(directory, version)
             checkout(src, tag)
 
             # replace the directory
--- a/testing/mozbase/setup_development.py
+++ b/testing/mozbase/setup_development.py
@@ -10,48 +10,53 @@ Setup mozbase packages for development.
 Packages may be specified as command line arguments.
 If no arguments are given, install all packages.
 
 See https://wiki.mozilla.org/Auto-tools/Projects/MozBase
 """
 
 import pkg_resources
 import os
+import subprocess
 import sys
 from optparse import OptionParser
-
 from subprocess import PIPE
 try:
     from subprocess import check_call as call
 except ImportError:
     from subprocess import call
 
 
 # directory containing this file
 here = os.path.dirname(os.path.abspath(__file__))
 
 # all python packages
-all_packages = [i for i in os.listdir(here)
-                if os.path.exists(os.path.join(here, i, 'setup.py'))]
+mozbase_packages = [i for i in os.listdir(here)
+                    if os.path.exists(os.path.join(here, i, 'setup.py'))]
+extra_packages = ["sphinx"]
 
 def cycle_check(order, dependencies):
     """ensure no cyclic dependencies"""
     order_dict = dict([(j, i) for i, j in enumerate(order)])
     for package, deps in dependencies.items():
         index = order_dict[package]
         for d in deps:
             assert index > order_dict[d], "Cyclic dependencies detected"
 
 def info(directory):
     "get the package setup.py information"
 
     assert os.path.exists(os.path.join(directory, 'setup.py'))
 
     # setup the egg info
-    call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE)
+    try:
+        call([sys.executable, 'setup.py', 'egg_info'], cwd=directory, stdout=PIPE)
+    except subprocess.CalledProcessError:
+        print "Error running setup.py in %s" % directory
+        raise
 
     # get the .egg-info directory
     egg_info = [entry for entry in os.listdir(directory)
                 if entry.endswith('.egg-info')]
     assert len(egg_info) == 1, 'Expected one .egg-info directory in %s, got: %s' % (directory, egg_info)
     egg_info = os.path.join(directory, egg_info[0])
     assert os.path.isdir(egg_info), "%s is not a directory" % egg_info
 
@@ -143,22 +148,25 @@ def main(args=sys.argv[1:]):
     parser = OptionParser(usage=usage, description=__doc__)
     parser.add_option('-d', '--dependencies', dest='list_dependencies',
                       action='store_true', default=False,
                       help="list dependencies for the packages")
     parser.add_option('--list', action='store_true', default=False,
                       help="list what will be installed")
     options, packages = parser.parse_args(args)
 
+    install_extra_packages = False
+
     if not packages:
         # install all packages
-        packages = sorted(all_packages)
+        packages = sorted(mozbase_packages)
+        install_extra_packages = True
 
     # ensure specified packages are in the list
-    assert set(packages).issubset(all_packages), "Packages should be in %s (You gave: %s)" % (all_packages, packages)
+    assert set(packages).issubset(mozbase_packages), "Packages should be in %s (You gave: %s)" % (mozbase_packages, packages)
 
     if options.list_dependencies:
         # list the package dependencies
         for package in packages:
             print '%s: %s' % get_dependencies(os.path.join(here, package))
         parser.exit()
 
     # gather dependencies
@@ -177,30 +185,30 @@ def main(args=sys.argv[1:]):
             alldeps[dependency_info(dep)['Name']] = ''.join(dep.split())
 
     # indirect dependencies
     flag = True
     while flag:
         flag = False
         for value in deps.values():
             for dep in value:
-                if dep in all_packages and dep not in deps:
+                if dep in mozbase_packages and dep not in deps:
                     key, value = get_dependencies(os.path.join(here, dep))
                     deps[key] = [sanitize_dependency(dep) for dep in value]
 
                     for dep in value:
                         alldeps[sanitize_dependency(dep)] = ''.join(dep.split())
                     mapping[package] = key
                     flag = True
                     break
             if flag:
                 break
 
     # get the remaining names for the mapping
-    for package in all_packages:
+    for package in mozbase_packages:
         if package in mapping:
             continue
         key, value = get_dependencies(os.path.join(here, package))
         mapping[package] = key
 
     # unroll dependencies
     unrolled = unroll_dependencies(deps)
 
@@ -211,26 +219,30 @@ def main(args=sys.argv[1:]):
     unrolled = [package for package in unrolled if package in reverse_mapping]
 
     if options.list:
         # list what will be installed
         for package in unrolled:
             print package
         parser.exit()
 
+    # set up the packages for development
+    for package in unrolled:
+        call([sys.executable, 'setup.py', 'develop', '--no-deps'],
+             cwd=os.path.join(here, reverse_mapping[package]))
+
     # install non-mozbase dependencies
-    # (currently none on modern python)
     # these need to be installed separately and the --no-deps flag
     # subsequently used due to a bug in setuptools; see
     # https://bugzilla.mozilla.org/show_bug.cgi?id=759836
     pypi_deps = dict([(i, j) for i,j in alldeps.items()
                       if i not in unrolled])
     for package, version in pypi_deps.items():
         # easy_install should be available since we rely on setuptools
         call(['easy_install', version])
 
-    # set up the packages for development
-    for package in unrolled:
-        call([sys.executable, 'setup.py', 'develop', '--no-deps'],
-             cwd=os.path.join(here, reverse_mapping[package]))
+    # install extra non-mozbase packages if desired
+    if install_extra_packages:
+        for package in extra_packages:
+            call(['easy_install', package])
 
 if __name__ == '__main__':
     main()