Bug 805190 - document new procedure for mozbase mirroring and versioning;r=wlach ; DONTBUILD because NPOTB
authorJeff Hammel <jhammel@mozilla.com>
Sat, 27 Oct 2012 08:44:30 -0700
changeset 111743 e2c6b346d1dfec97d9a1fef800ed97169d304942
parent 111742 08c38354ae7c243a247876c8688c26b979ecb4d2
child 111744 cf52e5234e911e55e9de514fcd7be8b729d9c21e
push id93
push usernmatsakis@mozilla.com
push dateWed, 31 Oct 2012 21:26:57 +0000
reviewerswlach
bugs805190
milestone19.0a1
Bug 805190 - document new procedure for mozbase mirroring and versioning;r=wlach ; DONTBUILD because NPOTB
testing/mozbase/generate_diff.py
--- a/testing/mozbase/generate_diff.py
+++ b/testing/mozbase/generate_diff.py
@@ -1,75 +1,94 @@
 #!/usr/bin/env python
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """
+Given a list of packages and the versions to mirror,
 generate a diff appropriate for mirroring
 https://github.com/mozilla/mozbase
 to http://hg.mozilla.org/mozilla-central/file/tip/testing/mozbase
 
+If a package version is not given, the latest version will be used.
+
 Note that this shells out to `cp` for simplicity, so you should run this
 somewhere that has the `cp` command available.
 
 Your mozilla-central repository must have no outstanding changes before this
 script is run.  The repository must also have no untracked
 files that show up in `hg st`.
 
 See: https://bugzilla.mozilla.org/show_bug.cgi?id=702832
 """
 
 from __future__ import with_statement
 
 import optparse
 import os
+import re
 import shutil
 import subprocess
 import sys
 import tempfile
 
 from subprocess import check_call as call
 
 # globals
 here = os.path.dirname(os.path.abspath(__file__))
 MOZBASE = 'git://github.com/mozilla/mozbase.git'
-
-# paths we don't want in m-c from mozbase's github repo
-git_excludes = ['.git',
-                '.gitignore',
-                'versionbump.py']
-
-# top-level paths we want to keep in m-c that aren't in the github repo
-keep = ['Makefile.in',
-        'generate_diff.py']
+version_regex = r"""PACKAGE_VERSION *= *['"]([0-9.]+)["'].*"""
 
 def error(msg):
     """err out with a message"""
     print >> sys.stdout, msg
     sys.exit(1)
 
 def remove(path):
     """remove a file or directory"""
     if os.path.isdir(path):
         shutil.rmtree(path)
     else:
         os.remove(path)
 
+### git functions
+
 def latest_commit(git_dir):
     """returns last commit hash from a git repository directory"""
     command = ['git', 'log', '--pretty=format:%H',  'HEAD^..HEAD']
     process = subprocess.Popen(command,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                cwd=git_dir)
     stdout, stderr = process.communicate()
     return stdout.strip()
 
+def tags(git_dir):
+    """return all tags in a git repository"""
+
+    command = ['git', 'tag']
+    process = subprocess.Popen(command,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE,
+                               cwd=git_dir)
+    stdout, stderr = process.communicate()
+    return [line.strip() for line in stdout.strip().splitlines()]
+
+def checkout(git_dir, tag):
+    """checkout a tagged version of a git repository"""
+
+    command = ['git', 'checkout', tag]
+    process = subprocess.Popen(command,
+                               cwd=git_dir)
+    process.communicate()
+
+### hg functions
+
 def untracked_files(hg_dir):
     """untracked files in an hg repository"""
     process = subprocess.Popen(['hg', 'st'],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                cwd=hg_dir)
     stdout, stderr = process.communicate()
     lines = [line.strip() for line in stdout.strip().splitlines()]
@@ -80,39 +99,62 @@ def revert(hg_dir, excludes=()):
     """revert a hg repository directory"""
     call(['hg', 'revert', '--no-backup', '--all'], cwd=hg_dir)
     newfiles = untracked_files(hg_dir)
     for f in newfiles:
         path = os.path.join(hg_dir, f)
         if path not in excludes:
             os.remove(path)
 
+
+### version-related functions
+
+def parse_versions(*args):
+    """return a list of 2-tuples of (directory, version)"""
+
+    retval = []
+    for arg in args:
+        if '=' in arg:
+            directory, version = arg.split('=', 1)
+        else:
+            directory = arg
+            version = None
+        retval.append((directory, version))
+    return retval
+
+def version_tag(directory, version):
+    return '%s-%s' % (directory, version)
+
+
+###
+
 def main(args=sys.argv[1:]):
     """command line entry point"""
 
     # parse command line options
-    usage = '%prog output'
+    usage = '%prog [options] package1[=version1] <package2=version2> <...>'
     class PlainDescriptionFormatter(optparse.IndentedHelpFormatter):
         """description formatter for console script entry point"""
         def format_description(self, description):
             if description:
                 return description.strip() + '\n'
             else:
                 return ''
     parser = optparse.OptionParser(usage=usage,
                                    description=__doc__,
                                    formatter=PlainDescriptionFormatter())
+    parser.add_option('-o', '--output', dest='output',
+                      help="specify the output file; otherwise will be in the current directory with a name based on the hash")
     options, args = parser.parse_args(args)
-    if len(args) > 1:
+    if args:
+        versions = parse_versions(*args)
+    else:
         parser.print_help()
         parser.exit()
-    if args:
-        output = args[0]
-    else:
-        output = None
+    output = options.output
 
     # calculate hg root
     hg_root = os.path.dirname(os.path.dirname(here))  # testing/mozbase
     hg_dir = os.path.join(hg_root, '.hg')
     assert os.path.exists(hg_dir) and os.path.isdir(hg_dir)
 
     # ensure there are no outstanding changes to m-c
     process = subprocess.Popen(['hg', 'diff'], cwd=here, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -131,33 +173,51 @@ def main(args=sys.argv[1:]):
         # download mozbase
         call(['git', 'clone', MOZBASE], cwd=tempdir)
         src = os.path.join(tempdir, 'mozbase')
         assert os.path.isdir(src)
         if output is None:
             commit_hash = latest_commit(src)
             output = os.path.join(os.getcwd(), '%s.diff' % commit_hash)
 
-        # remove files from github clone we don't want to copy
-        for path in git_excludes:
-            path = os.path.join(src, path)
-            if not os.path.exists(path):
-                continue
-            remove(path)
+        # get the tags
+        _tags = tags(src)
 
-        # remove the files from m-c we don't want to keep
-        for path in os.listdir(here):
-            if path in keep:
-                continue
-            path = os.path.join(here, path)
-            remove(path)
+        # ensure all directories and tags are available
+        for index, (directory, version) in enumerate(versions):
+            if not version:
+                # choose maximum version from setup.py
+                setup_py = os.path.join(src, directory, 'setup.py')
+                assert os.path.exists(setup_py), "'%s' not found" % setup_py
+                with file(setup_py) as f:
+                    for line in f.readlines():
+                        line = line.strip()
+                        match = re.match(version_regex, line)
+                        if match:
+                            version = match.groups()[0]
+                            versions[index] = (directory, version)
+                            print "Using %s=%s" % (directory, version)
+                            break
+                    else:
+                        error("Cannot find PACKAGE_VERSION in %s" % setup_py)
 
-        # copy mozbase to m-c
-        for path in os.listdir(src):
-            call(['cp', '-r', path, here], cwd=src)
+            tag = version_tag(directory, version)
+            if tag not in _tags:
+                error("Tag for '%s' -- %s -- not in tags")
+
+        # copy mozbase directories to m-c
+        for directory, version in versions:
+
+            # checkout appropriate revision of mozbase
+            tag = version_tag(directory, version)
+            checkout(src, tag)
+
+            # replace the directory
+            remove(os.path.join(here, directory))
+            call(['cp', '-r', directory, here], cwd=src)
 
         # generate the diff and write to output file
         call(['hg', 'addremove'], cwd=hg_root)
         process = subprocess.Popen(['hg', 'diff'],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    cwd=hg_root)
         stdout, stderr = process.communicate()