Bug 538528 - Allow client.py network attempts to restart itself in case of failures; (Av3d) Add a '--retries' option related to check_call_noisy(), Sort hg options, Fix program usage nits.
authorSerge Gautherie <sgautherie.bz@free.fr>
Wed, 17 Feb 2010 16:52:24 +0100
changeset 4931 c4bab0c78356e44c117b9b1f5c0d0ed7704fdee0
parent 4930 2b5f3bd9e71acb05191b69ec5ef0d2391ebae6f2
child 4932 1cdd680a41f6ed4572db76062a705556b1f976a9
push idunknown
push userunknown
push dateunknown
bugs538528
Bug 538528 - Allow client.py network attempts to restart itself in case of failures; (Av3d) Add a '--retries' option related to check_call_noisy(), Sort hg options, Fix program usage nits. r=(Callek, gozer) sr=Standard8. Initial patch by Justin Wood, rewritten by me.
client.py
--- a/client.py
+++ b/client.py
@@ -52,17 +52,17 @@ pyver = sys.version_info
 if pyver[0] <= 1 or (pyver[0] == 2 and pyver[1] < 4):
   sys.exit("ERROR: Python 2.4 or newer required")
 elif pyver[0] >= 3:
   sys.exit("ERROR: Python series 3 is not supported, use series 2 > 2.4")
 del pyver
 
 import os
 import datetime
-from optparse import OptionParser
+from optparse import OptionParser, OptionValueError
 
 topsrcdir = os.path.dirname(__file__)
 if topsrcdir == '':
     topsrcdir = '.'
 
 TREE_STATE_FILE = os.path.join(topsrcdir, '.treestate')
 
 try:
@@ -72,20 +72,54 @@ except ImportError:
     def check_call(*popenargs, **kwargs):
         retcode = subprocess.call(*popenargs, **kwargs)
         if retcode:
             cmd = kwargs.get("args")
             if cmd is None:
                 cmd = popenargs[0]
                 raise Exception("Command '%s' returned non-zero exit status %i" % (cmd, retcode))
 
-def check_call_noisy(cmd, *args, **kwargs):
+def check_call_noisy(cmd, retryMax=0, *args, **kwargs):
+  """Wrapper around execute_check_call() to allow retries before failing.
+
+  |cmd|, is the command to try and execute.
+  |retryMax|, is the maximum number of retries to attempt, 0 by default.
+  """
+
+  def execute_check_call(cmd, *args, **kwargs):
     print "Executing command:", cmd
     check_call(cmd, *args, **kwargs)
 
+  # By default (= no retries), simply pass the call on.
+  if retryMax == 0:
+    execute_check_call(cmd, *args, **kwargs)
+    return
+
+  # Loop (1 + retryMax) times, at most.
+  for r in range(0, 1 + retryMax):
+    # Add a retry header, not for initial call.
+    if r != 0:
+      print >> sys.stderr, "Retrying previous command: %d of %d time(s)" % (r, retryMax)
+    try:
+      # (Re-)Try the call.
+      execute_check_call(cmd, *args, **kwargs)
+      # If the call succeeded then no more retries.
+      break
+    except:
+      # Print the exception without its traceback.
+      # This traceback starts in the try block, which should be low value.
+      print >> sys.stderr, "The exception was:"
+      sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], None)
+      # Add a blank line.
+      print >> sys.stderr
+  else:
+    # Raise our own exception.
+    # This traceback starts at (= reports) the initial caller line.
+    raise Exception("Command '%s' failed %d time(s). Giving up." % (cmd, retryMax + 1))
+
 def repo_config():
     """Create/Update TREE_STATE_FILE as needed.
 
     switch_mozilla_repo() may be called if the branch that the mozilla repo is
     on needs changing
     """
 
     import ConfigParser
@@ -193,17 +227,18 @@ def switch_mozilla_repo():
             # Print the exception without its traceback.
             sys.excepthook(sys.exc_info()[0], sys.exc_info()[1], None)
             sys.exit("Error: Renaming old backup directory failed! You must recover manually.")
         # Let's leave the hgrc as it was, so any repo specific config is left
         # the same.
         return
 
     # Locally clone common repository history.
-    check_call_noisy([options.hg, 'clone', '-r', SWITCH_MOZILLA_BASE_REV] + hgopts + [backup_mozilla_path, mozilla_path])
+    check_call_noisy([options.hg, 'clone', '-r', SWITCH_MOZILLA_BASE_REV] + hgopts + [backup_mozilla_path, mozilla_path],
+                     retryMax=options.retries)
 
     # Rewrite hgrc for new local mozilla repo based on pre-existing hgrc
     # but with new values
     f = open(os.path.join(topsrcdir, 'mozilla', '.hg', 'hgrc'), 'w')
     try:
       config.write(f)
     finally:
       f.close()
@@ -233,51 +268,59 @@ def do_hg_pull(dir, repository, hg, rev)
         hgopts = options.hgopts.split()
     
     hgcloneopts = []
     if options.hgcloneopts:
         hgcloneopts = options.hgcloneopts.split()
 
     if not os.path.exists(fulldir):
         fulldir = os.path.join(topsrcdir, dir)
-        check_call_noisy([hg, 'clone'] + hgcloneopts + hgopts + [repository, fulldir])
+        check_call_noisy([hg, 'clone'] + hgcloneopts + hgopts + [repository, fulldir],
+                         retryMax=options.retries)
     else:
         cmd = [hg, 'pull', '-R', fulldir] + hgopts
         if repository is not None:
             cmd.append(repository)
-        check_call_noisy(cmd)
+        check_call_noisy(cmd, retryMax=options.retries)
     # update to specific revision
     if options.verbose:
         cmd = [hg, 'update', '-v', '-r', rev, '-R', fulldir ] + hgopts
     else:
         cmd = [hg, 'update', '-r', rev, '-R', fulldir ] + hgopts
-    check_call_noisy(cmd)
+    check_call_noisy(cmd, retryMax=options.retries)
     check_call([hg, 'parent', '-R', fulldir,
                 '--template=Updated to revision {node}.\n'])
 
 def do_cvs_checkout(modules, tag, cvsroot, cvs, checkoutdir):
     """Check out a CVS directory into the checkoutdir subdirectory.
     modules is a list of directories to check out, e.g. ['extensions/irc']
     """
     for module in modules:
         (parent, leaf) = os.path.split(module)
         print "CVS checkout begin: " + datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
         if tag == 'HEAD':
             check_call_noisy([cvs, '-d', cvsroot, '-q',
                               'checkout', '-P', '-A', '-d', leaf,
                               'mozilla/%s' % module],
-                             cwd=os.path.join(topsrcdir, checkoutdir, parent))
+                             cwd=os.path.join(topsrcdir, checkoutdir, parent),
+                             retryMax=options.retries)
         else:
             check_call_noisy([cvs, '-d', cvsroot, '-q',
                               'checkout', '-P', '-r', tag, '-d', leaf,
                               'mozilla/%s' % module],
-                             cwd=os.path.join(topsrcdir, checkoutdir, parent))
+                             cwd=os.path.join(topsrcdir, checkoutdir, parent),
+                             retryMax=options.retries)
         print "CVS checkout end: " + datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
 
-o = OptionParser(usage="client.py [options] checkout")
+def check_retries_option(option, opt_str, value, parser):
+  if value < 0:
+    raise OptionValueError("%s option value needs to be positive (not '%d')" % (opt_str, value))
+  setattr(parser.values, option.dest, value)
+
+o = OptionParser(usage="%prog [options] checkout")
 o.add_option("-m", "--comm-repo", dest="comm_repo",
              default=None,
              help="URL of comm (Calendar/Mail/Suite) repository to pull from (default: use hg default in .hg/hgrc)")
 o.add_option("--skip-comm", dest="skip_comm",
              action="store_true", default=False,
              help="Skip pulling the comm (Calendar/Mail/Suite) repository.")
 o.add_option("--comm-rev", dest="comm_rev",
              default=DEFAULT_COMM_REV,
@@ -312,42 +355,46 @@ o.add_option("--chatzilla-repo", dest = 
              help = "URL of ChatZilla repository to pull from (default: use hg default in mozilla/extensions/irc/.hg/hgrc; or if that file doesn't exist, use \"" + DEFAULT_CHATZILLA_REPO + "\".)")
 o.add_option("--skip-chatzilla", dest="skip_chatzilla",
              action="store_true", default=False,
              help="Skip pulling the ChatZilla repository.")
 o.add_option("--chatzilla-rev", dest = "chatzilla_rev",
              default = DEFAULT_CHATZILLA_REV,
              help = "Revision of ChatZilla repository to update to. Default: \"" + DEFAULT_CHATZILLA_REV + "\"")
 
-
 o.add_option("--venkman-repo", dest = "venkman_repo",
              default = None,
              help = "URL of Venkman repository to pull from (default: use hg default in mozilla/extensions/venkman/.hg/hgrc; or if that file doesn't exist, use \"" + DEFAULT_VENKMAN_REPO + "\".)")
 o.add_option("--skip-venkman", dest="skip_venkman",
              action="store_true", default=False,
              help="Skip pulling the Venkman repository.")
 o.add_option("--venkman-rev", dest = "venkman_rev",
              default = DEFAULT_VENKMAN_REV,
              help = "Revision of Venkman repository to update to. Default: \"" + DEFAULT_VENKMAN_REV + "\"")
 
 o.add_option("--hg", dest="hg", default=os.environ.get('HG', 'hg'),
              help="The location of the hg binary")
-o.add_option("--cvs", dest="cvs", default=os.environ.get('CVS', 'cvs'),
-             help="The location of the cvs binary")
-o.add_option("--cvsroot", dest="cvsroot",
-             default=os.environ.get('CVSROOT', ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot'),
-             help="The CVSROOT (default: :pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot")
 o.add_option("-v", "--verbose", dest="verbose",
              action="store_true", default=False,
              help="Enable verbose output on hg updates")
 o.add_option("--hg-options", dest="hgopts",
              help="Pass arbitrary options to hg commands (i.e. --debug, --time)")
 o.add_option("--hg-clone-options", dest="hgcloneopts",
              help="Pass arbitrary options to hg clone commands (i.e. --debug, --time)")
 
+o.add_option("--cvs", dest="cvs", default=os.environ.get('CVS', 'cvs'),
+             help="The location of the cvs binary")
+o.add_option("--cvsroot", dest="cvsroot",
+             default=os.environ.get('CVSROOT', ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot'),
+             help="The CVSROOT (default: :pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot")
+
+o.add_option("--retries", dest="retries", type="int", metavar="NUM",
+             default=1, help="Number of times to retry a failed command before giving up. (default: 1)",
+             action="callback", callback=check_retries_option)
+
 def fixup_comm_repo_options(options):
     """Check options.comm_repo value.
 
     options.comm_repo is normally None.
     This is fine -- our "hg pull" command will omit the repo URL.
     The exception is the initial checkout, which does an "hg clone".
     That command requires a repository URL.
     """