Bug 1118774 - Add retries to ssh/scp in upload.py; r=gps
authorMike Shal <mshal@mozilla.com>
Wed, 07 Jan 2015 15:48:13 -0500
changeset 223378 fff1eae2346fce938e64f070280a1528330a769c
parent 223377 a1654fa1847df3b85fc1d83879c9290332ea0325
child 223379 8e8ce6c0d19aca731410e3ef8b3810e2bfb2b799
push id13196
push userryanvm@gmail.com
push dateMon, 12 Jan 2015 20:57:04 +0000
treeherderb2g-inbound@afde7b62b332 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1118774
milestone37.0a1
Bug 1118774 - Add retries to ssh/scp in upload.py; r=gps Previously, buildbot was responsible for the retrying logic. Now that 'make upload' is part of mach build, we need to add retries here.
build/upload.py
--- a/build/upload.py
+++ b/build/upload.py
@@ -19,17 +19,18 @@
 #
 # All files to be uploaded should be passed as commandline arguments to this
 # script. The script takes one other parameter, --base-path, which you can use
 # to indicate that files should be uploaded including their paths relative
 # to the base path.
 
 import sys, os
 from optparse import OptionParser
-from subprocess import PIPE, Popen, check_call
+from subprocess import check_call, check_output
+import redo
 
 def RequireEnvironmentVariable(v):
     """Return the value of the environment variable named v, or print
     an error and exit if it's unset (or empty)."""
     if not v in os.environ or os.environ[v] == "":
         print "Error: required environment variable %s not set" % v
         sys.exit(1)
     return os.environ[v]
@@ -77,31 +78,35 @@ def AppendOptionalArgsToSSHCommandline(c
         cmdline.extend(["-o", "IdentityFile=%s" % ssh_key])
 
 def DoSSHCommand(command, user, host, port=None, ssh_key=None):
     """Execute command on user@host using ssh. Optionally use
     port and ssh_key, if provided."""
     cmdline = ["ssh"]
     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
     cmdline.extend(["%s@%s" % (user, host), command])
-    cmd = Popen(cmdline, stdout=PIPE)
-    retcode = cmd.wait()
-    if retcode != 0:
-        raise Exception("Command %s returned non-zero exit code: %i" % \
-          (cmdline, retcode))
-    return cmd.stdout.read().strip()
+
+    with redo.retrying(check_output, sleeptime=10) as f:
+        output = f(cmdline).strip()
+        return output
+
+    raise Exception("Command %s returned non-zero exit code" % cmdline)
 
 def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None):
     """Upload file to user@host:remote_path using scp. Optionally use
     port and ssh_key, if provided."""
     cmdline = ["scp"]
     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
     cmdline.extend([WindowsPathToMsysPath(file),
                     "%s@%s:%s" % (user, host, remote_path)])
-    check_call(cmdline)
+    with redo.retrying(check_call, sleeptime=10) as f:
+        f(cmdline)
+        return
+
+    raise Exception("Command %s returned non-zero exit code" % cmdline)
 
 def GetRemotePath(path, local_file, base_path):
     """Given a remote path to upload to, a full path to a local file, and an
     optional full path that is a base path of the local file, construct the
     full remote path to place the file in. If base_path is not None, include
     the relative path from base_path to file."""
     if base_path is None or not local_file.startswith(base_path):
         return path