Bug 1118774 - Add retries to ssh/scp in upload.py; r=gps
authorMike Shal <mshal@mozilla.com>
Wed, 07 Jan 2015 15:48:13 -0500
changeset 223311 fff1eae2346fce938e64f070280a1528330a769c
parent 223310 a1654fa1847df3b85fc1d83879c9290332ea0325
child 223312 8e8ce6c0d19aca731410e3ef8b3810e2bfb2b799
push id53878
push usermshal@mozilla.com
push dateMon, 12 Jan 2015 15:54:48 +0000
treeherdermozilla-inbound@fff1eae2346f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1118774
milestone37.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1118774 - Add retries to ssh/scp in upload.py; r=gps Previously, buildbot was responsible for the retrying logic. Now that 'make upload' is part of mach build, we need to add retries here.
build/upload.py
--- a/build/upload.py
+++ b/build/upload.py
@@ -19,17 +19,18 @@
 #
 # All files to be uploaded should be passed as commandline arguments to this
 # script. The script takes one other parameter, --base-path, which you can use
 # to indicate that files should be uploaded including their paths relative
 # to the base path.
 
 import sys, os
 from optparse import OptionParser
-from subprocess import PIPE, Popen, check_call
+from subprocess import check_call, check_output
+import redo
 
 def RequireEnvironmentVariable(v):
     """Return the value of the environment variable named v, or print
     an error and exit if it's unset (or empty)."""
     if not v in os.environ or os.environ[v] == "":
         print "Error: required environment variable %s not set" % v
         sys.exit(1)
     return os.environ[v]
@@ -77,31 +78,35 @@ def AppendOptionalArgsToSSHCommandline(c
         cmdline.extend(["-o", "IdentityFile=%s" % ssh_key])
 
 def DoSSHCommand(command, user, host, port=None, ssh_key=None):
     """Execute command on user@host using ssh. Optionally use
     port and ssh_key, if provided."""
     cmdline = ["ssh"]
     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
     cmdline.extend(["%s@%s" % (user, host), command])
-    cmd = Popen(cmdline, stdout=PIPE)
-    retcode = cmd.wait()
-    if retcode != 0:
-        raise Exception("Command %s returned non-zero exit code: %i" % \
-          (cmdline, retcode))
-    return cmd.stdout.read().strip()
+
+    with redo.retrying(check_output, sleeptime=10) as f:
+        output = f(cmdline).strip()
+        return output
+
+    raise Exception("Command %s returned non-zero exit code" % cmdline)
 
 def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None):
     """Upload file to user@host:remote_path using scp. Optionally use
     port and ssh_key, if provided."""
     cmdline = ["scp"]
     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
     cmdline.extend([WindowsPathToMsysPath(file),
                     "%s@%s:%s" % (user, host, remote_path)])
-    check_call(cmdline)
+    with redo.retrying(check_call, sleeptime=10) as f:
+        f(cmdline)
+        return
+
+    raise Exception("Command %s returned non-zero exit code" % cmdline)
 
 def GetRemotePath(path, local_file, base_path):
     """Given a remote path to upload to, a full path to a local file, and an
     optional full path that is a base path of the local file, construct the
     full remote path to place the file in. If base_path is not None, include
     the relative path from base_path to file."""
     if base_path is None or not local_file.startswith(base_path):
         return path