From 8a541c2cfd346baf428c2fbef5aa4ed1db1db283 Mon Sep 17 00:00:00 2001 From: Mike Shal Date: Wed, 7 Jan 2015 15:48:13 -0500 Subject: [PATCH] Bug 1118774 - Add retries to ssh/scp in upload.py; r=gps Previously, buildbot was responsible for the retrying logic. Now that 'make upload' is part of mach build, we need to add retries here. --- build/upload.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/build/upload.py b/build/upload.py index 9dae5d9be81..6b1112eed59 100644 --- a/build/upload.py +++ b/build/upload.py @@ -24,7 +24,8 @@ import sys, os from optparse import OptionParser -from subprocess import PIPE, Popen, check_call +from subprocess import check_call, check_output +import redo def RequireEnvironmentVariable(v): """Return the value of the environment variable named v, or print @@ -82,12 +83,12 @@ def DoSSHCommand(command, user, host, port=None, ssh_key=None): cmdline = ["ssh"] AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key) cmdline.extend(["%s@%s" % (user, host), command]) - cmd = Popen(cmdline, stdout=PIPE) - retcode = cmd.wait() - if retcode != 0: - raise Exception("Command %s returned non-zero exit code: %i" % \ - (cmdline, retcode)) - return cmd.stdout.read().strip() + + with redo.retrying(check_output, sleeptime=10) as f: + output = f(cmdline).strip() + return output + + raise Exception("Command %s returned non-zero exit code" % cmdline) def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None): """Upload file to user@host:remote_path using scp. Optionally use @@ -96,7 +97,11 @@ def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None): AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key) cmdline.extend([WindowsPathToMsysPath(file), "%s@%s:%s" % (user, host, remote_path)]) - check_call(cmdline) + with redo.retrying(check_call, sleeptime=10) as f: + f(cmdline) + return + + raise Exception("Command %s returned non-zero exit code" % cmdline) def GetRemotePath(path, local_file, base_path): """Given a remote path to upload to, a full path to a local file, and an