bug 762063: Fix winrs Proxy scripts so that the socket server is actually handling multiple requests at once
authorBrent Baker <brbaker@adobe.com>
Thu, 06 Dec 2012 06:47:48 -0800
changeset 7580 167a6c1b492e
parent 7579 a0c3dd2231e9
child 7581 c7b1844ed46d
push id4262
push userdschaffe@adobe.com
push dateWed, 30 Jan 2013 19:01:31 +0000
bugs762063, 1144100
bug 762063: Fix winrs Proxy scripts so that the socket server is actually handling multiple requests at once CL@1144100
build/buildbot/slaves/all/winrs-proxy.py
build/buildbot/slaves/all/winrs-shell-deployer.sh
build/buildbot/slaves/all/winrs-shell.sh
--- a/build/buildbot/slaves/all/winrs-proxy.py
+++ b/build/buildbot/slaves/all/winrs-proxy.py
@@ -5,16 +5,18 @@
 import sys,socket,os,getopt,time
 
 host=''
 port=0
 if os.environ.has_key("WINRSSERVERHOST"):
     host=os.environ.get("WINRSSERVERHOST")
 if os.environ.has_key("WINRSSERVERPORT"):
     port=int(os.environ.get("WINRSSERVERPORT"))
+expectNoOutput=os.environ.get("WINRSSERVEREXPECTNOOUTPUT", False)
+
 timeout=-1
 winrs_host=''
 winrs_user=''
 winrs_pass=''
 
 usage='''
 winrs.py args ...
 --port=
@@ -24,127 +26,103 @@ winrs.py args ...
 --file=
 --skipExit
 '''
 cmd='winrs '
 longoptions=['port=','host=','cmd=','timeout=','file=','skipexit']
 skipExit=False
 infile=None
 
-ctr=0
-while ctr<len(sys.argv):
-    arg=sys.argv[ctr]
-    if arg.startswith('-r:'):
-        winrs_host=arg[3:]
-        del sys.argv[ctr]
-    elif arg.startswith('-u:'):
-        winrs_user=arg[3:]
-        del sys.argv[ctr]
-    elif arg.startswith('-p:'):
-        winrs_pass=arg[3:]
-        del sys.argv[ctr]
-    else:
-        ctr+=1
-try:
-    opts,args=getopt.getopt(sys.argv[1:],[],longoptions)
-except:
-    print(usage)
-    sys.exit(1)
-for o,v in opts:
-    if o in ('--file'):
-        infile=v
-    if o in ('--port'):
-        port=int(v)
-    if o in ('--host'):
-        host=v
-    if o in ('--cmd'):
-        cmd=v
-    if o in ('--timeout'):
-        try:
-            timeout=int(v)
-        except:
-            None
-    if o in ('--skipexit'):
-        skipExit=True
-
-for arg in args:
-   cmd+=arg+' '
-
-if len(args)==0 and infile==None and cmd=='':
-    print("must specify an input file or a command")
-    print(usage)
-    sys.exit(1)
+cmd="winrs %s" % ' '.join(sys.argv[1:])
 
 if host=='':
    print("must specify host through --host or environment variable WINRSSERVERHOST")
    sys.exit(1)
 
 if port==0:
    print("must specify port through --port or environment variable WINRSSERVERPORT")
    sys.exit(1)
 
 
-start=time.time()
-s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-try:
-    s.connect((host,port))
-except:
-    print("could not connect to host=%s port=%s" % (host,port))
-    sys.exit(1)
+def run():
+    start=time.time()
+    s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        s.connect((host,port))
+    except:
+        print("could not connect to host=%s port=%s" % (host,port))
+        sys.exit(1)
 
-s.send("set timeout -1\n")
-start=time.time()
-result=''
-while True:
-    result+=s.recv(1024)
-    if result.find('$'):
-        break
-    if time.time()-start>30:
-        print("error failed to set timeout")
-        sys.exit(1)
-    time.sleep(.5)
+    s.send("set timeout -1\n")
+    start=time.time()
+    result=''
+    while True:
+        result+=s.recv(1024)
+        if result.find('$'):
+            break
+        if time.time()-start>30:
+            print("error failed to set timeout")
+            sys.exit(1)
+        time.sleep(.5)
 
 
-if infile!=None:
-    lines=open(infile)
-    count=0
-    for line in lines:
-        if line=='':
-            continue
-        s.send(line+'\n')
-        result=s.recv(1024)
-        if result.find("finished")==-1:
-            print("error: did not receive ack")
+    if infile!=None:
+        lines=open(infile)
+        count=0
+        for line in lines:
+            if line=='':
+                continue
+            s.send(line+'\n')
+            result=s.recv(1024)
+            if result.find("finished")==-1:
+                print("error: did not receive ack")
+                break
+            print('[%d] %s' % (count,line))
+            count+=1
+        print("done sending file")
+    else:
+        s.send(cmd+'\n')
+    result=''
+    exitSent=False
+    while True:
+        result+=s.recv(1024)
+        if result.find('$') and exitSent==False:
+            exitSent=True
+            s.send('exit\n')
+            if skipExit:
+                break
+        if result.find('closing connection')>-1:
             break
-        print('[%d] %s' % (count,line))
-        count+=1
-    print("done sending file")
-else:
-    s.send(cmd+'\n')
-result=''
-exitSent=False
-while True:
-    result+=s.recv(1024)
-    if result.find('$') and exitSent==False:
-        exitSent=True
-        s.send('exit\n')
-        if skipExit:
+        if timeout!=-1 and time.time()-start>timeout:
+            result+='timed out after %d' % timeout
             break
-    if result.find('closing connection')>-1:
-        break
-    if timeout!=-1 and time.time()-start>timeout:
-        result+='timed out after %d' % timeout
+        time.sleep(.5)
+    s.close()
+    if skipExit:
+        code=0
+    else:
+        fnd=result.find('closing connection')
+        code=1
+        if fnd>-1:
+            result=result[0:fnd-3]
+            code=0
+
+    return result, code
+
+
+tries=5
+attempts=0
+output=""
+code=0
+while (attempts < tries):
+    #print(":: Attempt %s" %attempts)
+    output,code = run()
+    #print(":: len(output) %s" % len(output))
+    #print(output)
+    if len(output) != 0 or expectNoOutput:
         break
-    time.sleep(.5)
-s.close()
-if skipExit:
-    code=0
-else:
-    fnd=result.find('closing connection')
-    code=1
-    if fnd>-1:
-        result=result[0:fnd-3]
-        code=0
-print(result)
+    attempts+=1
+
+print(output)
 #print("exitcode=%d" % code)
 sys.exit(code)
 
 
--- a/build/buildbot/slaves/all/winrs-shell-deployer.sh
+++ b/build/buildbot/slaves/all/winrs-shell-deployer.sh
@@ -92,68 +92,72 @@ do
         echo "Clean up the remote directory"
         ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "rmdir /s /q ${WINRS_SHELL_REMOTE_BASEDIR}"
         ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "mkdir ${WINRS_SHELL_REMOTE_BASEDIR}"
     fi
 
     echo""
     echo "Installing $filename on $WINRS_SHELL_REMOTE_HOST to $WINRS_SHELL_REMOTE_BASEDIR"
     # Setup the ftp script to get the acceptance media
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo open ${ftp_host} \> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo user \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo ftp \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo ftp \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo lcd ${WINRS_SHELL_REMOTE_BASEDIR} \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo cd ${branch}/${change}/windows \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo get ${filename} avmshell.exe \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo get winrs-shell-runner.bat \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo close \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo quit \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
+    export  WINRSSERVEREXPECTNOOUTPUT=True
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo open ${ftp_host} > ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo user >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo ftp >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo ftp >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo lcd ${WINRS_SHELL_REMOTE_BASEDIR} >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo cd ${branch}/${change}/windows >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo get ${filename} avmshell.exe >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo get winrs-shell-runner.bat >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo close >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo quit >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    unset WINRSSERVEREXPECTNOOUTPUT
 
     echo ""
     echo "Get the ${filename} from ftp server"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "ftp -n -s:${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "ftp -n -s:${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
 
     echo ""
-    echo "verify the deployed shell version if correct"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "${WINRS_SHELL_REMOTE_BASEDIR}\\\\winrs-shell-runner.bat" > /tmp/stdout
+    echo "verify the deployed shell version is correct"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "${WINRS_SHELL_REMOTE_BASEDIR}\\winrs-shell-runner.bat" > /tmp/stdout
     deploy_rev=`cat /tmp/stdout | grep "avmplus shell" | awk '{print $NF}'`
     if [ "$change_for_validation" != "${deploy_rev%:*}" ] || [ "$changeid" != "${deploy_rev#*:}" ];
     then
         # Could be possible that we are running from the hg mirror repo and using a binary that has a p4 changelist revision
         # The last line in the description is going to be "CL@12345"
         # so grab the last line and peel off the perforce changelist
         cl=`hg log -r $change --template {desc} | tail -n 1 | awk -F@  '{print $2}'`
         echo "found changelist in description: ${cl}"
         if [ "${cl}" != "${deploy_rev%:*}" ]; then
             echo $0 FAILED!!!
             echo requested build "$change_for_validation:$changeid" is not what is deployed "${deploy_rev%:*}:${deploy_rev#*:}"
             exit 1
         fi
     fi
 
     # Setup the ftp script to get the acceptance media
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo open ${ftp_host} \> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo user \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo ftp \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo ftp \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo lcd ${WINRS_SHELL_REMOTE_BASEDIR} \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo cd ${branch}/${change} \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo get acceptance-tests-abcs.zip \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo close \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo quit \>\> ${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
+    export  WINRSSERVEREXPECTNOOUTPUT=True
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo open ${ftp_host} > ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo user >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo ftp >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo ftp >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo lcd ${WINRS_SHELL_REMOTE_BASEDIR} >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo cd ${branch}/${change} >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo get acceptance-tests-abcs.zip >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo close >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "echo quit >> ${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
+    unset WINRSSERVEREXPECTNOOUTPUT
 
     echo ""
     echo "Get the acceptance media from ftp server"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "ftp -n -s:${WINRS_SHELL_REMOTE_BASEDIR}\\\\ftp_script"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "ftp -n -s:${WINRS_SHELL_REMOTE_BASEDIR}\\ftp_script"
 
     echo "unzipping acceptance-tests-abcs.zip..."
     echo ""
     echo "Unzip the acceptance media"
-    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "cscript c:\\\\tamarin-tools\\\\unzip.vbs ${WINRS_SHELL_REMOTE_BASEDIR}\\\\acceptance-tests-abcs.zip ${WINRS_SHELL_REMOTE_BASEDIR}"
+    ${winrs} -r:${WINRS_SHELL_REMOTE_HOST} -u:${WINRS_SHELL_REMOTE_USER} -p:${WINRS_SHELL_REMOTE_PASSWD} "cscript c:\\tamarin-tools\\unzip.vbs ${WINRS_SHELL_REMOTE_BASEDIR}\\acceptance-tests-abcs.zip ${WINRS_SHELL_REMOTE_BASEDIR}"
 
     # increment the counter, check if we are finished looping
     count=$[count+1]
     if [ "$count" = "$threadcount" ]
     then
         break
     fi
 done
--- a/build/buildbot/slaves/all/winrs-shell.sh
+++ b/build/buildbot/slaves/all/winrs-shell.sh
@@ -47,25 +47,35 @@ fi
 function try_command () {
     count=1
     while [ $count -le $MAX_RETRIES ]
     do
         # run the passed in command - not that "$@" (quoted) is used since that
         # is the only form that will work with quoted arguments containing spaces
         # see http://www.tldp.org/LDP/abs/html/internalvariables.html#APPREF2
         # for details
-        "$@" 2> ./stderr$id
+        "$@" 2>./stderr$id 1>./stdout_tmp$id
         ec=$?
-        if [ "$ec" -eq "$expectedExitCode" ]; then
+        # Either $ec is non-zero or there should be something in stdout
+        if [[ "$ec" -eq "0" && "`cat ./stdout_tmp$id`" == "" ]]; then
+            echo "stdout is empty and exitcode is $ec"
+            echo "Exit Code: $ec"
+            echo "Try $count of $MAX_RETRIES"
+            sleep 3
+        elif [ "$ec" -eq "$expectedExitCode" ]; then
             # command executed with expected exit code
             # Put captured stderr back into stderr so that it is handled properly by the test runner
             if [ -s ./stderr$id ]; then
                 echo "`cat ./stderr$id`" >&2
                 rm -f ./stderr$id
             fi
+            if [ -s ./stdout_tmp$id ]; then
+                echo "`cat ./stdout_tmp$id`" >&1
+                rm -f ./stdout_tmp$id
+            fi
             return 0
         else
             echo "Command Failed: $*"
             echo "Exit Code: $ec"
             echo "Expected Exit Code: $expectedExitCode"
             echo "Try $count of $MAX_RETRIES"
             sleep 3
         fi
@@ -73,16 +83,20 @@ function try_command () {
     done
     # command failed SSH_RETRIES times, report failure and exit
     echo "Reached max tries, exiting with exit code $ec ..."
     # Put captured stderr back into stderr so that it is handled properly by the test runner
     if [ -s ./stderr$id ]; then
         echo "`cat ./stderr$id`" >&2
         rm -f ./stderr$id
     fi
+    if [ -s ./stdout_tmp$id ]; then
+        echo "`cat ./stdout_tmp$id`" >&1
+        rm -f ./stdout_tmp$id
+    fi
     exit $ec
 }
 
 
 if [ "$1" = "" ]
 then
     # running the shell with no args prints the help and exits with exitcode 1
     expectedExitCode=0