Imported upstream patch c2fec250057daf8c85e673fc5172233c8d0c39e6
authorChris AtLee <catlee@mozilla.com>
Wed, 18 Aug 2010 06:56:20 -0400
changeset 82 298a2f5cfb3e7b1fea4356fa63ade653eabc331a
parent 81 ebdabf279a4c23e81ff1be2753ee67a23dbcafb5
child 83 9cc940a502192e9588c1725f413143c7ade2a8a7
push id31
push usercatlee@mozilla.com
push dateThu, 26 Aug 2010 12:50:22 +0000
Imported upstream patch c2fec250057daf8c85e673fc5172233c8d0c39e6 Adding new string transfer steps: StringDownload, JSONStringDownload, and JSONPropertiesDownload
master/buildbot/steps/transfer.py
master/buildbot/test/unit/test_steps_transfer.py
--- a/master/buildbot/steps/transfer.py
+++ b/master/buildbot/steps/transfer.py
@@ -1,17 +1,22 @@
 # -*- test-case-name: buildbot.test.test_transfer -*-
 
 import os.path, tarfile, tempfile
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
 from twisted.internet import reactor
 from twisted.spread import pb
 from twisted.python import log
 from buildbot.process.buildstep import RemoteCommand, BuildStep
 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED
 from buildbot.interfaces import BuildSlaveTooOldError
+from buildbot.util import json
 
 
 class _FileWriter(pb.Referenceable):
     """
     Helper class that acts as a file-object with write access
     """
 
     def __init__(self, destfile, maxsize, mode):
@@ -463,8 +468,125 @@ class FileDownload(_TransferBuildStep):
             'workdir': self._getWorkdir(),
             'mode': self.mode,
             }
 
         self.cmd = StatusRemoteCommand('downloadFile', args)
         d = self.runCommand(self.cmd)
         d.addCallback(self.finished).addErrback(self.failed)
 
+class StringDownload(_TransferBuildStep):
+    """
+    Download the first 'maxsize' bytes of a string, from the buildmaster to the
+    buildslave. Set the mode of the file
+
+    Arguments::
+
+     ['s']         string to transfer
+     ['slavedest'] filename of destination file at slave
+     ['workdir']   string with slave working directory relative to builder
+                   base dir, default 'build'
+     ['maxsize']   maximum size of the file, default None (=unlimited)
+     ['blocksize'] maximum size of each block being transfered
+     ['mode']      use this to set the access permissions of the resulting
+                   buildslave-side file. This is traditionally an octal
+                   integer, like 0644 to be world-readable (but not
+                   world-writable), or 0600 to only be readable by
+                   the buildslave account, or 0755 to be world-executable.
+                   The default (=None) is to leave it up to the umask of
+                   the buildslave process.
+    """
+    name = 'string_download'
+
+    def __init__(self, s, slavedest,
+                 workdir=None, maxsize=None, blocksize=16*1024, mode=None,
+                 **buildstep_kwargs):
+        BuildStep.__init__(self, **buildstep_kwargs)
+        self.addFactoryArguments(s=s,
+                                 slavedest=slavedest,
+                                 workdir=workdir,
+                                 maxsize=maxsize,
+                                 blocksize=blocksize,
+                                 mode=mode,
+                                 )
+
+        self.s = s
+        self.slavedest = slavedest
+        self.workdir = workdir
+        self.maxsize = maxsize
+        self.blocksize = blocksize
+        assert isinstance(mode, (int, type(None)))
+        self.mode = mode
+
+    def start(self):
+        properties = self.build.getProperties()
+
+        version = self.slaveVersion("downloadFile")
+        if not version:
+            m = "slave is too old, does not know about downloadFile"
+            raise BuildSlaveTooOldError(m)
+
+        # we are currently in the buildmaster's basedir, so any non-absolute
+        # paths will be interpreted relative to that
+        slavedest = properties.render(self.slavedest)
+        log.msg("StringDownload started, from master to slave %r" % slavedest)
+
+        self.step_status.setText(['downloading', "to",
+                                  os.path.basename(slavedest)])
+
+        # setup structures for reading the file
+        fp = StringIO(self.s)
+        fileReader = _FileReader(fp)
+
+        # default arguments
+        args = {
+            'slavedest': slavedest,
+            'maxsize': self.maxsize,
+            'reader': fileReader,
+            'blocksize': self.blocksize,
+            'workdir': self._getWorkdir(),
+            'mode': self.mode,
+            }
+
+        self.cmd = StatusRemoteCommand('downloadFile', args)
+        d = self.runCommand(self.cmd)
+        d.addCallback(self.finished).addErrback(self.failed)
+
+class JSONStringDownload(StringDownload):
+    """
+    Encode object o as a json string and save it on the buildslave
+
+    Arguments::
+
+     ['o']         object to encode and transfer
+    """
+    name = "json_download"
+    def __init__(self, o, slavedest, **buildstep_kwargs):
+        if 's' in buildstep_kwargs:
+            del buildstep_kwargs['s']
+        s = json.dumps(o)
+        StringDownload.__init__(self, s=s, slavedest=slavedest, **buildstep_kwargs)
+        self.addFactoryArguments(o=o)
+
+class JSONPropertiesDownload(StringDownload):
+    """
+    Download the current build properties as a json string and save it on the
+    buildslave
+    """
+    name = "json_properties_download"
+    def __init__(self, slavedest, **buildstep_kwargs):
+        self.super_class = StringDownload
+        if 's' in buildstep_kwargs:
+            del buildstep_kwargs['s']
+        StringDownload.__init__(self, s=None, slavedest=slavedest, **buildstep_kwargs)
+
+    def start(self):
+        properties = self.build.getProperties()
+        props = {}
+        for key, value, source in properties.asList():
+            props[key] = value
+
+        self.s = json.dumps(dict(
+                        properties=props,
+                        sourcestamp=self.build.getSourceStamp().asDict(),
+                    ),
+                )
+        return self.super_class.start(self)
new file mode 100644
--- /dev/null
+++ b/master/buildbot/test/unit/test_steps_transfer.py
@@ -0,0 +1,91 @@
+from twisted.trial import unittest
+
+from mock import Mock
+
+from buildbot.process.properties import Properties
+from buildbot.util import json
+from buildbot.steps.transfer import StringDownload, JSONStringDownload, JSONPropertiesDownload
+
+class TestStringDownload(unittest.TestCase):
+    def testBasic(self):
+        s = StringDownload("Hello World", "hello.txt")
+        s.build = Mock()
+        s.build.getProperties.return_value = Properties()
+        s.build.getSlaveCommandVersion.return_value = 1
+
+        s.step_status = Mock()
+        s.buildslave = Mock()
+        s.remote = Mock()
+
+        s.start()
+
+        for c in s.remote.method_calls:
+            name, command, args = c
+            commandName = command[3]
+            kwargs = command[-1]
+            if commandName == 'downloadFile':
+                self.assertEquals(kwargs['slavedest'], 'hello.txt')
+                reader = kwargs['reader']
+                data = reader.remote_read(100)
+                self.assertEquals(data, "Hello World")
+                break
+        else:
+            self.assert_(False, "No downloadFile command found")
+
+class TestJSONStringDownload(unittest.TestCase):
+    def testBasic(self):
+        msg = dict(message="Hello World")
+        s = JSONStringDownload(msg, "hello.json")
+        s.build = Mock()
+        s.build.getProperties.return_value = Properties()
+        s.build.getSlaveCommandVersion.return_value = 1
+
+        s.step_status = Mock()
+        s.buildslave = Mock()
+        s.remote = Mock()
+
+        s.start()
+
+        for c in s.remote.method_calls:
+            name, command, args = c
+            commandName = command[3]
+            kwargs = command[-1]
+            if commandName == 'downloadFile':
+                self.assertEquals(kwargs['slavedest'], 'hello.json')
+                reader = kwargs['reader']
+                data = reader.remote_read(100)
+                self.assertEquals(data, json.dumps(msg))
+                break
+        else:
+            self.assert_(False, "No downloadFile command found")
+
+class TestJSONPropertiesDownload(unittest.TestCase):
+    def testBasic(self):
+        s = JSONPropertiesDownload("props.json")
+        s.build = Mock()
+        props = Properties()
+        props.setProperty('key1', 'value1', 'test')
+        s.build.getProperties.return_value = props
+        s.build.getSlaveCommandVersion.return_value = 1
+        ss = Mock()
+        ss.asDict.return_value = dict(revision="12345")
+        s.build.getSourceStamp.return_value = ss
+
+        s.step_status = Mock()
+        s.buildslave = Mock()
+        s.remote = Mock()
+
+        s.start()
+
+        for c in s.remote.method_calls:
+            name, command, args = c
+            commandName = command[3]
+            kwargs = command[-1]
+            if commandName == 'downloadFile':
+                self.assertEquals(kwargs['slavedest'], 'props.json')
+                reader = kwargs['reader']
+                data = reader.remote_read(100)
+                self.assertEquals(data, json.dumps(dict(sourcestamp=ss.asDict(), properties={'key1': 'value1'})))
+                break
+        else:
+            self.assert_(False, "No downloadFile command found")