Bug 902711 - Make marionette_transport its own package, r=ato
authorJonathan Griffin <jgriffin@mozilla.com>
Mon, 14 Oct 2013 17:58:40 -0700
changeset 196876 cbe169081f1cce6e8cad4c9d0ec18013042bab24
parent 196875 824f4c321887550a291155d3bf28193d2812ea1c
child 196877 5b6e82e7bbbff6bb98bbc68745f1d8ad8352f3f5
child 196878 4b1495af7084a18a0c289ae299a3b28b68d59ade
push id3624
push userasasaki@mozilla.com
push dateMon, 09 Jun 2014 21:49:01 +0000
treeherdermozilla-beta@b1a5da15899a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato
bugs902711
milestone31.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 902711 - Make marionette_transport its own package, r=ato
testing/config/marionette_requirements.txt
testing/marionette/client/marionette/b2g_update_test.py
testing/marionette/client/marionette/client.py
testing/marionette/client/marionette/emulator.py
testing/marionette/client/marionette/marionette.py
testing/marionette/client/requirements.txt
testing/marionette/client/setup.py
testing/marionette/transport/README.md
testing/marionette/transport/marionette_transport/__init__.py
testing/marionette/transport/marionette_transport/transport.py
testing/marionette/transport/setup.py
testing/testsuite-targets.mk
--- a/testing/config/marionette_requirements.txt
+++ b/testing/config/marionette_requirements.txt
@@ -1,2 +1,3 @@
 -r mozbase_requirements.txt
+../marionette/transport
 ../marionette
--- a/testing/marionette/client/marionette/b2g_update_test.py
+++ b/testing/marionette/client/marionette/b2g_update_test.py
@@ -7,41 +7,41 @@ import imp
 import os
 import re
 import subprocess
 import time
 import types
 import weakref
 
 from b2ginstance import B2GInstance
-from client import MarionetteClient
 from errors import InvalidResponseException
 from marionette import Marionette
 from marionette_test import MarionetteTestCase
+from marionette_transport import MarionetteTransport
 from runtests import MarionetteTestRunner, cli
 
-class B2GUpdateMarionetteClient(MarionetteClient):
+class B2GUpdateMarionetteClient(MarionetteTransport):
     RETRY_TIMEOUT   = 5
     CONNECT_TIMEOUT = 30
     SEND_TIMEOUT    = 60 * 5
     MAX_RETRIES     = 24
 
     def __init__(self, addr, port, runner):
-        MarionetteClient.__init__(self, addr, port)
+        super(B2GUpdateMarionetteClient, self).__init__(addr, port)
         self.runner = runner
 
     def connect(self):
         """ When using ADB port forwarding, the ADB server will actually accept
             connections even though there isn't a listening socket open on the
             device. Here we add a retry loop since we have to restart the debug
             server / b2g process for update tests
         """
         for i in range(self.MAX_RETRIES):
             try:
-                MarionetteClient.connect(self, timeout=self.CONNECT_TIMEOUT)
+                MarionetteTransport.connect(self, timeout=self.CONNECT_TIMEOUT)
                 break
             except:
                 if i == self.MAX_RETRIES - 1:
                     raise
 
                 time.sleep(self.RETRY_TIMEOUT)
                 self.runner.port_forward()
 
deleted file mode 100644
--- a/testing/marionette/client/marionette/client.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import errno
-import json
-import socket
-
-from errors import InvalidResponseException, ErrorCodes
-
-
-class MarionetteClient(object):
-    """ The Marionette socket client.  This speaks the same protocol
-        as the remote debugger inside Gecko, in which messages are
-        always preceded by the message length and a colon, e.g.,
-
-        20:{'command': 'test'}
-    """
-
-    max_packet_length = 4096
-
-    def __init__(self, addr, port):
-        self.addr = addr
-        self.port = port
-        self.sock = None
-        self.traits = None
-        self.applicationType = None
-        self.actor = 'root'
-
-    def _recv_n_bytes(self, n):
-        """ Convenience method for receiving exactly n bytes from
-            self.sock (assuming it's open and connected).
-        """
-        data = ''
-        while len(data) < n:
-            chunk = self.sock.recv(n - len(data))
-            if chunk == '':
-                break
-            data += chunk
-        return data
-
-    def receive(self):
-        """ Receive the next complete response from the server, and return
-            it as a dict.  Each response from the server is prepended by
-            len(message) + ':'.
-        """
-        assert(self.sock)
-        response = self.sock.recv(10)
-        sep = response.find(':')
-        length = response[0:sep]
-        if length != '':
-            response = response[sep + 1:]
-            response += self._recv_n_bytes(int(length) + 1 + len(length) - 10)
-            return json.loads(response)
-        else:
-            raise InvalidResponseException("Could not communicate with Marionette server. "
-                                           "Is the Gecko process still running?",
-                                           status=ErrorCodes.INVALID_RESPONSE)
-
-    def connect(self, timeout=360.0):
-        """ Connect to the server and process the hello message we expect
-            to receive in response.
-        """
-        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.sock.settimeout(timeout)
-        try:
-            self.sock.connect((self.addr, self.port))
-        except:
-            # Unset self.sock so that the next attempt to send will cause
-            # another connection attempt.
-            self.sock = None
-            raise
-        hello = self.receive()
-        self.traits = hello.get('traits')
-        self.applicationType = hello.get('applicationType')
-
-        # get the marionette actor id
-        response = self.send({'to': 'root', 'name': 'getMarionetteID'})
-        self.actor = response['id']
-
-    def send(self, msg):
-        """ Send a message on the socket, prepending it with len(msg) + ':'.
-        """
-        if not self.sock:
-            self.connect()
-        if 'to' not in msg:
-            msg['to'] = self.actor
-        data = json.dumps(msg)
-        data = '%s:%s' % (len(data), data)
-
-        for packet in [data[i:i + self.max_packet_length] for i in
-                       range(0, len(data), self.max_packet_length)]:
-            try: 
-                self.sock.send(packet)
-            except IOError as e:
-                if e.errno == errno.EPIPE:
-                    raise IOError("%s: Connection to Marionette server is lost. Check gecko.log (desktop firefox) or logcat (b2g) for errors." % str(e))
-                else:
-                    raise e
-
-        response = self.receive()
-        return response
-
-    def close(self):
-        """ Close the socket.
-        """
-        self.sock.close()
-        self.sock = None
--- a/testing/marionette/client/marionette/emulator.py
+++ b/testing/marionette/client/marionette/emulator.py
@@ -288,17 +288,17 @@ waitFor(
             """)
         except ScriptTimeoutException:
             print 'timed out'
             # We silently ignore the timeout if it occurs, since
             # isSystemMessageListenerReady() isn't available on
             # older emulators.  45s *should* be enough of a delay
             # to allow telephony API's to work.
             pass
-        except InvalidResponseException:
+        except (InvalidResponseException, IOError):
             self.check_for_minidumps()
             raise
         print '...done'
 
     def connect(self):
         self.adb = B2GInstance.check_adb(self.homedir, emulator=True)
         self.start_adb()
 
--- a/testing/marionette/client/marionette/marionette.py
+++ b/testing/marionette/client/marionette/marionette.py
@@ -6,22 +6,22 @@ import ConfigParser
 import datetime
 import os
 import socket
 import sys
 import time
 import traceback
 
 from application_cache import ApplicationCache
-from client import MarionetteClient
 from decorators import do_crash_check
 from emulator import Emulator
 from emulator_screen import EmulatorScreen
 from errors import *
 from keys import Keys
+from marionette_transport import MarionetteTransport
 
 import geckoinstance
 
 class HTMLElement(object):
     """
     Represents a DOM Element.
     """
 
@@ -506,28 +506,28 @@ class Marionette(object):
 
         if connectToRunningEmulator:
             self.emulator = Emulator(homedir=homedir,
                                      logcat_dir=self.logcat_dir)
             self.emulator.connect()
             self.port = self.emulator.setup_port_forwarding(self.port)
             assert(self.emulator.wait_for_port()), "Timed out waiting for port!"
 
-        self.client = MarionetteClient(self.host, self.port)
+        self.client = MarionetteTransport(self.host, self.port)
 
         if emulator:
             self.emulator.setup(self,
                                 gecko_path=gecko_path,
                                 busybox=busybox)
 
     def cleanup(self):
         if self.session:
             try:
                 self.delete_session()
-            except (MarionetteException, socket.error):
+            except (MarionetteException, socket.error, IOError):
                 # These exceptions get thrown if the Marionette server
                 # hit an exception/died or the connection died. We can
                 # do no further server-side cleanup in this case.
                 pass
             self.session = None
         if self.emulator:
             self.emulator.close()
         if self.instance:
--- a/testing/marionette/client/requirements.txt
+++ b/testing/marionette/client/requirements.txt
@@ -1,8 +1,9 @@
+./transport
 manifestdestiny
 mozhttpd >= 0.5
 mozinfo >= 0.7
 mozprocess >= 0.9
 mozrunner >= 5.15
 mozdevice >= 0.22
 moznetwork >= 0.21
 mozcrash >= 0.5
--- a/testing/marionette/client/setup.py
+++ b/testing/marionette/client/setup.py
@@ -1,24 +1,42 @@
 import os
 from setuptools import setup, find_packages
+import sys
 
 version = '0.7.6'
 
 # get documentation from the README
 try:
     here = os.path.dirname(os.path.abspath(__file__))
     description = file(os.path.join(here, 'README.md')).read()
 except (OSError, IOError):
     description = ''
 
 # dependencies
 with open('requirements.txt') as f:
     deps = f.read().splitlines()
 
+# Requirements.txt contains a pointer to the local copy of marionette_transport;
+# if we're installing using setup.py, handle this locally or replace with a valid
+# pypi package reference.
+deps = [x for x in deps if 'transport' not in x]
+transport_dir = os.path.join(os.path.dirname(__file__), os.path.pardir, 'transport')
+method = [x for x in sys.argv if x in ('develop', 'install')]
+if os.path.exists(transport_dir) and method:
+    cmd = [sys.executable, 'setup.py', method[0]]
+    import subprocess
+    try:
+        subprocess.check_call(cmd, cwd=transport_dir)
+    except subprocess.CalledProcessError:
+        print "Error running setup.py in %s" % directory
+        raise
+else:
+    deps += ['marionette-transport == 0.1']
+
 setup(name='marionette_client',
       version=version,
       description="Marionette test automation client",
       long_description=description,
       classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
       keywords='mozilla',
       author='Jonathan Griffin',
       author_email='jgriffin@mozilla.com',
new file mode 100644
--- /dev/null
+++ b/testing/marionette/transport/README.md
@@ -0,0 +1,14 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+# Marionette Transport Layer
+
+[Marionette](https://developer.mozilla.org/en/Marionette) is a 
+Mozilla project to enable remote automation in Gecko-based projects,
+including desktop Firefox, mobile Firefox, and Firefox OS.   It's inspired
+by [Selenium Webdriver](http://www.seleniumhq.org/projects/webdriver/).
+
+This package defines the transport layer used by a Marionette client to
+communicate with the Marionette server embedded in Gecko.  It has no entry
+points; rather it's designed to be used by Marionette client implementations.
new file mode 100644
--- /dev/null
+++ b/testing/marionette/transport/marionette_transport/__init__.py
@@ -0,0 +1,1 @@
+from transport import MarionetteTransport
new file mode 100644
--- /dev/null
+++ b/testing/marionette/transport/marionette_transport/transport.py
@@ -0,0 +1,105 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import errno
+import json
+import socket
+
+
+class MarionetteTransport(object):
+    """ The Marionette socket client.  This speaks the same protocol
+        as the remote debugger inside Gecko, in which messages are
+        always preceded by the message length and a colon, e.g.,
+
+        20:{'command': 'test'}
+    """
+
+    max_packet_length = 4096
+    connection_lost_msg = "Connection to Marionette server is lost. Check gecko.log (desktop firefox) or logcat (b2g) for errors."
+
+    def __init__(self, addr, port):
+        self.addr = addr
+        self.port = port
+        self.sock = None
+        self.traits = None
+        self.applicationType = None
+        self.actor = 'root'
+
+    def _recv_n_bytes(self, n):
+        """ Convenience method for receiving exactly n bytes from
+            self.sock (assuming it's open and connected).
+        """
+        data = ''
+        while len(data) < n:
+            chunk = self.sock.recv(n - len(data))
+            if chunk == '':
+                break
+            data += chunk
+        return data
+
+    def receive(self):
+        """ Receive the next complete response from the server, and return
+            it as a dict.  Each response from the server is prepended by
+            len(message) + ':'.
+        """
+        assert(self.sock)
+        response = self.sock.recv(10)
+        sep = response.find(':')
+        length = response[0:sep]
+        if length != '':
+            response = response[sep + 1:]
+            response += self._recv_n_bytes(int(length) + 1 + len(length) - 10)
+            return json.loads(response)
+        else:
+            raise IOError(self.connection_lost_msg)
+
+    def connect(self, timeout=360.0):
+        """ Connect to the server and process the hello message we expect
+            to receive in response.
+        """
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.sock.settimeout(timeout)
+        try:
+            self.sock.connect((self.addr, self.port))
+        except:
+            # Unset self.sock so that the next attempt to send will cause
+            # another connection attempt.
+            self.sock = None
+            raise
+        hello = self.receive()
+        self.traits = hello.get('traits')
+        self.applicationType = hello.get('applicationType')
+
+        # get the marionette actor id
+        response = self.send({'to': 'root', 'name': 'getMarionetteID'})
+        self.actor = response['id']
+
+    def send(self, msg):
+        """ Send a message on the socket, prepending it with len(msg) + ':'.
+        """
+        if not self.sock:
+            self.connect()
+        if 'to' not in msg:
+            msg['to'] = self.actor
+        data = json.dumps(msg)
+        data = '%s:%s' % (len(data), data)
+
+        for packet in [data[i:i + self.max_packet_length] for i in
+                       range(0, len(data), self.max_packet_length)]:
+            try: 
+                self.sock.send(packet)
+            except IOError as e:
+                if e.errno == errno.EPIPE:
+                    raise IOError("%s: %s" % (str(e)), self.connection_lost_msg)
+                else:
+                    raise e
+
+        response = self.receive()
+        return response
+
+    def close(self):
+        """ Close the socket.
+        """
+        self.sock.close()
+        self.sock = None
new file mode 100644
--- /dev/null
+++ b/testing/marionette/transport/setup.py
@@ -0,0 +1,34 @@
+import os
+from setuptools import setup, find_packages
+
+version = '0.1'
+
+# get documentation from the README
+try:
+    here = os.path.dirname(os.path.abspath(__file__))
+    description = file(os.path.join(here, 'README.md')).read()
+except (OSError, IOError):
+    description = ''
+
+# dependencies
+deps = []
+
+setup(name='marionette-transport',
+      version=version,
+      description="Transport layer for Marionette client",
+      long_description=description,
+      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+      keywords='mozilla',
+      author='Mozilla Automation and Tools Team',
+      author_email='tools@lists.mozilla.org',
+      url='https://developer.mozilla.org/en-US/docs/Marionette',
+      license='MPL',
+      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
+      package_data={},
+      include_package_data=False,
+      zip_safe=False,
+      entry_points="""
+      """,
+      install_requires=deps,
+      )
+
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -524,17 +524,19 @@ stage-steeplechase: make-stage-dir
 	$(NSINSTALL) -D $(PKG_STAGE)/steeplechase/
 	cp -RL $(DEPTH)/_tests/steeplechase $(PKG_STAGE)/steeplechase/tests
 	cp -RL $(DIST)/xpi-stage/specialpowers $(PKG_STAGE)/steeplechase
 	cp -RL $(topsrcdir)/testing/profiles/prefs_general.js $(PKG_STAGE)/steeplechase
 
 MARIONETTE_DIR=$(PKG_STAGE)/marionette
 stage-marionette: make-stage-dir
 	$(NSINSTALL) -D $(MARIONETTE_DIR)/tests
-	@(cd $(topsrcdir)/testing/marionette/client && tar --exclude marionette/tests $(TAR_CREATE_FLAGS) - *) | (cd $(MARIONETTE_DIR) && tar -xf -)
+	$(NSINSTALL) -D $(MARIONETTE_DIR)/transport
+	@(cd $(topsrcdir)/testing/marionette/client && tar --exclude marionette/tests $(TAR_CREATE_FLAGS) - *) | (cd $(MARIONETTE_DIR)/ && tar -xf -)
+	@(cd $(topsrcdir)/testing/marionette/transport && tar $(TAR_CREATE_FLAGS) - *) | (cd $(MARIONETTE_DIR)/transport && tar -xf -)
 	$(PYTHON) $(topsrcdir)/testing/marionette/client/marionette/tests/print-manifest-dirs.py \
           $(topsrcdir) \
           $(topsrcdir)/testing/marionette/client/marionette/tests/unit-tests.ini \
           | (cd $(topsrcdir) && xargs tar $(TAR_CREATE_FLAGS) -) \
           | (cd $(MARIONETTE_DIR)/tests && tar -xf -)
 
 stage-mozbase: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/mozbase stage-package