Bug 936615 - Pandas should start buildbot when mozpool thinks the device is ready. r=armenzg
authorJustin Wood <Callek@gmail.com>
Tue, 12 Nov 2013 12:32:32 -0500
changeset 4179 9c1045ced0c64d919c6a915c6bd70bd90318e508
parent 4178 4e01c9d633bb53ccb5d9e2398e8ac85485c93c33
child 4180 9b153603b0d1b2f069b526e3585725bebb0d6d52
push id3034
push userCallek@gmail.com
push dateTue, 12 Nov 2013 17:33:16 +0000
reviewersarmenzg
bugs936615
Bug 936615 - Pandas should start buildbot when mozpool thinks the device is ready. r=armenzg
.hgignore
buildfarm/mobile/watch_devices.sh
lib/python/vendor/mozpoolclient-0.1.4/PKG-INFO
lib/python/vendor/mozpoolclient-0.1.4/README.txt
lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/PKG-INFO
lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/SOURCES.txt
lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/dependency_links.txt
lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/requires.txt
lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/top_level.txt
lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.py
lib/python/vendor/mozpoolclient-0.1.4/setup.cfg
lib/python/vendor/mozpoolclient-0.1.4/setup.py
lib/python/vendorlibs.pth
sut_tools/verify.py
--- a/.hgignore
+++ b/.hgignore
@@ -3,17 +3,17 @@ moz-n810-v
 moz-n900-v
 flasher-
 RX-44
 RX-51
 build/
 slavealloc\.db
 \..*\.swp
 twistd\.pid
-.*\.egg-info
+^[^\/]*\.egg-info
 dist/
 slavealloc.log
 release-runner.ini
 .coverage
 .tox
 coverage.xml
 lib/python/buildtools.egg-info/
 nosetests.xml
--- a/buildfarm/mobile/watch_devices.sh
+++ b/buildfarm/mobile/watch_devices.sh
@@ -64,17 +64,17 @@ function device_check() {
         log "removing $device error.flg (older than an hour) and trying again"
         rm -f /builds/$device/error.flg
       else
         death "Error flag less than an hour old, so exiting" 65
       fi
     fi
     export SUT_NAME=$device
     export SUT_IP=$deviceIP
-    if ! "${PYTHON}" /builds/sut_tools/verify.py $device; then
+    if ! "${PYTHON}" /builds/sut_tools/verify.py --success-if-mozpool-ready $device; then
       log "Verify procedure failed"
       if [ ! -f /builds/$device/error.flg ]; then
         log "error.flg file does not exist, so creating it..."
         echo "Unknown verify failure" | tee "/builds/$device/error.flg"
       fi
       death "Exiting due to verify failure" 66
     fi
     log "starting buildbot slave"
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/PKG-INFO
@@ -0,0 +1,33 @@
+Metadata-Version: 1.0
+Name: mozpoolclient
+Version: 0.1.4
+Summary: It allows you to interact with devices managed by Mozpool.
+Home-page: http://pypi.python.org/pypi/mozpoolclient/
+Author: Zambrano, Armen
+Author-email: armenzg@mozilla.com
+License: MPL
+Description: ==============
+        Mozpool Client
+        ==============
+        
+        The mozpool client allows you to interact with a Mozpool [1] instance through various
+        APIs. This allows you to interact with various mobile devices that are managed
+        by Mozpool. You can reboot the devices, query information about them
+        and reimage them among many other actions.
+        
+        
+        Introduction
+        ============
+        This code is extracted from the initial work from jhopkins in mozharness [2].
+        Not all of the API has been implemented as the initial work did but and the tests
+        have not yet been ported.
+        
+        Thanks to
+        =========
+        jhopkins for initial work in mozharness.
+        dustin, dividehex, ted and mcote for creating all the pieces in the Mozpool stack.
+        
+        * [1] https://github.com/djmitche/mozpool
+        * [2] http://hg.mozilla.org/build/mozharness/file/tip/mozharness/mozilla/testing/mozpool.py
+        
+Platform: UNKNOWN
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/README.txt
@@ -0,0 +1,23 @@
+==============
+Mozpool Client
+==============
+
+The mozpool client allows you to interact with a Mozpool [1] instance through various
+APIs. This allows you to interact with various mobile devices that are managed
+by Mozpool. You can reboot the devices, query information about them
+and reimage them among many other actions.
+
+
+Introduction
+============
+This code is extracted from the initial work from jhopkins in mozharness [2].
+Not all of the API has been implemented as the initial work did but and the tests
+have not yet been ported.
+
+Thanks to
+=========
+jhopkins for initial work in mozharness.
+dustin, dividehex, ted and mcote for creating all the pieces in the Mozpool stack.
+
+* [1] https://github.com/djmitche/mozpool
+* [2] http://hg.mozilla.org/build/mozharness/file/tip/mozharness/mozilla/testing/mozpool.py
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/PKG-INFO
@@ -0,0 +1,33 @@
+Metadata-Version: 1.0
+Name: mozpoolclient
+Version: 0.1.4
+Summary: It allows you to interact with devices managed by Mozpool.
+Home-page: http://pypi.python.org/pypi/mozpoolclient/
+Author: Zambrano, Armen
+Author-email: armenzg@mozilla.com
+License: MPL
+Description: ==============
+        Mozpool Client
+        ==============
+        
+        The mozpool client allows you to interact with a Mozpool [1] instance through various
+        APIs. This allows you to interact with various mobile devices that are managed
+        by Mozpool. You can reboot the devices, query information about them
+        and reimage them among many other actions.
+        
+        
+        Introduction
+        ============
+        This code is extracted from the initial work from jhopkins in mozharness [2].
+        Not all of the API has been implemented as the initial work did but and the tests
+        have not yet been ported.
+        
+        Thanks to
+        =========
+        jhopkins for initial work in mozharness.
+        dustin, dividehex, ted and mcote for creating all the pieces in the Mozpool stack.
+        
+        * [1] https://github.com/djmitche/mozpool
+        * [2] http://hg.mozilla.org/build/mozharness/file/tip/mozharness/mozilla/testing/mozpool.py
+        
+Platform: UNKNOWN
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/SOURCES.txt
@@ -0,0 +1,8 @@
+README.txt
+mozpoolclient.py
+setup.py
+mozpoolclient.egg-info/PKG-INFO
+mozpoolclient.egg-info/SOURCES.txt
+mozpoolclient.egg-info/dependency_links.txt
+mozpoolclient.egg-info/requires.txt
+mozpoolclient.egg-info/top_level.txt
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/dependency_links.txt
@@ -0,0 +1,1 @@
+
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/requires.txt
@@ -0,0 +1,1 @@
+requests >= 1.0.0
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.egg-info/top_level.txt
@@ -0,0 +1,1 @@
+mozpoolclient
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/mozpoolclient.py
@@ -0,0 +1,320 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+'''Interact with mozpool/lifeguard/bmm.
+'''
+import getpass
+import re
+import requests
+import socket
+import sys
+import time
+
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+JsonHeader = {'content-type': 'application/json'}
+
+# 200 OK
+# 201 Created
+# 202 Accepted
+# 300 Multiple Choices
+# 301 Moved Permanently
+# 302 Found
+# 304 Not Modified
+# 400 Bad Request
+# 401 Unauthorized
+# 403 Forbidden
+# 404 Not Found
+# 405 Method Not Allowed
+# 409 Conflict
+# 500 Server Error
+# 501 Not Implemented
+# 503 Service Unavailable
+
+class MozpoolException(Exception):
+    pass
+
+class MozpoolConflictException(MozpoolException):
+    pass
+
+def mozpool_status_ok(status):
+    if status in range(200,400):
+        return True
+    else:
+        return False
+
+def check_mozpool_status(status):
+    if not mozpool_status_ok(status):
+        if status == 409:
+            raise MozpoolConflictException()
+        import pprint
+        raise MozpoolException('mozpool status not ok, code %s' % pprint.pformat(status))
+
+class MozpoolHandler:
+    def __init__(self, mozpool_api_url, log_obj=None):
+        self.mozpool_api_url = mozpool_api_url
+        self.mozpool_timeout=10
+        self.user = getpass.getuser()
+        self.hostname = socket.gethostname()
+        if log_obj:
+            self.log_obj = log_obj
+        else:
+            import logging as log
+            self.log_obj = log
+
+    # Helper methods {{{2
+    def url_get(self, url, decode_json=True, **kwargs):
+        """Generic get output from a url method.
+
+        This could be moved to a generic url handler object.
+        """
+        self.log_obj.debug("Request GET %s..." % url)
+        if kwargs.get("timeout") is None:
+            kwargs["timeout"] = self.mozpool_timeout
+        num_retries = 10
+        try_num = 0
+        while try_num <= num_retries:
+            try_num += 1
+            try:
+                r = requests.get(url, **kwargs)
+                self.log_obj.debug("Status code: %s" % str(r.status_code))
+                if decode_json:
+                    j = self.decode_json(r.text)
+                    if j is not None:
+                        return (j, r.status_code)
+                    else:
+                        raise MozpoolException("Try %d: Can't decode json from %s!" % (try_num, url))
+                else:
+                    return (r.text, r.status_code)
+            except requests.exceptions.RequestException, e:
+                raise MozpoolException("Try %d: Can't get %s: %s!" % (try_num, url, str(e)))
+            if try_num <= num_retries:
+                sleep_time = 2 * try_num
+                self.log_obj.info("Sleeping %d..." % sleep_time)
+                time.sleep(sleep_time)
+
+    def decode_json(self, contents):
+        try:
+            return json.loads(contents, encoding="ascii")
+        except ValueError, e:
+            raise MozpoolException("Can't decode json: %s!" % str(e))
+        except TypeError, e:
+            raise MozpoolException("Can't decode json: %s!" % str(e))
+        else:
+            raise MozpoolException("Can't decode json: Unknown error!" % str(e))
+
+    def url_post(self, url, data, auth=None, params=None,
+                 good_statuses=None, decode_json=True, **kwargs):
+        """Generic post to a url method.
+
+        This could be moved to a generic url handler object.
+        """
+        self.log_obj.debug("Request POST %s..." % url)
+        if kwargs.get("timeout") is None:
+            kwargs["timeout"] = self.mozpool_timeout
+        num_retries = 10
+        if good_statuses is None:
+            good_statuses = [200, 201, 202, 204, 302]
+        try_num = 0
+        while try_num <= num_retries:
+            try_num += 1
+            try:
+                r = requests.post(url, data=data, **kwargs)
+                if r.status_code in good_statuses:
+                    self.log_obj.debug("Status code: %s" % str(r.status_code))
+                    if decode_json:
+                        j = self.decode_json(r.text)
+                        if j is not None:
+                            return (j, r.status_code)
+                        else:
+                            raise MozpoolException("Try %d: Can't decode json from %s!" % (try_num, url))
+                    else:
+                        return (r.text, r.status_code)
+                else:
+                    self.log_obj.critical("Bad return status from %s: %d!" % (url, r.status_code))
+                    return (None, r.status_code)
+            except requests.exceptions.RequestException, e:
+                self.log_obj.exception("Try %d: Can't get %s: %s!" % (try_num, url, str(e)))
+            if try_num <= num_retries:
+                sleep_time = 2 * try_num
+                self.log_obj.info("Sleeping %d..." % sleep_time)
+                time.sleep(sleep_time)
+
+    def partial_url_get(self, partial_url, **kwargs):
+        return self.url_get(self.mozpool_api_url + partial_url, **kwargs)
+
+    def partial_url_post(self, partial_url, **kwargs):
+        return self.url_post(self.mozpool_api_url + partial_url, **kwargs)
+
+    # Device queries {{{2
+    def query_device_status(self, device, **kwargs):
+        """ returns a JSON response body whose "status" key contains
+            a short string describing the last-known status of the device,
+            and whose "log" key contains an array of recent log entries
+            for the device.
+        """
+        response, status = self.partial_url_get("/api/device/%s/status/" % device, **kwargs)
+        check_mozpool_status(status)
+        return response
+
+    def query_device_state(self, device, **kwargs):
+        """ returns a JSON response body whose "state" key contains
+            a short string describing the last-known status of the device.
+        """
+        response, status = self.partial_url_get("/api/device/%s/state/" % device, **kwargs)
+        check_mozpool_status(status)
+        return response
+
+    def query_device_details(self, device, **kwargs):
+        bmm = self.mozpool_api_url
+        response, status = self.url_get("%s/api/device/list/?details=1" % bmm, **kwargs)
+        check_mozpool_status(status)
+        devices = response.get("devices")
+        if isinstance(devices, list):
+            matches = filter(lambda dd: dd['name'] == device, devices)
+            if len(matches) != 1:
+                self.log_obj.critical("Couldn't find %s in device list!" % device_id)
+                return
+            else:
+                return matches[0]
+        else:
+            # We shouldn't get here if query_all_device_details() FATALs...
+            raise MozpoolException("Invalid response from query_all_device_details()!")
+
+    def request_device(self, device, image, duration=30*60, assignee=None,
+            pxe_config=None, b2gbase=None, environment='any', **kwargs):
+        """ requests the given device. {id} may be "any" to let MozPool choose an
+            unassigned device. The body must be a JSON object with at least the keys
+            "requester", "duration", and "image". The value for "requester" takes an
+            email address, for human users, or a hostname, for machine users. "duration"
+            must be a value, in seconds, of the duration of the request (which can be
+            renewed; see below).
+
+            "image" specifies low-level configuration that should be done on the device
+            by mozpool. Some image types will require additional parameters. Currently
+            the only supported value is "b2g", for which a "b2gbase" key must also be
+            present. The value of "b2gbase" must be a URL to a b2g build directory
+            containing boot, system, and userdata tarballs.
+
+            If successful, returns 200 OK with a JSON object with the key "request".
+            The value of "request" is an object detailing the request, with the keys
+            "assigned_device" (which is blank if mozpool is still attempting to find
+            a device, "assignee", "expires", "id", "requested_device",
+            and "url". The "url" attribute contains a partial URL
+            for the request object and should be used in request calls, as detailed
+            below. If 'any' device was requested, always returns 200 OK, since it will
+            retry a few times if no devices are free. If a specific device is requested
+            but is already assigned, returns 409 Conflict; otherwise, returns 200 OK.
+
+            If a 200 OK code is returned, the client should then poll for the request's
+            state (using the value of request["url"] returned in the JSON object with
+            "status/" appended. A normal request will move through the states "new",
+            "find_device", "contact_lifeguard", "pending", and "ready", in that order.
+            When, and *only* when, the device is in the "ready" state, it is safe to be
+            used by the client. Other possible states are "expired", "closed",
+            "device_not_found", and "device_busy"; the assigned device (if any) is
+            returned to the pool when any of these states are entered.
+        """
+        if not assignee:
+            assignee = "%s-%s" % (self.user, self.hostname)
+        if image == 'b2g':
+            assert b2gbase is not None, "b2gbase must be supplied when image=='b2gbase'"
+        assert duration == int(duration)
+
+        data = {'assignee': assignee, 'duration': duration, 'image': image, 'environment': environment}
+        if pxe_config is not None:
+            data['pxe_config'] = pxe_config
+        data['b2gbase'] = b2gbase
+        response, status = self.url_post(
+                "%s/api/device/%s/request/" % (self.mozpool_api_url, device),
+                data=json.dumps(data), headers=JsonHeader)
+        check_mozpool_status(status)
+        return response
+
+    def renew_request(self, request_url, new_duration):
+        """ requests that the request's lifetime be updated. The request body
+            should be a JSON object with the key "duration", the value of which is the
+            *new* remaining time, in seconds, of the request. Returns 204 No Content.
+        """
+        request_url = request_url + 'renew/'
+        data = {'duration': new_duration}
+        response, status = self.url_post(request_url, data=json.dumps(data), headers=JsonHeader, decode_json=False)
+        check_mozpool_status(status)
+        return response
+
+    def close_request(self, request_url, **kwargs):
+        """ returns the device to the pool and deletes the request. Returns
+            204 No Content.
+        """
+        request_url = request_url + 'return/'
+        data = {}
+        response, status = self.url_post(request_url, data=json.dumps(data), headers=JsonHeader, decode_json=False)
+        check_mozpool_status(status)
+        return response
+
+    def device_power_cycle(self, device, duration, assignee=None, pxe_config=None):
+        """ initiate a power-cycle of this device. The POST body is a JSON object,
+            with optional keys `pxe_config` and `boot_config`. If `pxe_config` is
+            specified, then the device is configured to boot with that PXE config;
+            otherwise, the device boots from its internal storage. If `boot_config` is
+            supplied (as a string), it is stored for later use by the device via
+            `/api/device/{id}/config/`.
+        """
+        if assignee == None:
+            assignee = "%s-%s" % (self.user, self.hostname)
+        data = {'assignee': assignee, 'duration': duration}
+        if pxe_config is not None:
+            data['pxe_config'] = pxe_config
+        response, status = self.url_post(
+            "%s/api/device/%s/power-cycle/" % (self.mozpool_api_url, device),
+            data=json.dumps(data), headers=JsonHeader
+        )
+        check_mozpool_status(status)
+        return response
+
+    def device_ping(self, device, **kwargs):
+        """ ping this device. Returns a JSON object with a `success` key, and
+            value true or false. The ping happens synchronously, and takes around a
+            half-second.
+        """
+        response, status = self.url_get(
+            "%s/api/device/%s/ping" % (self.mozpool_api_url, device)
+        )
+        check_mozpool_status(status)
+        return response
+
+    def query_device_log(self, device, **kwargs):
+        """ get a list of recent log lines for this device. The return value has
+            a 'log' key containing a list of objects representing log lines.
+        """
+        response, status = self.url_get(
+                device, "/api/device/%s/log/" % device, **kwargs)
+        check_mozpool_status(status)
+        return response
+
+    def query_request_status(self, request_url, **kwargs):
+        """ returns a JSON response body with keys "log" and "state". Log objects
+            contain "message", "source", and "timestamp" keys. "state" is the name of
+            the current state, "ready" being the state in which it is safe to use the
+            device.
+        """
+        request_url = request_url + 'status/'
+        response, status = self.url_get(request_url, **kwargs)
+        check_mozpool_status(status)
+        return response
+
+    def query_request_details(self, request_url, **kwargs):
+        """ returns a JSON response body whose "request" key contains an object
+            representing the given request with the keys id, device_id, assignee,
+            expires, and status. The expires field is given as an ISO-formatted time.
+        """
+        request_url = request_url + 'details/'
+        response, status = self.url_get(request_url, **kwargs)
+        check_mozpool_status(status)
+        return response
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/setup.cfg
@@ -0,0 +1,5 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+tag_svn_revision = 0
+
new file mode 100644
--- /dev/null
+++ b/lib/python/vendor/mozpoolclient-0.1.4/setup.py
@@ -0,0 +1,17 @@
+from setuptools import setup
+
+setup(
+    name='mozpoolclient',
+    version='0.1.4',
+    author='Zambrano, Armen',
+    author_email='armenzg@mozilla.com',
+    py_modules=['mozpoolclient'],
+    scripts=[],
+    url='http://pypi.python.org/pypi/mozpoolclient/',
+    license='MPL',
+    description='It allows you to interact with devices managed by Mozpool.',
+    long_description=open('README.txt').read(),
+    install_requires=[
+        'requests >= 1.0.0',
+    ],
+)
--- a/lib/python/vendorlibs.pth
+++ b/lib/python/vendorlibs.pth
@@ -1,4 +1,5 @@
 vendor/poster-0.8.1/
 vendor/pefile-1.2.10-114/
 vendor/python-daemon-1.5.5/
 vendor/requests-0.10.8/
+vendor/mozpoolclient-0.1.4/
--- a/sut_tools/verify.py
+++ b/sut_tools/verify.py
@@ -10,21 +10,29 @@ import time
 
 import site
 site.addsitedir(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../lib/python"))
 
 from sut_lib import pingDevice, setFlag, connect, log
 from sut_lib.powermanagement import soft_reboot
 from mozdevice import devicemanagerSUT as devicemanager
 import updateSUT
+from mozpoolclient import MozpoolException, MozpoolHandler
 
 MAX_RETRIES = 5
 EXPECTED_DEVICE_SCREEN = 'X:1024 Y:768'
 EXPECTED_DEVICE_SCREEN_ARGS = {'width': 1024, 'height': 768, 'type': 'crt'}
 
+MOZPOOL_CNAME = "mobile-imaging"
+# CONSTS corresponding to mozpool check return codes.
+MOZPOOL_STATE_READY = 1
+MOZPOOL_STATE_UNKNOWN = 0
+MOZPOOL_STATE_ERROR = -1
+MOZPOOL_STATE_MISSING = -2
+
 errorFile = None
 dm = None
 
 watcherINI = "\r\n[watcher]\r\nPingTarget = bm-remote.build.mozilla.org\r\nstrikes = 0\r\n"
 
 
 def dmAlive(dm):
     """ Check that a devicemanager connection is still active
@@ -59,16 +67,51 @@ def canPing(device):
             else:
                 log.info("INFO: Unable to ping device after %s try. Sleeping for 90s then retrying" % curRetry)
                 time.sleep(90)
         else:
             break  # we're done here
     return True
 
 
+def isMozpoolReady(device):
+    """ Checks if the mozpool server is available and the device is in 'ready' state
+
+    Returns MOZPOOL_STATE_READY if Mozpool is indeed listing device as ready
+    Returns MOZPOOL_STATE_UNKNOWN if Mozpool claims the device is in a non-ready state
+    Returns MOZPOOL_STATE_ERROR if Mozpool reports some other error.
+    Returns MOZPOOL_STATE_MISSING if there is no Mozpool server found in DNS.
+    """
+    import socket
+    default_timeout = socket.getdefaulttimeout()
+    socket.setdefaulttimeout(5)  # Don't let networking delay us too long
+    try:
+        socket.gethostbyname(MOZPOOL_CNAME)
+    except:
+        log.info("No mozpool server in this VLAN")
+        return MOZPOOL_STATE_MISSING
+    finally:
+        # Set socket timeout back
+        socket.setdefaulttimeout(default_timeout)
+
+    mpc = MozpoolHandler("http://%s" % MOZPOOL_CNAME, log)
+    try:
+        result = mpc.query_device_state(device)
+    except MozpoolException as e:
+        log.error("Unable to get mozpool state, mozpool returned error: %s" % sys.exc_info()[1])
+        return MOZPOOL_STATE_ERROR
+
+    if result['state'] == "ready":
+        log.debug("Mozpool state is 'ready'")
+        return MOZPOOL_STATE_READY
+    else:
+        log.error("Mozpool state is '%s'" % result['state'])
+        return MOZPOOL_STATE_UNKNOWN
+
+
 def canTelnet(device):
     """ Checks if we can establish a Telnet session (via devicemanager)
 
     Sets global `dm`
     Returns False on failure, True on Success
     """
     global dm
     curRetry = 0
@@ -288,27 +331,43 @@ def setWatcherINI(dm):
         if watcherDataCurrent():
             return True
     except:
         pass
     setFlag(errorFile, "Unable to verify the updated watcher.ini")
     return False
 
 
-def verifyDevice(device, checksut=True, doCheckStalled=True, watcherINI=False):
+def verifyDevice(device, checksut=True, doCheckStalled=True, watcherINI=False,
+                 skipWhenMozpoolReady=False):
     # Returns False on failure, True on Success
     global dm, errorFile
     devicePath = os.path.join('/builds', device)
     errorFile = os.path.join(devicePath, 'error.flg')
 
     if doCheckStalled:
         if not cleanupFoopy(device):
             log.info("verifyDevice: failing to cleanup foopy")
             return False
 
+    mozpool_state = isMozpoolReady(device)
+    if skipWhenMozpoolReady and mozpool_state is MOZPOOL_STATE_READY:
+        log.info("Mozpool State is ready skipping device checks")
+        return True
+    elif mozpool_state is MOZPOOL_STATE_READY:
+        log.info("Mozpool claims device is ready, continuing to verify device...")
+    elif mozpool_state is MOZPOOL_STATE_UNKNOWN:
+        log.info("Mozpool knows about device, but claims we're not safe to continue")
+        return False
+    elif mozpool_state in (MOZPOOL_STATE_ERROR, MOZPOOL_STATE_MISSING):
+        log.info("Unable to determine state from Mozpool, falling back to device checks")
+    else:
+        log.info("Unexpected Mozpool State returned, hard stop.")
+        return False
+
     if not canPing(device):
         # TODO Reboot via PDU if ping fails
         log.info("verifyDevice: failing to ping")
         return False
 
     if not canTelnet(device):
         log.info("verifyDevice: failing to telnet")
         return False
@@ -335,26 +394,34 @@ def verifyDevice(device, checksut=True, 
         if not setWatcherINI(dm):
             log.info("verifyDevice: failing to set watcher.ini")
             return False
 
     return True
 
 if __name__ == '__main__':
     device_name = os.getenv('SUT_NAME')
-    if (len(sys.argv) != 2):
+    from optparse import OptionParser
+    parser = OptionParser(usage="usage: %prog [options] [device_name]")
+    parser.add_option("--success-if-mozpool-ready",
+                      action="store_true", dest="skipWhenMozpoolReady", default=False,
+                      help="if mozpool reports device is 'ready' skip all device checks")
+    (options, args) = parser.parse_args()
+    if (len(args) != 2):
         if device_name in (None, ''):
-            print "usage: verify.py [device name]"
+            parser.print_help()
             print "   Must have $SUT_NAME set in environ to omit device name"
             sys.exit(1)
         else:
             log.info(
                 "INFO: Using device '%s' found in env variable" % device_name)
     else:
-        device_name = sys.argv[1]
+        device_name = args[0]
 
-    # Only attempt updating the watcher if we run against a tegra.
+    # Only attempt updating the watcher INI if we run against a tegra.
     doWatcherUpdate = 'tegra' in device_name
 
-    if verifyDevice(device_name, watcherINI=doWatcherUpdate) is False:
+    if verifyDevice(device_name,
+                    watcherINI=doWatcherUpdate,
+                    skipWhenMozpoolReady=options.skipWhenMozpoolReady) is False:
         sys.exit(1)  # Not ok to proceed
 
     sys.exit(0)