Bug 812395 - Make emulator fail gracefully during errors for install_gecko, r=ahal, a=NPOTB,test-only
authorJonathan Griffin <jgriffin@mozilla.com>
Mon, 19 Nov 2012 09:29:34 -0800
changeset 117038 88f664cf0188a101db9b618644af44262fe3ca5f
parent 117035 65e445cc66555b05bbfa77bf9c936d2179358213
child 117041 7b8645ae4286940829cab149c39e82518ed877ff
push id1724
push userahalberstadt@mozilla.com
push dateWed, 21 Nov 2012 17:17:16 +0000
treeherdermozilla-beta@88f664cf0188 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal, NPOTB, test-only
bugs812395
milestone18.0
Bug 812395 - Make emulator fail gracefully during errors for install_gecko, r=ahal, a=NPOTB,test-only
layout/tools/reftest/runreftestb2g.py
testing/marionette/client/marionette/emulator.py
testing/marionette/client/marionette/errors.py
testing/marionette/client/marionette/marionette.py
testing/marionette/client/marionette/runtests.py
testing/mochitest/runtestsb2g.py
--- a/layout/tools/reftest/runreftestb2g.py
+++ b/layout/tools/reftest/runreftestb2g.py
@@ -484,17 +484,17 @@ def main(args=sys.argv[1:]):
     if options.emulator_res:
         kwargs['emulator_res'] = options.emulator_res
     if options.b2gPath:
         kwargs['homedir'] = options.b2gPath
     if options.marionette:
         host,port = options.marionette.split(':')
         kwargs['host'] = host
         kwargs['port'] = int(port)
-    marionette = Marionette(**kwargs)
+    marionette = Marionette.getMarionetteOrExit(**kwargs)
     auto.marionette = marionette
 
     # create the DeviceManager
     kwargs = {'adbPath': options.adbPath}
     if options.deviceIP:
         kwargs.update({'host': options.deviceIP,
                        'port': options.devicePort})
     dm = devicemanagerADB.DeviceManagerADB(**kwargs)
--- a/testing/marionette/client/marionette/emulator.py
+++ b/testing/marionette/client/marionette/emulator.py
@@ -7,19 +7,21 @@ from errors import *
 from mozdevice import devicemanagerADB, DMError
 from mozprocess import ProcessHandlerMixin
 import os
 import re
 import platform
 import shutil
 import socket
 import subprocess
+import sys
 from telnetlib import Telnet
 import tempfile
 import time
+import traceback
 
 from emulator_battery import EmulatorBattery
 from emulator_geo import EmulatorGeo
 from emulator_screen import EmulatorScreen
 
 
 class LogcatProc(ProcessHandlerMixin):
     """Process handler for logcat which saves all output to a logfile.
@@ -374,43 +376,53 @@ waitFor(
         """
         # See bug 800102.  We use this particular method of installing
         # gecko in order to avoid an adb bug in which adb will sometimes
         # hang indefinitely while copying large files to the system
         # partition.
         push_attempts = 10
 
         print 'installing gecko binaries...'
-        # need to remount so we can write to /system/b2g
-        self._run_adb(['remount'])
-        for root, dirs, files in os.walk(gecko_path):
-            for filename in files:
-                rel_path = os.path.relpath(os.path.join(root, filename), gecko_path)
-                system_b2g_file = os.path.join('/system/b2g', rel_path)
-                for retry in range(1, push_attempts+1):
-                    print 'pushing', system_b2g_file, '(attempt %s of %s)' % (retry, push_attempts)
-                    try:
-                        self.dm.pushFile(os.path.join(root, filename), system_b2g_file)
-                        break
-                    except DMError:
-                        if retry == push_attempts:
-                            raise
+
+        try:
+            # need to remount so we can write to /system/b2g
+            self._run_adb(['remount'])
+            for root, dirs, files in os.walk(gecko_path):
+                for filename in files:
+                    rel_path = os.path.relpath(os.path.join(root, filename), gecko_path)
+                    system_b2g_file = os.path.join('/system/b2g', rel_path)
+                    for retry in range(1, push_attempts+1):
+                        print 'pushing', system_b2g_file, '(attempt %s of %s)' % (retry, push_attempts)
+                        try:
+                            self.dm.pushFile(os.path.join(root, filename), system_b2g_file)
+                            break
+                        except DMError:
+                            if retry == push_attempts:
+                                raise
 
-        print 'restarting B2G'
-        # see bug 809437 for the path that lead to this madness
-        time.sleep(5)
-        self.dm.shellCheckOutput(['stop', 'b2g'])
-        time.sleep(10)
-        self.dm.shellCheckOutput(['start', 'b2g'])
-        time.sleep(5)
+            print 'restarting B2G'
+            # see bug 809437 for the path that lead to this madness
+            self.dm.shellCheckOutput(['stop', 'b2g'])
+            time.sleep(10)
+            self.dm.shellCheckOutput(['start', 'b2g'])
+
+            if not self.wait_for_port():
+                raise TimeoutException("Timeout waiting for marionette on port '%s'" % self.marionette_port)
+            self.wait_for_system_message(marionette)
 
-        if not self.wait_for_port():
-            raise TimeoutException("Timeout waiting for marionette on port '%s'" % self.marionette_port)
-        self.wait_for_system_message(marionette)
+        except (DMError, MarionetteException):
+            # Bug 812395 - raise a single exception type for these so we can
+            # explicitly catch them elsewhere.
 
+            # print exception, but hide from mozharness error detection
+            exc = traceback.format_exc()
+            exc = exc.replace('Traceback', '_traceback')
+            print exc
+
+            raise InstallGeckoError("unable to restart B2G after installing gecko")
 
     def rotate_log(self, srclog, index=1):
         """ Rotate a logfile, by recursively rotating logs further in the sequence,
             deleting the last file if necessary.
         """
         destlog = os.path.join(self.logcat_dir, 'emulator-%d.%d.log' % (self.port, index))
         if os.access(destlog, os.F_OK):
             if index == 3:
--- a/testing/marionette/client/marionette/errors.py
+++ b/testing/marionette/client/marionette/errors.py
@@ -12,16 +12,19 @@ class MarionetteException(Exception):
 
     def __str__(self):
         if self.stacktrace:
             return '%s\nstacktrace:\n%s' % (str(self.message),
                 ''.join(['\t%s\n' % x for x in self.stacktrace.split('\n')]))
         else:
             return str(self.message)
 
+class InstallGeckoError(MarionetteException):
+    pass
+
 class TimeoutException(MarionetteException):
     pass
 
 class NoSuchAttributeException(MarionetteException):
     pass
 
 class JavascriptException(MarionetteException):
     pass
--- a/testing/marionette/client/marionette/marionette.py
+++ b/testing/marionette/client/marionette/marionette.py
@@ -1,21 +1,24 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import socket
+import sys
+import traceback
 
 from client import MarionetteClient
 from application_cache import ApplicationCache
 from keys import Keys
 from errors import *
 from emulator import Emulator
 from geckoinstance import GeckoInstance
 
+
 class HTMLElement(object):
 
     CLASS = "class name"
     SELECTOR = "css selector"
     ID = "id"
     NAME = "name"
     LINK_TEXT = "link text"
     PARTIAL_LINK_TEXT = "partial link text"
@@ -150,16 +153,36 @@ class Marionette(object):
     def __del__(self):
         if self.emulator:
             self.emulator.close()
         if self.instance:
             self.instance.close()
         for qemu in self.extra_emulators:
             qemu.emulator.close()
 
+    @classmethod
+    def getMarionetteOrExit(cls, *args, **kwargs):
+        try:
+            m = cls(*args, **kwargs)
+            return m
+        except InstallGeckoError:
+            # Bug 812395 - the process of installing gecko into the emulator
+            # and then restarting B2G tickles some bug in the emulator/b2g
+            # that intermittently causes B2G to fail to restart.  To work
+            # around this in TBPL runs, we will fail gracefully from this
+            # error so that the mozharness script can try the run again.
+
+            # This string will get caught by mozharness and will cause it
+            # to retry the tests.
+            print "Error installing gecko!"
+
+            # Exit without a normal exception to prevent mozharness from
+            # flagging the error.
+            sys.exit()
+
     def _send_message(self, command, response_key, **kwargs):
         if not self.session and command not in ('newSession', 'getStatus'):
             raise MarionetteException(message="Please start a session")
 
         message = { 'type': command }
         if self.session:
             message['session'] = self.session
         if kwargs:
--- a/testing/marionette/client/marionette/runtests.py
+++ b/testing/marionette/client/marionette/runtests.py
@@ -3,23 +3,21 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from datetime import datetime
 import imp
 import inspect
 import logging
 from optparse import OptionParser
 import os
-import types
 import unittest
 import socket
 import sys
 import time
 import platform
-import weakref
 import xml.dom.minidom as dom
 
 from manifestparser import TestManifest
 from mozhttpd import iface, MozHttpd
 
 from marionette import Marionette
 from marionette_test import MarionetteJSTestCase, MarionetteTestCase
 
@@ -271,28 +269,30 @@ class MarionetteTestRunner(object):
                 host = 'localhost'
                 port = 2828
             self.marionette = Marionette(host=host, port=int(port),
                                          bin=self.bin, profile=self.profile,
                                          baseurl=self.baseurl)
         elif self.address:
             host, port = self.address.split(':')
             if self.emulator:
-                self.marionette = Marionette(host=host, port=int(port),
+                self.marionette = Marionette.getMarionetteOrExit(
+                                             host=host, port=int(port),
                                              connectToRunningEmulator=True,
                                              homedir=self.homedir,
                                              baseurl=self.baseurl,
                                              logcat_dir=self.logcat_dir,
                                              gecko_path=self.gecko_path)
             else:
                 self.marionette = Marionette(host=host,
                                              port=int(port),
                                              baseurl=self.baseurl)
         elif self.emulator:
-            self.marionette = Marionette(emulator=self.emulator,
+            self.marionette = Marionette.getMarionetteOrExit(
+                                         emulator=self.emulator,
                                          emulatorBinary=self.emulatorBinary,
                                          emulatorImg=self.emulatorImg,
                                          emulator_res=self.emulator_res,
                                          homedir=self.homedir,
                                          baseurl=self.baseurl,
                                          noWindow=self.noWindow,
                                          logcat_dir=self.logcat_dir,
                                          gecko_path=self.gecko_path)
--- a/testing/mochitest/runtestsb2g.py
+++ b/testing/mochitest/runtestsb2g.py
@@ -485,17 +485,18 @@ def main():
     if options.sdcard:
         kwargs['sdcard'] = options.sdcard
     if options.b2gPath:
         kwargs['homedir'] = options.b2gPath
     if options.marionette:
         host,port = options.marionette.split(':')
         kwargs['host'] = host
         kwargs['port'] = int(port)
-    marionette = Marionette(**kwargs)
+
+    marionette = Marionette.getMarionetteOrExit(**kwargs)
 
     auto.marionette = marionette
 
     # create the DeviceManager
     kwargs = {'adbPath': options.adbPath,
               'deviceRoot': B2GMochitest.testDir}
     if options.deviceIP:
         kwargs.update({'host': options.deviceIP,