Bug 883281 - Add ability to skip tests based on device, r=davehunt
authorJonathan Griffin <jgriffin@mozilla.com>
Wed, 19 Jun 2013 17:35:20 -0700
changeset 135741 f69a75b67d1ffc1b2af3aa465be462853bad1678
parent 135740 c50503ce86692f3906b86faff20cf4c4b40c7ccb
child 135742 903a455c5430edb53b5b39be6850c221aca626f5
push id24848
push useremorley@mozilla.com
push dateThu, 20 Jun 2013 07:59:05 +0000
treeherdermozilla-central@83aa31ec53d9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs883281
milestone24.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 883281 - Add ability to skip tests based on device, r=davehunt
testing/marionette/client/marionette/b2g_update_test.py
testing/marionette/client/marionette/runtests.py
testing/marionette/marionette-server.js
--- a/testing/marionette/client/marionette/b2g_update_test.py
+++ b/testing/marionette/client/marionette/b2g_update_test.py
@@ -65,17 +65,17 @@ class B2GUpdateTestRunner(MarionetteTest
 
         self.host, port = self.address.split(':')
         self.port = int(port)
         self.test_handlers.append(self)
 
         self.b2g = B2GInstance(homedir=kwargs.get('homedir'))
         self.update_tools = self.b2g.import_update_tools()
         self.adb = self.update_tools.AdbTool(path=self.b2g.adb_path,
-                                             device=self.device)
+                                             device=self.device_serial)
 
     def match(self, filename):
         return self.match_re.match(filename) is not None
 
     def add_tests_to_suite(self, mod_name, filepath, suite, testloader,
                            marionette, testvars):
         """ Here the runner itself is a handler so we can forward along the
             instance to test cases.
--- a/testing/marionette/client/marionette/runtests.py
+++ b/testing/marionette/client/marionette/runtests.py
@@ -173,18 +173,18 @@ class MarionetteTestRunner(object):
     textrunnerclass = MarionetteTextTestRunner
 
     def __init__(self, address=None, emulator=None, emulatorBinary=None,
                  emulatorImg=None, emulator_res='480x800', homedir=None,
                  app=None, bin=None, profile=None, autolog=False, revision=None,
                  es_server=None, rest_server=None, logger=None,
                  testgroup="marionette", noWindow=False, logcat_dir=None,
                  xml_output=None, repeat=0, gecko_path=None, testvars=None,
-                 tree=None, type=None, device=None, symbols_path=None, timeout=None,
-                 **kwargs):
+                 tree=None, type=None, device_serial=None, symbols_path=None,
+                 timeout=None, **kwargs):
         self.address = address
         self.emulator = emulator
         self.emulatorBinary = emulatorBinary
         self.emulatorImg = emulatorImg
         self.emulator_res = emulator_res
         self.homedir = homedir
         self.app = app
         self.bin = bin
@@ -202,19 +202,22 @@ class MarionetteTestRunner(object):
         self.logcat_dir = logcat_dir
         self.xml_output = xml_output
         self.repeat = repeat
         self.gecko_path = gecko_path
         self.testvars = {}
         self.test_kwargs = kwargs
         self.tree = tree
         self.type = type
-        self.device = device
+        self.device_serial = device_serial
         self.symbols_path = symbols_path
         self.timeout = timeout
+        self._device = None
+        self._capabilities = None
+        self._appName = None
 
         if testvars:
             if not os.path.exists(testvars):
                 raise Exception('--testvars file does not exist')
 
             import json
             with open(testvars) as f:
                 self.testvars = json.loads(f.read())
@@ -233,16 +236,42 @@ class MarionetteTestRunner(object):
         if self.logcat_dir:
             if not os.access(self.logcat_dir, os.F_OK):
                 os.mkdir(self.logcat_dir)
 
         # for XML output
         self.testvars['xml_output'] = self.xml_output
         self.results = []
 
+    @property
+    def capabilities(self):
+        if self._capabilities:
+            return self._capabilities
+
+        self.marionette.start_session()
+        self._capabilities = self.marionette.session_capabilities
+        self.marionette.delete_session()
+        return self._capabilities
+
+    @property
+    def device(self):
+        if self._device:
+            return self._device
+
+        self._device = self.capabilities.get('device')
+        return self._device
+
+    @property
+    def appName(self):
+        if self._appName:
+            return self._appName
+
+        self._appName = self.capabilities.get('browserName')
+        return self._appName
+
     def reset_test_stats(self):
         self.passed = 0
         self.failed = 0
         self.todo = 0
         self.failures = []
 
     def start_httpd(self):
         host = moznetwork.get_ip()
@@ -266,21 +295,22 @@ class MarionetteTestRunner(object):
                                          app=self.app,
                                          bin=self.bin,
                                          profile=self.profile,
                                          baseurl=self.baseurl,
                                          timeout=self.timeout)
         elif self.address:
             host, port = self.address.split(':')
             try:
-		#establish a telnet connection so we can vertify the data come back
-		tlconnection = Telnet(host, port)
-	    except:
-		raise Exception("could not connect to given marionette host/port")
-	    if self.emulator:
+                #establish a telnet connection so we can vertify the data come back
+                tlconnection = Telnet(host, port)
+            except:
+                raise Exception("could not connect to given marionette host/port")
+
+            if self.emulator:
                 self.marionette = Marionette.getMarionetteOrExit(
                                              host=host, port=int(port),
                                              connectToRunningEmulator=True,
                                              homedir=self.homedir,
                                              baseurl=self.baseurl,
                                              logcat_dir=self.logcat_dir,
                                              gecko_path=self.gecko_path,
                                              symbols_path=self.symbols_path,
@@ -416,17 +446,28 @@ class MarionetteTestRunner(object):
                     elif atype.startswith('-'):
                         testargs.update({ atype[1:]: 'false' })
                     else:
                         testargs.update({ atype: 'true' })
 
             manifest = TestManifest()
             manifest.read(filepath)
 
-            manifest_tests = manifest.active_tests(disabled=False)
+            all_tests = manifest.active_tests(disabled=False)
+            manifest_tests = manifest.active_tests(disabled=False,
+                                                   device=self.device,
+                                                   app=self.appName)
+            skip_tests = list(set([x['path'] for x in all_tests]) -
+                              set([x['path'] for x in manifest_tests]))
+            for skipped in skip_tests:
+                self.logger.info('TEST-SKIP | %s | device=%s, app=%s' %
+                                 (os.path.basename(skipped),
+                                  self.device,
+                                  self.appName))
+                self.todo += 1
 
             for i in manifest.get(tests=manifest_tests, **testargs):
                 self.run_test(i["path"])
                 if self.marionette.check_for_crash():
                     return
             return
 
         self.logger.info('TEST-START %s' % os.path.basename(test))
@@ -585,17 +626,17 @@ class MarionetteTestOptions(OptionParser
                         dest='logcat_dir',
                         action='store',
                         help='directory to store logcat dump files')
         self.add_option('--address',
                         dest='address',
                         action='store',
                         help='host:port of running Gecko instance to connect to')
         self.add_option('--device',
-                        dest='device',
+                        dest='device_serial',
                         action='store',
                         help='serial ID of a device to use for adb / fastboot')
         self.add_option('--type',
                         dest='type',
                         action='store',
                         default='browser+b2g',
                         help="the type of test to run, can be a combination of values defined in the manifest file; "
                              "individual values are combined with '+' or '-' characters. for example: 'browser+b2g' "
--- a/testing/marionette/marionette-server.js
+++ b/testing/marionette/marionette-server.js
@@ -37,28 +37,30 @@ let appName = Services.appinfo.name;
 // dumpn needed/used by dbg-transport.js
 this.dumpn = function dumpn(str) {
   logger.trace(str);
 }
 loader.loadSubScript("resource://gre/modules/devtools/DevToolsUtils.js");
 loader.loadSubScript("resource://gre/modules/devtools/server/transport.js");
 
 let bypassOffline = false;
+let qemu = "0";
+let device = null;
 
 try {
   XPCOMUtils.defineLazyGetter(this, "libcutils", function () {
     Cu.import("resource://gre/modules/systemlibs.js");
     return libcutils;
   });
   if (libcutils) {
-    let qemu = libcutils.property_get("ro.kernel.qemu");
+    qemu = libcutils.property_get("ro.kernel.qemu");
     logger.info("B2G emulator: " + (qemu == "1" ? "yes" : "no"));
-    let platform = libcutils.property_get("ro.product.device");
-    logger.info("Platform detected is " + platform);
-    bypassOffline = (qemu == "1" || platform == "panda");
+    device = libcutils.property_get("ro.product.device");
+    logger.info("Device detected is " + device);
+    bypassOffline = (qemu == "1" || device == "panda");
   }
 }
 catch(e) {}
 
 if (bypassOffline) {
   logger.info("Bypassing offline status.");
   Services.prefs.setBoolPref("network.gonk.manage-offline-status", false);
   Services.io.manageOfflineStatus = false;
@@ -572,16 +574,17 @@ MarionetteServerConnection.prototype = {
           'appBuildId' : Services.appinfo.appBuildID,
           'XULappId' : Services.appinfo.ID,
           'cssSelectorsEnabled': true,
           'browserName': appName,
           'handlesAlerts': false,
           'javascriptEnabled': true,
           'nativeEvents': false,
           'platform': Services.appinfo.OS,
+          'device': qemu == "1" ? "qemu" : (!device ? "desktop" : device),
           'rotatable': rotatable,
           'takesScreenshot': false,
           'version': Services.appinfo.version
     };
 
     this.sendResponse(value, this.command_id);
   },