Bug 791033 - Allow tests to provide custom pass/fail strings for is/isnot/ok assertions, r=mdas
authorJonathan Griffin <jgriffin@mozilla.com>
Fri, 14 Sep 2012 14:34:38 -0700
changeset 107293 1b9df2fa85a95d3a4d1dff92c465d28355a6af80
parent 107292 c2b7be27e41214a8abf43b2f9ed049b821d60050
child 107294 414b774c927b7c653757b0f34c154a3b3b537a1e
push id14960
push userjgriffin@mozilla.com
push dateMon, 17 Sep 2012 22:36:06 +0000
treeherdermozilla-inbound@1b9df2fa85a9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmdas
bugs791033
milestone18.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 791033 - Allow tests to provide custom pass/fail strings for is/isnot/ok assertions, r=mdas
testing/marionette/client/marionette/tests/unit/test_simpletest_fail.js
testing/marionette/client/marionette/tests/unit/test_simpletest_sanity.py
testing/marionette/marionette-actors.js
testing/marionette/marionette-listener.js
testing/marionette/marionette-simpletest.js
--- a/testing/marionette/client/marionette/tests/unit/test_simpletest_fail.js
+++ b/testing/marionette/client/marionette/tests/unit/test_simpletest_fail.js
@@ -2,15 +2,15 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 MARIONETTE_TIMEOUT = 1000;
 
 /* this test will fail */
 
 setTimeout(function() { 
-    is(1, 2); 
+    is(1, 2, "is(1,2) should fail", TEST_UNEXPECTED_FAIL, TEST_PASS); 
     finish();
 }, 100);
-isnot(1, 1);
-ok(1 == 2);
+isnot(1, 1, "isnot(1,1) should fail", TEST_UNEXPECTED_FAIL, TEST_PASS);
+ok(1 == 2, "ok(1==2) should fail", TEST_UNEXPECTED_FAIL, TEST_PASS);
 
 
--- a/testing/marionette/client/marionette/tests/unit/test_simpletest_sanity.py
+++ b/testing/marionette/client/marionette/tests/unit/test_simpletest_sanity.py
@@ -6,18 +6,18 @@ from marionette_test import MarionetteTe
 from errors import JavascriptException, MarionetteException, ScriptTimeoutException
 
 class SimpletestSanityTest(MarionetteTestCase):
 
     callFinish = "return finish();"
 
     def test_is(self):
         def runtests():
-            sentFail1 = "is(true, false, 'isTest1');" + self.callFinish
-            sentFail2 = "is(true, false, 'isTest2');" + self.callFinish
+            sentFail1 = "is(true, false, 'isTest1', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentFail2 = "is(true, false, 'isTest2', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
             sentPass1 = "is(true, true, 'isTest3');" + self.callFinish
             sentPass2 = "is(true, true, 'isTest4');" + self.callFinish
 
             self.assertEqual(1, self.marionette.execute_script(sentFail1)["failed"])
             self.assertEqual(0, self.marionette.execute_script(sentFail2)["passed"])
             self.assertEqual(1, self.marionette.execute_script(sentPass1)["passed"])
             self.assertEqual(0, self.marionette.execute_script(sentPass2)["failed"])
 
@@ -29,20 +29,20 @@ class SimpletestSanityTest(MarionetteTes
 
         self.marionette.set_context("content")
         runtests()
         self.marionette.set_context("chrome")
         runtests()
 
     def test_isnot(self):
         def runtests():
-           sentFail1 = "isnot(true, true, 'isTest3');" + self.callFinish
-           sentFail2 = "isnot(true, true, 'isTest4');" + self.callFinish
-           sentPass1 = "isnot(true, false, 'isTest1');" + self.callFinish
-           sentPass2 = "isnot(true, false, 'isTest2');" + self.callFinish
+           sentFail1 = "isnot(true, true, 'isnotTest3', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+           sentFail2 = "isnot(true, true, 'isnotTest4', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+           sentPass1 = "isnot(true, false, 'isnotTest1');" + self.callFinish
+           sentPass2 = "isnot(true, false, 'isnotTest2');" + self.callFinish
 
            self.assertEqual(1, self.marionette.execute_script(sentFail1)["failed"]);
            self.assertEqual(0, self.marionette.execute_script(sentFail2)["passed"]);
            self.assertEqual(0, self.marionette.execute_script(sentPass1)["failed"]);
            self.assertEqual(1, self.marionette.execute_script(sentPass2)["passed"]);
 
            self.marionette.set_script_timeout(1000)
            self.assertEqual(1, self.marionette.execute_async_script(sentFail1)["failed"]);
@@ -52,20 +52,20 @@ class SimpletestSanityTest(MarionetteTes
 
         self.marionette.set_context("content")
         runtests()
         self.marionette.set_context("chrome")
         runtests()
 
     def test_ok(self):
         def runtests():
-            sentFail1 = "ok(1==2, 'testOk');" + self.callFinish
-            sentFail2 = "ok(1==2, 'testOk');" + self.callFinish
-            sentPass1 = "ok(1==1, 'testOk');" + self.callFinish
-            sentPass2 = "ok(1==1, 'testOk');" + self.callFinish
+            sentFail1 = "ok(1==2, 'testOk1', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentFail2 = "ok(1==2, 'testOk2', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentPass1 = "ok(1==1, 'testOk3');" + self.callFinish
+            sentPass2 = "ok(1==1, 'testOk4');" + self.callFinish
 
             self.assertEqual(1, self.marionette.execute_script(sentFail1)["failed"]);
             self.assertEqual(0, self.marionette.execute_script(sentFail2)["passed"]);
             self.assertEqual(0, self.marionette.execute_script(sentPass1)["failed"]);
             self.assertEqual(1, self.marionette.execute_script(sentPass2)["passed"]);
 
             self.marionette.set_script_timeout(1000)
             self.assertEqual(1, self.marionette.execute_async_script(sentFail1)["failed"]);
--- a/testing/marionette/marionette-actors.js
+++ b/testing/marionette/marionette-actors.js
@@ -485,17 +485,22 @@ MarionetteDriverActor.prototype = {
 
     let _chromeSandbox = new Cu.Sandbox(aWindow,
        { sandboxPrototype: aWindow, wantXrays: false, sandboxName: ''});
     _chromeSandbox.__namedArgs = this.curBrowser.elementManager.applyNamedArgs(args);
     _chromeSandbox.__marionetteParams = args;
     _chromeSandbox.testUtils = utils;
 
     marionette.exports.forEach(function(fn) {
-      _chromeSandbox[fn] = marionette[fn].bind(marionette);
+      try {
+        _chromeSandbox[fn] = marionette[fn].bind(marionette);
+      }
+      catch(e) {
+        _chromeSandbox[fn] = marionette[fn];
+      }
     });
 
     if (specialPowers == true) {
       loader.loadSubScript("chrome://specialpowers/content/specialpowersAPI.js",
                            _chromeSandbox);
       loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserverAPI.js",
                            _chromeSandbox);
       loader.loadSubScript("chrome://specialpowers/content/ChromePowers.js",
--- a/testing/marionette/marionette-listener.js
+++ b/testing/marionette/marionette-listener.js
@@ -254,17 +254,22 @@ function createExecuteContentSandbox(aWi
   sandbox.document = sandbox.window.document;
   sandbox.navigator = sandbox.window.navigator;
   sandbox.__proto__ = sandbox.window;
   sandbox.testUtils = utils;
 
   let marionette = new Marionette(this, aWindow, "content", marionetteLogObj, marionettePerf);
   sandbox.marionette = marionette;
   marionette.exports.forEach(function(fn) {
-    sandbox[fn] = marionette[fn].bind(marionette);
+    try {
+      sandbox[fn] = marionette[fn].bind(marionette);
+    }
+    catch(e) {
+      sandbox[fn] = marionette[fn];
+    }
   });
 
   sandbox.SpecialPowers = new SpecialPowers(aWindow);
 
   sandbox.asyncComplete = function sandbox_asyncComplete(value, status) {
     if (Object.keys(_emu_cbs).length) {
       _emu_cbs = {};
       value = "Emulator callback still pending when finish() called";
--- a/testing/marionette/marionette-simpletest.js
+++ b/testing/marionette/marionette-simpletest.js
@@ -8,40 +8,49 @@
 function Marionette(scope, window, context, logObj, perfData) {
   this.scope = scope;
   this.window = window;
   this.tests = [];
   this.logObj = logObj;
   this.perfData = perfData;
   this.context = context;
   this.timeout = 0;
+  this.TEST_UNEXPECTED_FAIL = "TEST-UNEXPECTED-FAIL";
+  this.TEST_PASS = "TEST-PASS";
+  this.TEST_KNOWN_FAIL = "TEST-KNOWN-FAIL";
 }
 
 Marionette.prototype = {
   exports: ['ok', 'is', 'isnot', 'log', 'getLogs', 'generate_results', 'waitFor',
-            'runEmulatorCmd', 'addPerfData', 'getPerfData'],
+            'runEmulatorCmd', 'addPerfData', 'getPerfData', 'TEST_PASS',
+            'TEST_KNOWN_FAIL', 'TEST_UNEXPECTED_FAIL'],
 
-  ok: function Marionette__ok(condition, name, diag) {
+  ok: function Marionette__ok(condition, name, passString, failString, diag) {
+    if (typeof(diag) == "undefined") {
+      diag = this.repr(condition) + " was false, expected true";
+    }
     let test = {'result': !!condition, 'name': name, 'diag': diag};
-    this.logResult(test, "TEST-PASS", "TEST-UNEXPECTED-FAIL");
+    this.logResult(test,
+                   typeof(passString) == "undefined" ? this.TEST_PASS : passString,
+                   typeof(failString) == "undefined" ? this.TEST_UNEXPECTED_FAIL : failString);
     this.tests.push(test);
   },
 
-  is: function Marionette__is(a, b, name) {
+  is: function Marionette__is(a, b, name, passString, failString) {
     let pass = (a == b);
     let diag = pass ? this.repr(a) + " should equal " + this.repr(b)
                     : "got " + this.repr(a) + ", expected " + this.repr(b);
-    this.ok(pass, name, diag);
+    this.ok(pass, name, passString, failString, diag);
   },
 
-  isnot: function Marionette__isnot (a, b, name) {
+  isnot: function Marionette__isnot (a, b, name, passString, failString) {
     let pass = (a != b);
     let diag = pass ? this.repr(a) + " should not equal " + this.repr(b)
                     : "didn't expect " + this.repr(a) + ", but got it";
-    this.ok(pass, name, diag);
+    this.ok(pass, name, passString, failString, diag);
   },
 
   addPerfData: function Marionette__addPerfData(testSuite, testName, data) {
     this.perfData.addPerfData(testSuite, testName, data);
   },
 
   getPerfData: function Marionette__getPerfData() {
     return this.perfData.perfData;