Bug 820154 - Implement todo() in marionette-simpletest. r=jgriffin
authorJulien Pagès <j.parkouss@gmail.com>
Wed, 13 Aug 2014 07:19:00 -0400
changeset 199717 63cdf0eb0975332821c3a41237b8976aa29c050d
parent 199716 9422c84b143bd3a8c141ee820c02960c143ec7e3
child 199718 ba7754e8a2244d6b9ce09a47fb82914d45b83e04
push id47723
push userryanvm@gmail.com
push dateFri, 15 Aug 2014 15:25:09 +0000
treeherdermozilla-inbound@8a6bb8913ecb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgriffin
bugs820154
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 820154 - Implement todo() in marionette-simpletest. r=jgriffin
testing/marionette/client/marionette/marionette_test.py
testing/marionette/client/marionette/tests/unit/test_simpletest_sanity.py
testing/marionette/marionette-simpletest.js
--- a/testing/marionette/client/marionette/marionette_test.py
+++ b/testing/marionette/client/marionette/marionette_test.py
@@ -490,29 +490,46 @@ setReq.onerror = function() {
                                                         special_powers=True,
                                                         inactivity_timeout=inactivity_timeout,
                                                         filename=os.path.basename(self.jsFile))
 
             self.assertTrue(not 'timeout' in self.jsFile,
                             'expected timeout not triggered')
 
             if 'fail' in self.jsFile:
-                self.assertTrue(results['failed'] > 0,
+                self.assertTrue(len(results['failures']) > 0,
                                 "expected test failures didn't occur")
             else:
                 logger = get_default_logger()
                 for failure in results['failures']:
                     diag = "" if failure.get('diag') is None else failure['diag']
                     name = "got false, expected true" if failure.get('name') is None else failure['name']
                     logger.test_status(self.test_name, name, 'FAIL',
                                        message=diag)
-                self.assertEqual(0, results['failed'],
-                                 '%d tests failed' % (results['failed']))
+                for failure in results['expectedFailures']:
+                    diag = "" if failure.get('diag') is None else failure['diag']
+                    name = "got false, expected false" if failure.get('name') is None else failure['name']
+                    logger.test_status(self.test_name, name, 'FAIL',
+                                       expected='FAIL', message=diag)
+                for failure in results['unexpectedSuccesses']:
+                    diag = "" if failure.get('diag') is None else failure['diag']
+                    name = "got true, expected false" if failure.get('name') is None else failure['name']
+                    logger.test_status(self.test_name, name, 'PASS',
+                                       expected='FAIL', message=diag)
+                self.assertEqual(0, len(results['failures']),
+                                 '%d tests failed' % len(results['failures']))
+                if len(results['unexpectedSuccesses']) > 0:
+                    raise _UnexpectedSuccess('')
+                if len(results['expectedFailures']) > 0:
+                    raise _ExpectedFailure((AssertionError, AssertionError(''), None))
 
-            self.assertTrue(results['passed'] + results['failed'] > 0,
+            self.assertTrue(results['passed']
+                            + len(results['failures'])
+                            + len(results['expectedFailures'])
+                            + len(results['unexpectedSuccesses']) > 0,
                             'no tests run')
 
         except ScriptTimeoutException:
             if 'timeout' in self.jsFile:
                 # expected exception
                 pass
             else:
                 self.loglines = self.marionette.get_logs()
--- a/testing/marionette/client/marionette/tests/unit/test_simpletest_sanity.py
+++ b/testing/marionette/client/marionette/tests/unit/test_simpletest_sanity.py
@@ -11,70 +11,92 @@ class SimpletestSanityTest(MarionetteTes
 
     def test_is(self):
         def runtests():
             sentFail1 = "is(true, false, 'isTest1', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
             sentFail2 = "is(true, false, 'isTest2', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
             sentPass1 = "is(true, true, 'isTest3');" + self.callFinish
             sentPass2 = "is(true, true, 'isTest4');" + self.callFinish
 
-            self.assertEqual(1, self.marionette.execute_script(sentFail1)["failed"])
+            self.assertEqual(1, len(self.marionette.execute_script(sentFail1)["failures"]))
             self.assertEqual(0, self.marionette.execute_script(sentFail2)["passed"])
             self.assertEqual(1, self.marionette.execute_script(sentPass1)["passed"])
-            self.assertEqual(0, self.marionette.execute_script(sentPass2)["failed"])
+            self.assertEqual(0, len(self.marionette.execute_script(sentPass2)["failures"]))
 
             self.marionette.set_script_timeout(1000)
-            self.assertEqual(1, self.marionette.execute_async_script(sentFail1)["failed"])
+            self.assertEqual(1, len(self.marionette.execute_async_script(sentFail1)["failures"]))
             self.assertEqual(0, self.marionette.execute_async_script(sentFail2)["passed"])
             self.assertEqual(1, self.marionette.execute_async_script(sentPass1)["passed"])
-            self.assertEqual(0, self.marionette.execute_async_script(sentPass2)["failed"])
+            self.assertEqual(0, len(self.marionette.execute_async_script(sentPass2)["failures"]))
 
         self.marionette.set_context("content")
         runtests()
         self.marionette.set_context("chrome")
         runtests()
 
     def test_isnot(self):
         def runtests():
            sentFail1 = "isnot(true, true, 'isnotTest3', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
            sentFail2 = "isnot(true, true, 'isnotTest4', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
            sentPass1 = "isnot(true, false, 'isnotTest1');" + self.callFinish
            sentPass2 = "isnot(true, false, 'isnotTest2');" + self.callFinish
 
-           self.assertEqual(1, self.marionette.execute_script(sentFail1)["failed"]);
+           self.assertEqual(1, len(self.marionette.execute_script(sentFail1)["failures"]));
            self.assertEqual(0, self.marionette.execute_script(sentFail2)["passed"]);
-           self.assertEqual(0, self.marionette.execute_script(sentPass1)["failed"]);
+           self.assertEqual(0, len(self.marionette.execute_script(sentPass1)["failures"]));
            self.assertEqual(1, self.marionette.execute_script(sentPass2)["passed"]);
 
            self.marionette.set_script_timeout(1000)
-           self.assertEqual(1, self.marionette.execute_async_script(sentFail1)["failed"]);
+           self.assertEqual(1, len(self.marionette.execute_async_script(sentFail1)["failures"]));
            self.assertEqual(0, self.marionette.execute_async_script(sentFail2)["passed"]);
-           self.assertEqual(0, self.marionette.execute_async_script(sentPass1)["failed"]);
+           self.assertEqual(0, len(self.marionette.execute_async_script(sentPass1)["failures"]));
            self.assertEqual(1, self.marionette.execute_async_script(sentPass2)["passed"]);
 
         self.marionette.set_context("content")
         runtests()
         self.marionette.set_context("chrome")
         runtests()
 
     def test_ok(self):
         def runtests():
             sentFail1 = "ok(1==2, 'testOk1', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
             sentFail2 = "ok(1==2, 'testOk2', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
             sentPass1 = "ok(1==1, 'testOk3');" + self.callFinish
             sentPass2 = "ok(1==1, 'testOk4');" + self.callFinish
 
-            self.assertEqual(1, self.marionette.execute_script(sentFail1)["failed"]);
+            self.assertEqual(1, len(self.marionette.execute_script(sentFail1)["failures"]));
             self.assertEqual(0, self.marionette.execute_script(sentFail2)["passed"]);
-            self.assertEqual(0, self.marionette.execute_script(sentPass1)["failed"]);
+            self.assertEqual(0, len(self.marionette.execute_script(sentPass1)["failures"]));
             self.assertEqual(1, self.marionette.execute_script(sentPass2)["passed"]);
 
             self.marionette.set_script_timeout(1000)
-            self.assertEqual(1, self.marionette.execute_async_script(sentFail1)["failed"]);
+            self.assertEqual(1, len(self.marionette.execute_async_script(sentFail1)["failures"]));
             self.assertEqual(0, self.marionette.execute_async_script(sentFail2)["passed"]);
-            self.assertEqual(0, self.marionette.execute_async_script(sentPass1)["failed"]);
+            self.assertEqual(0, len(self.marionette.execute_async_script(sentPass1)["failures"]));
             self.assertEqual(1, self.marionette.execute_async_script(sentPass2)["passed"]);
 
         self.marionette.set_context("content")
         runtests()
         self.marionette.set_context("chrome")
         runtests()
 
+    def test_todo(self):
+        def runtests():
+            sentFail1 = "todo(1==1, 'testTodo1', TEST_UNEXPECTED_PASS, TEST_KNOWN_FAIL);" + self.callFinish
+            sentFail2 = "todo(1==1, 'testTodo2', TEST_UNEXPECTED_PASS, TEST_KNOWN_FAIL);" + self.callFinish
+            sentPass1 = "todo(1==2, 'testTodo3');" + self.callFinish
+            sentPass2 = "todo(1==2, 'testTodo4');" + self.callFinish
+
+            self.assertEqual(1, len(self.marionette.execute_script(sentFail1)["unexpectedSuccesses"]));
+            self.assertEqual(0, len(self.marionette.execute_script(sentFail2)["expectedFailures"]));
+            self.assertEqual(0, len(self.marionette.execute_script(sentPass1)["unexpectedSuccesses"]));
+            self.assertEqual(1, len(self.marionette.execute_script(sentPass2)["expectedFailures"]));
+
+            self.marionette.set_script_timeout(1000)
+            self.assertEqual(1, len(self.marionette.execute_async_script(sentFail1)["unexpectedSuccesses"]));
+            self.assertEqual(0, len(self.marionette.execute_async_script(sentFail2)["expectedFailures"]));
+            self.assertEqual(0, len(self.marionette.execute_async_script(sentPass1)["unexpectedSuccesses"]));
+            self.assertEqual(1, len(self.marionette.execute_async_script(sentPass2)["expectedFailures"]));
+
+        self.marionette.set_context("content")
+        runtests()
+        self.marionette.set_context("chrome")
+        runtests()
--- a/testing/marionette/marionette-simpletest.js
+++ b/testing/marionette/marionette-simpletest.js
@@ -11,51 +11,66 @@ this.Marionette = function Marionette(sc
   this.window = window;
   this.tests = [];
   this.logObj = logObj;
   this.context = context;
   this.timeout = timeout;
   this.heartbeatCallback = heartbeatCallback;
   this.testName = testName;
   this.TEST_UNEXPECTED_FAIL = "TEST-UNEXPECTED-FAIL";
+  this.TEST_UNEXPECTED_PASS = "TEST-UNEXPECTED-PASS";
   this.TEST_PASS = "TEST-PASS";
   this.TEST_KNOWN_FAIL = "TEST-KNOWN-FAIL";
 }
 
 Marionette.prototype = {
-  exports: ['ok', 'is', 'isnot', 'log', 'getLogs', 'generate_results', 'waitFor',
+  exports: ['ok', 'is', 'isnot', 'todo', 'log', 'getLogs', 'generate_results', 'waitFor',
             'runEmulatorCmd', 'runEmulatorShell', 'TEST_PASS', 'TEST_KNOWN_FAIL',
-            'TEST_UNEXPECTED_FAIL'],
+            'TEST_UNEXPECTED_FAIL', 'TEST_UNEXPECTED_PASS'],
 
-  ok: function Marionette__ok(condition, name, passString, failString, diag) {
-    this.heartbeatCallback();
-    if (typeof(diag) == "undefined") {
-      diag = this.repr(condition) + " was " + !!condition + ", expected true";
-    }
-    let test = {'result': !!condition, 'name': name, 'diag': diag};
+  addTest: function Marionette__addTest(condition, name, passString, failString, diag, state) {
+
+    let test = {'result': !!condition, 'name': name, 'diag': diag, 'state': state};
     this.logResult(test,
                    typeof(passString) == "undefined" ? this.TEST_PASS : passString,
                    typeof(failString) == "undefined" ? this.TEST_UNEXPECTED_FAIL : failString);
     this.tests.push(test);
   },
 
+  ok: function Marionette__ok(condition, name, passString, failString) {
+    this.heartbeatCallback();
+    let diag = this.repr(condition) + " was " + !!condition + ", expected true";
+    this.addTest(condition, name, passString, failString, diag);
+  },
+
   is: function Marionette__is(a, b, name, passString, failString) {
     this.heartbeatCallback();
     let pass = (a == b);
     let diag = pass ? this.repr(a) + " should equal " + this.repr(b)
                     : "got " + this.repr(a) + ", expected " + this.repr(b);
-    this.ok(pass, name, passString, failString, diag);
+    this.addTest(pass, name, passString, failString, diag);
   },
 
   isnot: function Marionette__isnot (a, b, name, passString, failString) {
     this.heartbeatCallback();
     let pass = (a != b);
     let diag = pass ? this.repr(a) + " should not equal " + this.repr(b)
                     : "didn't expect " + this.repr(a) + ", but got it";
-    this.ok(pass, name, passString, failString, diag);
+    this.addTest(pass, name, passString, failString, diag);
+  },
+
+  todo: function Marionette__todo(condition, name, passString, failString) {
+    this.heartbeatCallback();
+    let diag = this.repr(condition) + " was expected false";
+    this.addTest(!condition,
+                 name,
+                 typeof(passString) == "undefined" ? this.TEST_KNOWN_FAIL : passString,
+                 typeof(failString) == "undefined" ? this.TEST_UNEXPECTED_FAIL : failString,
+                 diag,
+                 "todo");
   },
 
   log: function Marionette__log(msg, level) {
     this.heartbeatCallback();
     dump("MARIONETTE LOG: " + (level ? level : "INFO") + ": " + msg + "\n");
     if (this.logObj != null) {
       this.logObj.log(msg, level);
     }
@@ -66,31 +81,42 @@ Marionette.prototype = {
     if (this.logObj != null) {
       this.logObj.getLogs();
     }
   },
 
   generate_results: function Marionette__generate_results() {
     this.heartbeatCallback();
     let passed = 0;
-    let failed = 0;
     let failures = [];
+    let expectedFailures = [];
+    let unexpectedSuccesses = [];
     for (let i in this.tests) {
+      let isTodo = (this.tests[i].state == "todo");
       if(this.tests[i].result) {
-        passed++;
+        if (isTodo) {
+          expectedFailures.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
+        }
+        else {
+          passed++;
+        }
       }
       else {
-        failed++;
-        failures.push({'name': this.tests[i].name,
-                       'diag': this.tests[i].diag});
+        if (isTodo) {
+          unexpectedSuccesses.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
+        }
+        else {
+          failures.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
+        }
       }
     }
     // Reset state in case this object is reused for more tests.
     this.tests = [];
-    return {"passed": passed, "failed": failed, "failures": failures};
+    return {"passed": passed, "failures": failures, "expectedFailures": expectedFailures,
+            "unexpectedSuccesses": unexpectedSuccesses};
   },
 
   logToFile: function Marionette__logToFile(file) {
     this.heartbeatCallback();
     //TODO
   },
 
   logResult: function Marionette__logResult(test, passString, failString) {