Backed out 3 changesets (bug 1368674) for flake8 failures.
authorRyan VanderMeulen <ryanvm@gmail.com>
Thu, 01 Jun 2017 16:54:10 -0400
changeset 361889 18b819dd5afc1b7af098076343487f4e5ef3d08e
parent 361888 51413258faef176d7ba0e9e52a97bed5c1078b21
child 361890 1d9b50fadc050ba8fa1f7c76f99f12b3be19dd00
push id31947
push userkwierso@gmail.com
push dateFri, 02 Jun 2017 00:13:09 +0000
treeherdermozilla-central@fec3a4b50acd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1368674
milestone55.0a1
backs out9aa183c8533e23762ce56370ccd18128eb1df27c
837ccbc38bfc1e9ad0437f0e50e47b5cb400c4d0
7f4f851da483b111feb4e84ac22fda0e286bfe6e
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 3 changesets (bug 1368674) for flake8 failures. Backed out changeset 9aa183c8533e (bug 1368674) Backed out changeset 837ccbc38bfc (bug 1368674) Backed out changeset 7f4f851da483 (bug 1368674)
testing/marionette/client/marionette_driver/marionette.py
testing/marionette/driver.js
testing/marionette/harness/marionette_harness/marionette_test/testcases.py
testing/marionette/harness/marionette_harness/runner/base.py
testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
testing/marionette/harness/marionette_harness/tests/unit/test_chrome_async_finish.js
testing/marionette/harness/marionette_harness/tests/unit/test_execute_script.py
testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_chrome.js
testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_fail.js
testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_pass.js
testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_sanity.py
testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_timeout.js
testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini
testing/marionette/jar.mn
testing/marionette/listener.js
testing/marionette/simpletest.js
--- a/testing/marionette/client/marionette_driver/marionette.py
+++ b/testing/marionette/client/marionette_driver/marionette.py
@@ -1331,16 +1331,17 @@ class Marionette(object):
         return self.session
 
     @property
     def test_name(self):
         return self._test_name
 
     @test_name.setter
     def test_name(self, test_name):
+        self._send_message("setTestName", {"value": test_name})
         self._test_name = test_name
 
     def delete_session(self, send_request=True, reset_session_id=False):
         """Close the current session and disconnect from the server.
 
         :param send_request: Optional, if `True` a request to close the session on
             the server side will be send. Use `False` in case of eg. in_app restart()
             or quit(), which trigger a deletion themselves. Defaults to `True`.
--- a/testing/marionette/driver.js
+++ b/testing/marionette/driver.js
@@ -30,16 +30,17 @@ Cu.import("chrome://marionette/content/e
 Cu.import("chrome://marionette/content/event.js");
 Cu.import("chrome://marionette/content/interaction.js");
 Cu.import("chrome://marionette/content/l10n.js");
 Cu.import("chrome://marionette/content/legacyaction.js");
 Cu.import("chrome://marionette/content/logging.js");
 Cu.import("chrome://marionette/content/modal.js");
 Cu.import("chrome://marionette/content/proxy.js");
 Cu.import("chrome://marionette/content/session.js");
+Cu.import("chrome://marionette/content/simpletest.js");
 Cu.import("chrome://marionette/content/wait.js");
 
 this.EXPORTED_SYMBOLS = ["GeckoDriver", "Context"];
 
 var FRAME_SCRIPT = "chrome://marionette/content/listener.js";
 
 const CLICK_TO_START_PREF = "marionette.debugging.clicktostart";
 const CONTENT_LISTENER_PREF = "marionette.contentListener";
@@ -888,16 +889,59 @@ GeckoDriver.prototype.execute_ = functio
       opts.timeout = timeout;
       let wargs = evaluate.fromJSON(args, this.curBrowser.seenEls, sb.window);
       let evaluatePromise = evaluate.sandbox(sb, script, wargs, opts);
       return evaluatePromise.then(res => evaluate.toJSON(res, this.curBrowser.seenEls));
   }
 };
 
 /**
+ * Execute pure JavaScript.  Used to execute simpletest harness tests,
+ * which are like mochitests only injected using Marionette.
+ *
+ * Scripts are expected to call the {@code finish} global when done.
+ */
+GeckoDriver.prototype.executeJSScript = function* (cmd, resp) {
+  let win = assert.window(this.getCurrentWindow());
+
+  let {script, args, scriptTimeout} = cmd.parameters;
+  scriptTimeout = scriptTimeout || this.timeouts.script;
+
+  let opts = {
+    filename: cmd.parameters.filename,
+    line: cmd.parameters.line,
+    async: cmd.parameters.async,
+  };
+
+  switch (this.context) {
+    case Context.CHROME:
+      let wargs = evaluate.fromJSON(args, this.curBrowser.seenEls, win);
+      let harness = new simpletest.Harness(
+          win,
+          Context.CHROME,
+          this.marionetteLog,
+          scriptTimeout,
+          function() {},
+          this.testName);
+
+      let sb = sandbox.createSimpleTest(win, harness);
+      // TODO(ato): Not sure this is needed:
+      sb = sandbox.augment(sb, new logging.Adapter(this.marionetteLog));
+
+      let res = yield evaluate.sandbox(sb, script, wargs, opts);
+      resp.body.value = evaluate.toJSON(res, this.curBrowser.seenEls);
+      break;
+
+    case Context.CONTENT:
+      resp.body.value = yield this.listener.executeSimpleTest(script, args, scriptTimeout, opts);
+      break;
+  }
+};
+
+/**
  * Navigate to given URL.
  *
  * Navigates the current browsing context to the given URL and waits for
  * the document to load or the session's page timeout duration to elapse
  * before returning.
  *
  * The command will return with a failure if there is an error loading
  * the document or the URL is blocked.  This can occur if it fails to
@@ -2343,16 +2387,25 @@ GeckoDriver.prototype.sendKeysToElement 
       break;
 
     case Context.CONTENT:
       yield this.listener.sendKeysToElement(id, text);
       break;
   }
 };
 
+/** Sets the test name.  The test name is used for logging purposes. */
+GeckoDriver.prototype.setTestName = function*(cmd, resp) {
+  assert.window(this.getCurrentWindow());
+
+  let val = cmd.parameters.value;
+  this.testName = val;
+  yield this.listener.setTestName({value: val});
+};
+
 /**
  * Clear the text of an element.
  *
  * @param {string} id
  *     Reference ID to the element that will be cleared.
  *
  * @throws {NoSuchWindowError}
  *     Top-level browsing context has been discarded.
--- a/testing/marionette/harness/marionette_harness/marionette_test/testcases.py
+++ b/testing/marionette/harness/marionette_harness/marionette_test/testcases.py
@@ -55,16 +55,23 @@ class MetaParameterized(type):
                         raise KeyError("{0} is already a defined method on {1}"
                                        .format(wrapper.__name__, name))
                     attrs[wrapper.__name__] = wrapper
                 del attrs[k]
 
         return type.__new__(cls, name, bases, attrs)
 
 
+class JSTest:
+    head_js_re = re.compile(r"MARIONETTE_HEAD_JS(\s*)=(\s*)['|\"](.*?)['|\"];")
+    context_re = re.compile(r"MARIONETTE_CONTEXT(\s*)=(\s*)['|\"](.*?)['|\"];")
+    timeout_re = re.compile(r"MARIONETTE_TIMEOUT(\s*)=(\s*)(\d+);")
+    inactivity_timeout_re = re.compile(r"MARIONETTE_INACTIVITY_TIMEOUT(\s*)=(\s*)(\d+);")
+
+
 class CommonTestCase(unittest.TestCase):
 
     __metaclass__ = MetaParameterized
     match_re = None
     failureException = AssertionError
     pydebugger = None
 
     def __init__(self, methodName, marionette_weakref, fixtures, **kwargs):
@@ -223,19 +230,22 @@ class CommonTestCase(unittest.TestCase):
     @classmethod
     def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette,
                            fixtures, testvars, **kwargs):
         """Add all the tests in the specified file to the specified suite."""
         raise NotImplementedError
 
     @property
     def test_name(self):
-        return '{0}.py {1}.{2}'.format(self.__class__.__module__,
-                                       self.__class__.__name__,
-                                       self._testMethodName)
+        if hasattr(self, 'jsFile'):
+            return os.path.basename(self.jsFile)
+        else:
+            return '{0}.py {1}.{2}'.format(self.__class__.__module__,
+                                           self.__class__.__name__,
+                                           self._testMethodName)
 
     def id(self):
         # TBPL starring requires that the "test name" field of a failure message
         # not differ over time. The test name to be used is passed to
         # mozlog via the test id, so this is overriden to maintain
         # consistency.
         return self.test_name
 
@@ -283,16 +293,137 @@ if (!testUtils.hasOwnProperty("specialPo
     .getService(Components.interfaces.mozIJSSubScriptLoader);
   loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserver.jsm",
     testUtils);
   testUtils.specialPowersObserver = new testUtils.SpecialPowersObserver();
   testUtils.specialPowersObserver.init();
 }
 """)
 
+    def run_js_test(self, filename, marionette=None):
+        """Run a JavaScript test file.
+
+        It collects its set of assertions into the current test's results.
+
+        :param filename: The path to the JavaScript test file to execute.
+                         May be relative to the current script.
+        :param marionette: The Marionette object in which to execute the test.
+                           Defaults to self.marionette.
+        """
+        marionette = marionette or self.marionette
+        if not os.path.isabs(filename):
+            # Find the caller's filename and make the path relative to that.
+            caller_file = sys._getframe(1).f_globals.get('__file__', '')
+            caller_file = os.path.abspath(caller_file)
+            filename = os.path.join(os.path.dirname(caller_file), filename)
+        self.assert_(os.path.exists(filename),
+                     'Script "{}" must exist' .format(filename))
+        original_test_name = self.marionette.test_name
+        self.marionette.test_name = os.path.basename(filename)
+        f = open(filename, 'r')
+        js = f.read()
+        args = []
+
+        head_js = JSTest.head_js_re.search(js)
+        if head_js:
+            head_js = head_js.group(3)
+            head = open(os.path.join(os.path.dirname(filename), head_js), 'r')
+            js = head.read() + js
+
+        context = JSTest.context_re.search(js)
+        if context:
+            context = context.group(3)
+        else:
+            context = 'content'
+
+        if 'SpecialPowers' in js:
+            self.setup_SpecialPowers_observer()
+
+            if context == 'content':
+                js = "var SpecialPowers = window.wrappedJSObject.SpecialPowers;\n" + js
+            else:
+                marionette.execute_script("""
+                if (typeof(SpecialPowers) == 'undefined') {
+                  let loader = Components.classes["@mozilla.org/moz/jssubscript-loader;1"]
+                    .getService(Components.interfaces.mozIJSSubScriptLoader);
+                  loader.loadSubScript("chrome://specialpowers/content/specialpowersAPI.js");
+                  loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserverAPI.js");
+                  loader.loadSubScript("chrome://specialpowers/content/ChromePowers.js");
+                }
+                """)
+
+        marionette.set_context(context)
+
+        if context != 'chrome':
+            marionette.navigate('data:text/html,<html>test page</html>')
+
+        timeout = JSTest.timeout_re.search(js)
+        if timeout:
+            ms = timeout.group(3)
+            marionette.timeout.script = int(ms) / 1000.0
+
+        inactivity_timeout = JSTest.inactivity_timeout_re.search(js)
+        if inactivity_timeout:
+            inactivity_timeout = int(inactivity_timeout.group(3))
+
+        try:
+            results = marionette.execute_js_script(
+                js,
+                args,
+                inactivity_timeout=inactivity_timeout,
+                filename=os.path.basename(filename)
+            )
+
+            self.assertTrue('timeout' not in filename,
+                            'expected timeout not triggered')
+
+            if 'fail' in filename:
+                self.assertTrue(len(results['failures']) > 0,
+                                "expected test failures didn't occur")
+            else:
+                for failure in results['failures']:
+                    diag = "" if failure.get('diag') is None else failure['diag']
+                    name = ("got false, expected true" if failure.get('name') is None else
+                            failure['name'])
+                    self.logger.test_status(self.test_name, name, 'FAIL',
+                                            message=diag)
+                for failure in results['expectedFailures']:
+                    diag = "" if failure.get('diag') is None else failure['diag']
+                    name = ("got false, expected false" if failure.get('name') is None else
+                            failure['name'])
+                    self.logger.test_status(self.test_name, name, 'FAIL',
+                                            expected='FAIL', message=diag)
+                for failure in results['unexpectedSuccesses']:
+                    diag = "" if failure.get('diag') is None else failure['diag']
+                    name = ("got true, expected false" if failure.get('name') is None else
+                            failure['name'])
+                    self.logger.test_status(self.test_name, name, 'PASS',
+                                            expected='FAIL', message=diag)
+                self.assertEqual(0, len(results['failures']),
+                                 '{} tests failed' .format(len(results['failures'])))
+                if len(results['unexpectedSuccesses']) > 0:
+                    raise _UnexpectedSuccess('')
+                if len(results['expectedFailures']) > 0:
+                    raise _ExpectedFailure((AssertionError, AssertionError(''), None))
+
+            self.assertTrue(results['passed'] +
+                            len(results['failures']) +
+                            len(results['expectedFailures']) +
+                            len(results['unexpectedSuccesses']) > 0,
+                            'no tests run')
+
+        except ScriptTimeoutException:
+            if 'timeout' in filename:
+                # expected exception
+                pass
+            else:
+                self.loglines = marionette.get_logs()
+                raise
+        self.marionette.test_name = original_test_name
+
 
 class MarionetteTestCase(CommonTestCase):
 
     match_re = re.compile(r"test_(.*)\.py$")
 
     def __init__(self, marionette_weakref, fixtures, methodName='runTest',
                  filepath='', **kwargs):
         self.filepath = filepath
@@ -331,31 +462,33 @@ class MarionetteTestCase(CommonTestCase)
                                       methodName=testname,
                                       filepath=filepath,
                                       testvars=testvars,
                                       **kwargs))
 
     def setUp(self):
         super(MarionetteTestCase, self).setUp()
         self.marionette.test_name = self.test_name
-        self.marionette.execute_script("log('TEST-START: {0}')"
-                                       .format(self.test_name),
+        self.marionette.execute_script("log('TEST-START: {0}:{1}')"
+                                       .format(self.filepath.replace('\\', '\\\\'),
+                                               self.methodName),
                                        sandbox="simpletest")
 
     def tearDown(self):
         # In the case no session is active (eg. the application was quit), start
         # a new session for clean-up steps.
         if not self.marionette.session:
             self.marionette.start_session()
 
         if not self.marionette.crashed:
             try:
                 self.marionette.clear_imported_scripts()
-                self.marionette.execute_script("log('TEST-END: {0}')"
-                                               .format(self.test_name),
+                self.marionette.execute_script("log('TEST-END: {0}:{1}')"
+                                               .format(self.filepath.replace('\\', '\\\\'),
+                                                       self.methodName),
                                                sandbox="simpletest")
                 self.marionette.test_name = None
             except (MarionetteException, IOError):
                 # We have tried to log the test end when there is no listener
                 # object that we can access
                 pass
 
         super(MarionetteTestCase, self).tearDown()
--- a/testing/marionette/harness/marionette_harness/runner/base.py
+++ b/testing/marionette/harness/marionette_harness/runner/base.py
@@ -178,16 +178,18 @@ class MarionetteTestResult(StructuredTes
         return test.test_name
 
     def getDescription(self, test):
         doc_first_line = test.shortDescription()
         if self.descriptions and doc_first_line:
             return '\n'.join((str(test), doc_first_line))
         else:
             desc = str(test)
+            if hasattr(test, 'jsFile'):
+                desc = "{0}, {1}".format(test.jsFile, desc)
             return desc
 
     def printLogs(self, test):
         for testcase in test._tests:
             if hasattr(testcase, 'loglines') and testcase.loglines:
                 # Don't dump loglines to the console if they only contain
                 # TEST-START and TEST-END.
                 skip_log = True
@@ -599,17 +601,17 @@ class BaseMarionetteTestRunner(object):
             self.gecko_log = gecko_log
 
         self.results = []
 
     @property
     def filename_pattern(self):
         if self._filename_pattern is None:
             self._filename_pattern = re.compile(
-                "^test(((_.+?)+?\.((py))))$")
+                "^test(((_.+?)+?\.((py)|(js)))|(([A-Z].*?)+?\.js))$")
 
         return self._filename_pattern
 
     @property
     def testvars(self):
         if self._testvars is not None:
             return self._testvars
 
@@ -796,17 +798,17 @@ class BaseMarionetteTestRunner(object):
     def _add_tests(self, tests):
         for test in tests:
             self.add_test(test)
 
         invalid_tests = [t['filepath'] for t in self.tests
                          if not self._is_filename_valid(t['filepath'])]
         if invalid_tests:
             raise Exception("Test file names must be of the form "
-                            "'test_something.py'."
+                            "'test_something.py', 'test_something.js', or 'testSomething.js'."
                             " Invalid test names:\n  {}".format('\n  '.join(invalid_tests)))
 
     def _is_filename_valid(self, filename):
         filename = os.path.basename(filename)
         return self.filename_pattern.match(filename)
 
     def _log_skipped_tests(self):
         for test in self.manifest_skipped_tests:
--- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
+++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py
@@ -277,29 +277,29 @@ def test_add_test_module(runner):
         assert expected in runner.tests
     # add_test doesn't validate module names; 'bad_test.py' gets through
     assert len(runner.tests) == 3
 
 
 def test_add_test_directory(runner):
     test_dir = 'path/to/tests'
     dir_contents = [
-        (test_dir, ('subdir',), ('test_a.py', 'bad_test_a.py')),
-        (test_dir + '/subdir', (), ('test_b.py', 'bad_test_b.py')),
+        (test_dir, ('subdir',), ('test_a.py', 'test_a.js', 'bad_test_a.py', 'bad_test_a.js')),
+        (test_dir + '/subdir', (), ('test_b.py', 'test_b.js', 'bad_test_b.py', 'bad_test_b.js')),
     ]
     tests = list(dir_contents[0][2] + dir_contents[1][2])
     assert len(runner.tests) == 0
     # Need to use side effect to make isdir return True for test_dir and False for tests
     with patch('os.path.isdir', side_effect=[True] + [False for t in tests]) as isdir:
         with patch('os.walk', return_value=dir_contents) as walk:
             runner.add_test(test_dir)
     assert isdir.called and walk.called
     for test in runner.tests:
         assert test_dir in test['filepath']
-    assert len(runner.tests) == 2
+    assert len(runner.tests) == 4
 
 
 @pytest.mark.parametrize("test_files_exist", [True, False])
 def test_add_test_manifest(mock_runner, manifest_with_tests, monkeypatch, test_files_exist):
     monkeypatch.setattr('marionette_harness.runner.base.TestManifest',
                         manifest_with_tests.manifest_class)
     mock_runner.marionette = mock_runner.driverclass()
     with patch('marionette_harness.runner.base.os.path.exists', return_value=test_files_exist):
@@ -399,20 +399,20 @@ def test_add_tests(mock_runner):
     fake_tests = ["test_" + i + ".py" for i in "abc"]
     mock_runner.run_tests(fake_tests)
     assert len(mock_runner.tests) == 3
     for (test_name, added_test) in zip(fake_tests, mock_runner.tests):
         assert added_test['filepath'].endswith(test_name)
 
 
 def test_catch_invalid_test_names(runner):
-    good_tests = [u'test_ok.py', u'test_is_ok.py']
-    bad_tests = [u'bad_test.py', u'testbad.py', u'_test_bad.py',
-                 u'test_bad.notpy', u'test_bad',
-                 u'test.py', u'test_.py']
+    good_tests = [u'test_ok.py', u'test_is_ok.py', u'test_is_ok.js', u'testIsOk.js']
+    bad_tests = [u'bad_test.py', u'testbad.py', u'_test_bad.py', u'testBad.notjs',
+                 u'test_bad.notpy', u'test_bad', u'testbad.js', u'badtest.js',
+                 u'test.py', u'test_.py', u'test.js', u'test_.js']
     with pytest.raises(Exception) as exc:
         runner._add_tests(good_tests + bad_tests)
     msg = exc.value.message
     assert "Test file names must be of the form" in msg
     for bad_name in bad_tests:
         assert bad_name in msg
     for good_name in good_tests:
         assert good_name not in msg
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_chrome_async_finish.js
@@ -0,0 +1,6 @@
+MARIONETTE_TIMEOUT = 60000;
+MARIONETTE_CONTEXT = "chrome";
+ok(true);
+(function () {
+  finish();
+})();
--- a/testing/marionette/harness/marionette_harness/tests/unit/test_execute_script.py
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_execute_script.py
@@ -24,16 +24,29 @@ globals = set([
               "btoa",
               "document",
               "navigator",
               "URL",
               "window",
               ])
 
 
+class TestExecuteSimpleTestContent(MarionetteTestCase):
+    def test_stack_trace(self):
+        try:
+            self.marionette.execute_js_script("""
+                let a = 1;
+                throwHere();
+                """, filename="file.js")
+            self.assertFalse(True)
+        except errors.JavascriptException as e:
+            self.assertIn("throwHere is not defined", e.message)
+            self.assertIn("@file.js:2", e.stacktrace)
+
+
 class TestExecuteContent(MarionetteTestCase):
 
     def assert_is_defined(self, property, sandbox="default"):
         self.assertTrue(self.marionette.execute_script(
             "return typeof arguments[0] != 'undefined'", [property], sandbox=sandbox),
             "property {} is undefined".format(property))
 
     def assert_is_web_element(self, element):
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_chrome.js
@@ -0,0 +1,12 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+MARIONETTE_TIMEOUT = 1000;
+MARIONETTE_CONTEXT = 'chrome';
+
+is(2, 2, "test for is()");
+isnot(2, 3, "test for isnot()");
+ok(2 == 2, "test for ok()");
+finish();
+
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_fail.js
@@ -0,0 +1,16 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+MARIONETTE_TIMEOUT = 1000;
+
+/* this test will fail */
+
+setTimeout(function() { 
+    is(1, 2, "is(1,2) should fail", TEST_UNEXPECTED_FAIL, TEST_PASS); 
+    finish();
+}, 100);
+isnot(1, 1, "isnot(1,1) should fail", TEST_UNEXPECTED_FAIL, TEST_PASS);
+ok(1 == 2, "ok(1==2) should fail", TEST_UNEXPECTED_FAIL, TEST_PASS);
+
+
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_pass.js
@@ -0,0 +1,12 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+MARIONETTE_TIMEOUT = 1000;
+
+is(2, 2, "test for is()");
+isnot(2, 3, "test for isnot()");
+ok(2 == 2, "test for ok()");
+
+setTimeout(finish, 100);
+
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_sanity.py
@@ -0,0 +1,107 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from marionette_harness import MarionetteTestCase
+
+
+class SimpletestSanityTest(MarionetteTestCase):
+    callFinish = "return finish();"
+
+    def run_sync(self, test):
+        return self.marionette.execute_js_script(test, async=False)
+
+    def run_async(self, test):
+        return self.marionette.execute_js_script(test)
+
+    def test_is(self):
+        def runtests():
+            sentFail1 = "is(true, false, 'isTest1', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentFail2 = "is(true, false, 'isTest2', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentPass1 = "is(true, true, 'isTest3');" + self.callFinish
+            sentPass2 = "is(true, true, 'isTest4');" + self.callFinish
+
+            self.assertEqual(1, len(self.run_sync(sentFail1)["failures"]))
+            self.assertEqual(0, self.run_sync(sentFail2)["passed"])
+            self.assertEqual(1, self.run_sync(sentPass1)["passed"])
+            self.assertEqual(0, len(self.run_sync(sentPass2)["failures"]))
+
+            self.marionette.timeout.script = 1
+            self.assertEqual(1, len(self.run_async(sentFail1)["failures"]))
+            self.assertEqual(0, self.run_async(sentFail2)["passed"])
+            self.assertEqual(1, self.run_async(sentPass1)["passed"])
+            self.assertEqual(0, len(self.run_async(sentPass2)["failures"]))
+
+        self.marionette.set_context("content")
+        runtests()
+        self.marionette.set_context("chrome")
+        runtests()
+
+    def test_isnot(self):
+        def runtests():
+           sentFail1 = "isnot(true, true, 'isnotTest3', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+           sentFail2 = "isnot(true, true, 'isnotTest4', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+           sentPass1 = "isnot(true, false, 'isnotTest1');" + self.callFinish
+           sentPass2 = "isnot(true, false, 'isnotTest2');" + self.callFinish
+
+           self.assertEqual(1, len(self.run_sync(sentFail1)["failures"]));
+           self.assertEqual(0, self.run_sync(sentFail2)["passed"]);
+           self.assertEqual(0, len(self.run_sync(sentPass1)["failures"]));
+           self.assertEqual(1, self.run_sync(sentPass2)["passed"]);
+
+           self.marionette.timeout.script = 1
+           self.assertEqual(1, len(self.run_async(sentFail1)["failures"]));
+           self.assertEqual(0, self.run_async(sentFail2)["passed"]);
+           self.assertEqual(0, len(self.run_async(sentPass1)["failures"]));
+           self.assertEqual(1, self.run_async(sentPass2)["passed"]);
+
+        self.marionette.set_context("content")
+        runtests()
+        self.marionette.set_context("chrome")
+        runtests()
+
+    def test_ok(self):
+        def runtests():
+            sentFail1 = "ok(1==2, 'testOk1', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentFail2 = "ok(1==2, 'testOk2', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentPass1 = "ok(1==1, 'testOk3');" + self.callFinish
+            sentPass2 = "ok(1==1, 'testOk4');" + self.callFinish
+
+            self.assertEqual(1, len(self.run_sync(sentFail1)["failures"]));
+            self.assertEqual(0, self.run_sync(sentFail2)["passed"]);
+            self.assertEqual(0, len(self.run_sync(sentPass1)["failures"]));
+            self.assertEqual(1, self.run_sync(sentPass2)["passed"]);
+
+            self.marionette.timeout.script = 1
+            self.assertEqual(1, len(self.run_async(sentFail1)["failures"]));
+            self.assertEqual(0, self.run_async(sentFail2)["passed"]);
+            self.assertEqual(0, len(self.run_async(sentPass1)["failures"]));
+            self.assertEqual(1, self.run_async(sentPass2)["passed"]);
+
+        self.marionette.set_context("content")
+        runtests()
+        self.marionette.set_context("chrome")
+        runtests()
+
+    def test_todo(self):
+        def runtests():
+            sentFail1 = "todo(1==1, 'testTodo1', TEST_UNEXPECTED_PASS, TEST_KNOWN_FAIL);" + self.callFinish
+            sentFail2 = "todo(1==1, 'testTodo2', TEST_UNEXPECTED_PASS, TEST_KNOWN_FAIL);" + self.callFinish
+            sentPass1 = "todo(1==2, 'testTodo3');" + self.callFinish
+            sentPass2 = "todo(1==2, 'testTodo4');" + self.callFinish
+
+            self.assertEqual(1, len(self.run_sync(sentFail1)["unexpectedSuccesses"]));
+            self.assertEqual(0, len(self.run_sync(sentFail2)["expectedFailures"]));
+            self.assertEqual(0, len(self.run_sync(sentPass1)["unexpectedSuccesses"]));
+            self.assertEqual(1, len(self.run_sync(sentPass2)["expectedFailures"]));
+
+            self.marionette.timeout.script = 1
+            self.assertEqual(1, len(self.run_async(sentFail1)["unexpectedSuccesses"]));
+            self.assertEqual(0, len(self.run_async(sentFail2)["expectedFailures"]));
+            self.assertEqual(0, len(self.run_async(sentPass1)["unexpectedSuccesses"]));
+            self.assertEqual(1, len(self.run_async(sentPass2)["expectedFailures"]));
+
+        self.marionette.set_context("content")
+        runtests()
+        self.marionette.set_context("chrome")
+        runtests()
new file mode 100644
--- /dev/null
+++ b/testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_timeout.js
@@ -0,0 +1,16 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+MARIONETTE_TIMEOUT = 100;
+
+/* this test will timeout */
+
+function do_test() {
+  is(1, 1);
+  isnot(1, 2);
+  ok(1 == 1);
+  finish();
+}
+
+setTimeout(do_test, 1000);
--- a/testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini
+++ b/testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini
@@ -28,28 +28,33 @@ skip-if = true # "Bug 896046"
 
 [test_clearing.py]
 [test_typing.py]
 
 [test_log.py]
 
 [test_execute_async_script.py]
 [test_execute_script.py]
+[test_simpletest_fail.js]
 [test_element_retrieval.py]
 [test_findelement_chrome.py]
 skip-if = appname == 'fennec'
 
 [test_get_current_url_chrome.py]
 [test_navigation.py]
 
 [test_timeouts.py]
 
 [test_single_finger_desktop.py]
 skip-if = appname == 'fennec' || os == "win" # Bug 1025040
 
+[test_simpletest_pass.js]
+[test_simpletest_sanity.py]
+[test_simpletest_chrome.js]
+[test_simpletest_timeout.js]
 [test_anonymous_content.py]
 skip-if = appname == 'fennec'
 [test_switch_frame.py]
 [test_switch_frame_chrome.py]
 skip-if = appname == 'fennec'
 [test_switch_remote_frame.py]
 skip-if = appname == 'fennec'
 [test_switch_window_chrome.py]
@@ -83,16 +88,17 @@ skip-if = appname == 'fennec'
 [test_window_type_chrome.py]
 skip-if = appname == 'fennec'
 [test_implicit_waits.py]
 [test_wait.py]
 [test_expected.py]
 [test_date_time_value.py]
 [test_getactiveframe_oop.py]
 skip-if = true # Bug 925688
+[test_chrome_async_finish.js]
 [test_screen_orientation.py]
 [test_errors.py]
 
 [test_execute_isolate.py]
 [test_click_scrolling.py]
 [test_profile_management.py]
 skip-if = manage_instance == false || appname == 'fennec' # Bug 1298921 and bug 1322993
 [test_quit_restart.py]
--- a/testing/marionette/jar.mn
+++ b/testing/marionette/jar.mn
@@ -8,16 +8,17 @@ marionette.jar:
   content/driver.js (driver.js)
   content/action.js (action.js)
   content/legacyaction.js (legacyaction.js)
   content/browser.js (browser.js)
   content/interaction.js (interaction.js)
   content/accessibility.js (accessibility.js)
   content/listener.js (listener.js)
   content/element.js (element.js)
+  content/simpletest.js (simpletest.js)
   content/frame.js (frame.js)
   content/cert.js (cert.js)
   content/event.js  (event.js)
   content/error.js (error.js)
   content/wait.js (wait.js)
   content/message.js (message.js)
   content/modal.js (modal.js)
   content/proxy.js (proxy.js)
--- a/testing/marionette/listener.js
+++ b/testing/marionette/listener.js
@@ -22,16 +22,17 @@ Cu.import("chrome://marionette/content/e
 Cu.import("chrome://marionette/content/evaluate.js");
 Cu.import("chrome://marionette/content/event.js");
 Cu.import("chrome://marionette/content/interaction.js");
 Cu.import("chrome://marionette/content/legacyaction.js");
 Cu.import("chrome://marionette/content/logging.js");
 Cu.import("chrome://marionette/content/navigate.js");
 Cu.import("chrome://marionette/content/proxy.js");
 Cu.import("chrome://marionette/content/session.js");
+Cu.import("chrome://marionette/content/simpletest.js");
 
 Cu.import("resource://gre/modules/FileUtils.jsm");
 Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 
 Cu.importGlobalProperties(["URL"]);
 
@@ -484,25 +485,27 @@ var performActionsFn = dispatch(performA
 var releaseActionsFn = dispatch(releaseActions);
 var actionChainFn = dispatch(actionChain);
 var multiActionFn = dispatch(multiAction);
 var addCookieFn = dispatch(addCookie);
 var deleteCookieFn = dispatch(deleteCookie);
 var deleteAllCookiesFn = dispatch(deleteAllCookies);
 var executeFn = dispatch(execute);
 var executeInSandboxFn = dispatch(executeInSandbox);
+var executeSimpleTestFn = dispatch(executeSimpleTest);
 var sendKeysToElementFn = dispatch(sendKeysToElement);
 
 /**
  * Start all message listeners
  */
 function startListeners() {
   addMessageListenerId("Marionette:newSession", newSession);
   addMessageListenerId("Marionette:execute", executeFn);
   addMessageListenerId("Marionette:executeInSandbox", executeInSandboxFn);
+  addMessageListenerId("Marionette:executeSimpleTest", executeSimpleTestFn);
   addMessageListenerId("Marionette:singleTap", singleTapFn);
   addMessageListenerId("Marionette:performActions", performActionsFn);
   addMessageListenerId("Marionette:releaseActions", releaseActionsFn);
   addMessageListenerId("Marionette:actionChain", actionChainFn);
   addMessageListenerId("Marionette:multiAction", multiActionFn);
   addMessageListenerId("Marionette:get", get);
   addMessageListenerId("Marionette:waitForPageLoaded", waitForPageLoaded);
   addMessageListenerId("Marionette:cancelRequest", cancelRequest);
@@ -527,16 +530,17 @@ function startListeners() {
   addMessageListenerId("Marionette:sendKeysToElement", sendKeysToElementFn);
   addMessageListenerId("Marionette:clearElement", clearElementFn);
   addMessageListenerId("Marionette:switchToFrame", switchToFrame);
   addMessageListenerId("Marionette:switchToParentFrame", switchToParentFrame);
   addMessageListenerId("Marionette:switchToShadowRoot", switchToShadowRootFn);
   addMessageListenerId("Marionette:deleteSession", deleteSession);
   addMessageListenerId("Marionette:sleepSession", sleepSession);
   addMessageListenerId("Marionette:getAppCacheStatus", getAppCacheStatus);
+  addMessageListenerId("Marionette:setTestName", setTestName);
   addMessageListenerId("Marionette:takeScreenshot", takeScreenshotFn);
   addMessageListenerId("Marionette:addCookie", addCookieFn);
   addMessageListenerId("Marionette:getCookies", getCookiesFn);
   addMessageListenerId("Marionette:deleteAllCookies", deleteAllCookiesFn);
   addMessageListenerId("Marionette:deleteCookie", deleteCookieFn);
 }
 
 /**
@@ -567,16 +571,17 @@ function restart(msg) {
 
 /**
  * Removes all listeners
  */
 function deleteSession(msg) {
   removeMessageListenerId("Marionette:newSession", newSession);
   removeMessageListenerId("Marionette:execute", executeFn);
   removeMessageListenerId("Marionette:executeInSandbox", executeInSandboxFn);
+  removeMessageListenerId("Marionette:executeSimpleTest", executeSimpleTestFn);
   removeMessageListenerId("Marionette:singleTap", singleTapFn);
   removeMessageListenerId("Marionette:performActions", performActionsFn);
   removeMessageListenerId("Marionette:releaseActions", releaseActionsFn);
   removeMessageListenerId("Marionette:actionChain", actionChainFn);
   removeMessageListenerId("Marionette:multiAction", multiActionFn);
   removeMessageListenerId("Marionette:get", get);
   removeMessageListenerId("Marionette:waitForPageLoaded", waitForPageLoaded);
   removeMessageListenerId("Marionette:cancelRequest", cancelRequest);
@@ -601,16 +606,17 @@ function deleteSession(msg) {
   removeMessageListenerId("Marionette:sendKeysToElement", sendKeysToElementFn);
   removeMessageListenerId("Marionette:clearElement", clearElementFn);
   removeMessageListenerId("Marionette:switchToFrame", switchToFrame);
   removeMessageListenerId("Marionette:switchToParentFrame", switchToParentFrame);
   removeMessageListenerId("Marionette:switchToShadowRoot", switchToShadowRootFn);
   removeMessageListenerId("Marionette:deleteSession", deleteSession);
   removeMessageListenerId("Marionette:sleepSession", sleepSession);
   removeMessageListenerId("Marionette:getAppCacheStatus", getAppCacheStatus);
+  removeMessageListenerId("Marionette:setTestName", setTestName);
   removeMessageListenerId("Marionette:takeScreenshot", takeScreenshotFn);
   removeMessageListenerId("Marionette:addCookie", addCookieFn);
   removeMessageListenerId("Marionette:getCookies", getCookiesFn);
   removeMessageListenerId("Marionette:deleteAllCookies", deleteAllCookiesFn);
   removeMessageListenerId("Marionette:deleteCookie", deleteCookieFn);
 
   seenEls.clear();
   // reset container frame to the top-most frame
@@ -752,16 +758,49 @@ function* executeInSandbox(script, args,
 
   let res = yield evaluatePromise;
   sendSyncMessage(
       "Marionette:shareData",
       {log: evaluate.toJSON(contentLog.get(), seenEls)});
   return evaluate.toJSON(res, seenEls);
 }
 
+function* executeSimpleTest(script, args, timeout, opts) {
+  opts.timeout = timeout;
+  let win = curContainer.frame;
+
+  let harness = new simpletest.Harness(
+      win,
+      "content",
+      contentLog,
+      timeout,
+      marionetteTestName);
+  let sb = sandbox.createSimpleTest(curContainer.frame, harness);
+  // TODO(ato): Not sure this is needed:
+  sb = sandbox.augment(sb, new logging.Adapter(contentLog));
+
+  let wargs = evaluate.fromJSON(
+      args, seenEls, curContainer.frame, curContainer.shadowRoot);
+  let evaluatePromise = evaluate.sandbox(sb, script, wargs, opts);
+
+  let res = yield evaluatePromise;
+  sendSyncMessage(
+      "Marionette:shareData",
+      {log: evaluate.toJSON(contentLog.get(), seenEls)});
+  return evaluate.toJSON(res, seenEls);
+}
+
+/**
+ * Sets the test name, used in logging messages.
+ */
+function setTestName(msg) {
+  marionetteTestName = msg.json.value;
+  sendOk(msg.json.command_id);
+}
+
 /**
  * This function creates a touch event given a touch type and a touch
  */
 function emitTouchEvent(type, touch) {
   if (!wasInterrupted()) {
     logger.info(`Emitting Touch event of type ${type} to element with id: ${touch.target.id} ` +
                 `and tag name: ${touch.target.tagName} at coordinates (${touch.clientX}, ` +
                 `${touch.clientY}) relative to the viewport`);
new file mode 100644
--- /dev/null
+++ b/testing/marionette/simpletest.js
@@ -0,0 +1,208 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {utils: Cu} = Components;
+
+Cu.import("chrome://marionette/content/error.js");
+
+this.EXPORTED_SYMBOLS = ["simpletest"];
+
+this.simpletest = {};
+
+/**
+ * The simpletest harness, exposed in the script evaluation sandbox.
+ */
+simpletest.Harness = class {
+  constructor(window, context, contentLogger, timeout, testName) {
+    this.window = window;
+    this.tests = [];
+    this.logger = contentLogger;
+    this.context = context;
+    this.timeout = timeout;
+    this.testName = testName;
+    this.TEST_UNEXPECTED_FAIL = "TEST-UNEXPECTED-FAIL";
+    this.TEST_UNEXPECTED_PASS = "TEST-UNEXPECTED-PASS";
+    this.TEST_PASS = "TEST-PASS";
+    this.TEST_KNOWN_FAIL = "TEST-KNOWN-FAIL";
+  }
+
+  get exports() {
+    return new Map([
+      ["ok", this.ok.bind(this)],
+      ["is", this.is.bind(this)],
+      ["isnot", this.isnot.bind(this)],
+      ["todo", this.todo.bind(this)],
+      ["log", this.log.bind(this)],
+      ["getLogs", this.getLogs.bind(this)],
+      ["generate_results", this.generate_results.bind(this)],
+      ["waitFor", this.waitFor.bind(this)],
+      ["TEST_PASS", this.TEST_PASS],
+      ["TEST_KNOWN_FAIL", this.TEST_KNOWN_FAIL],
+      ["TEST_UNEXPECTED_FAIL", this.TEST_UNEXPECTED_FAIL],
+      ["TEST_UNEXPECTED_PASS", this.TEST_UNEXPECTED_PASS],
+    ]);
+  }
+
+  addTest(condition, name, passString, failString, diag, state) {
+    let test = {
+      result: !!condition,
+      name: name,
+      diag: diag,
+      state: state
+    };
+    this.logResult(
+        test,
+        typeof passString == "undefined" ? this.TEST_PASS : passString,
+        typeof failString == "undefined" ? this.TEST_UNEXPECTED_FAIL : failString);
+    this.tests.push(test);
+  }
+
+  ok(condition, name, passString, failString) {
+    let diag = `${this.repr(condition)} was ${!!condition}, expected true`;
+    this.addTest(condition, name, passString, failString, diag);
+  }
+
+  is(a, b, name, passString, failString) {
+    let pass = (a == b);
+    let diag = pass ? this.repr(a) + " should equal " + this.repr(b)
+                    : "got " + this.repr(a) + ", expected " + this.repr(b);
+    this.addTest(pass, name, passString, failString, diag);
+  }
+
+  isnot(a, b, name, passString, failString) {
+    let pass = (a != b);
+    let diag = pass ? this.repr(a) + " should not equal " + this.repr(b)
+                    : "didn't expect " + this.repr(a) + ", but got it";
+    this.addTest(pass, name, passString, failString, diag);
+  }
+
+  todo(condition, name, passString, failString) {
+    let diag = this.repr(condition) + " was expected false";
+    this.addTest(!condition,
+                 name,
+                 typeof(passString) == "undefined" ? this.TEST_KNOWN_FAIL : passString,
+                 typeof(failString) == "undefined" ? this.TEST_UNEXPECTED_FAIL : failString,
+                 diag,
+                 "todo");
+  }
+
+  log(msg, level) {
+    dump("MARIONETTE LOG: " + (level ? level : "INFO") + ": " + msg + "\n");
+    if (this.logger) {
+      this.logger.log(msg, level);
+    }
+  }
+
+  // TODO(ato): Suspect this isn't used anywhere
+  getLogs() {
+    if (this.logger) {
+      return this.logger.get();
+    }
+  }
+
+  generate_results() {
+    let passed = 0;
+    let failures = [];
+    let expectedFailures = [];
+    let unexpectedSuccesses = [];
+    for (let i in this.tests) {
+      let isTodo = (this.tests[i].state == "todo");
+      if(this.tests[i].result) {
+        if (isTodo) {
+          expectedFailures.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
+        }
+        else {
+          passed++;
+        }
+      }
+      else {
+        if (isTodo) {
+          unexpectedSuccesses.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
+        }
+        else {
+          failures.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
+        }
+      }
+    }
+    // Reset state in case this object is reused for more tests.
+    this.tests = [];
+    return {
+      passed: passed,
+      failures: failures,
+      expectedFailures: expectedFailures,
+      unexpectedSuccesses: unexpectedSuccesses,
+    };
+  }
+
+  logToFile(file) {
+    //TODO
+  }
+
+  logResult(test, passString, failString) {
+    //TODO: dump to file
+    let resultString = test.result ? passString : failString;
+    let diagnostic = test.name + (test.diag ? " - " + test.diag : "");
+    let msg = resultString + " | " + this.testName + " | " + diagnostic;
+    dump("MARIONETTE TEST RESULT:" + msg + "\n");
+  }
+
+  repr(o) {
+    if (typeof o == "undefined") {
+      return "undefined";
+    } else if (o === null) {
+      return "null";
+    }
+
+    try {
+        if (typeof o.__repr__ == "function") {
+          return o.__repr__();
+        } else if (typeof o.repr == "function" && o.repr !== arguments.callee) {
+          return o.repr();
+        }
+   } catch (e) {}
+
+   try {
+      if (typeof o.NAME === "string" &&
+          (o.toString === Function.prototype.toString || o.toString === Object.prototype.toString)) {
+        return o.NAME;
+      }
+    } catch (e) {}
+
+    let ostring;
+    try {
+      ostring = (o + "");
+    } catch (e) {
+      return "[" + typeof(o) + "]";
+    }
+
+    if (typeof o == "function") {
+      o = ostring.replace(/^\s+/, "");
+      let idx = o.indexOf("{");
+      if (idx != -1) {
+        o = o.substr(0, idx) + "{...}";
+      }
+    }
+    return ostring;
+  }
+
+  waitFor(callback, test, timeout) {
+    if (test()) {
+      callback();
+      return;
+    }
+
+    let now = new Date();
+    let deadline = (timeout instanceof Date) ? timeout :
+        new Date(now.valueOf() + (typeof timeout == "undefined" ? this.timeout : timeout));
+    if (deadline <= now) {
+      dump("waitFor timeout: " + test.toString() + "\n");
+      // the script will timeout here, so no need to raise a separate
+      // timeout exception
+      return;
+    }
+    this.window.setTimeout(this.waitFor.bind(this), 100, callback, test, deadline);
+  }
+};