Bug 1540285 - Add a test generator script into `./mach addtest` r=ahal
authorBrian Grinstead <bgrinstead@mozilla.com>
Mon, 08 Apr 2019 19:09:42 +0000
changeset 468412 69eda9f7be573886154815681f2a94b089eddf87
parent 468411 5937ad352b2dd7bd092ea73348bb7f142db73050
child 468413 59ae6dc11ba316a178ebbb5ad65a809c9118d5a8
push id35837
push userrmaries@mozilla.com
push dateTue, 09 Apr 2019 03:43:40 +0000
treeherdermozilla-central@9eb55c9bf557 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1540285, 123456
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1540285 - Add a test generator script into `./mach addtest` r=ahal Instead of: perl gen_template.pl -b=123456 --type=plain > path/to/test_bug123456.html You can do: ./mach addtest --suite="mochitest-plain" > path/to/test_bug123456.html But you can also pass in a new file path and let it guess the suite/doc: ``` # mochitest-chrome tests ./mach addtest js/xpconnect/tests/chrome/test_chrome.html ./mach addtest js/xpconnect/tests/chrome/test_chrome.xhtml ./mach addtest js/xpconnect/tests/chrome/test_chrome.xul # mochitest-plain tests ./mach addtest js/xpconnect/tests/mochitest/test_plain.html ./mach addtest js/xpconnect/tests/mochitest/test_plain.xhtml ./mach addtest js/xpconnect/tests/mochitest/test_plain.xul # mochitest-browser tests ./mach addtest browser/base/content/test/alerts/browser_foo.js # xpcshell tests ./mach addtest browser/components/extensions/test/xpcshell/test_xpcshell.js ``` This also changes the mochitest template files in the following ways: - removes the bug # boilerplate - remove some unnecessary attributes in the template - removes the th.template - adds the browser.template for browser-chrome tests Differential Revision: https://phabricator.services.mozilla.com/D25482
testing/addtest.py
testing/mach_commands.py
testing/mochitest/gen_template.pl
testing/mochitest/moz.build
testing/mochitest/static/browser.template.txt
testing/mochitest/static/chrome.template.txt
testing/mochitest/static/chromehtml.template.txt
testing/mochitest/static/chromexhtml.template.txt
testing/mochitest/static/chromexul.template.txt
testing/mochitest/static/plainhtml.template.txt
testing/mochitest/static/plainxhtml.template.txt
testing/mochitest/static/plainxul.template.txt
testing/mochitest/static/test.template.txt
testing/mochitest/static/th.template.txt
testing/mochitest/static/xhtml.template.txt
testing/mochitest/static/xul.template.txt
new file mode 100644
--- /dev/null
+++ b/testing/addtest.py
@@ -0,0 +1,111 @@
+
+from __future__ import absolute_import, unicode_literals, print_function
+
+import os
+import manifestparser
+
+
+class XpcshellCreator():
+    template_body = """/* Any copyright is dedicated to the Public Domain.
+http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function test_TODO() {
+  ok(true, "TODO: implement the test");
+});
+"""
+
+    def get_template_contents(self, suite, doc):
+        return self.template_body
+
+    def add_test(self, test, suite, doc):
+        content = self.get_template_contents(suite, doc)
+        with open(test, "w") as f:
+            f.write(content)
+
+        manifest_file = os.path.join(os.path.dirname(test), "xpcshell.ini")
+        filename = os.path.basename(test)
+
+        if not os.path.isfile(manifest_file):
+            print('Could not open manifest file {}'.format(manifest_file))
+            return
+        write_to_ini_file(manifest_file, filename)
+
+
+class MochitestCreator():
+    def get_template_contents(self, suite, doc):
+        mochitest_templates = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), 'mochitest', 'static')
+        )
+        template_file_name = None
+        if suite == "mochitest-browser":
+            template_file_name = 'browser.template.txt'
+
+        if suite == "mochitest-plain":
+            template_file_name = 'plain{}.template.txt'.format(doc)
+
+        if suite == "mochitest-chrome":
+            template_file_name = 'chrome{}.template.txt'.format(doc)
+
+        if template_file_name is None:
+            return None
+
+        template_file = os.path.join(mochitest_templates, template_file_name)
+        if not os.path.isfile(template_file):
+            return None
+
+        with open(template_file) as f:
+            return f.read()
+
+    def add_test(self, test, suite, doc):
+        content = self.get_template_contents(suite, doc)
+        with open(test, "w") as f:
+            f.write(content)
+
+        # attempt to insert into the appropriate manifest
+        guessed_ini = {
+            "mochitest-plain": "mochitest.ini",
+            "mochitest-chrome": "chrome.ini",
+            "mochitest-browser": "browser.ini"
+        }[suite]
+        manifest_file = os.path.join(os.path.dirname(test), guessed_ini)
+        filename = os.path.basename(test)
+
+        if not os.path.isfile(manifest_file):
+            print('Could not open manifest file {}'.format(manifest_file))
+            return
+
+        write_to_ini_file(manifest_file, filename)
+
+
+def write_to_ini_file(manifest_file, filename):
+    # Insert a new test in the right place within a given manifest file
+    manifest = manifestparser.TestManifest(manifests=[manifest_file])
+    insert_before = None
+
+    if any(t['name'] == filename for t in manifest.tests):
+        print("{} is already in the manifest.".format(filename))
+        return
+
+    for test in manifest.tests:
+        if test.get('name') > filename:
+            insert_before = test.get('name')
+            break
+
+    with open(manifest_file, "r") as f:
+        contents = f.readlines()
+
+    filename = '[{}]\n'.format(filename)
+
+    if not insert_before:
+        contents.append(filename)
+    else:
+        insert_before = '[{}]'.format(insert_before)
+        for i in range(len(contents)):
+            if contents[i].startswith(insert_before):
+                contents.insert(i, filename)
+                break
+
+    with open(manifest_file, "w") as f:
+        f.write("".join(contents))
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -78,17 +78,138 @@ def get_test_parser():
     parser.add_argument('extra_args', default=None, nargs=argparse.REMAINDER,
                         help="Extra arguments to pass to the underlying test command(s). "
                              "If an underlying command doesn't recognize the argument, it "
                              "will fail.")
     add_logging_group(parser)
     return parser
 
 
+ADD_TEST_SUPPORTED_SUITES = ['mochitest-chrome', 'mochitest-plain', 'mochitest-browser',
+                             'xpcshell']
+ADD_TEST_SUPPORTED_DOCS = ['js', 'html', 'xhtml', 'xul']
+
+
 @CommandProvider
+class AddTest(MachCommandBase):
+    @Command('addtest', category='testing',
+             description='Generate tests based on templates')
+    @CommandArgument('--suite',
+                     choices=ADD_TEST_SUPPORTED_SUITES,
+                     help='suite for the test (currently only mochitests and xpcshell '
+                          'are supported). If you pass a `test` argument this will be determined'
+                          'based on the filename and the folder it is in')
+    @CommandArgument('-o', '--overwrite',
+                     action='store_true',
+                     help='Overwrite an existing file if it exists.')
+    @CommandArgument('--doc',
+                     choices=ADD_TEST_SUPPORTED_DOCS,
+                     help='Document type for the test (if applicable).'
+                          'If you pass a `test` argument this will be determined'
+                          'based on the filename.')
+    @CommandArgument('test',
+                     nargs='?',
+                     help=('Test to create.'))
+    def addtest(self, suite=None, doc=None, overwrite=False, test=None):
+        if test:
+            if not overwrite and os.path.isfile(os.path.abspath(test)):
+                print("Error: can't generate a test that already exists:", test)
+                return 1
+
+            abs_test = os.path.abspath(test)
+            if doc is None:
+                doc = self.guess_doc(abs_test)
+            if suite is None:
+                guessed_suite, err = self.guess_suite(abs_test)
+                if err:
+                    print(err)
+                    return 1
+                suite = guessed_suite
+
+        else:
+            test = None
+            if doc is None:
+                doc = "html"
+
+        if not suite:
+            print("We couldn't automatically determine a suite. "
+                  "Please specify `--suite` with one of the following options:\n{}\n"
+                  "If you'd like to add support to a new suite, please file a bug "
+                  "blocking https://bugzilla.mozilla.org/show_bug.cgi?id=1540285."
+                  .format(ADD_TEST_SUPPORTED_SUITES))
+            return 1
+
+        if doc not in ADD_TEST_SUPPORTED_DOCS:
+            print("Error: invalid `doc`. Either pass in a test with a valid extension"
+                  "({}) or pass in the `doc` argument".format(ADD_TEST_SUPPORTED_DOCS))
+            return 1
+
+        from addtest import (
+            MochitestCreator,
+            XpcshellCreator,
+        )
+        creator = None
+        if suite == "xpcshell":
+            creator = XpcshellCreator()
+        elif suite in ("mochitest-browser", "mochitest-chrome", "mochitest-plain"):
+            creator = MochitestCreator()
+        else:
+            print("Sorry, `addtest` doesn't currently know how to add {}".format(suite))
+            return 1
+
+        if (test):
+            print("Adding a test at {} (suite `{}`)".format(test, suite))
+
+            adding_error = creator.add_test(test, suite, doc)
+
+            if adding_error:
+                print("Error adding test: {}".format(adding_error))
+                return 1
+
+            mach_command = TEST_SUITES[suite]["mach_command"]
+            print('Please make sure to add the new test to your commit. '
+                  'You can now run the test with:\n    ./mach {} {}'.format(mach_command, test))
+        else:
+            # write to stdout if you passed only suite and doc and not a file path
+            print(creator.get_template_contents(suite, doc))
+        return 0
+
+    def guess_doc(self, abs_test):
+        filename = os.path.basename(abs_test)
+        return os.path.splitext(filename)[1].strip(".")
+
+    def guess_suite(self, abs_test):
+        # If you pass a abs_test, try to detect the type based on the name
+        # and folder. This detection can be skipped if you pass the `type` arg.
+        err = None
+        guessed_suite = None
+        parent = os.path.dirname(abs_test)
+        filename = os.path.basename(abs_test)
+
+        has_browser_ini = os.path.isfile(os.path.join(parent, "browser.ini"))
+        has_chrome_ini = os.path.isfile(os.path.join(parent, "chrome.ini"))
+        has_plain_ini = os.path.isfile(os.path.join(parent, "mochitest.ini"))
+        has_xpcshell_ini = os.path.isfile(os.path.join(parent, "xpcshell.ini"))
+
+        if filename.startswith("test_") and has_xpcshell_ini and self.guess_doc(abs_test) == "js":
+            guessed_suite = "xpcshell"
+        else:
+            if filename.startswith("browser_") and has_browser_ini:
+                guessed_suite = "mochitest-browser"
+            elif filename.startswith("test_"):
+                if has_chrome_ini and has_plain_ini:
+                    err = ("Error: directory contains both a chrome.ini and mochitest.ini. "
+                           "Please set --suite=mochitest-chrome or --suite=mochitest-plain.")
+                elif has_chrome_ini:
+                    guessed_suite = "mochitest-chrome"
+                elif has_plain_ini:
+                    guessed_suite = "mochitest-plain"
+        return guessed_suite, err
+
+
 class Test(MachCommandBase):
     @Command('test', category='testing',
              description='Run tests (detects the kind of test and runs it).',
              parser=get_test_parser)
     def test(self, what, extra_args, **log_args):
         """Run tests from names or paths.
 
         mach test accepts arguments specifying which tests to run. Each argument
deleted file mode 100644
--- a/testing/mochitest/gen_template.pl
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/perl
-
-# This script makes mochitest test case templates. See
-# https://developer.mozilla.org/en-US/docs/Mochitest#Test_templates
-#
-# It takes two arguments:
-#
-#   -b:     a bugnumber
-#   -type:  template type. One of {plain|xhtml|xul|th|chrome|chromexul}.
-#           Defaults to th (testharness.js).
-#
-# For example, this command:
-#
-#  perl gen_template.pl -b 345876 -type xul
-#
-# writes a XUL test case template for bug 345876 to stdout.
-
-use FindBin;
-use Getopt::Long;
-GetOptions("b=i"=> \$bug_number,
-           "type:s"=> \$template_type);
-
-if ($template_type eq "xul") {
-  $template_type = "$FindBin::RealBin/static/xul.template.txt";
-} elsif ($template_type eq "xhtml") {
-  $template_type = "$FindBin::RealBin/static/xhtml.template.txt";
-} elsif ($template_type eq "chrome") {
-  $template_type = "$FindBin::RealBin/static/chrome.template.txt";
-} elsif ($template_type eq "chromexul") {
-  $template_type = "$FindBin::RealBin/static/chromexul.template.txt";
-} elsif ($template_type eq "plain") {
-  $template_type = "$FindBin::RealBin/static/test.template.txt";
-} else {
-  $template_type = "$FindBin::RealBin/static/th.template.txt";
-}
-
-open(IN,$template_type) or die("Failed to open myfile for reading.");
-while((defined(IN)) && ($line = <IN>)) {
-        $line =~ s/{BUGNUMBER}/$bug_number/g;
-        print STDOUT $line;
-}
-close(IN);
--- a/testing/mochitest/moz.build
+++ b/testing/mochitest/moz.build
@@ -99,17 +99,16 @@ TEST_HARNESS_FILES.testing.mochitest += 
     '/build/valgrind/x86_64-pc-linux-gnu.sup',
     '/netwerk/test/httpserver/httpd.js',
     'bisection.py',
     'browser-harness.xul',
     'browser-test.js',
     'chrome-harness.js',
     'chunkifyTests.js',
     'favicon.ico',
-    'gen_template.pl',
     'harness.xul',
     'leaks.py',
     'mach_test_package_commands.py',
     'manifest.webapp',
     'manifestLibrary.js',
     'mochitest_options.py',
     'nested_setup.js',
     'pywebsocket_wrapper.py',
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/static/browser.template.txt
@@ -0,0 +1,8 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function test_TODO() {
+  ok(true, "TODO: implement the test");
+});
rename from testing/mochitest/static/chrome.template.txt
rename to testing/mochitest/static/chromehtml.template.txt
--- a/testing/mochitest/static/chrome.template.txt
+++ b/testing/mochitest/static/chromehtml.template.txt
@@ -1,31 +1,23 @@
 <!DOCTYPE HTML>
 <html>
-<!--
-https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}
--->
 <head>
   <meta charset="utf-8">
-  <title>Test for Bug {BUGNUMBER}</title>
-  <script type="application/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script>
-  <link rel="stylesheet" type="text/css" href="chrome://global/skin"/>
+  <title><!-- TODO: insert title here --></title>
+  <script src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script>
+  <script src="chrome://mochikit/content/tests/SimpleTest/AddTask.js"></script>
   <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"/>
-  <script type="application/javascript">
-
-  /** Test for Bug {BUGNUMBER} **/
-
-
-
-
-
+  <script>
+    add_task(async function test_TODO() {
+      ok(true, "TODO: implement the test");
+    });
   </script>
 </head>
 <body>
-<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}">Mozilla Bug {BUGNUMBER}</a>
 <p id="display"></p>
 <div id="content" style="display: none">
 
 </div>
 <pre id="test">
 </pre>
 </body>
 </html>
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/static/chromexhtml.template.txt
@@ -0,0 +1,23 @@
+<!DOCTYPE HTML>
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+  <meta charset="utf-8" />
+  <title><!-- TODO: insert title here --></title>
+  <script src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script>
+  <script src="chrome://mochikit/content/tests/SimpleTest/AddTask.js"></script>
+  <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"/>
+  <script><![CDATA[
+    add_task(async function test_TODO() {
+      ok(true, "TODO: implement the test");
+    });
+  ]]></script>
+</head>
+<body>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>
--- a/testing/mochitest/static/chromexul.template.txt
+++ b/testing/mochitest/static/chromexul.template.txt
@@ -1,26 +1,18 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/css" href="chrome://global/skin"?>
 <?xml-stylesheet type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"?>
-<!--
-https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}
--->
-<window title="Mozilla Bug {BUGNUMBER}"
+<window title="TODO: Insert title here"
         xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
-  <script type="application/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"/>
+  <script type="application/javascript"
+          src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js" />
+  <script type="application/javascript"
+          src="chrome://mochikit/content/tests/SimpleTest/AddTask.js" />
+  <script type="application/javascript"><![CDATA[
+    add_task(async function test_TODO() {
+      ok(true, "TODO: implement the test");
+    });
+  ]]></script>
 
-  <!-- test results are displayed in the html:body -->
   <body xmlns="http://www.w3.org/1999/xhtml">
-  <a href="https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}"
-     target="_blank">Mozilla Bug {BUGNUMBER}</a>
   </body>
-
-  <!-- test code goes here -->
-  <script type="application/javascript">
-  <![CDATA[
-  /** Test for Bug {BUGNUMBER} **/
-
-
-
-  ]]>
-  </script>
 </window>
rename from testing/mochitest/static/test.template.txt
rename to testing/mochitest/static/plainhtml.template.txt
--- a/testing/mochitest/static/test.template.txt
+++ b/testing/mochitest/static/plainhtml.template.txt
@@ -1,30 +1,20 @@
 <!DOCTYPE HTML>
 <html>
-<!--
-https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}
--->
 <head>
   <meta charset="utf-8">
-  <title>Test for Bug {BUGNUMBER}</title>
-  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <title><!-- TODO: insert title here --></title>
+  <script src="/tests/SimpleTest/SimpleTest.js"></script>
   <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
-  <script type="application/javascript">
-
-  /** Test for Bug {BUGNUMBER} **/
-
-
-
-
-
+  <script>
+    ok(true, "TODO: implement the test");
   </script>
 </head>
 <body>
-<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}">Mozilla Bug {BUGNUMBER}</a>
 <p id="display"></p>
 <div id="content" style="display: none">
 
 </div>
 <pre id="test">
 </pre>
 </body>
 </html>
rename from testing/mochitest/static/xhtml.template.txt
rename to testing/mochitest/static/plainxhtml.template.txt
--- a/testing/mochitest/static/xhtml.template.txt
+++ b/testing/mochitest/static/plainxhtml.template.txt
@@ -1,29 +1,20 @@
+<!DOCTYPE HTML>
 <html xmlns="http://www.w3.org/1999/xhtml">
-<!--
-https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}
--->
 <head>
-  <title>Test for Bug {BUGNUMBER}</title>
-  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <meta charset="utf-8" />
+  <title><!-- TODO: insert title here --></title>
+  <script src="/tests/SimpleTest/SimpleTest.js"></script>
   <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
-  <script type="application/javascript">
-  <![CDATA[
-
-  /** Test for Bug {BUGNUMBER} **/
-
-
-
-
-  ]]>
-</script>
+  <script><![CDATA[
+    ok(true, "TODO: implement the test");
+  ]]></script>
 </head>
 <body>
-<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}">Mozilla Bug {BUGNUMBER}</a>
 <p id="display"></p>
 <div id="content" style="display: none">
 
 </div>
 <pre id="test">
 </pre>
 </body>
 </html>
rename from testing/mochitest/static/xul.template.txt
rename to testing/mochitest/static/plainxul.template.txt
--- a/testing/mochitest/static/xul.template.txt
+++ b/testing/mochitest/static/plainxul.template.txt
@@ -1,28 +1,12 @@
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/css" href="chrome://global/skin"?>
 <?xml-stylesheet type="text/css" href="/tests/SimpleTest/test.css"?>
-<!--
-https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}
--->
-<window title="Mozilla Bug {BUGNUMBER}"
+<window title="TODO: Insert title here"
         xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
-  <script type="application/javascript"
-          src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"/>
-
-  <!-- test code goes here -->
-  <script type="application/javascript">
-  <![CDATA[
-
-  /** Test for Bug {BUGNUMBER} **/
-
-
-
-  ]]>
-  </script>
-
-  <!-- test results are displayed in the html:body -->
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="application/javascript"><![CDATA[
+    ok(true, "TODO: implement the test");
+  ]]></script>
   <body xmlns="http://www.w3.org/1999/xhtml">
-  <a href="https://bugzilla.mozilla.org/show_bug.cgi?id={BUGNUMBER}"
-     target="_blank">Mozilla Bug {BUGNUMBER}</a>
   </body>
 </window>
deleted file mode 100644
--- a/testing/mochitest/static/th.template.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-<!DOCTYPE html>
-<meta charset=utf-8>
-<title>Test for ...</title>
-<script src="/resources/testharness.js"></script>
-<script src="/resources/testharnessreport.js"></script>
-<div id="log"></div>
-<script>
-test(function() {
-
-}, "Description");
-</script>