Bug 1531202 - Part 1: Don't prepend reftest entries in test262 'raw' tests. r=jwalden
authorAndré Bargull <andre.bargull@gmail.com>
Thu, 07 Mar 2019 14:42:25 +0000
changeset 520766 c450f92d5e6219e962e8c5219a0d7c5348b813cb
parent 520765 0d69ee629ce50368c1dfead01fdd9878022380d9
child 520767 84f0b9acc4e9d12ceecd12e75481fd468b7c184a
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjwalden
bugs1531202
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1531202 - Part 1: Don't prepend reftest entries in test262 'raw' tests. r=jwalden - Add separate function to import fixture files instead of treating them like test files. This simplifies the code structure a bit and avoids unnecessary output spew, because the test262 test record parser no longer complains about missing yaml frontmatter. - Write reftest terms into a new test262/jstests.list file for raw tests. - Allow including nested jstests.list files in _parse_external_manifest. Differential Revision: https://phabricator.services.mozilla.com/D21677
js/src/tests/README.txt
js/src/tests/jstests.list
js/src/tests/lib/manifest.py
js/src/tests/test262-update.py
js/src/tests/test262/jstests.list
--- a/js/src/tests/README.txt
+++ b/js/src/tests/README.txt
@@ -53,16 +53,19 @@ Adjusting when and how a test runs
 
     When adding such comments to individual files is not feasible (e.g., for
     imported tests), reftest manifest entries can be added to jstests.list
     instead. Combining in-file comments with entries in this manifest file for
     the same files is not supported (the one from the manifest file will be
     used). Only the following two forms are supported:
         <failure-type> include <relative_path>
         <failure-type> script <relative_path>
+        include <relative_path>
     The <type> "include" indicates that <failure-type> should apply to all test
     cases within a directory. A statement for a nested directory or script
-    overrides one for an enclosing directory.
+    overrides one for an enclosing directory. The <type> "include" without a
+    <failure-type> recursively loads another jstests.list file for further
+    processing.
 
 Running tests
 -------------
 See
 https://developer.mozilla.org/en-US/docs/SpiderMonkey/Running_Automated_JavaScript_Tests
--- a/js/src/tests/jstests.list
+++ b/js/src/tests/jstests.list
@@ -17,16 +17,23 @@ skip script non262/fields/basic.js
 skip script non262/fields/error.js
 skip script non262/fields/field_types.js
 skip script non262/fields/literal.js
 skip script non262/fields/mixed_methods.js
 skip script non262/fields/quirks.js
 skip script non262/reflect-parse/class-fields.js
 
 
+###########################################################################
+# Generated jstests.list for test262 when inline |reftest| isn't possible #
+###########################################################################
+
+include test262/jstests.list
+
+
 #################################################################
 # Tests disabled due to intentional alternative implementations #
 #################################################################
 
 # Legacy "caller" and "arguments" implemented as accessor properties on Function.prototype.
 skip script test262/built-ins/Function/prototype/restricted-property-arguments.js
 skip script test262/built-ins/Function/prototype/restricted-property-caller.js
 skip script test262/built-ins/ThrowTypeError/unique-per-realm-function-proto.js
--- a/js/src/tests/lib/manifest.py
+++ b/js/src/tests/lib/manifest.py
@@ -326,38 +326,48 @@ def _parse_external_manifest(filename, r
                    within the test suite
     """
     if not os.path.exists(filename):
         return []
 
     entries = []
 
     with open(filename, 'r') as fp:
-        manifest_re = re.compile(r'^\s*(.*)\s+(include|script)\s+(\S+)$')
+        manifest_re = re.compile(r'^\s*(?P<terms>.*)\s+(?P<type>include|script)\s+(?P<path>\S+)$')
+        include_re = re.compile(r'^\s*include\s+(?P<path>\S+)$')
         for line in fp:
             line, _, comment = line.partition('#')
             line = line.strip()
             if not line:
                 continue
             matches = manifest_re.match(line)
             if not matches:
-                print('warning: unrecognized line in jstests.list:'
-                      ' {0}'.format(line))
+                matches = include_re.match(line)
+                if not matches:
+                    print('warning: unrecognized line in jstests.list:'
+                          ' {0}'.format(line))
+                    continue
+
+                include_file = matches.group('path')
+                include_filename = os.path.join(os.path.dirname(filename), include_file)
+                include_relpath = os.path.join(relpath, os.path.dirname(include_file))
+                include_entries = _parse_external_manifest(include_filename, include_relpath)
+                entries.extend(include_entries)
                 continue
 
-            path = os.path.normpath(os.path.join(relpath, matches.group(3)))
-            if matches.group(2) == 'include':
+            path = os.path.normpath(os.path.join(relpath, matches.group('path')))
+            if matches.group('type') == 'include':
                 # The manifest spec wants a reference to another manifest here,
                 # but we need just the directory. We do need the trailing
                 # separator so we don't accidentally match other paths of which
                 # this one is a prefix.
                 assert(path.endswith('jstests.list'))
                 path = path[:-len('jstests.list')]
 
-            entries.append({'path': path, 'terms': matches.group(1),
+            entries.append({'path': path, 'terms': matches.group('terms'),
                             'comment': comment.strip()})
 
     # if one directory name is a prefix of another, we want the shorter one
     # first
     entries.sort(key=lambda x: x["path"])
     return entries
 
 
--- a/js/src/tests/test262-update.py
+++ b/js/src/tests/test262-update.py
@@ -11,16 +11,17 @@ import contextlib
 import io
 import os
 import tempfile
 import shutil
 import sys
 
 from functools import partial
 from itertools import chain
+from operator import itemgetter
 
 # Skip all tests which use features not supported in SpiderMonkey.
 UNSUPPORTED_FEATURES = set([
     "tail-call-optimization",
     "class-fields-public",
     "class-static-fields-public",
     "class-fields-private",
     "class-static-fields-private",
@@ -86,18 +87,18 @@ def tryParseTestFile(test262parser, sour
     except Exception as err:
         print("Error '%s' in file: %s" % (err, testName), file=sys.stderr)
         print("Please report this error to the test262 GitHub repository!")
         return None
 
 
 def createRefTestEntry(skip, skipIf, error, isModule):
     """
-    Creates the |reftest| entry from the input list. Or the empty string if no
-    reftest entry is required.
+    Returns the |reftest| tuple (terms, comments) from the input arguments. Or a
+    tuple of empty strings if no reftest entry is required.
     """
 
     terms = []
     comments = []
 
     if skip:
         terms.append("skip")
         comments.extend(skip)
@@ -107,21 +108,28 @@ def createRefTestEntry(skip, skipIf, err
         comments.extend([comment for (_, comment) in skipIf])
 
     if error:
         terms.append("error:" + error)
 
     if isModule:
         terms.append("module")
 
-    line = " ".join(terms)
+    return (" ".join(terms), ", ".join(comments))
+
+
+def createRefTestLine(terms, comments):
+    """
+    Creates the |reftest| line using the given terms and comments.
+    """
+
+    refTest = terms
     if comments:
-        line += " -- " + ", ".join(comments)
-
-    return line
+        refTest += " -- " + comments
+    return refTest
 
 
 def createSource(testSource, refTest, prologue, epilogue):
     """
     Returns the post-processed source for |testSource|.
     """
 
     source = []
@@ -196,33 +204,29 @@ def writeShellAndBrowserFiles(test262Out
 
         # And additional local include files.
         map(partial(os.path.join, os.getcwd()), sorted(localIncludes))
     )))
 
     # Write the concatenated include sources to shell.js.
     with io.open(os.path.join(test262OutDir, relPath, "shell.js"), "wb") as shellFile:
         if includeSource:
-            shellFile.write("// GENERATED, DO NOT EDIT\n")
+            shellFile.write(b"// GENERATED, DO NOT EDIT\n")
             shellFile.write(includeSource)
 
     # The browser.js file is always empty for test262 tests.
     with io.open(os.path.join(test262OutDir, relPath, "browser.js"), "wb") as browserFile:
         browserFile.write(b"")
 
 
 def pathStartsWith(path, *args):
     prefix = os.path.join(*args)
     return os.path.commonprefix([path, prefix]) == prefix
 
 
-def fileNameEndsWith(filePath, suffix):
-    return os.path.splitext(os.path.basename(filePath))[0].endswith(suffix)
-
-
 def convertTestFile(test262parser, testSource, testName, includeSet, strictTests):
     """
     Convert a test262 test to a compatible jstests test file.
     """
 
     # The test record dictionary, its contents are explained in depth at
     # <https://github.com/tc39/test262/blob/master/INTERPRETING.md>.
     testRec = tryParseTestFile(test262parser, testSource.decode("utf-8"), testName)
@@ -268,21 +272,16 @@ def convertTestFile(test262parser, testS
     if "CanBlockIsFalse" in testRec:
         refTestSkipIf.append(("xulRuntime.shell", "shell can block main thread"))
 
     # CanBlockIsTrue is set when the test expects that the implementation
     # can block on the main thread.
     if "CanBlockIsTrue" in testRec:
         refTestSkipIf.append(("!xulRuntime.shell", "browser cannot block main thread"))
 
-    # Skip non-test files.
-    isSupportFile = fileNameEndsWith(testName, "FIXTURE")
-    if isSupportFile:
-        refTestSkip.append("not a test file")
-
     # Skip tests with unsupported features.
     if "features" in testRec:
         unsupported = [f for f in testRec["features"] if f in UNSUPPORTED_FEATURES]
         if unsupported:
             refTestSkip.append("%s is not supported" % ",".join(unsupported))
         else:
             releaseOrBeta = [f for f in testRec["features"] if f in RELEASE_OR_BETA]
             if releaseOrBeta:
@@ -304,46 +303,71 @@ def convertTestFile(test262parser, testS
     # Includes for every test file in a directory is collected in a single
     # shell.js file per directory level. This is done to avoid adding all
     # test harness files to the top level shell.js file.
     if "includes" in testRec:
         assert not raw, "Raw test with includes: %s" % testName
         includeSet.update(testRec["includes"])
 
     # Add reportCompare() after all positive, synchronous tests.
-    if not isNegative and not async and not isSupportFile:
+    if not isNegative and not async:
         testEpilogue = "reportCompare(0, 0);"
     else:
         testEpilogue = ""
 
-    refTest = createRefTestEntry(refTestSkip, refTestSkipIf, errorType, isModule)
+    (terms, comments) = createRefTestEntry(refTestSkip, refTestSkipIf, errorType, isModule)
+    if raw:
+        refTest = ""
+        externRefTest = (terms, comments)
+    else:
+        refTest = createRefTestLine(terms, comments)
+        externRefTest = None
 
-    # Don't write a strict-mode variant for raw, module or support files.
-    noStrictVariant = raw or isModule or isSupportFile
+    # Don't write a strict-mode variant for raw or module files.
+    noStrictVariant = raw or isModule
     assert not (noStrictVariant and (onlyStrict or noStrict)),\
         "Unexpected onlyStrict or noStrict attribute: %s" % testName
 
     # Write non-strict mode test.
     if noStrictVariant or noStrict or not onlyStrict:
         testPrologue = ""
         nonStrictSource = createSource(testSource, refTest, testPrologue, testEpilogue)
         testFileName = testName
-        yield (testFileName, nonStrictSource)
+        yield (testFileName, nonStrictSource, externRefTest)
 
     # Write strict mode test.
     if not noStrictVariant and (onlyStrict or (not noStrict and strictTests)):
         testPrologue = "'use strict';"
         strictSource = createSource(testSource, refTest, testPrologue, testEpilogue)
         testFileName = testName
         if not noStrict:
             testFileName = addSuffixToFileName(testFileName, "-strict")
-        yield (testFileName, strictSource)
+        yield (testFileName, strictSource, externRefTest)
 
 
-def process_test262(test262Dir, test262OutDir, strictTests):
+def convertFixtureFile(fixtureSource, fixtureName):
+    """
+    Convert a test262 fixture file to a compatible jstests test file.
+    """
+
+    # jsreftest meta data
+    refTestSkip = ["not a test file"]
+    refTestSkipIf = []
+    errorType = None
+    isModule = False
+
+    (terms, comments) = createRefTestEntry(refTestSkip, refTestSkipIf, errorType, isModule)
+    refTest = createRefTestLine(terms, comments)
+
+    source = createSource(fixtureSource, refTest, "", "")
+    externRefTest = None
+    yield (fixtureName, source, externRefTest)
+
+
+def process_test262(test262Dir, test262OutDir, strictTests, externManifests):
     """
     Process all test262 files and converts them into jstests compatible tests.
     """
 
     harnessDir = os.path.join(test262Dir, "harness")
     testDir = os.path.join(test262Dir, "test")
     test262parser = loadTest262Parser(test262Dir)
 
@@ -402,24 +426,39 @@ def process_test262(test262Dir, test262O
             testName = os.path.relpath(filePath, testDir)
 
             # Copy non-test files as is.
             (_, fileExt) = os.path.splitext(fileName)
             if fileExt != ".js":
                 shutil.copyfile(filePath, os.path.join(test262OutDir, testName))
                 continue
 
+            # Files ending with "_FIXTURE.js" are fixture files:
+            # https://github.com/tc39/test262/blob/master/INTERPRETING.md#modules
+            isFixtureFile = fileName.endswith("_FIXTURE.js")
+
             # Read the original test source and preprocess it for the jstests harness.
             with io.open(filePath, "rb") as testFile:
                 testSource = testFile.read()
 
-            for (newFileName, newSource) in convertTestFile(test262parser, testSource, testName,
-                                                            includeSet, strictTests):
+            if isFixtureFile:
+                convert = convertFixtureFile(testSource, testName)
+            else:
+                convert = convertTestFile(test262parser, testSource, testName,
+                                          includeSet, strictTests)
+
+            for (newFileName, newSource, externRefTest) in convert:
                 writeTestFile(test262OutDir, newFileName, newSource)
 
+                if externRefTest is not None:
+                    externManifests.append({
+                        "name": newFileName,
+                        "reftest": externRefTest,
+                    })
+
         # Add shell.js and browers.js files for the current directory.
         writeShellAndBrowserFiles(test262OutDir, harnessDir,
                                   includesMap, localIncludesMap, relPath)
 
 
 def fetch_local_changes(inDir, outDir, srcDir, strictTests):
     """
     Fetch the changes from a local clone of Test262.
@@ -501,17 +540,17 @@ def fetch_local_changes(inDir, outDir, s
     shutil.copytree(os.path.join(srcDir, "harness"), os.path.join(inDir, "harness"))
 
     # Reset any older directory in the output using the same branch name
     outDir = os.path.join(outDir, "local", branchName)
     if os.path.isdir(outDir):
         shutil.rmtree(outDir)
     os.makedirs(outDir)
 
-    process_test262(inDir, outDir, strictTests)
+    process_test262(inDir, outDir, strictTests, [])
 
 
 def fetch_pr_files(inDir, outDir, prNumber, strictTests):
     import requests
 
     prTestsOutDir = os.path.join(outDir, "prs", prNumber)
     if os.path.isdir(prTestsOutDir):
         print("Removing folder %s" % prTestsOutDir)
@@ -555,17 +594,17 @@ def fetch_pr_files(inDir, outDir, prNumb
         filePathDirs = os.path.join(inDir, *filename.split("/")[:-1])
 
         if not os.path.isdir(filePathDirs):
             os.makedirs(filePathDirs)
 
         with io.open(os.path.join(inDir, *filename.split("/")), "wb") as output_file:
             output_file.write(fileText.encode('utf8'))
 
-    process_test262(inDir, prTestsOutDir, strictTests)
+    process_test262(inDir, prTestsOutDir, strictTests, [])
 
 
 def general_update(inDir, outDir, strictTests):
     import subprocess
 
     restoreLocalTestsDir = False
     restorePrsTestsDir = False
     localTestsOutDir = os.path.join(outDir, "local")
@@ -590,17 +629,31 @@ def general_update(inDir, outDir, strict
     # Copy license file.
     shutil.copyfile(os.path.join(inDir, "LICENSE"), os.path.join(outDir, "LICENSE"))
 
     # Create the git info file.
     with io.open(os.path.join(outDir, "GIT-INFO"), "w", encoding="utf-8") as info:
         subprocess.check_call(["git", "-C", inDir, "log", "-1"], stdout=info)
 
     # Copy the test files.
-    process_test262(inDir, outDir, strictTests)
+    externManifests = []
+    process_test262(inDir, outDir, strictTests, externManifests)
+
+    # Create the external reftest manifest file.
+    with io.open(os.path.join(outDir, "jstests.list"), "wb") as manifestFile:
+        manifestFile.write(b"# GENERATED, DO NOT EDIT\n\n")
+        for externManifest in sorted(externManifests, key=itemgetter("name")):
+            (terms, comments) = externManifest["reftest"]
+            if terms:
+                entry = "%s script %s%s\n" % (
+                    terms,
+                    externManifest["name"],
+                    (" # %s" % comments) if comments else ""
+                )
+                manifestFile.write(entry.encode("utf-8"))
 
     # Move test262/local back.
     if restoreLocalTestsDir:
         shutil.move(os.path.join(inDir, "local"), outDir)
 
     # Restore test262/prs if necessary after a general Test262 update.
     if restorePrsTestsDir:
         shutil.move(os.path.join(inDir, "prs"), outDir)
new file mode 100644