Bug 1522756 - Part 1: Make it possible to encode specific files to BinAST, and add description. r=Yoric
authorTooru Fujisawa <arai_a@mac.com>
Fri, 25 Jan 2019 11:36:18 +0000
changeset 515425 1fd047e0fbb371be6fc04ca4e3f377af09dbc2f3
parent 515424 4694d772971d6dce45e4c47dde6b25c770b11d59
child 515426 3b8aae988f23624d43c48123987476f736a2720f
push id1953
push userffxbld-merge
push dateMon, 11 Mar 2019 12:10:20 +0000
treeherdermozilla-release@9c35dcbaa899 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersYoric
bugs1522756
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1522756 - Part 1: Make it possible to encode specific files to BinAST, and add description. r=Yoric Differential Revision: https://phabricator.services.mozilla.com/D17601
js/src/jsapi-tests/binast/README.md
js/src/jsapi-tests/binast/encode.py
--- a/js/src/jsapi-tests/binast/README.md
+++ b/js/src/jsapi-tests/binast/README.md
@@ -1,10 +1,113 @@
-To update all binjs files in tree, run the following command:
+# Overview
+
+This directory contains BinAST files for testing BinAST parser, and also
+a script to update BinAST test files in entire tree (including files in
+other directories).
+
+
+# Update BinAST files
+
+## Requirement
+
+To update BinAST files, you need to clone `binjs-ref` repository and
+build `binjs_encode` command, from the following:
+
+https://github.com/binast/binjs-ref
+
+
+## Update all files
+
+To update all BinAST files in tree, run the following command:
+
+```
+./encode.py \
+    --topsrcdir={path to mozilla-unified} \
+    --binjsdir={path to binjs-ref clone} \
+    --binjs_encode={path to binjs_encode}
+```
+
+For example, if `mozilla-unified` and `binjs-ref` are located in
+`~/somewhere`:
+
+```
+./encode.py \
+    --topsrcdir=~/somewhere/mozilla-unified \
+    --binjsdir=~/somewhere/binjs-ref \
+    --binjs_encode=~/somewhere/binjs-ref/target/debug/binjs_encode
+```
+
+
+## Update specific files
+
+
+To update specific BinAST files, run the following command,
+where "filter" is the substring of the path to the file:
 
 ```
 ./encode.py \
     --topsrcdir={path to mozilla-unified} \
     --binjsdir={path to binjs-ref clone} \
-    --binjs_encode={path to  binjs_encode} \
-    --babel={path to babel command} \
-    --babel-preset-env={path to babel-preset-env}
+    --binjs_encode={path to binjs_encode} \
+    {filter} {filter} ...
+```
+
+For example, to update the following files:
+
+ - js/src/jit-test/tests/binast/lazy/regexp_parse/Assertion.binjs
+ - js/src/jit-test/tests/binast/nonlazy/regexp_parse/Assertion.binjs
+ - js/src/jit-test/tests/binast/lazy/regexp_parse/Atom.binjs
+ - js/src/jit-test/tests/binast/nonlazy/regexp_parse/Atom.binjs
+
+Run the following command
+
+```
+./encode.py \
+    --topsrcdir=~/somewhere/mozilla-unified \
+    --binjsdir=~/somewhere/binjs-ref \
+    --binjs_encode=~/somewhere/binjs-ref/target/debug/binjs_encode \
+    regexp_parse/Assertion.binjs \
+    regexp_parse/Atom.binjs
 ```
+
+
+# jit-test
+
+BinAST files in jit-test are encoded from the corresponding plain JS files
+in js/src/jit-test/tests, recursively.
+
+For example js/src/jit-test/tests/FOO/BAR.js is encoded into the following
+2 files:
+
+ - js/src/jit-test/tests/binast/lazy/FOO/BAR.binjs
+ - js/src/jit-test/tests/binast/nonlazy/FOO/BAR.binjs
+
+Also, if FOO.js contains `|jit-test|` in the first line, that line is
+copied to the following 2 files:
+
+ - js/src/jit-test/tests/binast/lazy/FOO/BAR.dir
+ - js/src/jit-test/tests/binast/nonlazy/FOO/BAR.dir
+
+If the directory contains `directives.txt`, the file is copied to the
+following files.
+
+ - js/src/jit-test/tests/binast/lazy/FOO/directives.txt
+ - js/src/jit-test/tests/binast/nonlazy/FOO/directives.txt
+
+
+If a test is known to fail in BinAST for some reasons, putting that path in
+`jit-test.ignore` file in this directory skips encoding the file.
+
+
+# web-platform test
+
+BinAST files in web-platform tests are encoded from the corresponding plain
+JS files in the same directory.
+
+Files are directly listed in `convert_wpt` function in `encode.py`.
+
+
+# jsapi-test
+
+BinAST files in jsapi-test are encoded from the corresponding plain JS
+files in the same directory (js/src/jsapi-tests/binast/parser/multipart/),
+recursively.
--- a/js/src/jsapi-tests/binast/encode.py
+++ b/js/src/jsapi-tests/binast/encode.py
@@ -12,17 +12,17 @@ from sets import Set
 
 parser = OptionParser()
 parser.add_option('--topsrcdir', dest='topsrcdir',
                   help='path to mozilla-central')
 parser.add_option('--binjsdir', dest='binjsdir',
                   help='cwd when running binjs_encode')
 parser.add_option('--binjs_encode', dest='binjs_encode',
                   help='path to binjs_encode commad')
-(options, args) = parser.parse_args()
+(options, filters) = parser.parse_args()
 
 
 def ensure_dir(path, name):
     """ Ensure the given directory exists.
     If the directory doesn't exist, exits with non-zero status.
 
     :param path (string)
            The path to the directory
@@ -53,16 +53,31 @@ def ensure_file(path, name):
 ensure_dir(options.topsrcdir, 'topsrcdir')
 ensure_dir(options.binjsdir, 'binjsdir')
 ensure_file(options.binjs_encode, 'binjs_encode command')
 
 jittest_dir = os.path.join(options.topsrcdir, 'js', 'src', 'jit-test', 'tests')
 ensure_dir(jittest_dir, 'jit-test')
 
 
+def check_filter(outfile_path):
+    """ Check if the output file is the target.
+
+    :return (bool)
+            True if the file is target and should be written.
+    """
+    if len(filters) == 0:
+        return True
+
+    for pattern in filters:
+        if pattern in outfile_path:
+            return True
+    return False
+
+
 def encode(infile_path, outfile_path, binjs_encode_args=[],
            dir_path=None, ignore_fail=False):
     """ Encodes the given .js file into .binjs.
 
     :param infile_path (string)
            The path to the input .js file.
     :param outfile_path (string)
            The path to the output .binjs file.
@@ -72,18 +87,22 @@ def encode(infile_path, outfile_path, bi
            The path to the jit-test directive .dir file for the .binjs file.
            If specified and the .js file contains |jit-test| line, it's
            copied into this file.
     :param ignore_fail (bool)
             If true, ignore binjs_encode command's status.
             If false, exit if binjs_encode command exits with non-zero
             status.
     """
-    print(infile_path)
-    print(outfile_path)
+
+    if not check_filter(outfile_path):
+        return
+
+    print('encoding', infile_path)
+    print('      to', outfile_path)
 
     if dir_path:
         COOKIE = '|jit-test|'
         with open(infile_path) as infile:
             line = infile.readline()
             if COOKIE in line:
                 with open(dir_path, 'w') as dirfile:
                     dirfile.write(line)
@@ -147,16 +166,33 @@ def match_ignore(path, ignore_list):
     :return True if the file should be ignored.
     """
     for item in ignore_list:
         if path.startswith(item['pattern']):
             return True
     return False
 
 
+def copy_directive_file(dir_path, to_dir_path):
+    """ Copy single directives.txt file
+
+    :param dir_path (string)
+           The path to the source directives.txt file.
+    :param to_dir_path (string)
+           The path to the destination directives.txt file.
+    """
+
+    if not check_filter(to_dir_path):
+        return
+
+    print('copying', dir_path)
+    print('     to', to_dir_path)
+    shutil.copyfile(dir_path, to_dir_path)
+
+
 def encode_dir_impl(fromdir, get_todir,
                     ignore_list=None,
                     copy_jit_test_directive=False, **kwargs):
     """ Encode .js files in fromdir into .binjs files, recursively.
 
     :param fromdir (string)
            The path to the base directory that contains .js files.
     :param get_todir (function)
@@ -189,19 +225,17 @@ def encode_dir_impl(fromdir, get_todir,
                             **kwargs)
 
                 if copy_jit_test_directive:
                     # Copy directives.txt.
                     if todir not in copied_dir:
                         dir_path = os.path.join(root, dir_filename)
                         if os.path.exists(dir_path):
                             to_dir_path = os.path.join(todir, dir_filename)
-                            print(dir_path)
-                            print(to_dir_path)
-                            shutil.copyfile(dir_path, to_dir_path)
+                            copy_directive_file(dir_path, to_dir_path)
                             copied_dir.add(todir)
 
 
 def encode_dir(fromdir, todir, **kwargs):
     """ Encode .js files in fromdir into .binjs files in todir, recursively.
 
     :param fromdir (string)
            The path to the base directory that contains .js files.