Bug 937012 - Replace the busted find_vanilla_new_calls script with the much better check_vanilla_allocations.py. r=evilpie,gps.
authorNicholas Nethercote <nnethercote@mozilla.com>
Mon, 11 Nov 2013 18:37:55 +1100
changeset 156461 fd939ce2bcdc250615aff163fd1b605fe64a1b21
parent 156460 f30dfa0f184dc2a3a6cfde724dbcea874b215d1b
child 156462 d9e2cb0a838d337f8b45eb67a1efa9993bcd356e
push id36433
push usernnethercote@mozilla.com
push dateWed, 20 Nov 2013 08:21:47 +0000
treeherdermozilla-inbound@d9e2cb0a838d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersevilpie, gps
bugs937012
milestone28.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 937012 - Replace the busted find_vanilla_new_calls script with the much better check_vanilla_allocations.py. r=evilpie,gps.
config/check_vanilla_allocations.py
config/find_vanilla_new_calls
js/src/Makefile.in
js/src/config/check_vanilla_allocations.py
js/src/config/find_vanilla_new_calls
js/src/jsutil.cpp
new file mode 100644
--- /dev/null
+++ b/config/check_vanilla_allocations.py
@@ -0,0 +1,159 @@
+# vim: set ts=8 sts=4 et sw=4 tw=79:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#----------------------------------------------------------------------------
+# All heap allocations in SpiderMonkey must go through js_malloc, js_calloc,
+# js_realloc, and js_free.  This is so that any embedder who uses a custom
+# allocator (by defining JS_USE_CUSTOM_ALLOCATOR) will see all heap allocation
+# go through that custom allocator.
+#
+# Therefore, the presence of any calls to "vanilla" allocation/free functions
+# (e.g. malloc(), free()) is a bug.
+#
+# This script checks for the presence of such disallowed vanilla
+# allocation/free function in SpiderMonkey when it's built as a library.  It
+# relies on |nm| from the GNU binutils, and so only works on Linux, but one
+# platform is good enough to catch almost all violations.
+#
+# This checking is only 100% reliable in a JS_USE_CUSTOM_ALLOCATOR build in
+# which the default definitions of js_malloc et al (in Utility.h) -- which call
+# malloc et al -- are replaced with empty definitions.  This is because the
+# presence and possible inlining of the default js_malloc et al can cause
+# malloc/calloc/realloc/free calls show up in unpredictable places.
+#
+# Unfortunately, that configuration cannot be tested on Mozilla's standard
+# testing infrastructure.  Instead, by default this script only tests that none
+# of the other vanilla allocation/free functions (operator new, memalign, etc)
+# are present.  If given the --aggressive flag, it will also check for
+# malloc/calloc/realloc/free.
+#
+# Note:  We don't check for |operator delete| and |operator delete[]|.  These
+# can be present somehow due to virtual destructors, but this is not too
+# because vanilla delete/delete[] calls don't make sense without corresponding
+# vanilla new/new[] calls, and any explicit calls will be caught by Valgrind's
+# mismatched alloc/free checking.
+#----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import argparse
+import re
+import subprocess
+import sys
+
+# The obvious way to implement this script is to search for occurrences of
+# malloc et al, succeed if none are found, and fail is some are found.
+# However, "none are found" does not necessarily mean "none are present" --
+# this script could be buggy.  (Or the output format of |nm| might change in
+# the future.)
+#
+# So jsutil.cpp deliberately contains a (never-called) function that contains a
+# single use of all the vanilla allocation/free functions.  And this script
+# fails if it (a) finds uses of those functions in files other than jsutil.cpp,
+# *or* (b) fails to find them in jsutil.cpp.
+
+# Tracks overall success of the test.
+has_failed = False
+
+
+def fail(msg):
+    print('TEST-UNEXPECTED-FAIL | check_vanilla_allocations.py |', msg)
+    global has_failed
+    has_failed = True
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aggressive', action='store_true',
+                        help='also check for malloc, calloc, realloc and free')
+    parser.add_argument('file', type=str,
+                        help='name of the file to check')
+    args = parser.parse_args()
+
+    # Run |nm|.  Options:
+    # -u: show only undefined symbols
+    # -C: demangle symbol names
+    # -l: show a filename and line number for each undefined symbol
+    cmd = ['nm', '-u', '-C', '-l', args.file]
+    lines = subprocess.check_output(cmd, universal_newlines=True,
+                                    stderr=subprocess.PIPE).split('\n')
+
+    # alloc_fns contains all the vanilla allocation/free functions that we look
+    # for. Regexp chars are escaped appropriately.
+
+    alloc_fns = [
+        # Matches |operator new(unsigned T)|, where |T| is |int| or |long|.
+        r'operator new\(unsigned',
+
+        # Matches |operator new[](unsigned T)|, where |T| is |int| or |long|.
+        r'operator new\[\]\(unsigned',
+
+        r'memalign',
+        # These two aren't available on all Linux configurations.
+        #r'posix_memalign',
+        #r'aligned_alloc',
+        r'valloc',
+        r'strdup'
+    ]
+
+    if args.aggressive:
+        alloc_fns += [
+            r'malloc',
+            r'calloc',
+            r'realloc',
+            r'free'
+        ]
+
+    # This is like alloc_fns, but regexp chars are not escaped.
+    alloc_fns_unescaped = [fn.translate(None, r'\\') for fn in alloc_fns]
+
+    # This regexp matches the relevant lines in the output of |nm|, which look
+    # like the following.
+    #
+    #   U malloc  /path/to/objdir/dist/include/js/Utility.h:142
+    #
+    alloc_fns_re = r'U (' + r'|'.join(alloc_fns) + r').*\/([\w\.]+):(\d+)$'
+
+    # This tracks which allocation/free functions have been seen in jsutil.cpp.
+    jsutil_cpp = set([])
+
+    for line in lines:
+        m = re.search(alloc_fns_re, line)
+        if m is None:
+            continue
+
+        fn = m.group(1)
+        filename = m.group(2)
+        linenum = m.group(3)
+        if filename == 'jsutil.cpp':
+            jsutil_cpp.add(fn)
+        else:
+            # An allocation is present in a non-special file.  Fail!
+            fail("'" + fn + "' present at " + filename + ':' + linenum)
+
+
+    # Check that all functions we expect are used in jsutil.cpp.  (This will
+    # fail if the function-detection code breaks at any point.)
+    for fn in alloc_fns_unescaped:
+        if fn not in jsutil_cpp:
+            fail("'" + fn + "' isn't used as expected in jsutil.cpp")
+        else:
+            jsutil_cpp.remove(fn)
+
+    # This should never happen, but check just in case.
+    if jsutil_cpp:
+        fail('unexpected allocation fns used in jsutil.cpp: ' +
+             ', '.join(jsutil_cpp))
+
+    if has_failed:
+        sys.exit(1)
+
+    print('TEST-PASS | check_vanilla_allocations.py | ok')
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()
+
deleted file mode 100755
--- a/config/find_vanilla_new_calls
+++ /dev/null
@@ -1,80 +0,0 @@
-# /bin/bash
-
-#----------------------------------------------------------------------------
-# We must avoid using the vanilla new/new[] operators (and consequently, the
-# vanilla delete/delete[] operators) in SpiderMonkey, see bug 624878 for why.
-#
-# This script:
-# - Detects if any of the vanilla new/new[] operators are used in a file.
-#   Its exit code is 1 if it found some, and 0 if it didn't.
-# - Doesn't detect delete/delete[] because it appears they can be present
-#   somehow due to virtual destructors, but this is ok because vanilla
-#   delete/delete[] calls don't make sense without corresponding new/new[]
-#   calls, and any explicit calls will be caught by Valgrind's mismatched
-#   alloc/free checking.
-# - Doesn't detect the 'nothrow' variants, which are ok but probably still
-#   best avoided.
-# - Is designed to only run on Linux (though it may also work on Mac);  one
-#   platform will be enough to catch any violations.
-#
-# If this script fails:
-# - You need to find the uses of vanilla new/delete and replace them with
-#   {js::OffTheBooks,JSContext,JSRuntime}::{new_,/array_new}.
-# - Run this script on each of the .o files, that should narrow it down.
-# - After that, one way to find them is to run 'objdump -r -C' on the
-#   relevant .o files.  For example, you might search for 'operator new' and
-#   find a record like this:
-#
-#  RELOCATION RECORDS FOR [.text._ZN3JSC14ExecutablePool6createEj]:
-#  OFFSET   TYPE              VALUE
-#  00000009 R_386_PC32        __i686.get_pc_thunk.bx
-#  0000000f R_386_GOTPC       _GLOBAL_OFFSET_TABLE_
-#  0000001b R_386_PLT32       operator new(unsigned int)
-#  0000002e R_386_PC32        JSC::ExecutablePool::ExecutablePool(unsigned int)
-#  0000004a R_386_PC32        JSC::ExecutablePool::~ExecutablePool()
-#  00000052 R_386_PLT32       operator delete(void*)
-#
-#   This says that vanilla 'new' and 'delete' are both used in
-#   JSC::ExecutablePool::create(unsigned int).  This doesn't always work,
-#   though.  (Nb: use 'c++filt' to demangle names like
-#   _ZN3JSC14ExecutablePool6createEj.)
-#
-# If that doesn't work, use grep.
-#----------------------------------------------------------------------------
-
-if [ -z $1 ] ; then
-    echo "usage: find_vanilla_new_calls <file>"
-    exit 1
-fi
-
-file=$1
-
-if [ ! -f $file ] ; then
-    echo "TEST-UNEXPECTED-FAIL | find_vanilla_new_calls | file '$file' not found"
-    exit 1
-fi
-
-tmpfile1=`mktemp`
-tmpfile2=`mktemp`
-nm -C $file > $tmpfile1
-
-# Need to double-escape '[' and ']' to stop grep from interpreting them
-# specially.
-grep '^operator new(unsigned int)'        $tmpfile1 >> $tmpfile2
-grep '^operator new(unsigned long)'       $tmpfile1 >> $tmpfile2
-grep '^operator new\\[\\](unsigned int)'  $tmpfile1 >> $tmpfile2
-grep '^operator new\\[\\](unsigned long)' $tmpfile1 >> $tmpfile2
-rm -f $tmpfile1
-
-if [ -s $tmpfile2 ] ; then
-    echo "TEST-UNEXPECTED-FAIL | find_vanilla_new_calls | found calls are listed below"
-    cat $tmpfile2
-    echo
-    rm -f $tmpfile2
-    exit 1
-fi
-
-echo "TEST-PASS | find_vanilla_new_calls | ok"
-echo
-
-exit 0
--- a/js/src/Makefile.in
+++ b/js/src/Makefile.in
@@ -263,24 +263,28 @@ distclean clean::
 endif
 endif
 
 #
 # END ECMAScript Internationalization API
 #############################################
 
 
-# The "find any vanilla new/new[] calls" script is tailored to Linux, so
-# only run it there.  That should be enough to catch any such calls that
-# creep in.
-check-vanilla-new:
-	$(srcdir)/config/find_vanilla_new_calls $(LIBRARY)
+# check_vanilla_allocations.py is tailored to Linux, so only run it there.
+# That should be enough to catch any problems.
+check-vanilla-allocations:
+	$(PYTHON) $(srcdir)/config/check_vanilla_allocations.py $(REAL_LIBRARY)
+
+# The "aggressive" variant will likely fail on some compiler/platform
+# combinations, but is worth running by hand every once in a while.
+check-vanilla-allocations-aggressive:
+	$(PYTHON) $(srcdir)/config/check_vanilla_allocations.py --aggressive $(REAL_LIBRARY)
 
 ifeq ($(OS_ARCH),Linux)
-check:: check-vanilla-new
+check:: check-vanilla-allocations
 endif
 
 # Help ensure that the number of OOM errors in SpiderMonkey doesn't increase.
 # If the number of OOM errors changes, update the number below. We intend this
 # number to go down over time, by fixing OOMs.
 check-ooms:
 	$(wildcard $(RUN_TEST_PROGRAM)) $(PYTHON) -u $(srcdir)/config/find_OOM_errors.py --regression 125
 
new file mode 100644
--- /dev/null
+++ b/js/src/config/check_vanilla_allocations.py
@@ -0,0 +1,159 @@
+# vim: set ts=8 sts=4 et sw=4 tw=79:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#----------------------------------------------------------------------------
+# All heap allocations in SpiderMonkey must go through js_malloc, js_calloc,
+# js_realloc, and js_free.  This is so that any embedder who uses a custom
+# allocator (by defining JS_USE_CUSTOM_ALLOCATOR) will see all heap allocation
+# go through that custom allocator.
+#
+# Therefore, the presence of any calls to "vanilla" allocation/free functions
+# (e.g. malloc(), free()) is a bug.
+#
+# This script checks for the presence of such disallowed vanilla
+# allocation/free function in SpiderMonkey when it's built as a library.  It
+# relies on |nm| from the GNU binutils, and so only works on Linux, but one
+# platform is good enough to catch almost all violations.
+#
+# This checking is only 100% reliable in a JS_USE_CUSTOM_ALLOCATOR build in
+# which the default definitions of js_malloc et al (in Utility.h) -- which call
+# malloc et al -- are replaced with empty definitions.  This is because the
+# presence and possible inlining of the default js_malloc et al can cause
+# malloc/calloc/realloc/free calls show up in unpredictable places.
+#
+# Unfortunately, that configuration cannot be tested on Mozilla's standard
+# testing infrastructure.  Instead, by default this script only tests that none
+# of the other vanilla allocation/free functions (operator new, memalign, etc)
+# are present.  If given the --aggressive flag, it will also check for
+# malloc/calloc/realloc/free.
+#
+# Note:  We don't check for |operator delete| and |operator delete[]|.  These
+# can be present somehow due to virtual destructors, but this is not too
+# because vanilla delete/delete[] calls don't make sense without corresponding
+# vanilla new/new[] calls, and any explicit calls will be caught by Valgrind's
+# mismatched alloc/free checking.
+#----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import argparse
+import re
+import subprocess
+import sys
+
+# The obvious way to implement this script is to search for occurrences of
+# malloc et al, succeed if none are found, and fail is some are found.
+# However, "none are found" does not necessarily mean "none are present" --
+# this script could be buggy.  (Or the output format of |nm| might change in
+# the future.)
+#
+# So jsutil.cpp deliberately contains a (never-called) function that contains a
+# single use of all the vanilla allocation/free functions.  And this script
+# fails if it (a) finds uses of those functions in files other than jsutil.cpp,
+# *or* (b) fails to find them in jsutil.cpp.
+
+# Tracks overall success of the test.
+has_failed = False
+
+
+def fail(msg):
+    print('TEST-UNEXPECTED-FAIL | check_vanilla_allocations.py |', msg)
+    global has_failed
+    has_failed = True
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--aggressive', action='store_true',
+                        help='also check for malloc, calloc, realloc and free')
+    parser.add_argument('file', type=str,
+                        help='name of the file to check')
+    args = parser.parse_args()
+
+    # Run |nm|.  Options:
+    # -u: show only undefined symbols
+    # -C: demangle symbol names
+    # -l: show a filename and line number for each undefined symbol
+    cmd = ['nm', '-u', '-C', '-l', args.file]
+    lines = subprocess.check_output(cmd, universal_newlines=True,
+                                    stderr=subprocess.PIPE).split('\n')
+
+    # alloc_fns contains all the vanilla allocation/free functions that we look
+    # for. Regexp chars are escaped appropriately.
+
+    alloc_fns = [
+        # Matches |operator new(unsigned T)|, where |T| is |int| or |long|.
+        r'operator new\(unsigned',
+
+        # Matches |operator new[](unsigned T)|, where |T| is |int| or |long|.
+        r'operator new\[\]\(unsigned',
+
+        r'memalign',
+        # These two aren't available on all Linux configurations.
+        #r'posix_memalign',
+        #r'aligned_alloc',
+        r'valloc',
+        r'strdup'
+    ]
+
+    if args.aggressive:
+        alloc_fns += [
+            r'malloc',
+            r'calloc',
+            r'realloc',
+            r'free'
+        ]
+
+    # This is like alloc_fns, but regexp chars are not escaped.
+    alloc_fns_unescaped = [fn.translate(None, r'\\') for fn in alloc_fns]
+
+    # This regexp matches the relevant lines in the output of |nm|, which look
+    # like the following.
+    #
+    #   U malloc  /path/to/objdir/dist/include/js/Utility.h:142
+    #
+    alloc_fns_re = r'U (' + r'|'.join(alloc_fns) + r').*\/([\w\.]+):(\d+)$'
+
+    # This tracks which allocation/free functions have been seen in jsutil.cpp.
+    jsutil_cpp = set([])
+
+    for line in lines:
+        m = re.search(alloc_fns_re, line)
+        if m is None:
+            continue
+
+        fn = m.group(1)
+        filename = m.group(2)
+        linenum = m.group(3)
+        if filename == 'jsutil.cpp':
+            jsutil_cpp.add(fn)
+        else:
+            # An allocation is present in a non-special file.  Fail!
+            fail("'" + fn + "' present at " + filename + ':' + linenum)
+
+
+    # Check that all functions we expect are used in jsutil.cpp.  (This will
+    # fail if the function-detection code breaks at any point.)
+    for fn in alloc_fns_unescaped:
+        if fn not in jsutil_cpp:
+            fail("'" + fn + "' isn't used as expected in jsutil.cpp")
+        else:
+            jsutil_cpp.remove(fn)
+
+    # This should never happen, but check just in case.
+    if jsutil_cpp:
+        fail('unexpected allocation fns used in jsutil.cpp: ' +
+             ', '.join(jsutil_cpp))
+
+    if has_failed:
+        sys.exit(1)
+
+    print('TEST-PASS | check_vanilla_allocations.py | ok')
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()
+
deleted file mode 100755
--- a/js/src/config/find_vanilla_new_calls
+++ /dev/null
@@ -1,80 +0,0 @@
-# /bin/bash
-
-#----------------------------------------------------------------------------
-# We must avoid using the vanilla new/new[] operators (and consequently, the
-# vanilla delete/delete[] operators) in SpiderMonkey, see bug 624878 for why.
-#
-# This script:
-# - Detects if any of the vanilla new/new[] operators are used in a file.
-#   Its exit code is 1 if it found some, and 0 if it didn't.
-# - Doesn't detect delete/delete[] because it appears they can be present
-#   somehow due to virtual destructors, but this is ok because vanilla
-#   delete/delete[] calls don't make sense without corresponding new/new[]
-#   calls, and any explicit calls will be caught by Valgrind's mismatched
-#   alloc/free checking.
-# - Doesn't detect the 'nothrow' variants, which are ok but probably still
-#   best avoided.
-# - Is designed to only run on Linux (though it may also work on Mac);  one
-#   platform will be enough to catch any violations.
-#
-# If this script fails:
-# - You need to find the uses of vanilla new/delete and replace them with
-#   {js::OffTheBooks,JSContext,JSRuntime}::{new_,/array_new}.
-# - Run this script on each of the .o files, that should narrow it down.
-# - After that, one way to find them is to run 'objdump -r -C' on the
-#   relevant .o files.  For example, you might search for 'operator new' and
-#   find a record like this:
-#
-#  RELOCATION RECORDS FOR [.text._ZN3JSC14ExecutablePool6createEj]:
-#  OFFSET   TYPE              VALUE
-#  00000009 R_386_PC32        __i686.get_pc_thunk.bx
-#  0000000f R_386_GOTPC       _GLOBAL_OFFSET_TABLE_
-#  0000001b R_386_PLT32       operator new(unsigned int)
-#  0000002e R_386_PC32        JSC::ExecutablePool::ExecutablePool(unsigned int)
-#  0000004a R_386_PC32        JSC::ExecutablePool::~ExecutablePool()
-#  00000052 R_386_PLT32       operator delete(void*)
-#
-#   This says that vanilla 'new' and 'delete' are both used in
-#   JSC::ExecutablePool::create(unsigned int).  This doesn't always work,
-#   though.  (Nb: use 'c++filt' to demangle names like
-#   _ZN3JSC14ExecutablePool6createEj.)
-#
-# If that doesn't work, use grep.
-#----------------------------------------------------------------------------
-
-if [ -z $1 ] ; then
-    echo "usage: find_vanilla_new_calls <file>"
-    exit 1
-fi
-
-file=$1
-
-if [ ! -f $file ] ; then
-    echo "TEST-UNEXPECTED-FAIL | find_vanilla_new_calls | file '$file' not found"
-    exit 1
-fi
-
-tmpfile1=`mktemp`
-tmpfile2=`mktemp`
-nm -C $file > $tmpfile1
-
-# Need to double-escape '[' and ']' to stop grep from interpreting them
-# specially.
-grep '^operator new(unsigned int)'        $tmpfile1 >> $tmpfile2
-grep '^operator new(unsigned long)'       $tmpfile1 >> $tmpfile2
-grep '^operator new\\[\\](unsigned int)'  $tmpfile1 >> $tmpfile2
-grep '^operator new\\[\\](unsigned long)' $tmpfile1 >> $tmpfile2
-rm -f $tmpfile1
-
-if [ -s $tmpfile2 ] ; then
-    echo "TEST-UNEXPECTED-FAIL | find_vanilla_new_calls | found calls are listed below"
-    cat $tmpfile2
-    echo
-    rm -f $tmpfile2
-    exit 1
-fi
-
-echo "TEST-PASS | find_vanilla_new_calls | ok"
-echo
-
-exit 0
--- a/js/src/jsutil.cpp
+++ b/js/src/jsutil.cpp
@@ -156,16 +156,59 @@ JS_STATIC_ASSERT(sizeof(void *) == sizeo
 
 JS_PUBLIC_API(void)
 JS_Assert(const char *s, const char *file, int ln)
 {
     MOZ_ReportAssertionFailure(s, file, ln);
     MOZ_CRASH();
 }
 
+#ifdef __linux__
+
+#include <malloc.h>
+#include <stdlib.h>
+
+namespace js {
+
+// This function calls all the vanilla heap allocation functions.  It is never
+// called, and exists purely to help config/check_vanilla_allocations.py.  See
+// that script for more details.
+extern void
+AllTheNonBasicVanillaNewAllocations()
+{
+    // posix_memalign and aligned_alloc aren't available on all Linux
+    // configurations.
+    //char *q;
+    //posix_memalign((void**)&q, 16, 16);
+
+    intptr_t p =
+        intptr_t(malloc(16)) +
+        intptr_t(calloc(1, 16)) +
+        intptr_t(realloc(nullptr, 16)) +
+        intptr_t(new char) +
+        intptr_t(new char) +
+        intptr_t(new char) +
+        intptr_t(new char[16]) +
+        intptr_t(memalign(16, 16)) +
+        //intptr_t(q) +
+        //intptr_t(aligned_alloc(16, 16)) +
+        intptr_t(valloc(4096)) +
+        intptr_t(strdup("dummy"));
+
+    printf("%u\n", uint32_t(p));  // make sure |p| is not optimized away
+
+    free((int*)p);      // this would crash if ever actually called
+
+    MOZ_CRASH();
+}
+
+} // namespace js
+
+#endif // __linux__
+
 #ifdef JS_BASIC_STATS
 
 #include <math.h>
 
 /*
  * Histogram bins count occurrences of values <= the bin label, as follows:
  *
  *   linear:  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10 or more