Bug 1305970 - land NSS 0x457e3b2beeb9, r=me
authorFranziskus Kiefer <franziskuskiefer@gmail.com>
Mon, 07 Nov 2016 06:01:08 +0100
changeset 321373 d370d74e76d7783f664929eb6f32be9685e6ed41
parent 321372 4b1de4471e4b0f472c0b5c523b3f64cbd15ff30c
child 321374 b2b39f69581735e25f53b69ec51263f80a1a682a
push id30926
push userkwierso@gmail.com
push dateMon, 07 Nov 2016 22:02:19 +0000
treeherdermozilla-central@86f702229e32 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersme
bugs1305970
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1305970 - land NSS 0x457e3b2beeb9, r=me
security/nss/Makefile
security/nss/TAG-INFO
security/nss/automation/taskcluster/graph/src/extend.js
security/nss/automation/taskcluster/scripts/run_tests.sh
security/nss/build.sh
security/nss/coreconf/config.gypi
security/nss/coreconf/coreconf.dep
security/nss/coreconf/pkg_config.py
security/nss/gtests/ssl_gtest/manifest.mn
security/nss/gtests/ssl_gtest/ssl_fuzz_unittest.cc
security/nss/gtests/ssl_gtest/ssl_gtest.gyp
security/nss/lib/ssl/ssl.gyp
security/nss/lib/ssl/sslnonce.c
security/nss/readme.md
security/nss/tests/run.sh
security/nss/tests/ssl_gtests/ssl_gtests.sh
--- a/security/nss/Makefile
+++ b/security/nss/Makefile
@@ -121,31 +121,31 @@ USEABSPATH="NO"
 endif
 endif
 ifeq ($(USEABSPATH),"YES")
 NSPR_PREFIX = $(shell pwd)/../dist/$(OBJDIR_NAME)
 else
 NSPR_PREFIX = $$(topsrcdir)/../dist/$(OBJDIR_NAME)
 endif
 
-ifndef NSS_GYP
+ifndef NSS_GYP_PREFIX
 $(NSPR_CONFIG_STATUS): $(NSPR_CONFIGURE)
 	mkdir -p $(CORE_DEPTH)/../nspr/$(OBJDIR_NAME)
 	cd $(CORE_DEPTH)/../nspr/$(OBJDIR_NAME) ; \
 	$(NSPR_CONFIGURE_ENV) sh ../configure \
 	$(NSPR_CONFIGURE_OPTS) \
 	--with-dist-prefix='$(NSPR_PREFIX)' \
 	--with-dist-includedir='$(NSPR_PREFIX)/include'
 else
 $(NSPR_CONFIG_STATUS): $(NSPR_CONFIGURE)
 	mkdir -p $(CORE_DEPTH)/../nspr/$(OBJDIR_NAME)
 	cd $(CORE_DEPTH)/../nspr/$(OBJDIR_NAME) ; \
 	$(NSPR_CONFIGURE_ENV) sh ../configure \
 	$(NSPR_CONFIGURE_OPTS) \
-	--prefix='$(NSPR_PREFIX)'
+	--prefix='$(NSS_GYP_PREFIX)'
 endif
 
 build_nspr: $(NSPR_CONFIG_STATUS)
 	$(MAKE) -C $(CORE_DEPTH)/../nspr/$(OBJDIR_NAME)
 
 install_nspr: build_nspr
 	$(MAKE) -C $(CORE_DEPTH)/../nspr/$(OBJDIR_NAME) install
 
--- a/security/nss/TAG-INFO
+++ b/security/nss/TAG-INFO
@@ -1,1 +1,1 @@
-5eb5f52b7922
+457e3b2beeb9
--- a/security/nss/automation/taskcluster/graph/src/extend.js
+++ b/security/nss/automation/taskcluster/graph/src/extend.js
@@ -99,17 +99,16 @@ export default async function main() {
   });
 
   await scheduleLinux("Linux 64 (debug, gyp)", {
     command: [
       "/bin/bash",
       "-c",
       "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh"
     ],
-    env: {USE_64: "1"}, // This is only necessary for tests to work.
     platform: "linux64",
     collection: "gyp",
     image: LINUX_IMAGE
   });
 
   await scheduleLinux("Linux 64 (ASan, debug)", {
     env: {
       UBSAN_OPTIONS: "print_stacktrace=1",
@@ -235,18 +234,17 @@ async function scheduleLinux(name, base)
 /*****************************************************************************/
 
 async function scheduleFuzzing() {
   let base = {
     env: {
       ASAN_OPTIONS: "allocator_may_return_null=1",
       UBSAN_OPTIONS: "print_stacktrace=1",
       CC: "clang",
-      CCC: "clang++",
-      USE_64: "1" // This is only necessary for tests to work.
+      CCC: "clang++"
     },
     platform: "linux64",
     collection: "fuzz",
     image: LINUX_IMAGE
   };
 
   // Build base definition.
   let build_base = merge({
@@ -277,19 +275,19 @@ async function scheduleFuzzing() {
     parent: task_build,
     name: "Gtests",
     command: [
       "/bin/bash",
       "-c",
       "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
     ],
     env: {GTESTFILTER: "*Fuzz*"},
+    tests: "ssl_gtests gtests",
+    cycle: "standard",
     symbol: "Gtest",
-    tests: "gtests",
-    cycle: "standard",
     kind: "test"
   }));
 
   return queue.submit();
 }
 
 /*****************************************************************************/
 
--- a/security/nss/automation/taskcluster/scripts/run_tests.sh
+++ b/security/nss/automation/taskcluster/scripts/run_tests.sh
@@ -9,9 +9,9 @@ if [ $(id -u) = 0 ]; then
     # Drop privileges by re-running this script.
     exec su worker $0
 fi
 
 # Fetch artifact if needed.
 fetch_dist
 
 # Run tests.
-cd nss/tests && ./all.sh
+cd nss/tests && ./run.sh
--- a/security/nss/build.sh
+++ b/security/nss/build.sh
@@ -68,58 +68,68 @@ while [ $# -gt 0 ]; do
         --scan-build=?*) scanbuild=(scan-build -o "${1#*=}") ;;
         --opt|-o) opt_build=1; nspr_env+=(BUILD_OPT=1) ;;
         -m32|--m32) build_64=0 ;;
         *) show_help; exit ;;
     esac
     shift
 done
 
-# set paths
-cwd=$(cd $(dirname $0); pwd -P)
-obj_dir=$(USE_64=$build_64 make -s -C "$cwd" platform)
-dist_dir="$cwd/../dist/$obj_dir"
-
-# -c = clean first
-if [ "$clean" = 1 ]; then
-    rm -rf "$cwd/out"
-    rm -rf "$cwd/../nspr/$obj_dir"
-fi
-
 if [ "$opt_build" = "1" ]; then
     target=Release
 else
     target=Debug
 fi
 if [ "$build_64" == "1" ]; then
     target="${target}_x64"
     nspr_env+=(USE_64=1)
 else
     gyp_params+=(-Dtarget_arch=ia32)
 fi
+
+# set paths
+cwd=$(cd $(dirname $0); pwd -P)
+dist_dir="$cwd/../dist"
 target_dir="$cwd/out/$target"
 
+# get the realpath of $dist_dir
+dist_dir=$(mkdir -p $dist_dir; cd $dist_dir; pwd -P)
+
+# save the chosen target
+echo $target > $dist_dir/latest
+
+# get object directory
+obj_dir="$dist_dir/$target"
+gyp_params+=(-Dnss_dist_dir=$dist_dir)
+gyp_params+=(-Dnss_dist_obj_dir=$obj_dir)
+
+# -c = clean first
+if [ "$clean" = 1 ]; then
+    rm -rf "$cwd/out"
+    rm -rf "$cwd/../nspr/$target"
+fi
+
 # figure out the scan-build string
 if [ "${#scanbuild[@]}" -gt 0 ]; then
     if [ -n "$CC" ]; then
        scanbuild+=(--use-cc="$CC")
     fi
     if [ -n "$CCC" ]; then
        scanbuild+=(--use-c++="$CCC")
     fi
  fi
 
 # These steps can take a while, so don't overdo them.
 # Force a redo with -g.
 if [ "$rebuild_gyp" = 1 -o ! -d "$target_dir" ]; then
     # Build NSPR.
-    make "${nspr_env[@]}" -C "$cwd" NSS_GYP=1 install_nspr
+    make "${nspr_env[@]}" -C "$cwd" NSS_GYP_PREFIX=$obj_dir install_nspr
 
     # Run gyp.
-    PKG_CONFIG_PATH="$cwd/../nspr/$obj_dir/config" \
+    PKG_CONFIG_PATH="$obj_dir/lib/pkgconfig" \
         "${scanbuild[@]}" gyp -f ninja "${gyp_params[@]}" --depth="$cwd" \
           --generator-output="." "$cwd/nss.gyp"
 fi
 
 # Run ninja.
 if which ninja >/dev/null 2>&1; then
     ninja=(ninja)
 elif which ninja-build >/dev/null 2>&1; then
--- a/security/nss/coreconf/config.gypi
+++ b/security/nss/coreconf/config.gypi
@@ -25,29 +25,25 @@
           'target_arch%': '<(host_arch)',
         }],
         ['OS=="win"', {
           'use_system_zlib%': 0,
           'nspr_libs%': ['nspr4.lib', 'plc4.lib', 'plds4.lib'],
           #XXX: gyp breaks if these are empty!
           'nspr_lib_dir%': ' ',
           'nspr_include_dir%': ' ',
-          'nss_dist_obj_dir%': ' ',
-          'nss_dist_dir%': ' ',
           'zlib_libs%': [],
           #TODO
           'moz_debug_flags%': '',
           'dll_prefix': '',
           'dll_suffix': 'dll',
         }, {
           'nspr_libs%': ['-lplds4', '-lplc4', '-lnspr4'],
           'nspr_lib_dir%': '<!(<(python) <(DEPTH)/coreconf/pkg_config.py . --libs nspr)',
           'nspr_include_dir%': '<!(<(python) <(DEPTH)/coreconf/pkg_config.py . --cflags nspr)',
-          'nss_dist_obj_dir%': '<!(<(python) <(DEPTH)/coreconf/pkg_config.py ../.. --cflags nspr)',
-          'nss_dist_dir%': '<!(<(python) <(DEPTH)/coreconf/pkg_config.py ../../.. --cflags nspr)',
           'use_system_zlib%': 1,
         }],
         ['OS=="linux" or OS=="android"', {
           'zlib_libs%': ['<!@(<(python) <(DEPTH)/coreconf/pkg_config.py raw --libs zlib)'],
           'moz_debug_flags%': '-gdwarf-2',
           'optimize_flags%': '-O2',
           'dll_prefix': 'lib',
           'dll_suffix': 'so',
@@ -79,18 +75,16 @@
     'host_arch%': '<(host_arch)',
     'target_arch%': '<(target_arch)',
     'use_system_zlib%': '<(use_system_zlib)',
     'zlib_libs%': ['<@(zlib_libs)'],
     'moz_debug_flags%': '<(moz_debug_flags)',
     'nspr_libs%': ['<@(nspr_libs)'],
     'nspr_lib_dir%': '<(nspr_lib_dir)',
     'nspr_include_dir%': '<(nspr_include_dir)',
-    'nss_dist_obj_dir%': '<(nss_dist_obj_dir)',
-    'nss_dist_dir%': '<(nss_dist_dir)',
     'use_system_sqlite%': '<(use_system_sqlite)',
     'sqlite_libs%': ['-lsqlite3'],
     'dll_prefix': '<(dll_prefix)',
     'dll_suffix': '<(dll_suffix)',
     'freebl_name': '<(freebl_name)',
     'cc_is_clang%': '<(cc_is_clang)',
     # Some defaults
     'disable_tests%': 0,
--- a/security/nss/coreconf/coreconf.dep
+++ b/security/nss/coreconf/coreconf.dep
@@ -5,8 +5,9 @@
 
 /*
  * A dummy header file that is a dependency for all the object files.
  * Used to force a full recompilation of NSS in Mozilla's Tinderbox
  * depend builds.  See comments in rules.mk.
  */
 
 #error "Do not include this header file."
+
--- a/security/nss/coreconf/pkg_config.py
+++ b/security/nss/coreconf/pkg_config.py
@@ -12,17 +12,17 @@ def main():
     is_raw = sys.argv[1] == 'raw'
     stdout = None if is_raw else subprocess.PIPE
 
     if sys.argv[2] == '--libs':
         part_prefix = '-L'
     elif sys.argv[2] == '--cflags':
         part_prefix = '-I'
     else:
-        raise 'Specify either --libs or --cflags as the second argument.'
+        raise Exception('Specify either --libs or --cflags as the second argument.')
 
     try:
         process = subprocess.Popen(['pkg-config'] + sys.argv[2:], stdout=stdout, stderr=open(os.devnull, 'wb'))
 
     except OSError:
         # pkg-config is probably not installed
         return
 
--- a/security/nss/gtests/ssl_gtest/manifest.mn
+++ b/security/nss/gtests/ssl_gtest/manifest.mn
@@ -18,16 +18,17 @@ CPPSRCS = \
       ssl_cert_ext_unittest.cc \
       ssl_ciphersuite_unittest.cc \
       ssl_damage_unittest.cc \
       ssl_dhe_unittest.cc \
       ssl_drop_unittest.cc \
       ssl_ecdh_unittest.cc \
       ssl_ems_unittest.cc \
       ssl_extension_unittest.cc \
+      ssl_fuzz_unittest.cc \
       ssl_gtest.cc \
       ssl_hrr_unittest.cc \
       ssl_loopback_unittest.cc \
       ssl_record_unittest.cc \
       ssl_resumption_unittest.cc \
       ssl_skip_unittest.cc \
       ssl_staticrsa_unittest.cc \
       ssl_v2_client_hello_unittest.cc \
new file mode 100644
--- /dev/null
+++ b/security/nss/gtests/ssl_gtest/ssl_fuzz_unittest.cc
@@ -0,0 +1,24 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "ssl.h"
+#include "sslimpl.h"
+
+#include "gtest/gtest.h"
+
+namespace nss_test {
+
+#ifdef UNSAFE_FUZZER_MODE
+
+class TlsFuzzTest : public ::testing::Test {};
+
+// Ensure that ssl_Time() returns a constant value.
+TEST_F(TlsFuzzTest, Fuzz_SSL_Time_Constant) {
+  PRInt32 now = ssl_Time();
+  PR_Sleep(PR_SecondsToInterval(2));
+  EXPECT_EQ(ssl_Time(), now);
+}
+
+#endif
+}
--- a/security/nss/gtests/ssl_gtest/ssl_gtest.gyp
+++ b/security/nss/gtests/ssl_gtest/ssl_gtest.gyp
@@ -18,16 +18,17 @@
         'ssl_cert_ext_unittest.cc',
         'ssl_ciphersuite_unittest.cc',
         'ssl_damage_unittest.cc',
         'ssl_dhe_unittest.cc',
         'ssl_drop_unittest.cc',
         'ssl_ecdh_unittest.cc',
         'ssl_ems_unittest.cc',
         'ssl_extension_unittest.cc',
+        'ssl_fuzz_unittest.cc',
         'ssl_gtest.cc',
         'ssl_hrr_unittest.cc',
         'ssl_loopback_unittest.cc',
         'ssl_record_unittest.cc',
         'ssl_resumption_unittest.cc',
         'ssl_skip_unittest.cc',
         'ssl_staticrsa_unittest.cc',
         'ssl_v2_client_hello_unittest.cc',
--- a/security/nss/lib/ssl/ssl.gyp
+++ b/security/nss/lib/ssl/ssl.gyp
@@ -58,16 +58,21 @@
         [ 'ssl_enable_zlib==1', {
           'dependencies': [
             '<(DEPTH)/lib/zlib/zlib.gyp:nss_zlib'
           ],
           'defines': [
             'NSS_SSL_ENABLE_ZLIB',
           ],
         }],
+        [ 'fuzz==1', {
+          'defines': [
+            'UNSAFE_FUZZER_MODE',
+          ],
+        }],
       ],
       'dependencies': [
         '<(DEPTH)/exports.gyp:nss_exports',
         '<(DEPTH)/lib/freebl/freebl.gyp:freebl',
       ],
     },
     {
       'target_name': 'ssl3',
--- a/security/nss/lib/ssl/sslnonce.c
+++ b/security/nss/lib/ssl/sslnonce.c
@@ -435,16 +435,20 @@ SSL_ClearSessionCache(void)
         UncacheSID(cache);
     UNLOCK_CACHE;
 }
 
 /* returns an unsigned int containing the number of seconds in PR_Now() */
 PRUint32
 ssl_Time(void)
 {
+#ifdef UNSAFE_FUZZER_MODE
+    return 1234;
+#endif
+
     PRUint32 myTime;
 #if defined(XP_UNIX) || defined(XP_WIN) || defined(_WINDOWS) || defined(XP_BEOS)
     myTime = time(NULL); /* accurate until the year 2038. */
 #else
     /* portable, but possibly slower */
     PRTime now;
     PRInt64 ll;
 
new file mode 100644
--- /dev/null
+++ b/security/nss/readme.md
@@ -0,0 +1,99 @@
+# Network Security Services
+
+Network Security Services (NSS) is a set of libraries designed to support cross-platform development of security-enabled client and server applications. NSS supports SSL v3-TLS 1.2 (experimental TLS 1.3), PKCS #5, PKCS #7, PKCS #11, PKCS #12, S/MIME, X.509 v3 certificates, and other security standards.
+
+## Getting started
+In order to get started create a new directory on that you will be uses as your local work area, and check out NSS and NSPR. (Note that there's no git mirror of NSPR and you require mercurial to get the latest NSPR source.)
+
+    git clone https://github.com/nss-dev/nss.git
+    hg clone https://hg.mozilla.org/projects/nspr
+
+NSS can also be cloned with mercurial `
+    hg clone https://hg.mozilla.org/projects/nspr`
+
+## Building NSS
+*This build system is under development. It does not yet support all the features or platforms that NSS supports. To build on anything other than Mac or Linux please use the legacy build system as described below.*
+
+Build requirements:
+
+* [gyp](https://gyp.gsrc.io/)
+* [ninja](https://ninja-build.org/)
+
+After changing into the NSS directory a typical build is done as follows
+
+    ./build.sh
+
+Once the build is done the build output is found in the directory `../dist/*.OBJ`, where `*` will be a name dynamically derived from your system's architecture. Exported header files can be found in the `include` directory, library files in directory `lib`, and tools in directory `bin`. In order to run the tools, set your system environment to use the libraries of your build from the "lib" directory, e.g., using the `LD_LIBRARY_PATH` or `DYLD_LIBRARY_PATH`.
+
+---
+    Usage: build.sh [-hcgv] [-j <n>] [--test] [--fuzz] [--scan-build[=output]]
+                    [-m32] [--opt|-o]
+
+    -h            display this help and exit
+    -c            clean before build
+    -g            force a rebuild of gyp (and NSPR, because why not)
+    -j <n>        run at most <n> concurrent jobs
+    -v            verbose build
+    -m32          do a 32-bit build on a 64-bit system
+    --test        ignore map files and export everything we have
+    --fuzz        enable fuzzing mode. this always enables test builds
+    --scan-build  run the build with scan-build (scan-build has to be in the path)
+                  --scan-build=/out/path sets the output path for scan-build
+    --opt|-o      do an opt build
+
+## Building NSS (legacy build system)
+After changing into the NSS directory a typical build of 32-bit NSS is done as follows
+
+    make nss_build_all
+
+The following environment variables might be useful:
+* `BUILD_OPT=1` to get an optimised build
+* `USE_64=1` to get a 64-bit build (recommended)
+* `NSS_ENABLE_TLS_1_3=1` to enable TLS 1.3 support
+
+The complete list of environment variables can be found [here](https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/Reference/NSS_environment_variables).
+
+To clean the build directory run
+
+    make nss_clean_all
+
+## Tests
+### Setup
+Make sure that the address `$HOST.$DOMSUF` on your computer is available. This is necessary because NSS tests generate certificates and establish TLS connections, which requires a fully qualified domain name.
+You can test this by calling `ping $HOST.$DOMSUF`. If this is working, you're all set.
+If it's not, set or export
+
+    HOST=nss
+    DOMSUF=local
+Note that you might have to add `nss.local` to `/etc/hosts` if it's not there. The entry should look something like `127.0.0.1       nss.local   nss`.
+If you get name resolution errors, try to disable IPv6 on the loopback device, i.e. comment the lines starting with `::1` in your `/etc/hosts` .
+
+### Running tests
+*Runnning all tests will take a while!*
+
+    cd tests
+    ./all.sh
+Make sure that all environment variables set for the build are set while running the tests as well.
+Test results are published in the folder `../../test_results/`.
+Individual tests can be run with the `NSS_TESTS` environment variable, e.g. `NSS_TESTS=ssl_gtests ./all.sh` or by changing into the according directory and running the bash script there `cd ssl_gtests && ./ssl_gtests.sh`.  The following tests are available:
+
+    cipher lowhash libpkix cert dbtests tools fips sdr crmf smime ssl ocsp merge pkits chains ec gtests ssl_gtests bogo
+
+To make tests run faster it's recommended to set `NSS_CYCLES=standard` to run only the standard cycle.
+
+## Releases
+NSS releases can be found at [Mozilla's download server](https://ftp.mozilla.org/pub/security/nss/releases/). Because NSS depends on the base library NSPR you should download the archive that combines both NSS and NSPR.
+
+## Contributing
+[Bugzilla](https://bugzilla.mozilla.org/) is used to track NSS development and bugs. File new bugs in the NSS product.
+A list with good first bugs to start with are [listed here](https://bugzilla.mozilla.org/buglist.cgi?keywords=good-first-bug%2C%20&keywords_type=allwords&list_id=13238861&resolution=---&query_format=advanced&product=NSS).
+
+### NSS Folder Structure
+The nss directory contains the following important subdirectories:
+- `coreconf` contains the build logic.
+- `lib` contains all library code that is used to create the runtime libraries.
+- `cmd` contains a set of various tool programs that are built with NSS. Several tools are general purpose and can be used to inspect and manipulate the storage files that software using the NSS library creates and modifies. Other tools are only used for testing purposes.
+- `test` and `gtests` contain the NSS test suite. While `test` contains shell scripts to drive test programs in `cmd`, `gtests` holds a set of [gtests](https://github.com/google/googletest).
+
+A more comprehensible overview of the NSS folder structure and API guidelines can be found [here](https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/NSS_API_Guidelines).
+
new file mode 100755
--- /dev/null
+++ b/security/nss/tests/run.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+base=$(dirname $0)
+
+# Determine path to build target.
+obj_dir=$(cat $base/../../dist/latest 2>/dev/null)
+
+# Run tests.
+OBJDIR=$obj_dir $base/all.sh
--- a/security/nss/tests/ssl_gtests/ssl_gtests.sh
+++ b/security/nss/tests/ssl_gtests/ssl_gtests.sh
@@ -122,17 +122,18 @@ ssl_gtest_start()
   if [ ! -f ${BINDIR}/ssl_gtest ]; then
     html_unknown "Skipping ssl_gtest (not built)"
     return
   fi
 
   SSLGTESTREPORT="${SSLGTESTDIR}/report.xml"
   PARSED_REPORT="${SSLGTESTDIR}/report.parsed"
   echo "executing ssl_gtest"
-  ${BINDIR}/ssl_gtest -d "${SSLGTESTDIR}" --gtest_output=xml:"${SSLGTESTREPORT}"
+  ${BINDIR}/ssl_gtest -d "${SSLGTESTDIR}" --gtest_output=xml:"${SSLGTESTREPORT}" \
+                                          --gtest_filter="${GTESTFILTER-*}"
   html_msg $? 0 "ssl_gtest run successfully"
   echo "executing sed to parse the xml report"
   sed -f ${COMMON}/parsegtestreport.sed "${SSLGTESTREPORT}" > "${PARSED_REPORT}"
   echo "processing the parsed report"
   cat "${PARSED_REPORT}" | while read result name; do
     if [ "$result" = "notrun" ]; then
       echo "$name" SKIPPED
     elif [ "$result" = "run" ]; then