Bug 1485533 - Close gaps in taskcluster SSL testing. r=mt NSS_3_46_BETA2
authorKevin Jacobs <kjacobs@mozilla.com>
Tue, 27 Aug 2019 14:45:43 +0000
changeset 15273 24b0fc7002039af07f6976d46ee7e186970c5cdb
parent 15272 7f146eb7adacabb83dc7f77083776b28956c5528
child 15274 29cd579e74e4a36608d86ae4b05123a8864ad27f
push id3482
push userjjones@mozilla.com
push dateTue, 27 Aug 2019 14:51:23 +0000
reviewersmt
bugs1485533
Bug 1485533 - Close gaps in taskcluster SSL testing. r=mt This patch increases SSL testing on taskcluster, specifically, running an additional 395 tests on each SSL cycle (more for FIPS targets), and adding a new 'stress' cycle. Notable changes: 1) This patch removes SSL stress tests from the default `NSS_SSL_RUN` list in all.sh and ssl.sh. If stress tests are needed, this variable must be set to include. 2) The "normal_normal" case is added to `NSS_SSL_TESTS` for all targets. FIPS targets also run "normal_fips", "fips_normal", and "fips_fips". 3) `--enable-libpkix` is now set for all taskcluster "build.sh" builds in order to support a number of OCSP tests that were previously not run. Differential Revision: https://phabricator.services.mozilla.com/D43283
automation/taskcluster/graph/src/extend.js
automation/taskcluster/scripts/build_gyp.sh
automation/taskcluster/windows/build_gyp.sh
fuzz/fuzz.gyp
gtests/pk11_gtest/pk11_gtest.gyp
gtests/softoken_gtest/softoken_gtest.gyp
tests/all.sh
tests/ssl/ssl.sh
--- a/automation/taskcluster/graph/src/extend.js
+++ b/automation/taskcluster/graph/src/extend.js
@@ -116,22 +116,32 @@ queue.filter(task => {
 queue.map(task => {
   if (task.collection == "asan") {
     // CRMF and FIPS tests still leak, unfortunately.
     if (task.tests == "crmf") {
       task.env.ASAN_OPTIONS = "detect_leaks=0";
     }
   }
 
-  // We don't run FIPS SSL tests
   if (task.tests == "ssl") {
     if (!task.env) {
       task.env = {};
     }
-    task.env.NSS_SSL_TESTS = "crl iopr policy";
+
+    // Stress tests to not include other SSL tests
+    if (task.symbol == "stress") {
+      task.env.NSS_SSL_TESTS = "normal_normal";
+    } else {
+      task.env.NSS_SSL_TESTS = "crl iopr policy normal_normal";
+    }
+
+    // FIPS runs
+    if (task.collection == "fips") {
+      task.env.NSS_SSL_TESTS += " fips_fips fips_normal normal_fips";
+    }
 
     if (task.platform == "mac") {
       task.maxRunTime = 7200;
     }
   }
 
   // Windows is slow.
   if ((task.platform == "windows2012-32" || task.platform == "windows2012-64") &&
@@ -569,17 +579,17 @@ async function scheduleFuzzing() {
   };
 
   // Build base definition.
   let build_base = merge(base, {
     command: [
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz"
+      "nss/automation/taskcluster/scripts/build_gyp.sh --fuzz"
     ],
     artifacts: {
       public: {
         expires: 24 * 7,
         type: "directory",
         path: "/home/worker/artifacts"
       }
     },
@@ -596,17 +606,17 @@ async function scheduleFuzzing() {
   let task_build_tls = queue.scheduleTask(merge(build_base, {
     name: "Linux x64 (debug, TLS fuzz)",
     symbol: "B",
     group: "TLS",
     command: [
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls"
+      "nss/automation/taskcluster/scripts/build_gyp.sh --fuzz=tls"
     ],
   }));
 
   // Schedule tests.
   queue.scheduleTask(merge(base, {
     parent: task_build_tls,
     name: "Gtests",
     command: [
@@ -674,17 +684,17 @@ async function scheduleFuzzing32() {
   };
 
   // Build base definition.
   let build_base = merge(base, {
     command: [
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz -t ia32"
+      "nss/automation/taskcluster/scripts/build_gyp.sh --fuzz -t ia32"
     ],
     artifacts: {
       public: {
         expires: 24 * 7,
         type: "directory",
         path: "/home/worker/artifacts"
       }
     },
@@ -701,17 +711,17 @@ async function scheduleFuzzing32() {
   let task_build_tls = queue.scheduleTask(merge(build_base, {
     name: "Linux 32 (debug, TLS fuzz)",
     symbol: "B",
     group: "TLS",
     command: [
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls -t ia32"
+      "nss/automation/taskcluster/scripts/build_gyp.sh --fuzz=tls -t ia32"
     ],
   }));
 
   // Schedule tests.
   queue.scheduleTask(merge(base, {
     parent: task_build_tls,
     name: "Gtests",
     command: [
@@ -953,16 +963,20 @@ function scheduleTests(task_build, task_
     name: "SSL tests (pkix)", symbol: "pkix", cycle: "pkix"
   }));
   queue.scheduleTask(merge(ssl_base, {
     name: "SSL tests (sharedb)", symbol: "sharedb", cycle: "sharedb"
   }));
   queue.scheduleTask(merge(ssl_base, {
     name: "SSL tests (upgradedb)", symbol: "upgradedb", cycle: "upgradedb"
   }));
+  queue.scheduleTask(merge(ssl_base, {
+    name: "SSL tests (stress)", symbol: "stress", cycle: "sharedb",
+    env: {NSS_SSL_RUN: "stress"}
+  }));
 }
 
 /*****************************************************************************/
 
 async function scheduleTools() {
   let base = {
     platform: "nss-tools",
     kind: "test"
--- a/automation/taskcluster/scripts/build_gyp.sh
+++ b/automation/taskcluster/scripts/build_gyp.sh
@@ -1,17 +1,17 @@
 #!/usr/bin/env bash
 
 source $(dirname "$0")/tools.sh
 
 # Clone NSPR if needed.
 hg_clone https://hg.mozilla.org/projects/nspr ./nspr default
 
 # Build.
-nss/build.sh -g -v "$@"
+nss/build.sh -g -v --enable-libpkix "$@"
 
 # Package.
 if [[ $(uname) = "Darwin" ]]; then
   mkdir -p public
   tar cvfjh public/dist.tar.bz2 dist
 else
   mkdir artifacts
   tar cvfjh artifacts/dist.tar.bz2 dist
--- a/automation/taskcluster/windows/build_gyp.sh
+++ b/automation/taskcluster/windows/build_gyp.sh
@@ -27,12 +27,12 @@ export GYP_MSVS_VERSION=2015
 popd
 
 export PATH="${PATH}:${PWD}/ninja/bin:${PWD}/gyp/test-env/Scripts"
 
 # Clone NSPR.
 hg_clone https://hg.mozilla.org/projects/nspr nspr default
 
 # Build with gyp.
-./nss/build.sh -g -v "$@"
+./nss/build.sh -g -v --enable-libpkix "$@"
 
 # Package.
 7z a public/build/dist.7z dist
--- a/fuzz/fuzz.gyp
+++ b/fuzz/fuzz.gyp
@@ -38,16 +38,17 @@
         '<(DEPTH)/lib/base/base.gyp:nssb',
         '<(DEPTH)/lib/dev/dev.gyp:nssdev',
         '<(DEPTH)/lib/pki/pki.gyp:nsspki',
         '<(DEPTH)/lib/util/util.gyp:nssutil',
         '<(DEPTH)/lib/nss/nss.gyp:nss_static',
         '<(DEPTH)/lib/pkcs7/pkcs7.gyp:pkcs7',
         # This is a static build of pk11wrap, softoken, and freebl.
         '<(DEPTH)/lib/pk11wrap/pk11wrap.gyp:pk11wrap_static',
+        '<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
       ],
       'cflags_cc': [
         '-Wno-error=shadow',
       ],
       'conditions': [
         ['fuzz_oss==0', {
           'sources': [
             '<!@(ls <(DEPTH)/fuzz/libFuzzer/*.cpp)',
--- a/gtests/pk11_gtest/pk11_gtest.gyp
+++ b/gtests/pk11_gtest/pk11_gtest.gyp
@@ -42,16 +42,17 @@
             '<(DEPTH)/lib/certdb/certdb.gyp:certdb',
             '<(DEPTH)/lib/certhigh/certhigh.gyp:certhi',
             '<(DEPTH)/lib/cryptohi/cryptohi.gyp:cryptohi',
             '<(DEPTH)/lib/dev/dev.gyp:nssdev',
             '<(DEPTH)/lib/nss/nss.gyp:nss_static',
             '<(DEPTH)/lib/pk11wrap/pk11wrap.gyp:pk11wrap_static',
             '<(DEPTH)/lib/pki/pki.gyp:nsspki',
             '<(DEPTH)/lib/ssl/ssl.gyp:ssl',
+            '<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
           ],
         }, {
           'dependencies': [
             '<(DEPTH)/lib/nss/nss.gyp:nss3',
             '<(DEPTH)/lib/ssl/ssl.gyp:ssl3',
           ],
         }],
       ],
--- a/gtests/softoken_gtest/softoken_gtest.gyp
+++ b/gtests/softoken_gtest/softoken_gtest.gyp
@@ -25,16 +25,17 @@
             '<(DEPTH)/lib/pk11wrap/pk11wrap.gyp:pk11wrap_static',
             '<(DEPTH)/lib/cryptohi/cryptohi.gyp:cryptohi',
             '<(DEPTH)/lib/certhigh/certhigh.gyp:certhi',
             '<(DEPTH)/lib/certdb/certdb.gyp:certdb',
             '<(DEPTH)/lib/base/base.gyp:nssb',
             '<(DEPTH)/lib/dev/dev.gyp:nssdev',
             '<(DEPTH)/lib/pki/pki.gyp:nsspki',
             '<(DEPTH)/lib/ssl/ssl.gyp:ssl',
+            '<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
           ],
         }, {
           'dependencies': [
             '<(DEPTH)/lib/nss/nss.gyp:nss3',
             '<(DEPTH)/lib/ssl/ssl.gyp:ssl3',
           ],
         }],
       ],
--- a/tests/all.sh
+++ b/tests/all.sh
@@ -81,17 +81,17 @@
 #
 # Testing schema:
 # ---------------
 #                           all.sh                       ~  (main)
 #                              |                               |
 #          +------------+------------+-----------+       ~  run_cycles
 #          |            |            |           |             |
 #      standard       pkix       upgradedb     sharedb   ~  run_cycle_*
-#                       |                                      |
+#         ...           |           ...         ...            |
 #                +------+------+------+----->            ~  run_tests
 #                |      |      |      |                        |
 #              cert   tools   fips   ssl   ...           ~  . *.sh
 #
 # Special strings:
 # ----------------
 #   FIXME ... known problems, search for this string
 #   NOTE .... unexpected behavior
@@ -142,19 +142,16 @@ run_cycle_standard()
     TEST_MODE=STANDARD
 
     TESTS="${ALL_TESTS}"
     TESTS_SKIP="cipher libpkix sdr ocsp pkits"
 
     NSS_DEFAULT_DB_TYPE="dbm"
     export NSS_DEFAULT_DB_TYPE
 
-    NSS_SSL_TESTS=`echo "${NSS_SSL_TESTS}" | sed -e "s/normal//g" -e "s/fips//g" -e "s/_//g"`
-    NSS_SSL_RUN=`echo "${NSS_SSL_RUN}" | sed -e "s/cov//g" -e "s/auth//g"`
-
     run_tests
 }
 
 ############################ run_cycle_pkix ############################
 # run test suites with PKIX enabled
 ########################################################################
 run_cycle_pkix()
 {
@@ -169,17 +166,16 @@ run_cycle_pkix()
     init_directories
 
     NSS_ENABLE_PKIX_VERIFY="1"
     export NSS_ENABLE_PKIX_VERIFY
 
     TESTS="${ALL_TESTS}"
     TESTS_SKIP="cipher dbtests sdr crmf smime merge multinit"
 
-    NSS_SSL_TESTS=`echo "${NSS_SSL_TESTS}" | sed -e "s/normal//g" -e "s/fips//g" -e "s/_//g"`
     export -n NSS_SSL_RUN
 
     # use the default format. (unset for the shell, export -n for binaries)
     export -n NSS_DEFAULT_DB_TYPE
     unset NSS_DEFAULT_DB_TYPE
 
     run_tests
 }
@@ -217,19 +213,16 @@ run_cycle_upgrade_db()
 
     NSS_DEFAULT_DB_TYPE="sql"
     export NSS_DEFAULT_DB_TYPE
 
     # run the subset of tests with the upgraded database
     TESTS="${ALL_TESTS}"
     TESTS_SKIP="cipher libpkix cert dbtests sdr ocsp pkits chains"
 
-    NSS_SSL_TESTS=`echo "${NSS_SSL_TESTS}" | sed -e "s/normal//g" -e "s/fips//g" -e "s/_//g"`
-    NSS_SSL_RUN=`echo "${NSS_SSL_RUN}" | sed -e "s/cov//g" -e "s/auth//g"`
-
     run_tests
 }
 
 ########################## run_cycle_shared_db #########################
 # run test suites with certificate databases set to shareable format
 ########################################################################
 run_cycle_shared_db()
 {
@@ -313,17 +306,18 @@ TESTS=${NSS_TESTS:-$tests}
 ALL_TESTS=${TESTS}
 
 nss_ssl_tests="crl iopr policy normal_normal"
 if [ $NO_INIT_SUPPORT -eq 0 ]; then
     nss_ssl_tests="$nss_ssl_tests fips_normal normal_fips"
 fi
 NSS_SSL_TESTS="${NSS_SSL_TESTS:-$nss_ssl_tests}"
 
-nss_ssl_run="cov auth stapling signed_cert_timestamps stress scheme"
+# NOTE: 'stress' run is omitted by default
+nss_ssl_run="cov auth stapling signed_cert_timestamps scheme"
 NSS_SSL_RUN="${NSS_SSL_RUN:-$nss_ssl_run}"
 
 # NOTE:
 # Lists of enabled tests and other settings are stored to ${ENV_BACKUP}
 # file and are are restored after every test cycle.
 
 ENV_BACKUP=${HOSTDIR}/env.sh
 env_backup > ${ENV_BACKUP}
--- a/tests/ssl/ssl.sh
+++ b/tests/ssl/ssl.sh
@@ -5,17 +5,37 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 ########################################################################
 #
 # mozilla/security/nss/tests/ssl/ssl.sh
 #
 # Script to test NSS SSL
 #
-# needs to work on all Unix and Windows platforms
+# Needs to work on all Unix and Windows platforms
+#
+# Testing schema:
+# ---------------
+#                           all.sh                       ~  (main)
+#                              |                               |
+#          +------------+------------+-----------+       ~  run_cycles
+#          |            |            |           |             |
+#      standard       pkix       upgradedb     sharedb   ~  run_cycle_*
+#         ...           |           ...         ...            |
+#                +------+------+----->                   ~  run_tests
+#                |      |      |                               |
+#               ...    ssl    ...                        ~   ssl.sh
+#                       |                                      |
+#          +-------+-------+-----------------+           ~  ssl_run_tests
+#          |       |       |                 |                 |
+#         crl     iopr   policy    permute(normal,fips)  ~  ssl_run_test_*
+#                                         | | | |              |
+#         +------+------+------+------+---+-+-+-+---->   ~  ssl_run
+#         |      |      |      |      |      |                 |
+#    stapling   cov   auth  stress  dtls    ...          ~  ssl_run_*
 #
 # special strings
 # ---------------
 #   FIXME ... known problems, search for this string
 #   NOTE .... unexpected behavior
 #
 ########################################################################
 
@@ -59,17 +79,17 @@ ssl_init()
   PORT=${PORT-8443}
   # Avoid port conflicts when multiple tests are running on the same machine.
   if [ -n "$NSS_TASKCLUSTER_MAC" ]; then
     cwd=$(cd $(dirname $0); pwd -P)
     padd=$(echo $cwd | cut -d "/" -f4 | sed 's/[^0-9]//g')
     PORT=$(($PORT + $padd))
   fi
   NSS_SSL_TESTS=${NSS_SSL_TESTS:-normal_normal}
-  nss_ssl_run="stapling signed_cert_timestamps cov auth stress dtls scheme"
+  nss_ssl_run="stapling signed_cert_timestamps cov auth dtls scheme"
   NSS_SSL_RUN=${NSS_SSL_RUN:-$nss_ssl_run}
 
   # Test case files
   SSLCOV=${QADIR}/ssl/sslcov.txt
   SSLAUTH=${QADIR}/ssl/sslauth.txt
   SSLSTRESS=${QADIR}/ssl/sslstress.txt
   SSLPOLICY=${QADIR}/ssl/sslpolicy.txt
   REQUEST_FILE=${QADIR}/ssl/sslreq.dat
@@ -516,20 +536,20 @@ ssl_stapling_stress()
     SERVER_OPTIONS="${SERVER_OPTIONS} ${SO}"
 
     SAVE_P_R_SERVERDIR=${P_R_SERVERDIR}
     P_R_SERVERDIR=${P_R_SERVERDIR}/../stapling/
 
     echo "${testname}"
     start_selfserv
 
-    echo "strsclnt -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \\"
+    echo "strsclnt -4 -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \\"
     echo "         -c 1000 -V ssl3:tls1.2 -N -T $verbose ${HOSTADDR}"
     echo "strsclnt started at `date`"
-    ${PROFTOOL} ${BINDIR}/strsclnt -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \
+    ${PROFTOOL} ${BINDIR}/strsclnt -4 -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \
             -c 1000 -V ssl3:tls1.2 -N -T $verbose ${HOSTADDR}
     ret=$?
 
     echo "strsclnt completed at `date`"
     html_msg $ret $value \
             "${testname}" \
             "produced a returncode of $ret, expected is $value."
     kill_selfserv
@@ -646,20 +666,20 @@ ssl_stress()
           fi
 
           if [ "${NOLOGIN}" -eq 0 ] ; then
               dbdir=${P_R_NOLOGINDIR}
           else
               dbdir=${P_R_CLIENTDIR}
           fi
 
-          echo "strsclnt -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \\"
+          echo "strsclnt -4 -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \\"
           echo "         -V ssl3:tls1.2 $verbose ${HOSTADDR}"
           echo "strsclnt started at `date`"
-          ${PROFTOOL} ${BINDIR}/strsclnt -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \
+          ${PROFTOOL} ${BINDIR}/strsclnt -4 -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \
                    -V ssl3:tls1.2 $verbose ${HOSTADDR}
           ret=$?
           echo "strsclnt completed at `date`"
           html_msg $ret $value \
                    "${testname}" \
                    "produced a returncode of $ret, expected is $value. "
           if [ "`uname -n`" = "sjsu" ] ; then
               echo "debugging disapering selfserv... ps -ef | grep selfserv"
@@ -1270,19 +1290,19 @@ ssl_scheme_stress()
     schemes=("rsa_pkcs1_sha256" "rsa_pss_rsae_sha256" "rsa_pkcs1_sha256,rsa_pss_rsae_sha256")
     for sscheme in "${schemes[@]}"; do
         for cscheme in "${schemes[@]}"; do
             testname="ssl_scheme server='$sscheme' client='$cscheme'"
             echo "${testname}"
 
             start_selfserv -V tls1.2:tls1.2 -J "$sscheme"
 
-            echo "strsclnt -q -p ${PORT} -d ${P_R_CLIENTDIR} $verbose ${CLIENT_OPTIONS} \\"
+            echo "strsclnt -4 -q -p ${PORT} -d ${P_R_CLIENTDIR} $verbose ${CLIENT_OPTIONS} \\"
             echo "         -V tls1.2:tls1.2 -J "$cscheme" ${HOSTADDR} < ${REQUEST_FILE}"
-            ${PROFTOOL} ${BINDIR}/strsclnt -q -p ${PORT} ${CLIENT_OPTIONS} \
+            ${PROFTOOL} ${BINDIR}/strsclnt -4 -q -p ${PORT} ${CLIENT_OPTIONS} \
                         -d ${P_R_CLIENTDIR} $verbose -V tls1.2:tls1.2 -J "$cscheme" ${HOSTADDR} < ${REQUEST_FILE} 2>&1
             ret=$?
             # If both schemes include just one option and those options don't
             # match, then the test should fail; otherwise, assume that it works.
             if [ "${cscheme#*,}" = "$cscheme" -a \
                  "${sscheme#*,}" = "$sscheme" -a \
                  "$cscheme" != "$sscheme" ]; then
                 expected=1