Bug 1720232 SQLite calls could timeout in starvation situations.
authorRobert Relyea <rrelyea@redhat.com>
Thu, 15 Jul 2021 14:23:55 -0700
changeset 15961 b54b0d41e51bc302c58721c08525f85fd5e8d0f9
parent 15960 d1b9709d8861b0946e2d117602fc293ecc53c010
child 15962 d2ec946e601afb2d2406c7b5ab9cb713dccb8ae7
push id3995
push userrrelyea@redhat.com
push dateTue, 20 Jul 2021 18:54:26 +0000
bugs1720232
Bug 1720232 SQLite calls could timeout in starvation situations. Some of our servers could cause random failures when trying to generate many key pairs from multiple threads. This is caused because some threads would starve long enough for them to give up on getting a begin transaction on sqlite. sqlite only allows one transaction at a time. Also, there were some bugs in error handling of the broken transaction case where NSS would try to cancel a transation after the begin failed (most cases were correct, but one case in particular was problematic). Differential Revision: https://phabricator.services.mozilla.com/D120032
cmd/manifest.mn
cmd/sdbthreadtst/Makefile
cmd/sdbthreadtst/manifest.mn
cmd/sdbthreadtst/sdbthreadtst.c
cmd/sdbthreadtst/sdbthreadtst.gyp
lib/softoken/sdb.c
lib/softoken/sftkdb.c
nss.gyp
tests/dbtests/dbtests.sh
--- a/cmd/manifest.mn
+++ b/cmd/manifest.mn
@@ -60,16 +60,17 @@ NSS_SRCDIRS = \
  pk11ectest \
  pk11gcmtest \
  pk11mode \
  pk1sign  \
  pp  \
  pwdecrypt \
  rsaperf \
  rsapoptst \
+ sdbthreadtst \
  sdrtest \
  selfserv  \
  signtool \
  signver \
  smimetools  \
  ssltap  \
  strsclnt \
  symkeyutil \
new file mode 100644
--- /dev/null
+++ b/cmd/sdbthreadtst/Makefile
@@ -0,0 +1,48 @@
+#! gmake
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#######################################################################
+# (1) Include initial platform-independent assignments (MANDATORY).   #
+#######################################################################
+
+include manifest.mn
+
+#######################################################################
+# (2) Include "global" configuration information. (OPTIONAL)          #
+#######################################################################
+
+include $(CORE_DEPTH)/coreconf/config.mk
+
+#######################################################################
+# (3) Include "component" configuration information. (OPTIONAL)       #
+#######################################################################
+
+#######################################################################
+# (4) Include "local" platform-dependent assignments (OPTIONAL).      #
+#######################################################################
+
+include ../platlibs.mk
+
+
+#######################################################################
+# (5) Execute "global" rules. (OPTIONAL)                              #
+#######################################################################
+
+include $(CORE_DEPTH)/coreconf/rules.mk
+
+#######################################################################
+# (6) Execute "component" rules. (OPTIONAL)                           #
+#######################################################################
+
+
+
+#######################################################################
+# (7) Execute "local" rules. (OPTIONAL).                              #
+#######################################################################
+
+
+include ../platrules.mk
+
new file mode 100644
--- /dev/null
+++ b/cmd/sdbthreadtst/manifest.mn
@@ -0,0 +1,22 @@
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+CORE_DEPTH = ../..
+
+DEFINES += -DNSPR20
+
+# MODULE public and private header  directories are implicitly REQUIRED.
+MODULE = nss
+
+CSRCS = \
+	sdbthreadtst.c \
+	$(NULL)
+
+# The MODULE is always implicitly required.
+# Listing it here in REQUIRES makes it appear twice in the cc command line.
+
+PROGRAM = sdbthreadtst
+
+# USE_STATIC_LIBS = 1
new file mode 100644
--- /dev/null
+++ b/cmd/sdbthreadtst/sdbthreadtst.c
@@ -0,0 +1,213 @@
+#if defined(XP_UNIX)
+#include <unistd.h>
+#endif
+#include <stdio.h>
+#include <nss.h>
+#include <prtypes.h>
+#include <prerr.h>
+#include <prerror.h>
+#include <prthread.h>
+#include <pk11pub.h>
+#include <keyhi.h>
+
+#define MAX_THREAD_COUNT 100
+
+/* globals */
+int THREAD_COUNT = 30;
+int FAILED = 0;
+int ERROR = 0;
+int LOOP_COUNT = 100;
+int KEY_SIZE = 3072;
+int STACK_SIZE = 0;
+int VERBOSE = 0;
+char *NSSDIR = ".";
+PRBool ISTOKEN = PR_TRUE;
+CK_MECHANISM_TYPE MECHANISM = CKM_RSA_PKCS_KEY_PAIR_GEN;
+
+void
+usage(char *prog, char *error)
+{
+    if (error) {
+        fprintf(stderr, "Bad Arguments: %s", error);
+    }
+    fprintf(stderr, "usage: %s [-l loop_count] [-t thread_count] "
+                    "[-k key_size] [-s stack_size] [-d nss_dir] [-e] [-v] [-h]\n",
+            prog);
+    fprintf(stderr, "    loop_count   -- "
+                    "number of keys to generate on each thread (default=%d)\n",
+            LOOP_COUNT);
+    fprintf(stderr, "    thread_count -- "
+                    "number of of concurrent threads to run (def=%d,max=%d)\n",
+            THREAD_COUNT, MAX_THREAD_COUNT);
+    fprintf(stderr, "    key_size     -- "
+                    "rsa key size in bits (default=%d)\n",
+            KEY_SIZE);
+    fprintf(stderr, "    stack_size     -- "
+                    "thread stack size in bytes, 0=optimal (default=%d)\n",
+            STACK_SIZE);
+    fprintf(stderr, "    nss_dir     -- "
+                    "location of the nss directory (default=%s)\n",
+            NSSDIR);
+    fprintf(stderr, "    -e use session keys rather than token keys\n");
+    fprintf(stderr, "    -v verbose, print progress indicators\n");
+    fprintf(stderr, "    -h print this message\n");
+    exit(2);
+}
+
+void
+create_key_loop(void *arg)
+{
+    int i;
+    PK11SlotInfo *slot = PK11_GetInternalKeySlot();
+    PK11RSAGenParams param;
+    int threadnumber = *(int *)arg;
+    int failures = 0;
+    int progress = 5;
+    PRIntervalTime epoch = PR_IntervalNow();
+    param.keySizeInBits = KEY_SIZE;
+    param.pe = 0x10001L;
+    printf(" - thread %d starting\n", threadnumber);
+    progress = 30 / THREAD_COUNT;
+    if (progress < 2)
+        progress = 2;
+    for (i = 0; i < LOOP_COUNT; i++) {
+        SECKEYPrivateKey *privKey;
+        SECKEYPublicKey *pubKey;
+        privKey = PK11_GenerateKeyPair(slot, MECHANISM, &param, &pubKey,
+                                       ISTOKEN, PR_TRUE, NULL);
+        if (privKey == NULL) {
+            fprintf(stderr,
+                    "keypair gen in thread %d failed %s\n", threadnumber,
+                    PORT_ErrorToString(PORT_GetError()));
+            FAILED++;
+            failures++;
+        }
+        if (VERBOSE && (i % progress) == 0) {
+            PRIntervalTime current = PR_IntervalNow();
+            PRIntervalTime interval = current - epoch;
+            int seconds = (interval / PR_TicksPerSecond());
+            int mseconds = ((interval * 1000) / PR_TicksPerSecond()) - (seconds * 1000);
+            epoch = current;
+            printf(" - thread %d @ %d iterations %d.%03d sec\n", threadnumber,
+                   i, seconds, mseconds);
+        }
+        if (ISTOKEN && privKey) {
+            SECKEY_DestroyPublicKey(pubKey);
+            SECKEY_DestroyPrivateKey(privKey);
+        }
+    }
+    PK11_FreeSlot(slot);
+    printf(" * thread %d ending with %d failures\n", threadnumber, failures);
+    return;
+}
+
+int
+main(int argc, char **argv)
+{
+    PRThread *thread[MAX_THREAD_COUNT];
+    int threadnumber[MAX_THREAD_COUNT];
+    int i;
+    PRStatus status;
+    SECStatus rv;
+    char *prog = *argv++;
+    char buf[2048];
+    char *arg;
+
+    while ((arg = *argv++) != NULL) {
+        if (*arg == '-') {
+            switch (arg[1]) {
+                case 'l':
+                    if (*argv == NULL)
+                        usage(prog, "missing loop count");
+                    LOOP_COUNT = atoi(*argv++);
+                    break;
+                case 'k':
+                    if (*argv == NULL)
+                        usage(prog, "missing key size");
+                    KEY_SIZE = atoi(*argv++);
+                    break;
+                case 's':
+                    if (*argv == NULL)
+                        usage(prog, "missing stack size");
+                    STACK_SIZE = atoi(*argv++);
+                    break;
+                case 't':
+                    if (*argv == NULL)
+                        usage(prog, "missing thread count");
+                    THREAD_COUNT = atoi(*argv++);
+                    if (THREAD_COUNT > MAX_THREAD_COUNT) {
+                        usage(prog, "max thread count exceeded");
+                    }
+                    break;
+                case 'v':
+                    VERBOSE = 1;
+                    break;
+                case 'd':
+                    if (*argv == NULL)
+                        usage(prog, "missing directory");
+                    NSSDIR = *argv++;
+                    break;
+                case 'e':
+                    ISTOKEN = PR_FALSE;
+                    break;
+                case 'h':
+                    usage(prog, NULL);
+                    break;
+                default:
+                    sprintf(buf, "unknown option %c", arg[1]);
+                    usage(prog, buf);
+            }
+        } else {
+            sprintf(buf, "unknown argument %s", arg);
+            usage(prog, buf);
+        }
+    }
+    /* initialize NSS */
+    rv = NSS_InitReadWrite(NSSDIR);
+    if (rv != SECSuccess) {
+        fprintf(stderr,
+                "NSS_InitReadWrite(%s) failed(%s)\n", NSSDIR,
+                PORT_ErrorToString(PORT_GetError()));
+        exit(2);
+    }
+
+    /* need to initialize the database here if it's not already */
+
+    printf("creating %d threads\n", THREAD_COUNT);
+    for (i = 0; i < THREAD_COUNT; i++) {
+        threadnumber[i] = i;
+        thread[i] = PR_CreateThread(PR_USER_THREAD, create_key_loop,
+                                    &threadnumber[i], PR_PRIORITY_NORMAL,
+                                    PR_GLOBAL_THREAD,
+                                    PR_JOINABLE_THREAD, STACK_SIZE);
+        if (thread[i] == NULL) {
+            ERROR++;
+            fprintf(stderr,
+                    "PR_CreateThread failed iteration %d, %s\n", i,
+                    PORT_ErrorToString(PORT_GetError()));
+        }
+    }
+    printf("waiting on %d threads\n", THREAD_COUNT);
+    for (i = 0; i < THREAD_COUNT; i++) {
+        if (thread[i] == NULL) {
+            continue;
+        }
+        status = PR_JoinThread(thread[i]);
+        if (status != PR_SUCCESS) {
+            ERROR++;
+            fprintf(stderr,
+                    "PR_CreateThread filed iteration %d, %s]n", i,
+                    PORT_ErrorToString(PORT_GetError()));
+        }
+    }
+    printf("%d failures and %d errors found\n", FAILED, ERROR);
+    /* clean up */
+    NSS_Shutdown();
+    if (FAILED) {
+        exit(1);
+    }
+    if (ERROR) {
+        exit(2);
+    }
+    exit(0);
+}
new file mode 100644
--- /dev/null
+++ b/cmd/sdbthreadtst/sdbthreadtst.gyp
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+{
+  'includes': [
+    '../../coreconf/config.gypi',
+    '../../cmd/platlibs.gypi'
+  ],
+  'targets': [
+    {
+      'target_name': 'sdbthreadtst',
+      'type': 'executable',
+      'sources': [
+        'sdbthreadtst.c'
+      ],
+      'dependencies': [
+        '<(DEPTH)/exports.gyp:nss_exports'
+      ]
+    }
+  ],
+  'target_defaults': {
+    'defines': [
+      'NSPR20'
+    ]
+  },
+  'variables': {
+    'module': 'nss'
+  }
+}
--- a/lib/softoken/sdb.c
+++ b/lib/softoken/sdb.c
@@ -77,22 +77,22 @@ typedef enum {
  *
  * SDB_SQLITE_BUSY_TIMEOUT affects all opertions, both manual
  *   (prepare/step/reset/finalize) and automatic (sqlite3_exec()).
  * SDB_BUSY_RETRY_TIME and SDB_MAX_BUSY_RETRIES only affect manual operations
  *
  * total wait time for automatic operations:
  *   1 second (SDB_SQLITE_BUSY_TIMEOUT/1000).
  * total wait time for manual operations:
- *   (1 second + 5 seconds) * 10 = 60 seconds.
+ *   (1 second + SDB_BUSY_RETRY_TIME) * 30 = 30 seconds.
  * (SDB_SQLITE_BUSY_TIMEOUT/1000 + SDB_BUSY_RETRY_TIME)*SDB_MAX_BUSY_RETRIES
  */
 #define SDB_SQLITE_BUSY_TIMEOUT 1000 /* milliseconds */
-#define SDB_BUSY_RETRY_TIME 5        /* seconds */
-#define SDB_MAX_BUSY_RETRIES 10
+#define SDB_BUSY_RETRY_TIME 5        /* 'ticks', varies by platforms */
+#define SDB_MAX_BUSY_RETRIES 30
 
 /*
  * known attributes
  */
 static const CK_ATTRIBUTE_TYPE known_attributes[] = {
     CKA_CLASS, CKA_TOKEN, CKA_PRIVATE, CKA_LABEL, CKA_APPLICATION,
     CKA_VALUE, CKA_OBJECT_ID, CKA_CERTIFICATE_TYPE, CKA_ISSUER,
     CKA_SERIAL_NUMBER, CKA_AC_ISSUER, CKA_OWNER, CKA_ATTR_TYPES, CKA_TRUSTED,
@@ -996,16 +996,17 @@ sdb_GetValidAttributeValueNoLock(SDB *sd
                     }
                     PORT_Memcpy(template[i].pValue, blobData, blobSize);
                 }
                 template[i].ulValueLen = blobSize;
             }
             found = 1;
         }
     } while (!sdb_done(sqlerr, &retry));
+
     sqlite3_reset(stmt);
     sqlite3_finalize(stmt);
     stmt = NULL;
 
 loser:
     /* fix up the error if necessary */
     if (error == CKR_OK) {
         error = sdb_mapSQLError(sdb_p->type, sqlerr);
@@ -1519,16 +1520,18 @@ sdb_Begin(SDB *sdb)
 
     sqlerr = sqlite3_prepare_v2(sqlDB, BEGIN_CMD, -1, &stmt, NULL);
 
     do {
         sqlerr = sqlite3_step(stmt);
         if (sqlerr == SQLITE_BUSY) {
             PR_Sleep(SDB_BUSY_RETRY_TIME);
         }
+        /* don't retry BEGIN transaction*/
+        retry = 0;
     } while (!sdb_done(sqlerr, &retry));
 
     if (stmt) {
         sqlite3_reset(stmt);
         sqlite3_finalize(stmt);
     }
 
 loser:
@@ -2256,16 +2259,17 @@ sdb_init(char *dbname, char *table, sdbD
                 /* Record the ULONG attribute value. */
                 char *val = (char *)sqlite3_column_text(stmt, 1);
                 if (val && val[0] == 'a') {
                     CK_ATTRIBUTE_TYPE attr = strtoul(&val[1], NULL, 16);
                     sdb_p->schemaAttrs[backedAttrs++] = attr;
                 }
             }
         } while (!sdb_done(sqlerr, &retry));
+
         if (sqlerr != SQLITE_DONE) {
             goto loser;
         }
         sqlerr = sqlite3_reset(stmt);
         if (sqlerr != SQLITE_OK) {
             goto loser;
         }
         sqlerr = sqlite3_finalize(stmt);
--- a/lib/softoken/sftkdb.c
+++ b/lib/softoken/sftkdb.c
@@ -1521,17 +1521,17 @@ sftkdb_DestroyObject(SFTKDBHandle *handl
     if (handle == NULL) {
         return CKR_TOKEN_WRITE_PROTECTED;
     }
     db = SFTK_GET_SDB(handle);
     objectID &= SFTK_OBJ_ID_MASK;
 
     crv = (*db->sdb_Begin)(db);
     if (crv != CKR_OK) {
-        goto loser;
+        return crv;
     }
     crv = (*db->sdb_DestroyObject)(db, objectID);
     if (crv != CKR_OK) {
         goto loser;
     }
     /* if the database supports meta data, delete any old signatures
      * that we may have added */
     if ((db->sdb_flags & SDB_HAS_META) == SDB_HAS_META) {
@@ -2456,17 +2456,17 @@ sftkdb_Update(SFTKDBHandle *handle, SECI
         return CKR_OK;
     }
     /*
      * put the whole update under a transaction. This allows us to handle
      * any possible race conditions between with the updateID check.
      */
     crv = (*handle->db->sdb_Begin)(handle->db);
     if (crv != CKR_OK) {
-        goto loser;
+        return crv;
     }
     inTransaction = PR_TRUE;
 
     /* some one else has already updated this db */
     if (sftkdb_hasUpdate(sftkdb_TypeString(handle),
                          handle->db, handle->updateID)) {
         crv = CKR_OK;
         goto done;
--- a/nss.gyp
+++ b/nss.gyp
@@ -184,16 +184,17 @@
             'cmd/pk11ectest/pk11ectest.gyp:pk11ectest',
             'cmd/pk11gcmtest/pk11gcmtest.gyp:pk11gcmtest',
             'cmd/pk11mode/pk11mode.gyp:pk11mode',
             'cmd/pk11importtest/pk11importtest.gyp:pk11importtest',
             'cmd/pk1sign/pk1sign.gyp:pk1sign',
             'cmd/pp/pp.gyp:pp',
             'cmd/rsaperf/rsaperf.gyp:rsaperf',
             'cmd/rsapoptst/rsapoptst.gyp:rsapoptst',
+            'cmd/sdbthreadtst/sdbthreadtst.gyp:sdbthreadtst',
             'cmd/sdrtest/sdrtest.gyp:sdrtest',
             'cmd/selfserv/selfserv.gyp:selfserv',
             'cmd/shlibsign/mangle/mangle.gyp:mangle',
             'cmd/strsclnt/strsclnt.gyp:strsclnt',
             'cmd/tests/tests.gyp:baddbdir',
             'cmd/tests/tests.gyp:conflict',
             'cmd/tests/tests.gyp:dertimetest',
             'cmd/tests/tests.gyp:encodeinttest',
--- a/tests/dbtests/dbtests.sh
+++ b/tests/dbtests/dbtests.sh
@@ -3,17 +3,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 ########################################################################
 #
 # mozilla/security/nss/tests/dbtest/dbtest.sh
 #
-# Certificate generating and handeling for NSS QA, can be included 
+# Certificate generating and handeling for NSS QA, can be included
 # multiple times from all.sh and the individual scripts
 #
 # needs to work on all Unix and Windows platforms
 #
 # included from (don't expect this to be up to date)
 # --------------------------------------------------
 #   all.sh
 #   ssl.sh
@@ -45,106 +45,107 @@ dbtest_init()
       cd ../cert
       . ./cert.sh
   fi
 
   SCRIPTNAME="dbtests.sh"
   RONLY_DIR=${HOSTDIR}/ronlydir
   EMPTY_DIR=${HOSTDIR}/emptydir
   CONFLICT_DIR=${HOSTDIR}/conflictdir
+  THREAD_DIR=${HOSTDIR}/threadir
 
   html_head "CERT and Key DB Tests"
 
 }
 
 ############################## dbtest_cleanup ############################
 # local shell function to finish this script (no exit since it might be
 # sourced)
 ########################################################################
 dbtest_cleanup()
 {
-  html "</TABLE><BR>" 
+  html "</TABLE><BR>"
   cd ${QADIR}
   chmod a+rw $RONLY_DIR
   . common/cleanup.sh
 }
 
 Echo()
 {
     echo
     echo "---------------------------------------------------------------"
     echo "| $*"
     echo "---------------------------------------------------------------"
 }
 dbtest_main()
 {
     cd ${HOSTDIR}
 
-    
+
     Echo "test opening the database read/write in a nonexisting directory"
     ${BINDIR}/certutil -L -X -d ./non_existent_dir
     ret=$?
     if [ $ret -ne 255 ]; then
       html_failed "Certutil succeeded in a nonexisting directory $ret"
     else
-      html_passed "Certutil didn't work in a nonexisting dir $ret" 
+      html_passed "Certutil didn't work in a nonexisting dir $ret"
     fi
     ${BINDIR}/dbtest -r -d ./non_existent_dir
     ret=$?
     if [ $ret -ne 46 ]; then
       html_failed "Dbtest readonly succeeded in a nonexisting directory $ret"
     else
-      html_passed "Dbtest readonly didn't work in a nonexisting dir $ret" 
+      html_passed "Dbtest readonly didn't work in a nonexisting dir $ret"
     fi
 
     Echo "test force opening the database in a nonexisting directory"
     ${BINDIR}/dbtest -f -d ./non_existent_dir
     ret=$?
     if [ $ret -ne 0 ]; then
       html_failed "Dbtest force failed in a nonexisting directory $ret"
     else
       html_passed "Dbtest force succeeded in a nonexisting dir $ret"
     fi
 
     Echo "test opening the database readonly in an empty directory"
     mkdir $EMPTY_DIR
-    ${BINDIR}/tstclnt -h  ${HOST}  -d $EMPTY_DIR 
+    ${BINDIR}/tstclnt -h  ${HOST}  -d $EMPTY_DIR
     ret=$?
     if [ $ret -ne 1 ]; then
       html_failed "Tstclnt succeded in an empty directory $ret"
     else
       html_passed "Tstclnt didn't work in an empty dir $ret"
     fi
     ${BINDIR}/dbtest -r -d $EMPTY_DIR
     ret=$?
     if [ $ret -ne 46 ]; then
       html_failed "Dbtest readonly succeeded in an empty directory $ret"
     else
-      html_passed "Dbtest readonly didn't work in an empty dir $ret" 
+      html_passed "Dbtest readonly didn't work in an empty dir $ret"
     fi
     rm -rf $EMPTY_DIR/* 2>/dev/null
     ${BINDIR}/dbtest -i -d $EMPTY_DIR
     ret=$?
     if [ $ret -ne 0 ]; then
       html_failed "Dbtest logout after empty DB Init loses key $ret"
     else
-      html_passed "Dbtest logout after empty DB Init has key" 
+      html_passed "Dbtest logout after empty DB Init has key"
     fi
     rm -rf $EMPTY_DIR/* 2>/dev/null
     ${BINDIR}/dbtest -i -p pass -d $EMPTY_DIR
     ret=$?
     if [ $ret -ne 0 ]; then
       html_failed "Dbtest password DB Init loses needlogin state $ret"
     else
-      html_passed "Dbtest password DB Init maintains needlogin state" 
+      html_passed "Dbtest password DB Init maintains needlogin state"
     fi
     rm -rf $EMPTY_DIR/* 2>/dev/null
     ${BINDIR}/certutil -D -n xxxx -d $EMPTY_DIR #created DB
     ret=$?
-    if [ $ret -ne 255 ]; then 
+    if [ $ret -ne 255 ]; then
         html_failed "Certutil succeeded in deleting a cert in an empty directory $ret"
     else
         html_passed "Certutil didn't work in an empty dir $ret"
     fi
     rm -rf $EMPTY_DIR/* 2>/dev/null
     Echo "test force opening the database  readonly in a empty directory"
     ${BINDIR}/dbtest -r -f -d $EMPTY_DIR
     ret=$?
@@ -171,41 +172,41 @@ dbtest_main()
     # skipping the next two tests when user is root,
     # otherwise they would fail due to rooty powers
     if [ $UID -ne 0 ]; then
       ${BINDIR}/dbtest -d $RONLY_DIR
     ret=$?
     if [ $ret -ne 46 ]; then
       html_failed "Dbtest r/w succeeded in a readonly directory $ret"
     else
-      html_passed "Dbtest r/w didn't work in an readonly dir $ret" 
+      html_passed "Dbtest r/w didn't work in an readonly dir $ret"
     fi
     else
       html_passed "Skipping Dbtest r/w in a readonly dir because user is root"
     fi
     if [ $UID -ne 0 ]; then
       ${BINDIR}/certutil -D -n "TestUser" -d .
     ret=$?
     if [ $ret -ne 255 ]; then
       html_failed "Certutil succeeded in deleting a cert in a readonly directory $ret"
     else
       html_passed "Certutil didn't work in an readonly dir $ret"
     fi
     else
-        html_passed "Skipping Certutil delete cert in a readonly directory test because user is root" 
+        html_passed "Skipping Certutil delete cert in a readonly directory test because user is root"
     fi
-    
+
     Echo "test opening the database ronly in a readonly directory"
 
     ${BINDIR}/dbtest -d $RONLY_DIR -r
     ret=$?
     if [ $ret -ne 0 ]; then
       html_failed "Dbtest readonly failed in a readonly directory $ret"
     else
-      html_passed "Dbtest readonly succeeded in a readonly dir $ret" 
+      html_passed "Dbtest readonly succeeded in a readonly dir $ret"
     fi
 
     Echo "test force opening the database  r/w in a readonly directory"
     ${BINDIR}/dbtest -d $RONLY_DIR -f
     ret=$?
     if [ $ret -ne 0 ]; then
       html_failed "Dbtest force failed in a readonly directory $ret"
     else
@@ -218,17 +219,17 @@ dbtest_main()
     mkdir ${CONFLICT_DIR}
     Echo "test creating a new cert with a conflicting nickname"
     cd ${CONFLICT_DIR}
     pwd
     ${BINDIR}/certutil -N -d ${CONFLICT_DIR} -f ${R_PWFILE}
     ret=$?
     if [ $ret -ne 0 ]; then
       html_failed "Nicknane conflict test failed, couldn't create database $ret"
-    else 
+    else
       ${BINDIR}/certutil -A -n alice -t ,, -i ${R_ALICEDIR}/Alice.cert -d ${CONFLICT_DIR}
       ret=$?
       if [ $ret -ne 0 ]; then
         html_failed "Nicknane conflict test failed, couldn't import alice cert $ret"
       else
         ${BINDIR}/certutil -A -n alice -t ,, -i ${R_BOBDIR}/Bob.cert -d ${CONFLICT_DIR}
         ret=$?
         if [ $ret -eq 0 ]; then
@@ -247,24 +248,48 @@ dbtest_main()
     # the old one should still be there...
     ${BINDIR}/certutil -L -n bob -d ${CONFLICT_DIR}
     ret=$?
     if [ $ret -ne 0 ]; then
       html_failed "Nicknane conflict test-setting nickname conflict incorrectly worked"
     else
       html_passed "Nicknane conflict test-setting nickname conflict was correctly rejected"
     fi
-    # import a token private key and make sure the corresponding public key is 
+    # import a token private key and make sure the corresponding public key is
     # created
     ${BINDIR}/pk11importtest -d ${CONFLICT_DIR} -f ${R_PWFILE}
     ret=$?
     if [ $ret -ne 0 ]; then
       html_failed "Importing Token Private Key does not create the corrresponding Public Key"
     else
       html_passed "Importing Token Private Key correctly creates the corrresponding Public Key"
     fi
+
+
+    if [ "${NSS_DEFAULT_DB_TYPE}" = "sql" ] ; then
+      LOOPS=${NSS_SDB_THREAD_LOOPS-7}
+      THREADS=${NSS_SDB_THREAD_THREADS-30}
+      mkdir -p ${THREAD_DIR}
+      Echo "testing for thread starvation while creating keys"
+      ${BINDIR}/certutil -N -d ${THREAD_DIR} --empty-password
+      ${BINDIR}/sdbthreadtst -l ${LOOPS} -t ${THREADS} -d ${THREAD_DIR}
+      ret=$?
+      case "$ret" in
+      "0")
+         html_passed "Successfully completed ${LOOPS} loops in ${THREADS} threads without failure."
+         ;;
+      "2")
+         html_failed "sdbthreadtst failed for some environment reason (like lack of memory)"
+         ;;
+      "1")
+         html_failed "sdbthreadtst failed do to starvation using ${LOOPS} loops and ${THREADS} threads."
+         ;;
+      *)
+         html_failed "sdbthreadtst failed with an unrecognized error code."
+      esac
+    fi
 }
 
 ################## main #################################################
 
-dbtest_init 
+dbtest_init
 dbtest_main 2>&1
 dbtest_cleanup