Bug 521849, SSL broken, may crash when FIPS softtoken checksum verification fails
authorKai Engert <kaie@kuix.de>
Tue, 09 Nov 2010 17:10:40 +0100
changeset 57159 60fa0e6a4365
parent 57158 74efbd8a5184
child 57160 7aeaf6d005ec
push id16819
push userkaie@kuix.de
push dateTue, 09 Nov 2010 16:12:49 +0000
treeherdermozilla-central@60fa0e6a4365 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs521849
milestone2.0b8pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 521849, SSL broken, may crash when FIPS softtoken checksum verification fails r=honzab.moz, a=blocking2.0-final+=bsmedberg
security/manager/ssl/src/nsNSSComponent.cpp
security/manager/ssl/src/nsNSSComponent.h
--- a/security/manager/ssl/src/nsNSSComponent.cpp
+++ b/security/manager/ssl/src/nsNSSComponent.cpp
@@ -278,20 +278,25 @@ nsTokenEventRunnable::Run()
   nsresult rv;
   nsCOMPtr<nsINSSComponent> nssComponent(do_GetService(kNSSComponentCID, &rv));
   if (NS_FAILED(rv))
     return rv;
 
   return nssComponent->DispatchEvent(mType, mTokenName);
 }
 
+PRBool nsPSMInitPanic::isPanic = PR_FALSE;
+
 // We must ensure that the nsNSSComponent has been loaded before
 // creating any other components.
 PRBool EnsureNSSInitialized(EnsureNSSOperator op)
 {
+  if (nsPSMInitPanic::GetPanic())
+    return PR_FALSE;
+
 #ifdef MOZ_IPC
   if (GeckoProcessType_Default != XRE_GetProcessType())
   {
     if (op == nssEnsureOnChromeOnly)
     {
       // If the component needs PSM/NSS initialized only on the chrome process,
       // pretend we successfully initiated it but in reality we bypass it.
       // It's up to the programmer to check for process type in such components
@@ -1572,18 +1577,20 @@ nsNSSComponent::InitializeNSS(PRBool sho
     nsCOMPtr<nsIFile> profilePath;
 
     rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
                                 getter_AddRefs(profilePath));
     if (NS_FAILED(rv)) {
       PR_LOG(gPIPNSSLog, PR_LOG_ERROR, ("Unable to get profile directory\n"));
       ConfigureInternalPKCS11Token();
       SECStatus init_rv = NSS_NoDB_Init(NULL);
-      if (init_rv != SECSuccess)
+      if (init_rv != SECSuccess) {
+        nsPSMInitPanic::SetPanic();
         return NS_ERROR_NOT_AVAILABLE;
+      }
     }
     else
     {
 
   // XP_MAC == CFM
   // XP_MACOSX == MachO
 
   #if defined(XP_MAC) && defined(XP_MACOSX)
@@ -1602,36 +1609,40 @@ nsNSSComponent::InitializeNSS(PRBool sho
     // on CFM, cfmSecurityPath and profilePath point to the same oject
     profilePath->Create(nsIFile::DIRECTORY_TYPE, 0); //This is for Mac, don't worry about
                                                      //permissions.
   #elif defined(XP_MACOSX)
     // On MachO, we need to access both directories,
     // and therefore need separate nsIFile instances.
     // Keep cfmSecurityPath instance, obtain new instance for MachO profilePath.
     rv = cfmSecurityPath->GetParent(getter_AddRefs(profilePath));
-    if (NS_FAILED(rv))
+    if (NS_FAILED(rv)) {
+      nsPSMInitPanic::SetPanic();
       return rv;
+    }
   #endif
 
     const char *dbdir_override = getenv("MOZPSM_NSSDBDIR_OVERRIDE");
     if (dbdir_override && strlen(dbdir_override)) {
       profileStr = dbdir_override;
     }
     else {
   #if defined(XP_WIN)
       // Native path will drop Unicode characters that cannot be mapped to system's
       // codepage, using short (canonical) path as workaround.
       nsCOMPtr<nsILocalFileWin> profilePathWin(do_QueryInterface(profilePath, &rv));
       if (profilePathWin)
         rv = profilePathWin->GetNativeCanonicalPath(profileStr);
   #else
       rv = profilePath->GetNativePath(profileStr);
   #endif
-      if (NS_FAILED(rv)) 
+      if (NS_FAILED(rv)) {
+        nsPSMInitPanic::SetPanic();
         return rv;
+      }
     }
 
     hashTableCerts = PL_NewHashTable( 0, certHashtable_keyHash, certHashtable_keyCompare,
       certHashtable_valueCompare, 0, 0 );
 
   #if defined(XP_MACOSX)
     // function may modify the parameters
     // ignore return code from conversion, we continue anyway
@@ -1681,18 +1692,20 @@ nsNSSComponent::InitializeNSS(PRBool sho
       init_rv = ::NSS_Initialize(profileStr.get(), "", "",
                                  SECMOD_DB, init_flags);
 
       if (init_rv != SECSuccess) {
         PR_LOG(gPIPNSSLog, PR_LOG_DEBUG, ("can not init in r/o either\n"));
         which_nss_problem = problem_no_security_at_all;
 
         init_rv = NSS_NoDB_Init(profileStr.get());
-        if (init_rv != SECSuccess)
+        if (init_rv != SECSuccess) {
+          nsPSMInitPanic::SetPanic();
           return NS_ERROR_NOT_AVAILABLE;
+        }
       }
     } // have profile dir
     } // lock
 
     // init phase 3, only if phase 2 was successful
 
     if (problem_no_security_at_all != which_nss_problem) {
 
@@ -1772,17 +1785,17 @@ nsNSSComponent::InitializeNSS(PRBool sho
 
       LaunchSmartCardThreads();
 
       PR_LOG(gPIPNSSLog, PR_LOG_DEBUG, ("NSS Initialization done\n"));
     }
   }
 
   if (problem_none != which_nss_problem) {
-    nsString message;
+    nsPSMInitPanic::SetPanic();
 
     PR_LOG(gPIPNSSLog, PR_LOG_DEBUG, ("NSS problem, trying to bring up GUI error message\n"));
 
     // We might want to use different messages, depending on what failed.
     // For now, let's use the same message.
     if (showWarningBox) {
       ShowAlert(ai_nss_init_problem);
     }
--- a/security/manager/ssl/src/nsNSSComponent.h
+++ b/security/manager/ssl/src/nsNSSComponent.h
@@ -375,10 +375,19 @@ class nsNSSErrors
 public:
   static const char *getDefaultErrorStringName(PRInt32 err);
   static const char *getOverrideErrorStringName(PRInt32 aErrorCode);
   static nsresult getErrorMessageFromCode(PRInt32 err,
                                           nsINSSComponent *component,
                                           nsString &returnedMessage);
 };
 
+class nsPSMInitPanic
+{
+private:
+  static PRBool isPanic;
+public:
+  static void SetPanic() {isPanic = PR_TRUE;}
+  static PRBool GetPanic() {return isPanic;}
+};
+
 #endif // _nsNSSComponent_h_