Bug 1204155 - Account for OOM when setting or copying CryptoKeys. r=mt, a=sylvestre
authorTim Taubert <ttaubert@mozilla.com>
Mon, 14 Sep 2015 11:19:16 +0200
changeset 291112 79ae2fb834bbdc5da2741e08e0ecbd5d6a0bd864
parent 291111 e93e54b574c469cbf4f785a401275c0d6778bea5
child 291113 effd8a2c9afff0bc20db04a1d6c35f5b4786126b
push id934
push userraliiev@mozilla.com
push dateMon, 26 Oct 2015 12:58:05 +0000
treeherdermozilla-release@05704e35c1d0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmt, sylvestre
bugs1204155
milestone42.0
Bug 1204155 - Account for OOM when setting or copying CryptoKeys. r=mt, a=sylvestre From fb9c43d93da266c527c9e0ef67c0f1f673c58f7b Mon Sep 17 00:00:00 2001
dom/crypto/CryptoKey.cpp
dom/crypto/CryptoKey.h
dom/crypto/WebCryptoTask.cpp
--- a/dom/crypto/CryptoKey.cpp
+++ b/dom/crypto/CryptoKey.cpp
@@ -379,41 +379,51 @@ CryptoKey::AllUsagesRecognized(const Seq
   for (uint32_t i = 0; i < aUsages.Length(); ++i) {
     if (!IsRecognizedUsage(aUsages[i])) {
       return false;
     }
   }
   return true;
 }
 
-void CryptoKey::SetSymKey(const CryptoBuffer& aSymKey)
+nsresult CryptoKey::SetSymKey(const CryptoBuffer& aSymKey)
 {
-  mSymKey = aSymKey;
+  if (!mSymKey.Assign(aSymKey)) {
+    return NS_ERROR_OUT_OF_MEMORY;
+  }
+
+  return NS_OK;
 }
 
-void
+nsresult
 CryptoKey::SetPrivateKey(SECKEYPrivateKey* aPrivateKey)
 {
   nsNSSShutDownPreventionLock locker;
+
   if (!aPrivateKey || isAlreadyShutDown()) {
     mPrivateKey = nullptr;
-    return;
+    return NS_OK;
   }
+
   mPrivateKey = SECKEY_CopyPrivateKey(aPrivateKey);
+  return mPrivateKey ? NS_OK : NS_ERROR_OUT_OF_MEMORY;
 }
 
-void
+nsresult
 CryptoKey::SetPublicKey(SECKEYPublicKey* aPublicKey)
 {
   nsNSSShutDownPreventionLock locker;
+
   if (!aPublicKey || isAlreadyShutDown()) {
     mPublicKey = nullptr;
-    return;
+    return NS_OK;
   }
+
   mPublicKey = SECKEY_CopyPublicKey(aPublicKey);
+  return mPublicKey ? NS_OK : NS_ERROR_OUT_OF_MEMORY;
 }
 
 const CryptoBuffer&
 CryptoKey::GetSymKey() const
 {
   return mSymKey;
 }
 
@@ -1257,18 +1267,18 @@ CryptoKey::ReadStructuredClone(JSStructu
               ReadBuffer(aReader, sym) &&
               ReadBuffer(aReader, priv) &&
               ReadBuffer(aReader, pub) &&
               mAlgorithm.ReadStructuredClone(aReader);
   if (!read) {
     return false;
   }
 
-  if (sym.Length() > 0)  {
-    mSymKey = sym;
+  if (sym.Length() > 0 && !mSymKey.Assign(sym))  {
+    return false;
   }
   if (priv.Length() > 0) {
     mPrivateKey = CryptoKey::PrivateKeyFromPkcs8(priv, locker);
   }
   if (pub.Length() > 0)  {
     mPublicKey = CryptoKey::PublicKeyFromSpki(pub, locker);
   }
 
--- a/dom/crypto/CryptoKey.h
+++ b/dom/crypto/CryptoKey.h
@@ -122,19 +122,19 @@ public:
   nsresult AddUsageIntersecting(const nsString& aUsage, uint32_t aUsageMask);
   void AddUsage(KeyUsage aUsage);
   bool HasAnyUsage();
   bool HasUsage(KeyUsage aUsage);
   bool HasUsageOtherThan(uint32_t aUsages);
   static bool IsRecognizedUsage(const nsString& aUsage);
   static bool AllUsagesRecognized(const Sequence<nsString>& aUsages);
 
-  void SetSymKey(const CryptoBuffer& aSymKey);
-  void SetPrivateKey(SECKEYPrivateKey* aPrivateKey);
-  void SetPublicKey(SECKEYPublicKey* aPublicKey);
+  nsresult SetSymKey(const CryptoBuffer& aSymKey);
+  nsresult SetPrivateKey(SECKEYPrivateKey* aPrivateKey);
+  nsresult SetPublicKey(SECKEYPublicKey* aPublicKey);
 
   // Accessors for the keys themselves
   // Note: GetPrivateKey and GetPublicKey return copies of the internal
   // key handles, which the caller must free with SECKEY_DestroyPrivateKey
   // or SECKEY_DestroyPublicKey.
   const CryptoBuffer& GetSymKey() const;
   SECKEYPrivateKey* GetPrivateKey() const;
   SECKEYPublicKey* GetPublicKey() const;
--- a/dom/crypto/WebCryptoTask.cpp
+++ b/dom/crypto/WebCryptoTask.cpp
@@ -1283,17 +1283,21 @@ public:
         return;
       }
       mDataIsJwk = true;
     }
   }
 
   void SetKeyData(const CryptoBuffer& aKeyData)
   {
-    mKeyData = aKeyData;
+    if (!mKeyData.Assign(aKeyData)) {
+      mEarlyRv = NS_ERROR_DOM_OPERATION_ERR;
+      return;
+    }
+
     mDataIsJwk = false;
 
     if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_JWK)) {
       SetJwkFromKeyData();
     }
   }
 
   void SetJwkFromKeyData()
@@ -1463,17 +1467,20 @@ public:
       if (mDataIsJwk && mJwk.mUse.WasPassed() &&
           !mJwk.mUse.Value().EqualsLiteral(JWK_USE_SIG)) {
         return NS_ERROR_DOM_DATA_ERR;
       }
     } else {
       return NS_ERROR_DOM_NOT_SUPPORTED_ERR;
     }
 
-    mKey->SetSymKey(mKeyData);
+    if (NS_FAILED(mKey->SetSymKey(mKeyData))) {
+      return NS_ERROR_DOM_OPERATION_ERR;
+    }
+
     mKey->SetType(CryptoKey::SECRET);
 
     if (mDataIsJwk && !JwkCompatible(mJwk, mKey)) {
       return NS_ERROR_DOM_DATA_ERR;
     }
 
     mEarlyComplete = true;
     return NS_OK;
@@ -1569,33 +1576,39 @@ private:
       } else {
         pubKey = CryptoKey::PublicKeyFromJwk(mJwk, locker);
       }
 
       if (!pubKey) {
         return NS_ERROR_DOM_DATA_ERR;
       }
 
-      mKey->SetPublicKey(pubKey.get());
+      if (NS_FAILED(mKey->SetPublicKey(pubKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PUBLIC);
     } else if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_PKCS8) ||
         (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_JWK) &&
          mJwk.mD.WasPassed())) {
       // Private key import
       if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_PKCS8)) {
         privKey = CryptoKey::PrivateKeyFromPkcs8(mKeyData, locker);
       } else {
         privKey = CryptoKey::PrivateKeyFromJwk(mJwk, locker);
       }
 
       if (!privKey) {
         return NS_ERROR_DOM_DATA_ERR;
       }
 
-      mKey->SetPrivateKey(privKey.get());
+      if (NS_FAILED(mKey->SetPrivateKey(privKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PRIVATE);
       pubKey = SECKEY_ConvertToPublicKey(privKey.get());
       if (!pubKey) {
         return NS_ERROR_DOM_UNKNOWN_ERR;
       }
     } else {
       // Invalid key format
       return NS_ERROR_DOM_SYNTAX_ERR;
@@ -1703,17 +1716,20 @@ private:
     nsNSSShutDownPreventionLock locker;
     if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_JWK) && mJwk.mD.WasPassed()) {
       // Private key import
       privKey = CryptoKey::PrivateKeyFromJwk(mJwk, locker);
       if (!privKey) {
         return NS_ERROR_DOM_DATA_ERR;
       }
 
-      mKey->SetPrivateKey(privKey.get());
+      if (NS_FAILED(mKey->SetPrivateKey(privKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PRIVATE);
     } else if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_RAW) ||
                mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_SPKI) ||
                (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_JWK) &&
                 !mJwk.mD.WasPassed())) {
       // Public key import
       if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_RAW)) {
         pubKey = CryptoKey::PublicECKeyFromRaw(mKeyData, mNamedCurve, locker);
@@ -1740,17 +1756,20 @@ private:
         oid.data = pubKey->u.ec.DEREncodedParams.data + 2;
 
         // Find a matching and supported named curve.
         if (!MapOIDTagToNamedCurve(SECOID_FindOIDTag(&oid), mNamedCurve)) {
           return NS_ERROR_DOM_NOT_SUPPORTED_ERR;
         }
       }
 
-      mKey->SetPublicKey(pubKey.get());
+      if (NS_FAILED(mKey->SetPublicKey(pubKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PUBLIC);
     } else {
       return NS_ERROR_DOM_NOT_SUPPORTED_ERR;
     }
 
     // Extract 'crv' parameter from JWKs.
     if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_JWK)) {
       if (!NormalizeToken(mJwk.mCrv.Value(), mNamedCurve)) {
@@ -1862,17 +1881,20 @@ private:
         return NS_ERROR_DOM_DATA_ERR;
       }
 
       if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_SPKI)) {
         ATTEMPT_BUFFER_ASSIGN(mPrime, &pubKey->u.dh.prime);
         ATTEMPT_BUFFER_ASSIGN(mGenerator, &pubKey->u.dh.base);
       }
 
-      mKey->SetPublicKey(pubKey.get());
+      if (NS_FAILED(mKey->SetPublicKey(pubKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PUBLIC);
     } else {
       return NS_ERROR_DOM_NOT_SUPPORTED_ERR;
     }
 
     return NS_OK;
   }
 
@@ -2148,18 +2170,21 @@ private:
     // just refers to a buffer managed by symKey.  The assignment copies the
     // data, so mKeyData manages one copy, while symKey manages another.
     ATTEMPT_BUFFER_ASSIGN(mKeyData, PK11_GetKeyData(symKey));
     return NS_OK;
   }
 
   virtual void Resolve() override
   {
-    mKey->SetSymKey(mKeyData);
-    mResultPromise->MaybeResolve(mKey);
+    if (NS_SUCCEEDED(mKey->SetSymKey(mKeyData))) {
+      mResultPromise->MaybeResolve(mKey);
+    } else {
+      mResultPromise->MaybeReject(NS_ERROR_DOM_OPERATION_ERR);
+    }
   }
 
   virtual void Cleanup() override
   {
     mKey = nullptr;
   }
 };
 
@@ -2373,23 +2398,25 @@ GenerateAsymmetricKeyTask::DoCrypto()
   SECKEYPublicKey* pubKey = nullptr;
   mPrivateKey = PK11_GenerateKeyPair(slot.get(), mMechanism, param, &pubKey,
                                      PR_FALSE, PR_FALSE, nullptr);
   mPublicKey = pubKey;
   if (!mPrivateKey.get() || !mPublicKey.get()) {
     return NS_ERROR_DOM_UNKNOWN_ERR;
   }
 
-  mKeyPair.mPrivateKey.get()->SetPrivateKey(mPrivateKey);
-  mKeyPair.mPublicKey.get()->SetPublicKey(mPublicKey);
+  nsresult rv = mKeyPair.mPrivateKey.get()->SetPrivateKey(mPrivateKey);
+  NS_ENSURE_SUCCESS(rv, NS_ERROR_DOM_OPERATION_ERR);
+  rv = mKeyPair.mPublicKey.get()->SetPublicKey(mPublicKey);
+  NS_ENSURE_SUCCESS(rv, NS_ERROR_DOM_OPERATION_ERR);
 
   // PK11_GenerateKeyPair() does not set a CKA_EC_POINT attribute on the
   // private key, we need this later when exporting to PKCS8 and JWK though.
   if (mMechanism == CKM_EC_KEY_PAIR_GEN) {
-    nsresult rv = mKeyPair.mPrivateKey->AddPublicKeyData(mPublicKey);
+    rv = mKeyPair.mPrivateKey->AddPublicKeyData(mPublicKey);
     NS_ENSURE_SUCCESS(rv, NS_ERROR_DOM_OPERATION_ERR);
   }
 
   return NS_OK;
 }
 
 void
 GenerateAsymmetricKeyTask::Resolve()