Backout bug 1018259 / dcc6a567b46e due to windows linker bustage CLOSED TREE
authorMark Banner <standard8@mozilla.com>
Wed, 03 Sep 2014 17:08:15 +0100
changeset 20913 1575d08812cb7032866262a1bb7c5de93cdc65ba
parent 20912 9f8ef4f974ea28b76e5d3a9045031f410b3ef01a
child 20914 5540d3b4eb234376fcd74a6a9022f57bba8dc36d
push id1259
push usermbanner@mozilla.com
push dateFri, 28 Nov 2014 16:21:42 +0000
treeherdercomm-beta@219e12aa22c1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1018259
Backout bug 1018259 / dcc6a567b46e due to windows linker bustage CLOSED TREE
mailnews/extensions/smime/src/nsMsgComposeSecure.cpp
mailnews/mime/public/nsICMSMessage.idl
mailnews/mime/src/nsCMS.cpp
--- a/mailnews/extensions/smime/src/nsMsgComposeSecure.cpp
+++ b/mailnews/extensions/smime/src/nsMsgComposeSecure.cpp
@@ -1,20 +1,17 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
  *
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "nsMsgComposeSecure.h"
 
-#include "cert.h"
-#include "keyhi.h"
 #include "msgCore.h"
-#include "nsICryptoHash.h"
 #include "nsIMsgCompFields.h"
 #include "nsIMsgIdentity.h"
 #include "nsIX509CertDB.h"
 #include "nsMimeTypes.h"
 #include "nsMsgMimeCID.h"
 #include "nspr.h"
 #include "nsComponentManagerUtils.h"
 #include "nsServiceManagerUtils.h"
@@ -37,25 +34,24 @@ using namespace mozilla;
 
 // It doesn't make sense to encode the message because the message will be
 // displayed only if the MUA doesn't support MIME.
 // We need to consider what to do in case the server doesn't support 8BITMIME.
 // In short, we can't use non-ASCII characters here.
 static const char crypto_multipart_blurb[] = "This is a cryptographically signed message in MIME format.";
 
 static void mime_crypto_write_base64 (void *closure, const char *buf,
-                                      unsigned long size);
+              unsigned long size);
 static nsresult mime_encoder_output_fn(const char *buf, int32_t size,
                                        void *closure);
 static nsresult mime_nested_encoder_output_fn(const char *buf, int32_t size,
                                               void *closure);
 static nsresult make_multipart_signed_header_string(bool outer_p,
-                                                    char **header_return,
-                                                    char **boundary_return,
-                                                    int16_t hash_type);
+                  char **header_return,
+                  char **boundary_return);
 static char *mime_make_separator(const char *prefix);
 
 
 static void
 GenerateGlobalRandomBytes(unsigned char *buf, int32_t len)
 {
   static bool      firstTime = true;
   
@@ -135,17 +131,16 @@ NS_IMETHODIMP nsMsgSMIMEComposeFields::G
 NS_IMPL_ISUPPORTS(nsMsgComposeSecure, nsIMsgComposeSecure)
 
 nsMsgComposeSecure::nsMsgComposeSecure()
 {
   /* member initializers and constructor code */
   mMultipartSignedBoundary  = 0;
   mBuffer = 0;
   mBufferedBytes = 0;
-  mHashType = 0;
 }
 
 nsMsgComposeSecure::~nsMsgComposeSecure()
 {
   /* destructor code */
   if (mEncryptionContext) {
     if (mBufferedBytes) {
       mEncryptionContext->Update(mBuffer, mBufferedBytes);
@@ -309,97 +304,16 @@ nsresult nsMsgComposeSecure::ExtractEncr
   testrv = aIdentity->GetBoolAttribute("sign_mail", aSignMessage);
   if (NS_FAILED(testrv))
   {
     *aSignMessage = false;
   }
   return NS_OK;
 }
 
-// Select a hash algorithm to sign message
-// based on subject public key type and size.
-static nsresult
-GetSigningHashFunction(nsIX509Cert *aSigningCert, int16_t *hashType)
-{
-  // Get the signing certificate
-  CERTCertificate *scert = nullptr;
-  if (aSigningCert) {
-    scert = aSigningCert->GetCert();
-  }
-  if (!scert) {
-    return NS_ERROR_FAILURE;
-  }
-
-  mozilla::ScopedSECKEYPublicKey scertPublicKey(CERT_ExtractPublicKey(scert));
-  if (!scertPublicKey) {
-    return mozilla::MapSECStatus(SECFailure);
-  }
-  KeyType subjectPublicKeyType = SECKEY_GetPublicKeyType(scertPublicKey);
-
-  // Get the length of the signature in bits.
-  unsigned siglen = SECKEY_SignatureLen(scertPublicKey) * 8;
-  if (!siglen) {
-    return mozilla::MapSECStatus(SECFailure);
-  }
-
-  // Select a hash function for signature generation whose security strength
-  // meets or exceeds the security strength of the public key, using NIST
-  // Special Publication 800-57, Recommendation for Key Management - Part 1:
-  // General (Revision 3), where Table 2 specifies the security strength of
-  // the public key and Table 3 lists acceptable hash functions. (The security
-  // strength of the hash (for digital signatures) is half the length of the
-  // output.)
-  // [SP 800-57 is available at http://csrc.nist.gov/publications/PubsSPs.html.]
-  if (subjectPublicKeyType == rsaKey) {
-    // For RSA, siglen is the same as the length of the modulus.
-
-    // SHA-1 provides equivalent security strength for up to 1024 bits
-    // SHA-256 provides equivalent security strength for up to 3072 bits
-
-    if (siglen > 3072) {
-      *hashType = nsICryptoHash::SHA512;
-    } else if (siglen > 1024) {
-      *hashType = nsICryptoHash::SHA256;
-    } else {
-      *hashType = nsICryptoHash::SHA1;
-    }
-  } else if (subjectPublicKeyType == dsaKey) {
-    // For DSA, siglen is twice the length of the q parameter of the key.
-    // The security strength of the key is half the length (in bits) of
-    // the q parameter of the key.
-
-    // NSS only supports SHA-1, SHA-224, and SHA-256 for DSA signatures.
-    // The S/MIME code does not support SHA-224.
-
-    if (siglen >= 512) { // 512-bit signature = 256-bit q parameter
-      *hashType = nsICryptoHash::SHA256;
-    } else {
-      *hashType = nsICryptoHash::SHA1;
-    }
-  } else if (subjectPublicKeyType == ecKey) {
-    // For ECDSA, siglen is twice the length of the field size. The security
-    // strength of the key is half the length (in bits) of the field size.
-
-    if (siglen >= 1024) { // 1024-bit signature = 512-bit field size
-      *hashType = nsICryptoHash::SHA512;
-    } else if (siglen >= 768) { // 768-bit signature = 384-bit field size
-      *hashType = nsICryptoHash::SHA384;
-    } else if (siglen >= 512) { // 512-bit signature = 256-bit field size
-      *hashType = nsICryptoHash::SHA256;
-    } else {
-      *hashType = nsICryptoHash::SHA1;
-    }
-  } else {
-    // Unknown key type
-    *hashType = nsICryptoHash::SHA256;
-    NS_WARNING("GetSigningHashFunction: Subject public key type unknown.");
-  }
-  return NS_OK;
-}
-
 /* void beginCryptoEncapsulation (in nsOutputFileStream aStream, in boolean aEncrypt, in boolean aSign, in string aRecipeints, in boolean aIsDraft); */
 NS_IMETHODIMP nsMsgComposeSecure::BeginCryptoEncapsulation(nsIOutputStream * aStream,
                                                            const char * aRecipients,
                                                            nsIMsgCompFields * aCompFields,
                                                            nsIMsgIdentity * aIdentity,
                                                            nsIMsgSendReport *sendReport,
                                                            bool aIsDraft)
 {
@@ -427,21 +341,16 @@ NS_IMETHODIMP nsMsgComposeSecure::BeginC
   aIdentity->GetUnicharAttribute("signing_cert_name", mSigningCertName);
   aIdentity->GetUnicharAttribute("encryption_cert_name", mEncryptionCertName);
 
   rv = MimeCryptoHackCerts(aRecipients, sendReport, encryptMessages, signMessage);
   if (NS_FAILED(rv)) {
     goto FAIL;
   }
 
-  if (signMessage && mSelfSigningCert) {
-    rv = GetSigningHashFunction(mSelfSigningCert, &mHashType);
-    NS_ENSURE_SUCCESS(rv, rv);
-  }
-
   switch (mCryptoState)
   {
   case mime_crypto_clear_signed:
     rv = MimeInitMultipartSigned(true, sendReport);
     break;
   case mime_crypto_opaque_signed:
     PR_ASSERT(0);    /* #### no api for this yet */
     rv = NS_ERROR_NOT_IMPLEMENTED;
@@ -500,18 +409,17 @@ nsresult nsMsgComposeSecure::MimeInitMul
 {
   /* First, construct and write out the multipart/signed MIME header data.
    */
   nsresult rv = NS_OK;
   char *header = 0;
   uint32_t L;
 
   rv = make_multipart_signed_header_string(aOuter, &header,
-    &mMultipartSignedBoundary, mHashType);
-
+                    &mMultipartSignedBoundary);
   NS_ENSURE_SUCCESS(rv, rv);
 
   L = strlen(header);
 
   if (aOuter){
     /* If this is the outer block, write it to the file. */
     uint32_t n;
     rv = mStream->Write(header, L, &n);
@@ -526,16 +434,18 @@ nsresult nsMsgComposeSecure::MimeInitMul
 
   PR_Free(header);
   NS_ENSURE_SUCCESS(rv, rv);
 
   /* Now initialize the crypto library, so that we can compute a hash
    on the object which we are signing.
    */
 
+  mHashType = nsICryptoHash::SHA1;
+
   PR_SetError(0,0);
   mDataHash = do_CreateInstance("@mozilla.org/security/hash;1", &rv);
   NS_ENSURE_SUCCESS(rv, rv);
 
   rv = mDataHash->Init(mHashType);
   NS_ENSURE_SUCCESS(rv, rv);
 
   PR_SetError(0,0);
@@ -724,23 +634,22 @@ nsresult nsMsgComposeSecure::MimeFinishM
     rv = MimeCryptoWriteBlock (header, L);
   }
 
   PR_Free(header);
 
   /* Create the signature...
    */
 
-  NS_ASSERTION(mHashType, "Hash function for signature has not been set.");
+  PR_ASSERT(mHashType == nsICryptoHash::SHA1);
 
   PR_ASSERT (mSelfSigningCert);
   PR_SetError(0,0);
 
-  rv = cinfo->CreateSigned(mSelfSigningCert, mSelfEncryptionCert,
-    (unsigned char*)hashString.get(), hashString.Length(), mHashType);
+  rv = cinfo->CreateSigned(mSelfSigningCert, mSelfEncryptionCert, (unsigned char*)hashString.get(), hashString.Length());
   if (NS_FAILED(rv))  {
     SetError(sendReport, MOZ_UTF16("ErrorCanNotSignMail"));
     goto FAIL;
   }
 
   // Initialize the base64 encoder for the signature data.
   MOZ_ASSERT(!mSigEncoder, "Shouldn't already have a mSigEncoder");
   mSigEncoder = MimeEncoder::GetBase64Encoder(
@@ -1032,53 +941,34 @@ NS_IMETHODIMP nsMsgComposeSecure::MimeCr
 
 /* Returns a string consisting of a Content-Type header, and a boundary
    string, suitable for moving from the header block, down into the body
    of a multipart object.  The boundary itself is also returned (so that
    the caller knows what to write to close it off.)
  */
 static nsresult
 make_multipart_signed_header_string(bool outer_p,
-                                    char **header_return,
-                                    char **boundary_return,
-                                    int16_t hash_type)
+									char **header_return,
+									char **boundary_return)
 {
-  const char *hashStr;
   *header_return = 0;
   *boundary_return = mime_make_separator("ms");
 
   if (!*boundary_return)
 	return NS_ERROR_OUT_OF_MEMORY;
 
-  switch (hash_type) {
-  case nsICryptoHash::SHA1:
-    hashStr = PARAM_MICALG_SHA1;
-    break;
-  case nsICryptoHash::SHA256:
-    hashStr = PARAM_MICALG_SHA256;
-    break;
-  case nsICryptoHash::SHA384:
-    hashStr = PARAM_MICALG_SHA384;
-    break;
-  case nsICryptoHash::SHA512:
-    hashStr = PARAM_MICALG_SHA512;
-    break;
-  default:
-    return NS_ERROR_INVALID_ARG;
-  }
-
   *header_return = PR_smprintf(
         "Content-Type: " MULTIPART_SIGNED "; "
         "protocol=\"" APPLICATION_PKCS7_SIGNATURE "\"; "
-				"micalg=%s; "
+				"micalg=" PARAM_MICALG_SHA1 "; "
 				"boundary=\"%s\"" CRLF
 				CRLF
 				"%s%s"
 				"--%s" CRLF,
-				hashStr,
+
 				*boundary_return,
 				(outer_p ? crypto_multipart_blurb : ""),
 				(outer_p ? CRLF CRLF : ""),
 				*boundary_return);
 
   if (!*header_return) {
 	  PR_Free(*boundary_return);
 	  *boundary_return = 0;
--- a/mailnews/mime/public/nsICMSMessage.idl
+++ b/mailnews/mime/public/nsICMSMessage.idl
@@ -13,27 +13,23 @@
 
 interface nsIX509Cert;
 interface nsIArray;
 
 /**
  * nsICMSMessage
  *  Interface to a CMS Message
  */
-[uuid(c6d51c22-73e9-4dad-86b9-bde584e33c63)]
+[uuid(cfd6af3d-8ac6-401a-afc8-3f94275c1c11)]
 interface nsICMSMessage : nsISupports
 {
   void contentIsSigned(out boolean aSigned);
   void contentIsEncrypted(out boolean aEncrypted);
   void getSignerCommonName(out string aName);
   void getSignerEmailAddress(out string aEmail);
   void getSignerCert(out nsIX509Cert scert);
   void getEncryptionCert(out nsIX509Cert ecert);
   void verifySignature();
   void verifyDetachedSignature(in UnsignedCharPtr aDigestData, in unsigned long aDigestDataLen);
   void CreateEncrypted(in nsIArray aRecipientCerts);
-
-  /* The parameter aDigestType must be one of the values in nsICryptoHash */
-  void CreateSigned(in nsIX509Cert scert, in nsIX509Cert ecert,
-                    in UnsignedCharPtr aDigestData,
-                    in unsigned long aDigestDataLen, in int16_t aDigestType);
+  void CreateSigned(in nsIX509Cert scert, in nsIX509Cert ecert, in UnsignedCharPtr aDigestData, in unsigned long aDigestDataLen);
 };
 
--- a/mailnews/mime/src/nsCMS.cpp
+++ b/mailnews/mime/src/nsCMS.cpp
@@ -1,33 +1,33 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "nsCMS.h"
 
+#include "nsIX509CertDB.h"
 #include "CertVerifier.h"
+#include "CryptoTask.h"
+#include "mozilla/RefPtr.h"
+#include "pkix/pkixtypes.h"
+#include "nsISupports.h"
+#include "nsNSSHelper.h"
+#include "nsNSSCertificate.h"
+#include "ScopedNSSTypes.h"
+#include "smime.h"
 #include "cms.h"
-#include "CryptoTask.h"
+#include "nsNSSComponent.h"
+#include "nsICMSMessageErrors.h"
+#include "nsIArray.h"
 #include "nsArrayUtils.h"
 #include "nsCertVerificationThread.h"
-#include "nsIArray.h"
-#include "nsICMSMessageErrors.h"
-#include "nsICryptoHash.h"
-#include "nsISupports.h"
-#include "nsIX509CertDB.h"
-#include "nsNSSCertificate.h"
-#include "nsNSSComponent.h"
-#include "nsNSSHelper.h"
 #include "nsServiceManagerUtils.h"
 #include "mozilla/RefPtr.h"
-#include "pkix/pkixtypes.h"
-#include "ScopedNSSTypes.h"
-#include "smime.h"
 
 #include "prlog.h"
 
 using namespace mozilla;
 using namespace mozilla::psm;
 using namespace mozilla::pkix;
 
 #ifdef PR_LOGGING
@@ -614,20 +614,17 @@ loser:
   if (m_cmsMsg) {
     NSS_CMSMessage_Destroy(m_cmsMsg);
     m_cmsMsg = nullptr;
   }
 
   return rv;
 }
 
-NS_IMETHODIMP
-nsCMSMessage::CreateSigned(nsIX509Cert* aSigningCert, nsIX509Cert* aEncryptCert,
-                           unsigned char* aDigestData, uint32_t aDigestDataLen,
-                           int16_t aDigestType)
+NS_IMETHODIMP nsCMSMessage::CreateSigned(nsIX509Cert* aSigningCert, nsIX509Cert* aEncryptCert, unsigned char* aDigestData, uint32_t aDigestDataLen)
 {
   NS_ENSURE_ARG(aSigningCert);
   nsNSSShutDownPreventionLock locker;
   if (isAlreadyShutDown())
     return NS_ERROR_NOT_AVAILABLE;
 
   PR_LOG(gPIPNSSLog, PR_LOG_DEBUG, ("nsCMSMessage::CreateSigned\n"));
   NSSCMSContentInfo *cinfo;
@@ -640,34 +637,16 @@ nsCMSMessage::CreateSigned(nsIX509Cert* 
   if (!scert) {
     return NS_ERROR_FAILURE;
   }
 
   if (aEncryptCert) {
     ecert = aEncryptCert->GetCert();
   }
 
-  SECOidTag digestType;
-  switch (aDigestType) {
-    case nsICryptoHash::SHA1:
-      digestType = SEC_OID_SHA1;
-      break;
-    case nsICryptoHash::SHA256:
-      digestType = SEC_OID_SHA256;
-      break;
-    case nsICryptoHash::SHA384:
-      digestType = SEC_OID_SHA384;
-      break;
-    case nsICryptoHash::SHA512:
-      digestType = SEC_OID_SHA512;
-      break;
-    default:
-      return NS_ERROR_INVALID_ARG;
-  }
-
   /*
    * create the message object
    */
   m_cmsMsg = NSS_CMSMessage_Create(nullptr); /* create a message on its own pool */
   if (!m_cmsMsg) {
     PR_LOG(gPIPNSSLog, PR_LOG_DEBUG, ("nsCMSMessage::CreateSigned - can't create new message\n"));
     rv = NS_ERROR_OUT_OF_MEMORY;
     goto loser;
@@ -694,18 +673,18 @@ nsCMSMessage::CreateSigned(nsIX509Cert* 
           != SECSuccess) {
     PR_LOG(gPIPNSSLog, PR_LOG_DEBUG, ("nsCMSMessage::CreateSigned - can't set content data\n"));
     goto loser;
   }
 
   /* 
    * create & attach signer information
    */
-  signerinfo = NSS_CMSSignerInfo_Create(m_cmsMsg, scert.get(), digestType);
-  if (!signerinfo) {
+  if ((signerinfo = NSS_CMSSignerInfo_Create(m_cmsMsg, scert.get(), SEC_OID_SHA1)) 
+          == nullptr) {
     PR_LOG(gPIPNSSLog, PR_LOG_DEBUG, ("nsCMSMessage::CreateSigned - can't create signer info\n"));
     goto loser;
   }
 
   /* we want the cert chain included for this one */
   if (NSS_CMSSignerInfo_IncludeCerts(signerinfo, NSSCMSCM_CertChain, 
                                        certUsageEmailSigner) 
           != SECSuccess) {
@@ -757,17 +736,17 @@ nsCMSMessage::CreateSigned(nsIX509Cert* 
 
   // Finally, add the pre-computed digest if passed in
   if (aDigestData) {
     SECItem digest;
 
     digest.data = aDigestData;
     digest.len = aDigestDataLen;
 
-    if (NSS_CMSSignedData_SetDigestValue(sigd, digestType, &digest) != SECSuccess) {
+    if (NSS_CMSSignedData_SetDigestValue(sigd, SEC_OID_SHA1, &digest)) {
       PR_LOG(gPIPNSSLog, PR_LOG_DEBUG, ("nsCMSMessage::CreateSigned - can't set digest value\n"));
       goto loser;
     }
   }
 
   return NS_OK;
 loser:
   if (m_cmsMsg) {