Bug 850905 - shlibsign fails digest/signature verification unless default keysize is used, r=rrelyea
authorElio Maldonado <emaldona@redhat.com>
Tue, 09 Apr 2013 20:55:49 -0700
changeset 10727 b786ac75c1c7fc57bc85fb82ae0a7fddda69878c
parent 10726 4b43c609072d21ddec6738f698ab51fa7e31bff2
child 10728 351b2d508a32b42e2a3bd70f4537953aa12fe0ca
push id37
push useremaldona@redhat.com
push dateWed, 10 Apr 2013 04:50:02 +0000
reviewersrrelyea
bugs850905
Bug 850905 - shlibsign fails digest/signature verification unless default keysize is used, r=rrelyea
cmd/shlibsign/shlibsign.c
--- a/cmd/shlibsign/shlibsign.c
+++ b/cmd/shlibsign/shlibsign.c
@@ -44,31 +44,32 @@
 #include "plstr.h"
 #include "prerror.h"
 
 /* softoken headers */
 #include "pkcs11.h"
 #include "pkcs11t.h"
 
 /* freebl headers */
+#include "blapit.h"
 #include "shsign.h"
 
 #define NUM_ELEM(array) (sizeof(array)/sizeof(array[0]))
 CK_BBOOL true = CK_TRUE;
 CK_BBOOL false = CK_FALSE;
 static PRBool verbose = PR_FALSE;
 
 static void
 usage (const char *program_name)
 {
     PRFileDesc *debug_out = PR_GetSpecialFD(PR_StandardError);
     PR_fprintf (debug_out,
                 "type %s -H for more detail information.\n", program_name);
     PR_fprintf (debug_out,
-                "Usage: %s [-v] [-V] [-o outfile] [-d dbdir] [-f pwfile]\n"
+                "Usage: %s [-v] [-V] [-k keysize] [-o outfile] [-d dbdir] [-f pwfile]\n"
                 "          [-F] [-p pwd] -[P dbprefix ] "
                 "-i shared_library_name\n",
                 program_name);
     exit(1);
 }
 
 static void 
 long_usage(const char *program_name) 
@@ -79,16 +80,17 @@ long_usage(const char *program_name)
     PR_fprintf(debug_out, "\t-o <outfile> checksum outfile\n");
     PR_fprintf(debug_out, "\t-d <path>    database path location\n");
     PR_fprintf(debug_out, "\t-P <prefix>  database prefix\n");
     PR_fprintf(debug_out, "\t-f <file>    password File : echo pw > file \n");
     PR_fprintf(debug_out, "\t-F           FIPS mode\n"); 
     PR_fprintf(debug_out, "\t-p <pwd>     password\n");
     PR_fprintf(debug_out, "\t-v           verbose output\n");
     PR_fprintf(debug_out, "\t-V           perform Verify operations\n");
+    PR_fprintf(debug_out, "\t-k <key_size>  key size in bits, default 2048\n");
     PR_fprintf(debug_out, "\t-?           short help message\n");
     PR_fprintf(debug_out, "\t-h           short help message\n");
     PR_fprintf(debug_out, "\t-H           this help message\n");
     PR_fprintf(debug_out, "\n\n\tNote: Use of FIPS mode requires your ");
     PR_fprintf(debug_out, "library path is using \n");
     PR_fprintf(debug_out, "\t      pre-existing libraries with generated ");
     PR_fprintf(debug_out, "checksum files\n");
     PR_fprintf(debug_out, "\t      and database in FIPS mode \n");
@@ -720,23 +722,26 @@ int main(int argc, char **argv)
     CK_TOKEN_INFO tokenInfo;
     CK_FUNCTION_LIST_PTR pFunctionList = NULL;
     CK_RV crv = CKR_OK;
     CK_SESSION_HANDLE hRwSession;
     CK_SLOT_ID *pSlotList = NULL;
     CK_ULONG slotIndex = 0; 
     CK_MECHANISM digestmech;
     CK_ULONG digestLen = 0;
-    CK_BYTE digest[32]; /* SHA256_LENGTH */
-    CK_BYTE sign[64];   /* DSA SIGNATURE LENGTH */
+    CK_BYTE digest[HASH_LENGTH_MAX];
+    CK_BYTE sign[DSA_MAX_SIGNATURE_LEN];
     CK_ULONG signLen = 0 ;
     CK_MECHANISM signMech = {
         CKM_DSA, NULL, 0
     };
 
+    int expectedDigestLen;
+    int expectedSignatureLen;
+
     /*** DSA Key ***/
 
     CK_MECHANISM dsaKeyPairGenMech;
     CK_ATTRIBUTE dsaPubKeyTemplate[5];
     CK_ATTRIBUTE dsaPrivKeyTemplate[5];
     CK_OBJECT_HANDLE hDSApubKey = CK_INVALID_HANDLE;
     CK_OBJECT_HANDLE hDSAprivKey = CK_INVALID_HANDLE;
 
@@ -908,20 +913,28 @@ int main(int argc, char **argv)
 
 	if (keySize && (mechInfo.ulMaxKeySize < keySize)) {
 	    PR_fprintf(PR_STDERR, 
 		"token doesn't support DSA2 (Max key size=%d)\n", 
 		mechInfo.ulMaxKeySize);
 	    goto cleanup;
 	}
 
-	if ((keySize == 0) && mechInfo.ulMaxKeySize >=2048 ) {
-	    keySize = 2048;
-	} else {
-	    keySize = 1024;
+	if (keySize == 0) {
+	    if (mechInfo.ulMaxKeySize >=2048) {
+		keySize = 2048;
+	    } else {
+		keySize = 1024;
+	    }
+	}
+	if (keySize > mechInfo.ulMaxKeySize) {
+	    PR_fprintf(PR_STDERR, 
+		"Requested key size of %d bits exceeds the mechanism's maximum key size of %d bits\n",
+		keySize, mechInfo.ulMaxKeySize);
+	    goto cleanup;
 	}
     }
 
     /* DSA key init */
     if (keySize == 1024) {
 	dsaPubKeyTemplate[0].type       = CKA_PRIME;
 	dsaPubKeyTemplate[0].pValue     = (CK_VOID_PTR) &prime;
 	dsaPubKeyTemplate[0].ulValueLen = sizeof(prime);
@@ -929,32 +942,40 @@ int main(int argc, char **argv)
 	dsaPubKeyTemplate[1].pValue = (CK_VOID_PTR) &subprime;
 	dsaPubKeyTemplate[1].ulValueLen = sizeof(subprime);
 	dsaPubKeyTemplate[2].type = CKA_BASE;
 	dsaPubKeyTemplate[2].pValue = (CK_VOID_PTR) &base;
 	dsaPubKeyTemplate[2].ulValueLen = sizeof(base);
 	digestmech.mechanism = CKM_SHA_1;
 	digestmech.pParameter = NULL;
 	digestmech.ulParameterLen = 0;
+
+	expectedDigestLen = SHA1_LENGTH;
+	expectedSignatureLen = sizeof(subprime)*2;   /* length of q*2 */;
+
     } else if (keySize == 2048) {
 	dsaPubKeyTemplate[0].type       = CKA_PRIME;
 	dsaPubKeyTemplate[0].pValue     = (CK_VOID_PTR) &prime2;
 	dsaPubKeyTemplate[0].ulValueLen = sizeof(prime2);
 	dsaPubKeyTemplate[1].type = CKA_SUBPRIME;
 	dsaPubKeyTemplate[1].pValue = (CK_VOID_PTR) &subprime2;
 	dsaPubKeyTemplate[1].ulValueLen = sizeof(subprime2);
 	dsaPubKeyTemplate[2].type = CKA_BASE;
 	dsaPubKeyTemplate[2].pValue = (CK_VOID_PTR) &base2;
 	dsaPubKeyTemplate[2].ulValueLen = sizeof(base2);
 	digestmech.mechanism = CKM_SHA256;
 	digestmech.pParameter = NULL;
 	digestmech.ulParameterLen = 0;
+
+	expectedDigestLen = SHA256_LENGTH;
+	expectedSignatureLen = sizeof(subprime2)*2; /* length of q*2 */
+
     } else {
 	/* future - generate pqg */
-        PR_fprintf(PR_STDERR, "Only keysizes 1024 and 2048 are supported");
+        PR_fprintf(PR_STDERR, "Only keysizes 1024 and 2048 are supported\n");
 	goto cleanup;
     }
     dsaPubKeyTemplate[3].type = CKA_TOKEN;
     dsaPubKeyTemplate[3].pValue = &false; /* session object */
     dsaPubKeyTemplate[3].ulValueLen = sizeof(false);
     dsaPubKeyTemplate[4].type = CKA_VERIFY;
     dsaPubKeyTemplate[4].pValue = &true;
     dsaPubKeyTemplate[4].ulValueLen = sizeof(true);
@@ -1099,19 +1120,19 @@ int main(int argc, char **argv)
     digestLen = sizeof(digest);
     crv = pFunctionList->C_DigestFinal(hRwSession, (CK_BYTE_PTR)digest,
                                        &digestLen);
     if (crv != CKR_OK) {
         pk11error("C_DigestFinal failed", crv);
         goto cleanup;
     }
 
-    if (digestLen != sizeof(digest)) {
+    if (digestLen != expectedDigestLen) {
         PR_fprintf(PR_STDERR, "digestLen has incorrect length %lu "
-                "it should be %lu \n",digestLen, sizeof(digest));
+                "it should be %lu \n",digestLen, expectedDigestLen);
         goto cleanup;
     }
 
     /* sign the hash */
     memset(sign, 0, sizeof(sign));
     /* SignUpdate  */
     crv = pFunctionList->C_SignInit(hRwSession, &signMech, hDSAprivKey);
     if (crv != CKR_OK) {
@@ -1122,19 +1143,19 @@ int main(int argc, char **argv)
     signLen = sizeof(sign);
     crv = pFunctionList->C_Sign(hRwSession, (CK_BYTE * ) digest, digestLen,
                                 sign, &signLen);
     if (crv != CKR_OK) {
         pk11error("C_Sign failed", crv);
         goto cleanup;
     }
 
-    if (signLen != sizeof(sign)) {
+    if (signLen != expectedSignatureLen) {
         PR_fprintf(PR_STDERR, "signLen has incorrect length %lu "
-                    "it should be %lu \n", signLen, sizeof(sign));
+                    "it should be %lu \n", signLen, expectedSignatureLen);
         goto cleanup;
     }
 
     if (verify) {
         crv = pFunctionList->C_VerifyInit(hRwSession, &signMech, hDSApubKey);
         if (crv != CKR_OK) {
             pk11error("C_VerifyInit failed", crv);
             goto cleanup;