Bug 1221620 - DER integer decoding update, r=mt NSS_3_21_BRANCH
authorFranziskus Kiefer <franziskuskiefer@gmail.com>
Mon, 18 Jan 2016 13:08:44 +0100
branchNSS_3_21_BRANCH
changeset 12723 9061e00062e5df90e39f332cdcd4e81932c3a23d
parent 12722 cb438f27902d4259a8425e705c2db1833d7786ee
child 12724 e51a1972bd8cd21e2041b1180c5604c599c0c0a6
push id1672
push userkaie@kuix.de
push dateMon, 17 Oct 2016 13:17:00 +0000
reviewersmt
bugs1221620
Bug 1221620 - DER integer decoding update, r=mt
lib/util/dersubr.c
--- a/lib/util/dersubr.c
+++ b/lib/util/dersubr.c
@@ -174,43 +174,51 @@ DER_SetUInteger(PLArenaPool *arena, SECI
 /*
 ** Convert a der encoded *signed* integer into a machine integral value.
 ** If an underflow/overflow occurs, sets error code and returns min/max.
 */
 long
 DER_GetInteger(const SECItem *it)
 {
     long ival = 0;
-    unsigned len = it->len;
+    PRBool negative = PR_FALSE;
+    unsigned int len = it->len;
+    unsigned int originalLength = len;
     unsigned char *cp = it->data;
     unsigned long overflow = 0x1ffUL << (((sizeof(ival) - 1) * 8) - 1);
-    unsigned long ofloinit;
+    unsigned long mask = 1;
 
     PORT_Assert(len);
     if (!len) {
 	PORT_SetError(SEC_ERROR_INPUT_LEN);
 	return 0;
     }
 
-    if (*cp & 0x80)
-    	ival = -1L;
-    ofloinit = ival & overflow;
+    if (*cp & 0x80) {
+	negative = PR_TRUE;
+	overflow <<= 1;
+    }
 
     while (len) {
-	if ((ival & overflow) != ofloinit) {
+	if ((ival & overflow) != 0) {
 	    PORT_SetError(SEC_ERROR_BAD_DER);
-	    if (ival < 0) {
+	    if (negative) {
 		return LONG_MIN;
 	    }
 	    return LONG_MAX;
 	}
 	ival = ival << 8;
 	ival |= *cp++;
 	--len;
     }
+    if (negative && ival && (overflow & ival) == 0) {
+	mask <<=  ((originalLength  * 8) - 1);
+	ival &= ~mask;
+	ival -= mask;
+    }
     return ival;
 }
 
 /*
 ** Convert a der encoded *unsigned* integer into a machine integral value.
 ** If an overflow occurs, sets error code and returns max.
 */
 unsigned long