Bug 1221620 - DER integer decoding update, r=mt
authorFranziskus Kiefer <franziskuskiefer@gmail.com>
Mon, 18 Jan 2016 13:08:44 +0100
changeset 11826 8d78a5ae260a7271c8dfb5b38c496f1800d74c34
parent 11825 7bbcda0719dc50278bc16d3eb9fc41f24965b9dd
child 11827 99beadb152436d9243c57e87cf9931a99fe1c0b5
push id941
push usermartin.thomson@gmail.com
push dateMon, 01 Feb 2016 00:51:26 +0000
reviewersmt
bugs1221620
Bug 1221620 - DER integer decoding update, r=mt
lib/util/dersubr.c
--- a/lib/util/dersubr.c
+++ b/lib/util/dersubr.c
@@ -174,43 +174,51 @@ DER_SetUInteger(PLArenaPool *arena, SECI
 /*
 ** Convert a der encoded *signed* integer into a machine integral value.
 ** If an underflow/overflow occurs, sets error code and returns min/max.
 */
 long
 DER_GetInteger(const SECItem *it)
 {
     long ival = 0;
-    unsigned len = it->len;
+    PRBool negative = PR_FALSE;
+    unsigned int len = it->len;
+    unsigned int originalLength = len;
     unsigned char *cp = it->data;
     unsigned long overflow = 0x1ffUL << (((sizeof(ival) - 1) * 8) - 1);
-    unsigned long ofloinit;
+    unsigned long mask = 1;
 
     PORT_Assert(len);
     if (!len) {
 	PORT_SetError(SEC_ERROR_INPUT_LEN);
 	return 0;
     }
 
-    if (*cp & 0x80)
-    	ival = -1L;
-    ofloinit = ival & overflow;
+    if (*cp & 0x80) {
+	negative = PR_TRUE;
+	overflow <<= 1;
+    }
 
     while (len) {
-	if ((ival & overflow) != ofloinit) {
+	if ((ival & overflow) != 0) {
 	    PORT_SetError(SEC_ERROR_BAD_DER);
-	    if (ival < 0) {
+	    if (negative) {
 		return LONG_MIN;
 	    }
 	    return LONG_MAX;
 	}
 	ival = ival << 8;
 	ival |= *cp++;
 	--len;
     }
+    if (negative && ival && (overflow & ival) == 0) {
+	mask <<=  ((originalLength  * 8) - 1);
+	ival &= ~mask;
+	ival -= mask;
+    }
     return ival;
 }
 
 /*
 ** Convert a der encoded *unsigned* integer into a machine integral value.
 ** If an overflow occurs, sets error code and returns max.
 */
 unsigned long