Bug 1344380 - gtests for b64 bug and some fixes, r=ttaubert NSS_3_21_BRANCH
authorFranziskus Kiefer <franziskuskiefer@gmail.com>
Wed, 05 Apr 2017 16:13:01 +0200
branchNSS_3_21_BRANCH
changeset 13277 99a86619eac9
parent 13276 e126381a3c29
child 13278 390e643fc21a
push id2137
push userfranziskuskiefer@gmail.com
push date2017-04-05 14:31 +0000
reviewersttaubert
bugs1344380
Bug 1344380 - gtests for b64 bug and some fixes, r=ttaubert
lib/util/nssb64d.c
lib/util/nssb64e.c
--- a/lib/util/nssb64d.c
+++ b/lib/util/nssb64d.c
@@ -368,17 +368,17 @@ pl_base64_decode_flush (PLBase64Decoder 
 
 /*
  * The maximum space needed to hold the output of the decoder given
  * input data of length "size".
  */
 static PRUint32
 PL_Base64MaxDecodedLength (PRUint32 size)
 {
-    return ((size * 3) / 4);
+    return size * 0.75;
 }
 
 
 /*
  * A distinct internal creation function for the buffer version to use.
  * (It does not want to specify an output_fn, and we want the normal
  * Create function to require that.)  If more common initialization
  * of the decoding context needs to be done, it should be done *here*.
--- a/lib/util/nssb64e.c
+++ b/lib/util/nssb64e.c
@@ -280,16 +280,21 @@ pl_base64_encode_flush (PLBase64Encoder 
  * line_length bytes (we will add it at nearest lower multiple of 4).
  * There is no trailing CRLF.
  */
 static PRUint32
 PL_Base64MaxEncodedLength (PRUint32 size, PRUint32 line_length)
 {
     PRUint32 tokens, tokens_per_line, full_lines, line_break_chars, remainder;
 
+    /* This is the maximum length we support. */
+    if (size > 0x3fffffff) {
+        return 0;
+    }
+
     tokens = (size + 2) / 3;
 
     if (line_length == 0)
 	return tokens * 4;
 
     if (line_length < 4)	/* too small! */
 	line_length = 4;
 
@@ -456,16 +461,20 @@ PL_Base64EncodeBuffer (const unsigned ch
     PR_ASSERT(srclen > 0);
     if (srclen == 0)
 	return dest;
 
     /*
      * How much space could we possibly need for encoding this input?
      */
     need_length = PL_Base64MaxEncodedLength (srclen, line_length);
+    if (need_length == 0) {
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        return NULL;
+    }
 
     /*
      * Make sure we have at least that much, if output buffer provided.
      */
     if (dest != NULL) {
 	PR_ASSERT(maxdestlen >= need_length);
 	if (maxdestlen < need_length) {
 	    PR_SetError(PR_BUFFER_OVERFLOW_ERROR, 0);
@@ -638,16 +647,20 @@ NSSBase64_EncodeItem (PLArenaPool *arena
 
     PORT_Assert(inItem != NULL && inItem->data != NULL && inItem->len != 0);
     if (inItem == NULL || inItem->data == NULL || inItem->len == 0) {
 	PORT_SetError (SEC_ERROR_INVALID_ARGS);
 	return NULL;
     }
 
     max_out_len = PL_Base64MaxEncodedLength (inItem->len, 64);
+    if (max_out_len == 0) {
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        return NULL;
+    }
 
     if (arenaOpt != NULL)
 	mark = PORT_ArenaMark (arenaOpt);
 
     if (out_string == NULL) {
 	if (arenaOpt != NULL)
 	    out_string = PORT_ArenaAlloc (arenaOpt, max_out_len + 1);
 	else