Bug 1539788 - UBSAN fixup for 128b counter. r=mt,jcj
authorKevin Jacobs <kjacobs@mozilla.com>
Wed, 14 Aug 2019 15:35:35 +0000
changeset 15253 9d1f5e71773d4e3146524096d74cb96c8df51abe
parent 15252 dfd6996fe7425eb0437346d11a01082f16fcfe34
child 15254 b03ff661491e412bf42d15d2ebe6f6ea7d9e4b9c
push id3464
push userjjones@mozilla.com
push dateWed, 14 Aug 2019 15:36:52 +0000
reviewersmt, jcj
bugs1539788
Bug 1539788 - UBSAN fixup for 128b counter. r=mt,jcj Differential Revision: https://phabricator.services.mozilla.com/D41884
lib/freebl/ctr.c
--- a/lib/freebl/ctr.c
+++ b/lib/freebl/ctr.c
@@ -124,17 +124,18 @@ CTR_Update(CTRContext *ctr, unsigned cha
            unsigned int *outlen, unsigned int maxout,
            const unsigned char *inbuf, unsigned int inlen,
            unsigned int blocksize)
 {
     unsigned int tmp;
     SECStatus rv;
 
     // Limit block count to 2^counterBits - 2
-    if (inlen > ((1ULL << ctr->counterBits) - 2) * AES_BLOCK_SIZE) {
+    if (ctr->counterBits < (sizeof(unsigned int) * 8) &&
+        inlen > ((1 << ctr->counterBits) - 2) * AES_BLOCK_SIZE) {
         PORT_SetError(SEC_ERROR_INPUT_LEN);
         return SECFailure;
     }
     if (maxout < inlen) {
         *outlen = inlen;
         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
         return SECFailure;
     }
@@ -200,17 +201,18 @@ CTR_Update_HW_AES(CTRContext *ctr, unsig
                   const unsigned char *inbuf, unsigned int inlen,
                   unsigned int blocksize)
 {
     unsigned int fullblocks;
     unsigned int tmp;
     SECStatus rv;
 
     // Limit block count to 2^counterBits - 2
-    if (inlen > ((1ULL << ctr->counterBits) - 2) * AES_BLOCK_SIZE) {
+    if (ctr->counterBits < (sizeof(unsigned int) * 8) &&
+        inlen > ((1 << ctr->counterBits) - 2) * AES_BLOCK_SIZE) {
         PORT_SetError(SEC_ERROR_INPUT_LEN);
         return SECFailure;
     }
     if (maxout < inlen) {
         *outlen = inlen;
         PORT_SetError(SEC_ERROR_OUTPUT_LEN);
         return SECFailure;
     }