Bug 534136 Part 6: Properly handle invalid UTF8 string being passed to NS_NewAtom/do_GetAtom. r=jst
authorJonas Sicking <jonas@sicking.cc>
Mon, 08 Mar 2010 07:45:00 -0800
changeset 39104 9cb12f3ff15679669d5f45600a533dec6afa1cd3
parent 39103 62bc77046ac9bbf386f2258766f18bf382fcde36
child 39105 f8dd7b5b02ef6059451f4ebfe379466ed392fdd9
push id12012
push usersicking@mozilla.com
push dateMon, 08 Mar 2010 15:47:04 +0000
treeherdermozilla-central@f8dd7b5b02ef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjst
bugs534136
milestone1.9.3a3pre
Bug 534136 Part 6: Properly handle invalid UTF8 string being passed to NS_NewAtom/do_GetAtom. r=jst
xpcom/ds/nsAtomTable.cpp
xpcom/ds/nsCRT.cpp
xpcom/ds/nsCRT.h
xpcom/tests/TestUTF.cpp
--- a/xpcom/ds/nsAtomTable.cpp
+++ b/xpcom/ds/nsAtomTable.cpp
@@ -102,17 +102,25 @@ struct AtomTableKey
 };
 
 static PLDHashNumber
 AtomTableGetHash(PLDHashTable *table, const void *key)
 {
   const AtomTableKey *k = static_cast<const AtomTableKey*>(key);
 
   if (k->mUTF8String) {
-    return nsCRT::HashCodeAsUTF16(k->mUTF8String, k->mLength);;
+    PRBool err;
+    PRUint32 hash = nsCRT::HashCodeAsUTF16(k->mUTF8String, k->mLength, &err);
+    if (err) {
+      AtomTableKey* mutableKey = const_cast<AtomTableKey*>(k);
+      mutableKey->mUTF8String = nsnull;
+      mutableKey->mLength = 0;
+      hash = 0;
+    }
+    return hash;
   }
 
   return nsCRT::HashCode(k->mUTF16String, k->mLength);
 }
 
 static PRBool
 AtomTableMatchKey(PLDHashTable *table, const PLDHashEntryHdr *entry,
                   const void *key)
--- a/xpcom/ds/nsCRT.cpp
+++ b/xpcom/ds/nsCRT.cpp
@@ -248,27 +248,29 @@ PRUint32 nsCRT::HashCode(const PRUnichar
   while ( s < end ) {
     c = *s++;
     ADD_TO_HASHVAL(h, c);
   }
 
   return h;
 }
 
-PRUint32 nsCRT::HashCodeAsUTF16(const char* start, PRUint32 length)
+PRUint32 nsCRT::HashCodeAsUTF16(const char* start, PRUint32 length,
+                                PRBool* err)
 {
   PRUint32 h = 0;
   const char* s = start;
   const char* end = start + length;
 
+  *err = PR_FALSE;
+
   while ( s < end )
     {
-      PRBool err;
-      PRUint32 ucs4 = UTF8CharEnumerator::NextChar(&s, end, &err);
-      if (err) {
+      PRUint32 ucs4 = UTF8CharEnumerator::NextChar(&s, end, err);
+      if (*err) {
 	return 0;
       }
 
       if (ucs4 < PLANE1_BASE) {
         ADD_TO_HASHVAL(h, ucs4);
       }
       else {
         ADD_TO_HASHVAL(h, H_SURROGATE(ucs4));
--- a/xpcom/ds/nsCRT.h
+++ b/xpcom/ds/nsCRT.h
@@ -237,17 +237,18 @@ public:
                            PRUint32* resultingStrLen = nsnull);
 
   // Computes the hashcode for a buffer with a specified length.
   static PRUint32 HashCode(const PRUnichar* str, PRUint32 strLen);
 
   // Computes a hashcode for a length number of UTF8
   // characters. Returns the same hash code as the HashCode method
   // taking a |PRUnichar*| would if the string were converted to UTF16.
-  static PRUint32 HashCodeAsUTF16(const char* start, PRUint32 length);
+  static PRUint32 HashCodeAsUTF16(const char* start, PRUint32 length,
+                                  PRBool* err);
 
   // String to longlong
   static PRInt64 atoll(const char *str);
   
   static char ToUpper(char aChar) { return NS_ToUpper(aChar); }
   static char ToLower(char aChar) { return NS_ToLower(aChar); }
   
   static PRBool IsUpper(char aChar) { return NS_IsUpper(aChar); }
--- a/xpcom/tests/TestUTF.cpp
+++ b/xpcom/tests/TestUTF.cpp
@@ -140,31 +140,37 @@ test_malformed8()
   return PR_TRUE;
 }
 
 PRBool
 test_hashas16()
 {
   for (unsigned int i = 0; i < NS_ARRAY_LENGTH(ValidStrings); ++i) {
     nsDependentCString str8(ValidStrings[i].m8);
+    PRBool err;
     if (nsCRT::HashCode(ValidStrings[i].m16) !=
-        nsCRT::HashCodeAsUTF16(str8.get(), str8.Length()))
+        nsCRT::HashCodeAsUTF16(str8.get(), str8.Length(), &err) ||
+        err)
       return PR_FALSE;
   }
 
   for (unsigned int i = 0; i < NS_ARRAY_LENGTH(Invalid8Strings); ++i) {
     nsDependentCString str8(Invalid8Strings[i].m8);
+    PRBool err;
     if (nsCRT::HashCode(Invalid8Strings[i].m16) !=
-        nsCRT::HashCodeAsUTF16(str8.get(), str8.Length()))
+        nsCRT::HashCodeAsUTF16(str8.get(), str8.Length(), &err) ||
+        err)
       return PR_FALSE;
   }
 
   for (unsigned int i = 0; i < NS_ARRAY_LENGTH(Malformed8Strings); ++i) {
     nsDependentCString str8(Malformed8Strings[i]);
-    if (nsCRT::HashCodeAsUTF16(str8.get(), str8.Length()) != 0)
+    PRBool err;
+    if (nsCRT::HashCodeAsUTF16(str8.get(), str8.Length(), &err) != 0 ||
+        !err)
       return PR_FALSE;
   }
 
   return PR_TRUE;
 }
 
 typedef PRBool (*TestFunc)();