remove landed patches
authorDaniel Holbert <dholbert@cs.stanford.edu>
Tue, 04 May 2010 13:44:42 -0700
changeset 1561 13878b706dbd4e25ff932ff301f72d9502b6c061
parent 1560 8319ac6b9ef9a77cfc45cb6e9380725e2f4aa822
child 1562 70a80121927daf521d67d8e4e028bb61cf4736c8
push id1258
push userdholbert@mozilla.com
push dateTue, 04 May 2010 20:44:47 +0000
remove landed patches
series
tokenizer_allowOtherChars
tokenizer_allowSeparatorToBeOptional
tokenizer_extendCstringVersion
tokenizer_fixOldUses
tokenizer_removeEmptylineWhitespace
tokenizer_renameFile
tokenizer_useInSVGAndSMIL
--- a/series
+++ b/series
@@ -1,14 +1,7 @@
 smil_infallibleMisc #+WIP
-tokenizer_renameFile
-tokenizer_allowOtherChars
-tokenizer_allowSeparatorToBeOptional
-tokenizer_fixOldUses
-tokenizer_useInSVGAndSMIL
-tokenizer_extendCstringVersion
-tokenizer_removeEmptylineWhitespace
 smil_541297ChangeAbortToWarning
 smil_541297Fix
 smil_debuggingCode #+debug
 smil_misc #+nits
 smil_cacheTextZoom #+notsure
 smil_alwaysRecomposeDebugging #+debug
deleted file mode 100644
--- a/tokenizer_allowOtherChars
+++ /dev/null
@@ -1,169 +0,0 @@
-From: Daniel Holbert <dholbert@cs.stanford.edu>
-Bug 562310 Part 2: Convert nsCommaSeparatedTokenizer class into a custom-SeparatorChar version, nsCharSeparatedTokenizer. r=roc
-
-diff --git a/content/html/content/src/nsHTMLMediaElement.cpp b/content/html/content/src/nsHTMLMediaElement.cpp
---- a/content/html/content/src/nsHTMLMediaElement.cpp
-+++ b/content/html/content/src/nsHTMLMediaElement.cpp
-@@ -1324,17 +1324,17 @@ static CanPlayStatus GetCanPlay(const ns
-   PRBool expectMoreTokens = PR_FALSE;
-   while (tokenizer.hasMoreTokens()) {
-     const nsSubstring& token = tokenizer.nextToken();
- 
-     if (!CodecListContains(supportedCodecs, token)) {
-       // Totally unsupported codec
-       return CANPLAY_NO;
-     }
--    expectMoreTokens = tokenizer.lastTokenEndedWithComma();
-+    expectMoreTokens = tokenizer.lastTokenEndedWithSeparator();
-   }
-   if (expectMoreTokens) {
-     // Last codec name was empty
-     return CANPLAY_NO;
-   }
-   return result;
- }
- 
-diff --git a/xpcom/ds/nsCharSeparatedTokenizer.h b/xpcom/ds/nsCharSeparatedTokenizer.h
---- a/xpcom/ds/nsCharSeparatedTokenizer.h
-+++ b/xpcom/ds/nsCharSeparatedTokenizer.h
-@@ -37,107 +37,123 @@
-  * ***** END LICENSE BLOCK ***** */
- 
- #ifndef __nsCharSeparatedTokenizer_h
- #define __nsCharSeparatedTokenizer_h
- 
- #include "nsDependentSubstring.h"
- 
- /**
-- * This parses a comma separated string into tokens. Whitespace surrounding
-- * tokens are not treated as part of tokens, however whitespace inside a token
-- * is. If the final token is the empty string it is not returned.
-+ * This parses a SeparatorChar-separated string into tokens.
-+ * Whitespace surrounding tokens is not treated as part of tokens, however
-+ * whitespace inside a token is. If the final token is the empty string, it is
-+ * not returned.
-  *
-- * Some examples:
-+ * Some examples, with SeparatorChar = ',':
-  *
-  * "foo, bar, baz" ->      "foo" "bar" "baz"
-  * "foo,bar,baz" ->        "foo" "bar" "baz"
-  * "foo , bar hi , baz" -> "foo" "bar hi" "baz"
-  * "foo, ,bar,baz" ->      "foo" "" "bar" "baz"
-  * "foo,,bar,baz" ->       "foo" "" "bar" "baz"
-  * "foo,bar,baz," ->       "foo" "bar" "baz"
-  */
--
--class nsCommaSeparatedTokenizer
-+class nsCharSeparatedTokenizer
- {
- public:
--    nsCommaSeparatedTokenizer(const nsSubstring& aSource)
-+    nsCharSeparatedTokenizer(const nsSubstring& aSource,
-+                             PRUnichar aSeparatorChar)
-+        : mLastTokenEndedWithSeparator(PR_FALSE),
-+          mSeparatorChar(aSeparatorChar)
-     {
-         aSource.BeginReading(mIter);
-         aSource.EndReading(mEnd);
- 
-+        // Skip initial whitespace
-         while (mIter != mEnd && isWhitespace(*mIter)) {
-             ++mIter;
-         }
-     }
- 
-     /**
-      * Checks if any more tokens are available.
-      */
-     PRBool hasMoreTokens()
-     {
-         NS_ASSERTION(mIter == mEnd || !isWhitespace(*mIter),
-                      "Should be at beginning of token if there is one");
- 
-         return mIter != mEnd;
-     }
- 
--    PRBool lastTokenEndedWithComma()
-+    PRBool lastTokenEndedWithSeparator()
-     {
--        return mLastTokenEndedWithComma;
-+        return mLastTokenEndedWithSeparator;
-     }
- 
-     /**
-      * Returns the next token.
-      */
-     const nsDependentSubstring nextToken()
-     {
-         nsSubstring::const_char_iterator end = mIter, begin = mIter;
- 
-         NS_ASSERTION(mIter == mEnd || !isWhitespace(*mIter),
-                      "Should be at beginning of token if there is one");
- 
--        // Search until we hit comma or end
--        while (mIter != mEnd && *mIter != ',') {
--          while (mIter != mEnd && !isWhitespace(*mIter) && *mIter != ',') {
-+        // Search until we hit separator or end.
-+        while (mIter != mEnd && *mIter != mSeparatorChar) {
-+          // Skip to end of current word.
-+          while (mIter != mEnd &&
-+                 !isWhitespace(*mIter) && *mIter != mSeparatorChar) {
-               ++mIter;
-           }
-           end = mIter;
- 
-+          // Skip whitespace after current word.
-           while (mIter != mEnd && isWhitespace(*mIter)) {
-               ++mIter;
-           }
-         }
--        mLastTokenEndedWithComma = mIter != mEnd;
-+        mLastTokenEndedWithSeparator = mIter != mEnd;
- 
--        // Skip comma
--        if (mLastTokenEndedWithComma) {
--            NS_ASSERTION(*mIter == ',', "Ended loop too soon");
-+        // Skip separator (and any whitespace after it).
-+        if (mLastTokenEndedWithSeparator) {
-+            NS_ASSERTION(*mIter == mSeparatorChar, "Ended loop too soon");
-             ++mIter;
- 
-             while (mIter != mEnd && isWhitespace(*mIter)) {
-                 ++mIter;
-             }
-         }
-         
-         return Substring(begin, end);
-     }
- 
- private:
-     nsSubstring::const_char_iterator mIter, mEnd;
--    PRPackedBool mLastTokenEndedWithComma;
-+    PRPackedBool mLastTokenEndedWithSeparator;
-+    PRUnichar mSeparatorChar;
- 
-     PRBool isWhitespace(PRUnichar aChar)
-     {
-         return aChar <= ' ' &&
-                (aChar == ' ' || aChar == '\n' ||
-                 aChar == '\r'|| aChar == '\t');
-     }
- };
- 
-+class nsCommaSeparatedTokenizer : public nsCharSeparatedTokenizer
-+{
-+public:
-+    nsCommaSeparatedTokenizer(const nsSubstring& aSource)
-+        : nsCharSeparatedTokenizer(aSource, ',') {}
-+};
-+
-+
- class nsCCommaSeparatedTokenizer
- {
- public:
-     nsCCommaSeparatedTokenizer(const nsCSubstring& aSource)
-     {
-         aSource.BeginReading(mIter);
-         aSource.EndReading(mEnd);
- 
deleted file mode 100644
--- a/tokenizer_allowSeparatorToBeOptional
+++ /dev/null
@@ -1,104 +0,0 @@
-From: Daniel Holbert <dholbert@cs.stanford.edu>
-Bug 562310 Part 3: Extend nsCharSeparatedTokenizer to allow separator to be optional. r=roc
-
-diff --git a/xpcom/ds/nsCharSeparatedTokenizer.h b/xpcom/ds/nsCharSeparatedTokenizer.h
---- a/xpcom/ds/nsCharSeparatedTokenizer.h
-+++ b/xpcom/ds/nsCharSeparatedTokenizer.h
-@@ -54,20 +54,28 @@
-  * "foo , bar hi , baz" -> "foo" "bar hi" "baz"
-  * "foo, ,bar,baz" ->      "foo" "" "bar" "baz"
-  * "foo,,bar,baz" ->       "foo" "" "bar" "baz"
-  * "foo,bar,baz," ->       "foo" "bar" "baz"
-  */
- class nsCharSeparatedTokenizer
- {
- public:
-+    // Flags -- only one for now. If we need more, they should be defined to
-+    // be 1<<1, 1<<2, etc. (They're masks, and aFlags/mFlags are bitfields.)
-+    enum {
-+        SEPARATOR_OPTIONAL = 1
-+    };
-+
-     nsCharSeparatedTokenizer(const nsSubstring& aSource,
--                             PRUnichar aSeparatorChar)
-+                             PRUnichar aSeparatorChar,
-+                             PRUint32  aFlags = 0)
-         : mLastTokenEndedWithSeparator(PR_FALSE),
--          mSeparatorChar(aSeparatorChar)
-+          mSeparatorChar(aSeparatorChar),
-+          mFlags(aFlags)
-     {
-         aSource.BeginReading(mIter);
-         aSource.EndReading(mEnd);
- 
-         // Skip initial whitespace
-         while (mIter != mEnd && isWhitespace(*mIter)) {
-             ++mIter;
-         }
-@@ -94,49 +102,62 @@ public:
-      */
-     const nsDependentSubstring nextToken()
-     {
-         nsSubstring::const_char_iterator end = mIter, begin = mIter;
- 
-         NS_ASSERTION(mIter == mEnd || !isWhitespace(*mIter),
-                      "Should be at beginning of token if there is one");
- 
--        // Search until we hit separator or end.
-+        // Search until we hit separator or end (or whitespace, if separator
-+        // isn't required -- see clause with 'break' below).
-         while (mIter != mEnd && *mIter != mSeparatorChar) {
-           // Skip to end of current word.
-           while (mIter != mEnd &&
-                  !isWhitespace(*mIter) && *mIter != mSeparatorChar) {
-               ++mIter;
-           }
-           end = mIter;
- 
-           // Skip whitespace after current word.
-           while (mIter != mEnd && isWhitespace(*mIter)) {
-               ++mIter;
-           }
-+          if (mFlags & SEPARATOR_OPTIONAL) {
-+            // We've hit (and skipped) whitespace, and that's sufficient to end
-+            // our token, regardless of whether we've reached a SeparatorChar.
-+            break;
-+          } // (else, we'll keep looping until we hit mEnd or SeparatorChar)
-         }
--        mLastTokenEndedWithSeparator = mIter != mEnd;
- 
--        // Skip separator (and any whitespace after it).
-+        mLastTokenEndedWithSeparator = (mIter != mEnd &&
-+                                        *mIter == mSeparatorChar);
-+        NS_ASSERTION((mFlags & SEPARATOR_OPTIONAL) ||
-+                     (mLastTokenEndedWithSeparator == (mIter != mEnd)),
-+                     "If we require a separator and haven't hit the end of "
-+                     "our string, then we shouldn't have left the loop "
-+                     "unless we hit a separator");
-+
-+        // Skip separator (and any whitespace after it), if we're at one.
-         if (mLastTokenEndedWithSeparator) {
--            NS_ASSERTION(*mIter == mSeparatorChar, "Ended loop too soon");
-             ++mIter;
- 
-             while (mIter != mEnd && isWhitespace(*mIter)) {
-                 ++mIter;
-             }
-         }
-         
-         return Substring(begin, end);
-     }
- 
- private:
-     nsSubstring::const_char_iterator mIter, mEnd;
-     PRPackedBool mLastTokenEndedWithSeparator;
-     PRUnichar mSeparatorChar;
-+    PRUint32  mFlags;
- 
-     PRBool isWhitespace(PRUnichar aChar)
-     {
-         return aChar <= ' ' &&
-                (aChar == ' ' || aChar == '\n' ||
-                 aChar == '\r'|| aChar == '\t');
-     }
- };
deleted file mode 100644
--- a/tokenizer_extendCstringVersion
+++ /dev/null
@@ -1,186 +0,0 @@
-From: Daniel Holbert <dholbert@cs.stanford.edu>
-Bug 562310 Part 6: Convert C-string version (nsCCommaSeparatedTokenizer) into generic nsCCharSeparatedTokenizer class, too. r=roc
-
-diff --git a/content/base/src/nsCrossSiteListenerProxy.cpp b/content/base/src/nsCrossSiteListenerProxy.cpp
---- a/content/base/src/nsCrossSiteListenerProxy.cpp
-+++ b/content/base/src/nsCrossSiteListenerProxy.cpp
-@@ -286,17 +286,17 @@ nsCrossSiteListenerProxy::CheckRequestAp
-   if (mIsPreflight) {
-     nsCAutoString headerVal;
-     // The "Access-Control-Allow-Methods" header contains a comma separated
-     // list of method names.
-     http->GetResponseHeader(NS_LITERAL_CSTRING("Access-Control-Allow-Methods"),
-                             headerVal);
-     PRBool foundMethod = mPreflightMethod.EqualsLiteral("GET") ||
-       mPreflightMethod.EqualsLiteral("POST");
--    nsCCommaSeparatedTokenizer methodTokens(headerVal);
-+    nsCCharSeparatedTokenizer methodTokens(headerVal, ',');
-     while(methodTokens.hasMoreTokens()) {
-       const nsDependentCSubstring& method = methodTokens.nextToken();
-       if (method.IsEmpty()) {
-         continue;
-       }
-       if (!IsValidHTTPToken(method)) {
-         return NS_ERROR_DOM_BAD_URI;
-       }
-@@ -305,17 +305,17 @@ nsCrossSiteListenerProxy::CheckRequestAp
-     NS_ENSURE_TRUE(foundMethod, NS_ERROR_DOM_BAD_URI);
- 
-     // The "Access-Control-Allow-Headers" header contains a comma separated
-     // list of header names.
-     headerVal.Truncate();
-     http->GetResponseHeader(NS_LITERAL_CSTRING("Access-Control-Allow-Headers"),
-                             headerVal);
-     nsTArray<nsCString> headers;
--    nsCCommaSeparatedTokenizer headerTokens(headerVal);
-+    nsCCharSeparatedTokenizer headerTokens(headerVal, ',');
-     while(headerTokens.hasMoreTokens()) {
-       const nsDependentCSubstring& header = headerTokens.nextToken();
-       if (header.IsEmpty()) {
-         continue;
-       }
-       if (!IsValidHTTPToken(header)) {
-         return NS_ERROR_DOM_BAD_URI;
-       }
-diff --git a/content/base/src/nsXMLHttpRequest.cpp b/content/base/src/nsXMLHttpRequest.cpp
---- a/content/base/src/nsXMLHttpRequest.cpp
-+++ b/content/base/src/nsXMLHttpRequest.cpp
-@@ -377,17 +377,17 @@ nsACProxyListener::AddResultToCache(nsIR
-     return;
-   }
- 
-   // The "Access-Control-Allow-Methods" header contains a comma separated
-   // list of method names.
-   http->GetResponseHeader(NS_LITERAL_CSTRING("Access-Control-Allow-Methods"),
-                           headerVal);
- 
--  nsCCommaSeparatedTokenizer methods(headerVal);
-+  nsCCharSeparatedTokenizer methods(headerVal, ',');
-   while(methods.hasMoreTokens()) {
-     const nsDependentCSubstring& method = methods.nextToken();
-     if (method.IsEmpty()) {
-       continue;
-     }
-     PRUint32 i;
-     for (i = 0; i < entry->mMethods.Length(); ++i) {
-       if (entry->mMethods[i].token.Equals(method)) {
-@@ -407,17 +407,17 @@ nsACProxyListener::AddResultToCache(nsIR
-     }
-   }
- 
-   // The "Access-Control-Allow-Headers" header contains a comma separated
-   // list of method names.
-   http->GetResponseHeader(NS_LITERAL_CSTRING("Access-Control-Allow-Headers"),
-                           headerVal);
- 
--  nsCCommaSeparatedTokenizer headers(headerVal);
-+  nsCCharSeparatedTokenizer headers(headerVal, ',');
-   while(headers.hasMoreTokens()) {
-     const nsDependentCSubstring& header = headers.nextToken();
-     if (header.IsEmpty()) {
-       continue;
-     }
-     PRUint32 i;
-     for (i = 0; i < entry->mHeaders.Length(); ++i) {
-       if (entry->mHeaders[i].token.Equals(header)) {
-diff --git a/security/manager/ssl/src/nsNSSIOLayer.cpp b/security/manager/ssl/src/nsNSSIOLayer.cpp
---- a/security/manager/ssl/src/nsNSSIOLayer.cpp
-+++ b/security/manager/ssl/src/nsNSSIOLayer.cpp
-@@ -2258,17 +2258,17 @@ void nsSSLIOLayerHelpers::setRenegoUnres
-   }
- 
-   mRenegoUnrestrictedSites = new nsCStringHashSet();
-   if (!mRenegoUnrestrictedSites)
-     return;
-   
-   mRenegoUnrestrictedSites->Init(1);
-   
--  nsCCommaSeparatedTokenizer toker(str);
-+  nsCCharSeparatedTokenizer toker(str, ',');
- 
-   while (toker.hasMoreTokens()) {
-     const nsCSubstring &host = toker.nextToken();
-     if (!host.IsEmpty()) {
-       mRenegoUnrestrictedSites->Put(host);
-     }
-   }
- }
-diff --git a/xpcom/ds/nsCharSeparatedTokenizer.h b/xpcom/ds/nsCharSeparatedTokenizer.h
---- a/xpcom/ds/nsCharSeparatedTokenizer.h
-+++ b/xpcom/ds/nsCharSeparatedTokenizer.h
-@@ -157,20 +157,22 @@ private:
-     PRBool isWhitespace(PRUnichar aChar)
-     {
-         return aChar <= ' ' &&
-                (aChar == ' ' || aChar == '\n' ||
-                 aChar == '\r'|| aChar == '\t');
-     }
- };
- 
--class nsCCommaSeparatedTokenizer
-+class nsCCharSeparatedTokenizer
- {
- public:
--    nsCCommaSeparatedTokenizer(const nsCSubstring& aSource)
-+    nsCCharSeparatedTokenizer(const nsCSubstring& aSource,
-+                              char aSeparatorChar)
-+        : mSeparatorChar(aSeparatorChar)
-     {
-         aSource.BeginReading(mIter);
-         aSource.EndReading(mEnd);
- 
-         while (mIter != mEnd && isWhitespace(*mIter)) {
-             ++mIter;
-         }
-     }
-@@ -185,43 +187,45 @@ public:
- 
-     /**
-      * Returns the next token.
-      */
-     const nsDependentCSubstring nextToken()
-     {
-         nsCSubstring::const_char_iterator end = mIter, begin = mIter;
- 
--        // Search until we hit comma or end
--        while (mIter != mEnd && *mIter != ',') {
--          while (mIter != mEnd && !isWhitespace(*mIter) && *mIter != ',') {
-+        // Search until we hit separator or end.
-+        while (mIter != mEnd && *mIter != mSeparatorChar) {
-+          while (mIter != mEnd &&
-+                 !isWhitespace(*mIter) && *mIter != mSeparatorChar) {
-               ++mIter;
-           }
-           end = mIter;
- 
-           while (mIter != mEnd && isWhitespace(*mIter)) {
-               ++mIter;
-           }
-         }
-         
--        // Skip comma
-+        // Skip separator (and any whitespace after it).
-         if (mIter != mEnd) {
--            NS_ASSERTION(*mIter == ',', "Ended loop too soon");
-+            NS_ASSERTION(*mIter == mSeparatorChar, "Ended loop too soon");
-             ++mIter;
- 
-             while (mIter != mEnd && isWhitespace(*mIter)) {
-                 ++mIter;
-             }
-         }
-         
-         return Substring(begin, end);
-     }
- 
- private:
-     nsCSubstring::const_char_iterator mIter, mEnd;
-+    char mSeparatorChar;
- 
-     PRBool isWhitespace(unsigned char aChar)
-     {
-         return aChar <= ' ' &&
-                (aChar == ' ' || aChar == '\n' ||
-                 aChar == '\r'|| aChar == '\t');
-     }
- };
deleted file mode 100644
--- a/tokenizer_fixOldUses
+++ /dev/null
@@ -1,105 +0,0 @@
-From: Daniel Holbert <dholbert@cs.stanford.edu>
-Bug 562310 Part 4: Remove nsCommaSeparatedTokenizer typedef, and replace its uses with nsCharSeparatedTokenizer(','). r=roc
-
-diff --git a/content/html/content/src/nsHTMLMediaElement.cpp b/content/html/content/src/nsHTMLMediaElement.cpp
---- a/content/html/content/src/nsHTMLMediaElement.cpp
-+++ b/content/html/content/src/nsHTMLMediaElement.cpp
-@@ -1315,17 +1315,17 @@ static CanPlayStatus GetCanPlay(const ns
-   rv = parser.GetParameter("codecs", codecs);
-   if (NS_FAILED(rv))
-     // Parameter not found or whatever
-     return CANPLAY_MAYBE;
- 
-   CanPlayStatus result = CANPLAY_YES;
-   // See http://www.rfc-editor.org/rfc/rfc4281.txt for the description
-   // of the 'codecs' parameter
--  nsCommaSeparatedTokenizer tokenizer(codecs);
-+  nsCharSeparatedTokenizer tokenizer(codecs, ',');
-   PRBool expectMoreTokens = PR_FALSE;
-   while (tokenizer.hasMoreTokens()) {
-     const nsSubstring& token = tokenizer.nextToken();
- 
-     if (!CodecListContains(supportedCodecs, token)) {
-       // Totally unsupported codec
-       return CANPLAY_NO;
-     }
-diff --git a/content/svg/content/src/nsSVGFeatures.cpp b/content/svg/content/src/nsSVGFeatures.cpp
---- a/content/svg/content/src/nsSVGFeatures.cpp
-+++ b/content/svg/content/src/nsSVGFeatures.cpp
-@@ -108,21 +108,21 @@ nsSVGFeatures::HaveExtensions(const nsSu
- }
- 
- /*static*/ PRBool
- nsSVGFeatures::MatchesLanguagePreferences(const nsSubstring& aAttribute,
-                                           const nsSubstring& aAcceptLangs) 
- {
-   const nsDefaultStringComparator defaultComparator;
- 
--  nsCommaSeparatedTokenizer attributeTokenizer(aAttribute);
-+  nsCharSeparatedTokenizer attributeTokenizer(aAttribute, ',');
- 
-   while (attributeTokenizer.hasMoreTokens()) {
-     const nsSubstring &attributeToken = attributeTokenizer.nextToken();
--    nsCommaSeparatedTokenizer languageTokenizer(aAcceptLangs);
-+    nsCharSeparatedTokenizer languageTokenizer(aAcceptLangs, ',');
-     while (languageTokenizer.hasMoreTokens()) {
-       if (nsStyleUtil::DashMatchCompare(attributeToken,
-                                         languageTokenizer.nextToken(),
-                                         defaultComparator)) {
-         return PR_TRUE;
-       }
-     }
-   }
-@@ -130,22 +130,22 @@ nsSVGFeatures::MatchesLanguagePreference
- }
- 
- /*static*/ PRInt32
- nsSVGFeatures::GetBestLanguagePreferenceRank(const nsSubstring& aAttribute,
-                                              const nsSubstring& aAcceptLangs) 
- {
-   const nsDefaultStringComparator defaultComparator;
- 
--  nsCommaSeparatedTokenizer attributeTokenizer(aAttribute);
-+  nsCharSeparatedTokenizer attributeTokenizer(aAttribute, ',');
-   PRInt32 lowestRank = -1;
- 
-   while (attributeTokenizer.hasMoreTokens()) {
-     const nsSubstring &attributeToken = attributeTokenizer.nextToken();
--    nsCommaSeparatedTokenizer languageTokenizer(aAcceptLangs);
-+    nsCharSeparatedTokenizer languageTokenizer(aAcceptLangs, ',');
-     PRInt32 index = 0;
-     while (languageTokenizer.hasMoreTokens()) {
-       const nsSubstring &languageToken = languageTokenizer.nextToken();
-       PRBool exactMatch = (languageToken == attributeToken);
-       PRBool prefixOnlyMatch =
-         !exactMatch &&
-         nsStyleUtil::DashMatchCompare(attributeToken,
-                                       languageTokenizer.nextToken(),
-diff --git a/xpcom/ds/nsCharSeparatedTokenizer.h b/xpcom/ds/nsCharSeparatedTokenizer.h
---- a/xpcom/ds/nsCharSeparatedTokenizer.h
-+++ b/xpcom/ds/nsCharSeparatedTokenizer.h
-@@ -157,24 +157,16 @@ private:
-     PRBool isWhitespace(PRUnichar aChar)
-     {
-         return aChar <= ' ' &&
-                (aChar == ' ' || aChar == '\n' ||
-                 aChar == '\r'|| aChar == '\t');
-     }
- };
- 
--class nsCommaSeparatedTokenizer : public nsCharSeparatedTokenizer
--{
--public:
--    nsCommaSeparatedTokenizer(const nsSubstring& aSource)
--        : nsCharSeparatedTokenizer(aSource, ',') {}
--};
--
--
- class nsCCommaSeparatedTokenizer
- {
- public:
-     nsCCommaSeparatedTokenizer(const nsCSubstring& aSource)
-     {
-         aSource.BeginReading(mIter);
-         aSource.EndReading(mEnd);
- 
deleted file mode 100644
--- a/tokenizer_removeEmptylineWhitespace
+++ /dev/null
@@ -1,55 +0,0 @@
-From: Daniel Holbert <dholbert@cs.stanford.edu>
-Bug 562310 followup: remove whitespace on empty lines, in nsCharSeparatedTokenizer.h. (whitespace-only changes)
-
-diff --git a/xpcom/ds/nsCharSeparatedTokenizer.h b/xpcom/ds/nsCharSeparatedTokenizer.h
---- a/xpcom/ds/nsCharSeparatedTokenizer.h
-+++ b/xpcom/ds/nsCharSeparatedTokenizer.h
-@@ -139,17 +139,17 @@ public:
-         // Skip separator (and any whitespace after it), if we're at one.
-         if (mLastTokenEndedWithSeparator) {
-             ++mIter;
- 
-             while (mIter != mEnd && isWhitespace(*mIter)) {
-                 ++mIter;
-             }
-         }
--        
-+
-         return Substring(begin, end);
-     }
- 
- private:
-     nsSubstring::const_char_iterator mIter, mEnd;
-     PRPackedBool mLastTokenEndedWithSeparator;
-     PRUnichar mSeparatorChar;
-     PRUint32  mFlags;
-@@ -199,27 +199,27 @@ public:
-               ++mIter;
-           }
-           end = mIter;
- 
-           while (mIter != mEnd && isWhitespace(*mIter)) {
-               ++mIter;
-           }
-         }
--        
-+
-         // Skip separator (and any whitespace after it).
-         if (mIter != mEnd) {
-             NS_ASSERTION(*mIter == mSeparatorChar, "Ended loop too soon");
-             ++mIter;
- 
-             while (mIter != mEnd && isWhitespace(*mIter)) {
-                 ++mIter;
-             }
-         }
--        
-+
-         return Substring(begin, end);
-     }
- 
- private:
-     nsCSubstring::const_char_iterator mIter, mEnd;
-     char mSeparatorChar;
- 
-     PRBool isWhitespace(unsigned char aChar)
deleted file mode 100644
--- a/tokenizer_renameFile
+++ /dev/null
@@ -1,172 +0,0 @@
-From: Daniel Holbert <dholbert@cs.stanford.edu>
-Bug 562310 Part 1: Rename file nsCommaSeparatedTokenizer.h to nsCharSeparatedTokenizer.h (and update #includes). r=roc
-
-diff --git a/content/base/src/nsCrossSiteListenerProxy.cpp b/content/base/src/nsCrossSiteListenerProxy.cpp
---- a/content/base/src/nsCrossSiteListenerProxy.cpp
-+++ b/content/base/src/nsCrossSiteListenerProxy.cpp
-@@ -47,17 +47,17 @@
- #include "nsICharsetAlias.h"
- #include "nsMimeTypes.h"
- #include "nsIStreamConverterService.h"
- #include "nsStringStream.h"
- #include "nsParserUtils.h"
- #include "nsGkAtoms.h"
- #include "nsWhitespaceTokenizer.h"
- #include "nsIChannelEventSink.h"
--#include "nsCommaSeparatedTokenizer.h"
-+#include "nsCharSeparatedTokenizer.h"
- #include "nsXMLHttpRequest.h"
- 
- static PRBool gDisableCORS = PR_FALSE;
- static PRBool gDisableCORSPrivateData = PR_FALSE;
- 
- class nsChannelCanceller
- {
- public:
-diff --git a/content/base/src/nsXMLHttpRequest.cpp b/content/base/src/nsXMLHttpRequest.cpp
---- a/content/base/src/nsXMLHttpRequest.cpp
-+++ b/content/base/src/nsXMLHttpRequest.cpp
-@@ -89,17 +89,17 @@
- #include "nsDOMError.h"
- #include "nsIHTMLDocument.h"
- #include "nsIDOM3Document.h"
- #include "nsIMultiPartChannel.h"
- #include "nsIScriptObjectPrincipal.h"
- #include "nsIStorageStream.h"
- #include "nsIPromptFactory.h"
- #include "nsIWindowWatcher.h"
--#include "nsCommaSeparatedTokenizer.h"
-+#include "nsCharSeparatedTokenizer.h"
- #include "nsIConsoleService.h"
- #include "nsIChannelPolicy.h"
- #include "nsChannelPolicy.h"
- #include "nsIContentSecurityPolicy.h"
- 
- #define LOAD_STR "load"
- #define ERROR_STR "error"
- #define ABORT_STR "abort"
-diff --git a/content/html/content/src/nsHTMLMediaElement.cpp b/content/html/content/src/nsHTMLMediaElement.cpp
---- a/content/html/content/src/nsHTMLMediaElement.cpp
-+++ b/content/html/content/src/nsHTMLMediaElement.cpp
-@@ -66,17 +66,17 @@
- #include "nsIRenderingContext.h"
- #include "nsITimer.h"
- 
- #include "nsEventDispatcher.h"
- #include "nsIDOMDocumentEvent.h"
- #include "nsIDOMProgressEvent.h"
- #include "nsHTMLMediaError.h"
- #include "nsICategoryManager.h"
--#include "nsCommaSeparatedTokenizer.h"
-+#include "nsCharSeparatedTokenizer.h"
- #include "nsMediaStream.h"
- 
- #include "nsIDOMHTMLVideoElement.h"
- #include "nsIContentPolicy.h"
- #include "nsContentPolicyUtils.h"
- #include "nsContentErrors.h"
- #include "nsCrossSiteListenerProxy.h"
- #include "nsCycleCollectionParticipant.h"
-diff --git a/content/svg/content/src/nsSVGFeatures.cpp b/content/svg/content/src/nsSVGFeatures.cpp
---- a/content/svg/content/src/nsSVGFeatures.cpp
-+++ b/content/svg/content/src/nsSVGFeatures.cpp
-@@ -46,17 +46,17 @@
-  *   http://www.w3.org/TR/SVG11/struct.html#ConditionalProcessing
-  */
- 
- #include "nsSVGFeatures.h"
- #include "nsGkAtoms.h"
- #include "nsIContent.h"
- #include "nsContentUtils.h"
- #include "nsWhitespaceTokenizer.h"
--#include "nsCommaSeparatedTokenizer.h"
-+#include "nsCharSeparatedTokenizer.h"
- #include "nsStyleUtil.h"
- #include "nsSVGUtils.h"
- 
- /*static*/ PRBool
- nsSVGFeatures::HaveFeature(const nsAString& aFeature)
- {
-   if (!NS_SVGEnabled()) {
-     return PR_FALSE;
-diff --git a/security/manager/ssl/src/nsNSSIOLayer.cpp b/security/manager/ssl/src/nsNSSIOLayer.cpp
---- a/security/manager/ssl/src/nsNSSIOLayer.cpp
-+++ b/security/manager/ssl/src/nsNSSIOLayer.cpp
-@@ -82,17 +82,17 @@
- #include "nsThreadUtils.h"
- #include "nsIDocShell.h"
- #include "nsIDocShellTreeItem.h"
- #include "nsISecureBrowserUI.h"
- #include "nsProxyRelease.h"
- #include "nsIClassInfoImpl.h"
- #include "nsIProgrammingLanguage.h"
- #include "nsIArray.h"
--#include "nsCommaSeparatedTokenizer.h"
-+#include "nsCharSeparatedTokenizer.h"
- 
- #include "ssl.h"
- #include "secerr.h"
- #include "sslerr.h"
- #include "secder.h"
- #include "secasn1.h"
- #include "certdb.h"
- #include "cert.h"
-diff --git a/xpcom/ds/Makefile.in b/xpcom/ds/Makefile.in
---- a/xpcom/ds/Makefile.in
-+++ b/xpcom/ds/Makefile.in
-@@ -108,17 +108,17 @@ EXPORTS		= \
- 		nsStaticAtom.h \
- 		nsSupportsArray.h \
- 		nsSupportsPrimitives.h \
- 		nsTime.h \
- 		nsVariant.h \
- 		nsStringEnumerator.h \
- 		nsHashPropertyBag.h \
- 		nsWhitespaceTokenizer.h \
--		nsCommaSeparatedTokenizer.h \
-+		nsCharSeparatedTokenizer.h \
- 		$(NULL)			
- 
- XPIDLSRCS	= \
- 		nsIAtom.idl \
- 		nsIAtomService.idl \
- 		nsICollection.idl \
- 		nsIEnumerator.idl \
- 		nsIINIParser.idl \
-diff --git a/xpcom/ds/nsCommaSeparatedTokenizer.h b/xpcom/ds/nsCharSeparatedTokenizer.h
-rename from xpcom/ds/nsCommaSeparatedTokenizer.h
-rename to xpcom/ds/nsCharSeparatedTokenizer.h
---- a/xpcom/ds/nsCommaSeparatedTokenizer.h
-+++ b/xpcom/ds/nsCharSeparatedTokenizer.h
-@@ -31,18 +31,18 @@
-  * use your version of this file under the terms of the MPL, indicate your
-  * decision by deleting the provisions above and replace them with the notice
-  * and other provisions required by the GPL or the LGPL. If you do not delete
-  * the provisions above, a recipient may use your version of this file under
-  * the terms of any one of the MPL, the GPL or the LGPL.
-  *
-  * ***** END LICENSE BLOCK ***** */
- 
--#ifndef __nsCommaSeparatedTokenizer_h
--#define __nsCommaSeparatedTokenizer_h
-+#ifndef __nsCharSeparatedTokenizer_h
-+#define __nsCharSeparatedTokenizer_h
- 
- #include "nsDependentSubstring.h"
- 
- /**
-  * This parses a comma separated string into tokens. Whitespace surrounding
-  * tokens are not treated as part of tokens, however whitespace inside a token
-  * is. If the final token is the empty string it is not returned.
-  *
-@@ -192,9 +192,9 @@ private:
-     PRBool isWhitespace(unsigned char aChar)
-     {
-         return aChar <= ' ' &&
-                (aChar == ' ' || aChar == '\n' ||
-                 aChar == '\r'|| aChar == '\t');
-     }
- };
- 
--#endif /* __nsWhitespaceTokenizer_h */
-+#endif /* __nsCharSeparatedTokenizer_h */
deleted file mode 100644
--- a/tokenizer_useInSVGAndSMIL
+++ /dev/null
@@ -1,883 +0,0 @@
-From: Daniel Holbert <dholbert@cs.stanford.edu>
-Bug 562310 Part 5: Use nsCharSeparatedTokenizer to simplify/fix existing parsing code in SVG and SMIL. r=roc
-
-diff --git a/content/smil/nsSMILParserUtils.cpp b/content/smil/nsSMILParserUtils.cpp
---- a/content/smil/nsSMILParserUtils.cpp
-+++ b/content/smil/nsSMILParserUtils.cpp
-@@ -43,16 +43,17 @@
- #include "nsSMILTypes.h"
- #include "nsSMILRepeatCount.h"
- #include "nsContentUtils.h"
- #include "nsString.h"
- #include "prdtoa.h"
- #include "nsCRT.h"
- #include "nsCOMPtr.h"
- #include "prlong.h"
-+#include "nsCharSeparatedTokenizer.h"
- 
- //------------------------------------------------------------------------------
- // Helper functions and Constants
- 
- namespace {
- 
- const PRUint32 MSEC_PER_SEC  = 1000;
- const PRUint32 MSEC_PER_MIN  = 1000 * 60;
-@@ -590,55 +591,34 @@ nsSMILParserUtils::ParseValues(const nsA
-                               &aValuesArray, &aCanCache);
-   return ParseValuesGeneric(aSpec, valueParser);
- }
- 
- nsresult
- nsSMILParserUtils::ParseValuesGeneric(const nsAString& aSpec,
-                                       GenericValueParser& aParser)
- {
--  nsresult rv = NS_ERROR_FAILURE;
--
--  const PRUnichar* start = aSpec.BeginReading();
--  const PRUnichar* end = aSpec.EndReading();
--  const PRUnichar* substrEnd = nsnull;
--  const PRUnichar* next = nsnull;
--
--  while (start != end) {
--    rv = NS_ERROR_FAILURE;
--
--    SkipBeginWsp(start, end);
--
--    if (start == end || *start == ';')
--      break;
--
--    substrEnd = start;
--
--    while (substrEnd != end && *substrEnd != ';') {
--      ++substrEnd;
--    }
--
--    next = substrEnd;
--    if (*substrEnd == ';') {
--      ++next;
--      if (next == end)
--        break;
--    }
--
--    while (substrEnd != start && NS_IS_SPACE(*(substrEnd-1)))
--      --substrEnd;
--
--    rv = aParser.Parse(Substring(start, substrEnd));
--    if (NS_FAILED(rv))
--      break;
--
--    start = next;
-+  nsCharSeparatedTokenizer tokenizer(aSpec, ';');
-+  if (!tokenizer.hasMoreTokens()) { // Empty list
-+    return NS_ERROR_FAILURE;
-   }
- 
--  return rv;
-+  while (tokenizer.hasMoreTokens()) {
-+    nsresult rv = aParser.Parse(tokenizer.nextToken());
-+    if (NS_FAILED(rv)) {
-+      return NS_ERROR_FAILURE;
-+    }
-+  }
-+
-+  // Disallow ;-terminated values lists.
-+  if (tokenizer.lastTokenEndedWithSeparator()) {
-+    return NS_ERROR_FAILURE;
-+  }
-+
-+  return NS_OK;
- }
- 
- nsresult
- nsSMILParserUtils::ParseRepeatCount(const nsAString& aSpec,
-                                     nsSMILRepeatCount& aResult)
- {
-   nsresult rv = NS_OK;
- 
-diff --git a/content/smil/test/smilAnimateMotionValueLists.js b/content/smil/test/smilAnimateMotionValueLists.js
---- a/content/smil/test/smilAnimateMotionValueLists.js
-+++ b/content/smil/test/smilAnimateMotionValueLists.js
-@@ -43,24 +43,26 @@ const gValidValues = [
-   "   10   10em  ",
-   "1 2  ; 3,4",
-   "1,2;3,4",
-   "0 0",
-   "0,0",
- ];
- 
- const gInvalidValues = [
-+  ";10 10",
-   "10 10;",  // We treat semicolon-terminated value-lists as failure cases
-+  "10 10;;",
-   "1 2 3",
-   "1 2 3 4",
-   "1,2;3,4 ,",
--  ",", ";", "a", "", " ",
--];
--
--const gInvalidValuesTodo = [
-+  ",", " , ",
-+  ";", " ; ",
-+  "a", " a; ", ";a;",
-+  "", " ",
-   "1,2;3,4,",
-   "1,,2",
-   ",1,2",
- ];
- 
- const gValidRotate = [
-   "10",
-   "20.1",
-@@ -78,36 +80,34 @@ const gInvalidRotate = [
-   "10 rad    ",
-   "aaa",
-   " 10.1 ",
- ];
- 
- const gValidToBy = [
-  "0 0",
-  "1em,2",
-- "50.3em 0.2in"
-+ "50.3em 0.2in",
-+ " 1,2",
-+ "1 2 "
- ];
- 
- const gInvalidToBy = [
-  "0 0 0",
-  "0 0,0",
-  "0,0,0",
-  "1emm 2",
-  "1 2;",
-+ "1 2,",
-  " 1,2 ,",
-  "abc",
-  ",",
-- ""
--];
--
--const gInvalidToByTodo = [
-+ "",
-  "1,,2",
-- "1,2,",
-- " 1,2",
-- "1 2 "
-+ "1,2,"
- ];
- 
- const gValidPath = [
-  "m0 0     L30 30",
-  "M20,20L10    10",
-  "M20,20 L30, 30h20",
-  "m50 50", "M50 50",
-  "m0 0", "M0, 0"
-diff --git a/content/smil/test/test_smilAnimateMotionInvalidValues.xhtml b/content/smil/test/test_smilAnimateMotionInvalidValues.xhtml
---- a/content/smil/test/test_smilAnimateMotionInvalidValues.xhtml
-+++ b/content/smil/test/test_smilAnimateMotionInvalidValues.xhtml
-@@ -140,28 +140,25 @@ function main()
- 
-   // Start out with document paused
-   var svg = SMILUtil.getSVGRoot();
-   ok(svg.animationsPaused(), "should be paused by <svg> load handler");
-   is(svg.getCurrentTime(), 0, "should be paused at 0 in <svg> load handler");
- 
-   testAttr("values", gValidValues, true, false);
-   testAttr("values", gInvalidValues, false, false);
--  testAttr("values", gInvalidValuesTodo, false, true);
- 
-   testAttr("rotate", gValidRotate, true, false);
-   testAttr("rotate", gInvalidRotate, false, false);
- 
-   testAttr("to", gValidToBy, true, false);
-   testAttr("to", gInvalidToBy, false, false);
--  testAttr("to", gInvalidToByTodo, false, true);
- 
-   testAttr("by", gValidToBy, true, false);
-   testAttr("by", gInvalidToBy, false, false);
--  testAttr("by", gInvalidToByTodo, false, true);
- 
-   testAttr("path", gValidPath, true, false);
-   testAttr("path", gInvalidPath, false, false);
- 
-   testMpathElem(gValidPath, true, false);
-   testMpathElem(gInvalidPath, false, false);
- 
-   SimpleTest.finish();
-diff --git a/content/svg/content/src/SVGMotionSMILPathUtils.cpp b/content/svg/content/src/SVGMotionSMILPathUtils.cpp
---- a/content/svg/content/src/SVGMotionSMILPathUtils.cpp
-+++ b/content/svg/content/src/SVGMotionSMILPathUtils.cpp
-@@ -34,17 +34,17 @@
-  * the terms of any one of the MPL, the GPL or the LGPL.
-  *
-  * ***** END LICENSE BLOCK ***** */
- 
- #include "SVGMotionSMILPathUtils.h"
- #include "nsSVGElement.h"
- #include "nsSVGLength2.h"
- #include "nsContentCreatorFunctions.h" // For NS_NewSVGElement
--
-+#include "nsCharSeparatedTokenizer.h"
- 
- namespace mozilla {
- 
- //----------------------------------------------------------------------
- // PathGenerator methods
- 
- // For the dummy 'from' value used in pure by-animation & to-animation
- void
-@@ -112,71 +112,51 @@ already_AddRefed<gfxFlattenedPath>
- SVGMotionSMILPathUtils::PathGenerator::GetResultingPath()
- {
-   return mGfxContext.GetFlattenedPath();
- }
- 
- //----------------------------------------------------------------------
- // Helper / protected methods
- 
--static PRBool
--ParseOneCoordinate(char** aRest, nsSVGLength2& aLengthVal)
--{
--  aLengthVal.Init();
--
--  // Grab token, up to next delimiter
--  // XXXdholbert Ideally we'd like to know if the delimeter we found was a
--  // comma (and if so, fail if we come across any more commas before the next
--  // value). Current behavior will accept mutliple commas in between values,
--  // and that's not technically spec-correct, but it's simpler and it matches
--  // our behavior elsewhere where we use strtok with SVG_COMMA_WSP_DELIM.
--  char* token = nsCRT::strtok(*aRest, SVG_COMMA_WSP_DELIM, aRest);
--  if (!token) {
--    return PR_FALSE;
--  }
--
--  // Parse token into value + unit
--  nsresult rv = aLengthVal.SetBaseValueString(NS_ConvertASCIItoUTF16(token),
--                                              nsnull, PR_FALSE);
--  return NS_SUCCEEDED(rv);
--}
--
- PRBool
- SVGMotionSMILPathUtils::PathGenerator::
-   ParseCoordinatePair(const nsAString& aCoordPairStr,
-                       float& aXVal, float& aYVal)
- {
--  nsSVGLength2 xLength, yLength;
--  
--  char* str = ToNewCString(aCoordPairStr);
--  char* rest = str;
--  PRBool success = PR_FALSE;
-+  nsCharSeparatedTokenizer
-+    tokenizer(aCoordPairStr, ',',
-+              nsCharSeparatedTokenizer::SEPARATOR_OPTIONAL);
- 
--  if (ParseOneCoordinate(&rest, xLength) &&
--      ParseOneCoordinate(&rest, yLength)) {
-+  nsSVGLength2 x, y;
-+  nsresult rv;
- 
--    // Check for any non-whitespace characters remaining at the end.
--    PRBool foundTrailingNonWhitespace = PR_FALSE;
--    while (*rest != '\0') {
--      if (!IsSVGWhitespace(*rest)) {
--        foundTrailingNonWhitespace = PR_TRUE;
--        break;
--      }
--    }
--    if (!foundTrailingNonWhitespace) {
--      success = PR_TRUE;
--    }
-+  if (!tokenizer.hasMoreTokens()) { // No 1st token
-+    return PR_FALSE;
-   }
--  nsMemory::Free(str);
-+  // Parse X value
-+  x.Init();
-+  rv = x.SetBaseValueString(tokenizer.nextToken(), nsnull, PR_FALSE);
-+  if (NS_FAILED(rv) ||                // 1st token failed to parse.
-+      !tokenizer.hasMoreTokens()) {   // No 2nd token.
-+    return PR_FALSE;
-+  }
- 
--  if (success) {
--    aXVal = xLength.GetBaseValue(mSVGElement);
--    aYVal = yLength.GetBaseValue(mSVGElement);
-+  // Parse Y value
-+  y.Init();
-+  rv = y.SetBaseValueString(tokenizer.nextToken(), nsnull, PR_FALSE);
-+  if (NS_FAILED(rv) ||                           // 2nd token failed to parse.
-+      tokenizer.lastTokenEndedWithSeparator() || // Trailing comma.
-+      tokenizer.hasMoreTokens()) {               // More text remains
-+    return PR_FALSE;
-   }
--  return success;
-+
-+  aXVal = x.GetBaseValue(mSVGElement);
-+  aYVal = y.GetBaseValue(mSVGElement);
-+  return PR_TRUE;
- }
- 
- //----------------------------------------------------------------------
- // MotionValueParser methods
- nsresult
- SVGMotionSMILPathUtils::MotionValueParser::
-   Parse(const nsAString& aValueStr)
- {
-diff --git a/content/svg/content/src/SVGMotionSMILPathUtils.h b/content/svg/content/src/SVGMotionSMILPathUtils.h
---- a/content/svg/content/src/SVGMotionSMILPathUtils.h
-+++ b/content/svg/content/src/SVGMotionSMILPathUtils.h
-@@ -84,19 +84,16 @@ public:
-     // Accessor to get the finalized path
-     already_AddRefed<gfxFlattenedPath> GetResultingPath();
- 
-   protected:
-     // Helper methods
-     PRBool ParseCoordinatePair(const nsAString& aStr,
-                                float& aXVal, float& aYVal);
- 
--    PRBool AppendCommand(const nsACString& aCommandStr,
--                         const nsAString&  aCoordPairStr);
--
-     // Member data
-     nsSVGElement* mSVGElement; // context for converting out of relative units
-     gfxContext    mGfxContext;
-     PRPackedBool  mHaveReceivedCommands;
-   };
- 
-   // Class to assist in passing each subcomponent of a |values| attribute to
-   // a PathGenerator, for generating a corresponding gfxFlattenedPath.
-diff --git a/content/svg/content/src/nsSVGPointList.cpp b/content/svg/content/src/nsSVGPointList.cpp
---- a/content/svg/content/src/nsSVGPointList.cpp
-+++ b/content/svg/content/src/nsSVGPointList.cpp
-@@ -34,16 +34,17 @@
-  * the provisions above, a recipient may use your version of this file under
-  * the terms of any one of the MPL, the GPL or the LGPL.
-  *
-  * ***** END LICENSE BLOCK ***** */
- 
- #include "nsSVGPointList.h"
- #include "nsSVGPoint.h"
- #include "nsSVGUtils.h"
-+#include "nsCharSeparatedTokenizer.h"
- #include "nsDOMError.h"
- #include "prdtoa.h"
- #include "nsReadableUtils.h"
- #include "nsTextFormatter.h"
- #include "nsCRT.h"
- #include "nsCOMArray.h"
- #include "nsContentUtils.h"
- 
-@@ -158,66 +159,65 @@ NS_INTERFACE_MAP_END
- 
- 
- //----------------------------------------------------------------------
- // nsISVGValue methods:
- 
- NS_IMETHODIMP
- nsSVGPointList::SetValueString(const nsAString& aValue)
- {
--  nsresult rv = NS_OK;
-+  nsCharSeparatedTokenizer
-+    tokenizer(aValue, ',',
-+              nsCharSeparatedTokenizer::SEPARATOR_OPTIONAL);
-+  nsCOMArray<nsIDOMSVGPoint> points;
- 
--  char* str = ToNewCString(aValue);
--  
--  char* rest = str;
--  char* token1;
--  char* token2;
--  nsCOMArray<nsIDOMSVGPoint> points;
--  
--  while ( (token1 = nsCRT::strtok(rest, SVG_COMMA_WSP_DELIM, &rest)) &&
--          (token2 = nsCRT::strtok(rest, SVG_COMMA_WSP_DELIM, &rest)) ) {
-+  while (tokenizer.hasMoreTokens()) {
-+    // Parse 2 tokens
-+    NS_ConvertUTF16toUTF8 utf8String1(tokenizer.nextToken());
-+    const char *token1 = utf8String1.get();
-+    if (!tokenizer.hasMoreTokens() ||  // No 2nd token.
-+        *token1 == '\0') {             // 1st token is empty string.
-+      return NS_ERROR_DOM_SYNTAX_ERR;
-+    }
-+    NS_ConvertUTF16toUTF8 utf8String2(tokenizer.nextToken());
-+    const char *token2 = utf8String2.get();
-+    if (*token2 == '\0') {             // 2nd token is empty string.
-+      return NS_ERROR_DOM_SYNTAX_ERR;
-+    }
- 
-+    // Convert parsed tokens to float values.
-     char *end;
--    
-     float x = float(PR_strtod(token1, &end));
-     if (*end != '\0' || !NS_FloatIsFinite(x)) {
--      rv = NS_ERROR_DOM_SYNTAX_ERR;
--      break; // parse error
-+      return NS_ERROR_DOM_SYNTAX_ERR;
-     }
-     float y = float(PR_strtod(token2, &end));
-     if (*end != '\0' || !NS_FloatIsFinite(y)) {
--      rv = NS_ERROR_DOM_SYNTAX_ERR;
--      break; // parse error
-+      return NS_ERROR_DOM_SYNTAX_ERR;
-     }
--    
-+
-+    // Build a point from our parsed float values.
-     nsCOMPtr<nsIDOMSVGPoint> point;
--    NS_NewSVGPoint(getter_AddRefs(point), x, y);
--    if (!point) {
--      rv = NS_ERROR_OUT_OF_MEMORY;
--      break;
--    }
-+    NS_NewSVGPoint(getter_AddRefs(point), x, y); // uses infallible 'new'.
-     points.AppendObject(point);
-   }
- 
--  if (token1 || NS_FAILED(rv)) {
--    // there was a parse error or we ran out of memory
--    rv = NS_ERROR_DOM_SYNTAX_ERR;
--  } else {
--    WillModify();
--    ReleasePoints();
--    PRInt32 count = points.Count();
--    for (PRInt32 i=0; i<count; ++i) {
--      AppendElement(points.ObjectAt(i));
--    }
--    DidModify();
-+  if (tokenizer.lastTokenEndedWithSeparator()) { // Reject trailing comma
-+    return NS_ERROR_DOM_SYNTAX_ERR;
-   }
- 
--  nsMemory::Free(str);
--  
--  return rv;
-+  WillModify();
-+  ReleasePoints();
-+  PRInt32 count = points.Count();
-+  for (PRInt32 i = 0; i < count; ++i) {
-+    AppendElement(points.ObjectAt(i));
-+  }
-+  DidModify();
-+
-+  return NS_OK;
- }
- 
- NS_IMETHODIMP
- nsSVGPointList::GetValueString(nsAString& aValue)
- {
-   aValue.Truncate();
- 
-   PRUint32 count = mPoints.Length();
-diff --git a/content/svg/content/src/nsSVGViewBox.cpp b/content/svg/content/src/nsSVGViewBox.cpp
---- a/content/svg/content/src/nsSVGViewBox.cpp
-+++ b/content/svg/content/src/nsSVGViewBox.cpp
-@@ -35,21 +35,23 @@
-  * the terms of any one of the MPL, the GPL or the LGPL.
-  *
-  * ***** END LICENSE BLOCK ***** */
- 
- #include "nsSVGViewBox.h"
- #include "nsSVGUtils.h"
- #include "prdtoa.h"
- #include "nsTextFormatter.h"
-+#include "nsCharSeparatedTokenizer.h"
- #ifdef MOZ_SMIL
- #include "nsSMILValue.h"
- #include "SVGViewBoxSMILType.h"
- #endif // MOZ_SMIL
- 
-+#define NUM_VIEWBOX_COMPONENTS 4
- using namespace mozilla;
- 
- /* Implementation of nsSVGViewBoxRect methods */
- 
- PRBool
- nsSVGViewBoxRect::operator==(const nsSVGViewBoxRect& aOther) const
- {
-   if (&aOther == this)
-@@ -135,47 +137,47 @@ nsSVGViewBox::SetBaseValue(float aX, flo
-     aSVGElement->AnimationNeedsResample();
-   }
- #endif
- }
- 
- static nsresult
- ToSVGViewBoxRect(const nsAString& aStr, nsSVGViewBoxRect *aViewBox)
- {
--  nsresult rv = NS_OK;
--
--  char *str = ToNewUTF8String(aStr);
--
--  char *rest = str;
--  char *token;
--
--  float vals[4];
-+  nsCharSeparatedTokenizer
-+    tokenizer(aStr, ',',
-+              nsCharSeparatedTokenizer::SEPARATOR_OPTIONAL);
-+  float vals[NUM_VIEWBOX_COMPONENTS];
-   PRUint32 i;
--  for (i = 0; i < 4; ++i) {
--    if (!(token = nsCRT::strtok(rest, SVG_COMMA_WSP_DELIM, &rest)))
--      break; // parse error
-+  for (i = 0; i < NUM_VIEWBOX_COMPONENTS && tokenizer.hasMoreTokens(); ++i) {
-+    NS_ConvertUTF16toUTF8 utf8Token(tokenizer.nextToken());
-+    const char *token = utf8Token.get();
-+    if (*token == '\0') {
-+      return NS_ERROR_DOM_SYNTAX_ERR; // empty string (e.g. two commas in a row)
-+    }
- 
-     char *end;
-     vals[i] = float(PR_strtod(token, &end));
--    if (*end != '\0' || !NS_FloatIsFinite(vals[i]))
--      break; // parse error
-+    if (*end != '\0' || !NS_FloatIsFinite(vals[i])) {
-+      return NS_ERROR_DOM_SYNTAX_ERR; // parse error
-+    }
-   }
--  if (i != 4 || (nsCRT::strtok(rest, SVG_COMMA_WSP_DELIM, &rest) != 0)) {
--    // there was a parse error.
--    rv = NS_ERROR_DOM_SYNTAX_ERR;
-+
-+  if (i != NUM_VIEWBOX_COMPONENTS ||              // Too few values.
-+      tokenizer.hasMoreTokens() ||                // Too many values.
-+      tokenizer.lastTokenEndedWithSeparator()) {  // Trailing comma.
-+    return NS_ERROR_DOM_SYNTAX_ERR;
-   } else {
-     aViewBox->x = vals[0];
-     aViewBox->y = vals[1];
-     aViewBox->width = vals[2];
-     aViewBox->height = vals[3];
-   }
- 
--  nsMemory::Free(str);
--
--  return rv;
-+  return NS_OK;
- }
- 
- nsresult
- nsSVGViewBox::SetBaseValueString(const nsAString& aValue,
-                                  nsSVGElement *aSVGElement,
-                                  PRBool aDoSetAttr)
- {
-   nsSVGViewBoxRect viewBox;
-diff --git a/layout/reftests/svg/polyline-points-invalid-01.svg b/layout/reftests/svg/polyline-points-invalid-01.svg
-new file mode 100644
---- /dev/null
-+++ b/layout/reftests/svg/polyline-points-invalid-01.svg
-@@ -0,0 +1,44 @@
-+<svg xmlns="http://www.w3.org/2000/svg" version="1.1">
-+  <title>Testing invalid values for |points| attribute</title>
-+  <style>
-+    polyline {
-+      fill: red;
-+      stroke: orange;
-+      stroke-width: 4;
-+   }
-+  </style>
-+  <rect fill="lime" height="100%" width="100%"/>
-+
-+  <!-- First row: Initial comma(s) -->
-+  <g transform="translate(0, 0)">
-+    <polyline points=",0,0 10,20 20,0"    />
-+    <polyline points=",,0,0 10,20 20,0"   transform="translate( 40, 0)"/>
-+    <polyline points=", ,0,0 10,20 20,0"  transform="translate( 80, 0)"/>
-+    <polyline points=", 0,0 10,20 20,0"   transform="translate(120, 0)"/>
-+    <polyline points=",,, 0,0 10,20 20,0" transform="translate(160, 0)"/>
-+  </g>
-+  <!-- Second row: Trailing comma(s) -->
-+  <g transform="translate(0, 40)">
-+    <polyline points="0,0 10,20 20,0,"    />
-+    <polyline points="0,0 10,20 20,0 ,"   transform="translate( 40, 0)"/>
-+    <polyline points="0,0 10,20 20,0,,"   transform="translate( 80, 0)"/>
-+    <polyline points="0,0 10,20 20,0, ,"  transform="translate(120, 0)"/>
-+    <polyline points="0,0 10,20 20,0 ,,," transform="translate(160, 0)"/>
-+  </g>
-+  <!-- Third row: Repeated commas in the middle -->
-+  <g transform="translate(0, 80)">
-+    <polyline points="0,,0 10,20 20,0"    />
-+    <polyline points="0,,,0,10,20 20,0"   transform="translate( 40, 0)"/>
-+    <polyline points="0, ,0 10,20 20,0"   transform="translate( 80, 0)"/>
-+    <polyline points="0 0,, 10,20 20,0"   transform="translate(120, 0)"/>
-+    <polyline points="0,0,,10,20 20,0"    transform="translate(160, 0)"/>
-+  </g>
-+  <!-- Fourth row: wrong number of values (at least, to get anything drawn)  -->
-+  <g transform="translate(0, 120)">
-+    <polyline points=""                   />
-+    <polyline points="0,0"                transform="translate( 40, 0)"/>
-+    <polyline points="0,0 10"             transform="translate( 80, 0)"/>
-+    <polyline points="0,0 10,20 20"       transform="translate(120, 0)"/>
-+    <polyline points="0,0 10,20 20,0 0"   transform="translate(160, 0)"/>
-+  </g>
-+</svg>
-diff --git a/layout/reftests/svg/reftest.list b/layout/reftests/svg/reftest.list
---- a/layout/reftests/svg/reftest.list
-+++ b/layout/reftests/svg/reftest.list
-@@ -110,16 +110,17 @@ fails == inline-in-xul-basic-01.xul pass
- == objectBoundingBox-and-fePointLight-01.svg objectBoundingBox-and-fePointLight-01-ref.svg
- == objectBoundingBox-and-mask.svg pass.svg
- == objectBoundingBox-and-pattern-01a.svg objectBoundingBox-and-pattern-01-ref.svg
- == objectBoundingBox-and-pattern-01b.svg objectBoundingBox-and-pattern-01-ref.svg
- == objectBoundingBox-and-pattern-01c.svg objectBoundingBox-and-pattern-01-ref.svg
- == opacity-and-gradient-01.svg pass.svg
- == opacity-and-gradient-02.svg opacity-and-gradient-02-ref.svg
- == opacity-and-pattern-01.svg pass.svg
-+== polyline-points-invalid-01.svg pass.svg
- == path-01.svg path-01-ref.svg
- == path-02.svg pass.svg
- == pattern-live-01a.svg pattern-live-01-ref.svg
- == pattern-live-01b.svg pattern-live-01-ref.svg
- == pattern-live-01c.svg pattern-live-01-ref.svg
- == pseudo-classes-01.svg pass.svg
- # This test depends on :visited styles (which are asynchronous), so we run
- # it in layout/style/test/test_visited_reftests.html instead of using the
-@@ -154,14 +155,16 @@ random-if(MOZ_WIDGET_TOOLKIT=="gtk2") !=
- == text-style-01a.svg text-style-01-ref.svg
- == text-style-01b.svg text-style-01-ref.svg
- == text-style-01c.svg text-style-01-ref.svg
- == text-style-01d.svg text-style-01-ref.svg
- == text-style-01e.svg text-style-01-ref.svg
- == thin-stroke-01.svg pass.svg
- == userSpaceOnUse-and-pattern-01.svg userSpaceOnUse-and-pattern-01-ref.svg
- == viewBox-and-pattern-01.svg pass.svg
-+== viewBox-invalid-01.svg pass.svg
-+== viewBox-valid-01.svg pass.svg
- == viewport-percent-graphic-user-01.svg pass.svg
- 
- == svg-effects-area-unzoomed.xhtml svg-effects-area-unzoomed-ref.xhtml
- fails == svg-effects-area-zoomed-in.xhtml svg-effects-area-zoomed-in-ref.xhtml # Bug 541270
- fails == svg-effects-area-zoomed-out.xhtml svg-effects-area-zoomed-out-ref.xhtml # Bug 541270
- == href-attr-change-restyles.svg href-attr-change-restyles-ref.svg
-diff --git a/layout/reftests/svg/viewBox-invalid-01.svg b/layout/reftests/svg/viewBox-invalid-01.svg
-new file mode 100644
---- /dev/null
-+++ b/layout/reftests/svg/viewBox-invalid-01.svg
-@@ -0,0 +1,127 @@
-+<svg xmlns="http://www.w3.org/2000/svg" version="1.1"
-+     xmlns:xlink="http://www.w3.org/1999/xlink">
-+  <title>Testing invalid values for |viewBox| attribute</title>
-+  <defs>
-+    <rect id="redRect"  fill="red"  height="10" width="10"/>
-+    <rect id="limeRect" fill="lime" height="10" width="10"/>
-+  </defs>
-+  <rect fill="lime" height="100%" width="100%"/>
-+
-+  <!-- SUMMARY: Each <svg> subdocument below has an invalid viewBox. If we
-+       honor the invalid viewBox (ignoring typos), that will make us scale
-+       the <use>'d redRect to appear larger than the limeRect, and we'll have
-+       some red showing (and we'll fail the test). -->
-+
-+  <!-- First row: Initial comma(s) -->
-+  <g transform="translate(0, 0)">
-+    <g transform="translate(0, 0)">
-+      <svg width="20" height="20" viewBox=",0,0,1,1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(40, 0)">
-+      <svg width="20" height="20" viewBox=",,0,0,1,1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(80, 0)">
-+      <svg width="20" height="20" viewBox=", ,0,0,1,1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(120, 0)">
-+      <svg width="20" height="20" viewBox=", 0,0,1,1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(160, 0)">
-+      <svg width="20" height="20" viewBox=",,, 0,0,1,1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+  </g>
-+  <!-- Second row: Trailing comma(s) -->
-+  <g transform="translate(0, 40)">
-+    <g transform="translate(0, 0)">
-+      <svg width="20" height="20" viewBox="0,0,1,1,">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(40, 0)">
-+      <svg width="20" height="20" viewBox="0,0,1,1 ,">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(80, 0)">
-+      <svg width="20" height="20" viewBox="0,0,1,1,,">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(120, 0)">
-+      <svg width="20" height="20" viewBox="0,0,1,1, ,">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(160, 0)">
-+      <svg width="20" height="20" viewBox="0,0,1,1 ,,,">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+  </g>
-+  <!-- Third row: Repeated commas in the middle -->
-+  <g transform="translate(0, 80)">
-+    <g transform="translate(0, 0)">
-+      <svg width="20" height="20" viewBox="0,,0,1,1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(40, 0)">
-+      <svg width="20" height="20" viewBox="0,,,0,1,1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(80, 0)">
-+      <svg width="20" height="20" viewBox="0, ,0,1,1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(120, 0)">
-+      <svg width="20" height="20" viewBox="0 0,, 1,1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(160, 0)">
-+      <svg width="20" height="20" viewBox="0 0,,1 1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+  </g>
-+  <!-- Fourth row: wrong number of values -->
-+  <g transform="translate(0, 120)">
-+    <g transform="translate(0, 0)">
-+      <svg width="20" height="20" viewBox="">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(40, 0)">
-+      <svg width="20" height="20" viewBox="0">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(80, 0)">
-+      <svg width="20" height="20" viewBox="0 0">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(120, 0)">
-+      <svg width="20" height="20" viewBox="0 0 1">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+    <g transform="translate(160, 0)">
-+      <svg width="20" height="20" viewBox="0 0 1 1 1 ">
-+        <use xlink:href="#redRect"/>
-+      </svg><use xlink:href="#limeRect"/>
-+    </g>
-+  </g>
-+</svg>
-diff --git a/layout/reftests/svg/viewBox-valid-01.svg b/layout/reftests/svg/viewBox-valid-01.svg
-new file mode 100644
---- /dev/null
-+++ b/layout/reftests/svg/viewBox-valid-01.svg
-@@ -0,0 +1,96 @@
-+<svg xmlns="http://www.w3.org/2000/svg" version="1.1"
-+     xmlns:xlink="http://www.w3.org/1999/xlink">
-+  <title>Testing valid values for |viewBox| attribute</title>
-+  <defs>
-+    <rect id="redRect"  fill="red"  height="20" width="20"/>
-+    <rect id="limeRect" fill="lime" height="10" width="10"/>
-+  </defs>
-+  <rect fill="lime" height="100%" width="100%"/>
-+
-+  <!-- SUMMARY: Each <svg> subdocument below has a valid viewBox.  If we honor
-+       the valid viewBox (ignoring typos), that will make us scale the <use>'d
-+       limeRect to appear as big as than the redRect, and we'll have no red
-+       showing (and we'll pass the test). -->
-+
-+  <!-- First row: no commas at all -->
-+  <g transform="translate(0, 0)">
-+    <g transform="translate(0, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="0 0 10 10">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+    <g transform="translate(40, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox=" 0 0 10 10">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+    <g transform="translate(80, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="0 0 10 10 ">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+    <g transform="translate(120, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="  0   0    10     10      ">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+  </g>
-+  <!-- Second row: some commas -->
-+  <g transform="translate(0, 40)">
-+    <g transform="translate(0, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="0,0 10 10">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+    <g transform="translate(40, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="0 0,10 10">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+    <g transform="translate(80, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="0 0 10,10">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+    <g transform="translate(120, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="0,0,10,10">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+  </g>
-+  <!-- Third row: commas & whitespace mixed -->
-+  <g transform="translate(0, 80)">
-+    <g transform="translate(0, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="0,  0 10 10">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+    <g transform="translate(40, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="0 0  ,  10 10">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+    <g transform="translate(80, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="0 0 10   ,10">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+    <g transform="translate(120, 0)">
-+      <use xlink:href="#redRect"/>
-+      <svg width="20" height="20" viewBox="  0  ,0,  10,10  ">
-+        <use xlink:href="#limeRect"/>
-+      </svg>
-+    </g>
-+  </g>
-+</svg>