Bug 1265787 - add javascript CSS lexer to devtools; r=pbro draft
authorTom Tromey <tom@tromey.com>
Fri, 29 Apr 2016 11:21:22 -0600
changeset 363819 0bf5a3aed855ab5ddfd48a9021220ec6bd570688
parent 363818 ce2501a5f268283105549cc5316a35ca938c099f
child 520135 64ba53c64d7be1d15f9b31d0c2157dd720d4bfc4
push id17306
push userbmo:ttromey@mozilla.com
push dateThu, 05 May 2016 16:01:46 +0000
reviewerspbro
bugs1265787
milestone49.0a1
Bug 1265787 - add javascript CSS lexer to devtools; r=pbro MozReview-Commit-ID: BgbACHTTMtX
.eslintignore
devtools/client/inspector/shared/utils.js
devtools/client/shared/css-color.js
devtools/client/shared/css-parsing-utils.js
devtools/client/shared/output-parser.js
devtools/client/shared/widgets/CubicBezierWidget.js
devtools/client/shared/widgets/MdnDocsWidget.js
devtools/shared/css-lexer.js
devtools/shared/inspector/css-logic.js
devtools/shared/moz.build
devtools/shared/tests/unit/test_csslexer.js
devtools/shared/tests/unit/xpcshell.ini
--- a/.eslintignore
+++ b/.eslintignore
@@ -113,30 +113,32 @@ devtools/client/sourceeditor/**
 devtools/client/webaudioeditor/**
 devtools/client/webconsole/**
 !devtools/client/webconsole/panel.js
 !devtools/client/webconsole/jsterm.js
 devtools/client/webide/**
 devtools/server/**
 !devtools/server/actors/webbrowser.js
 devtools/shared/*.js
+!devtools/shared/css-lexer.js
 devtools/shared/*.jsm
 devtools/shared/apps/**
 devtools/shared/client/**
 devtools/shared/discovery/**
 devtools/shared/gcli/**
 devtools/shared/heapsnapshot/**
 devtools/shared/inspector/**
 devtools/shared/layout/**
 devtools/shared/locales/**
 devtools/shared/performance/**
 devtools/shared/qrcode/**
 devtools/shared/security/**
 devtools/shared/shims/**
 devtools/shared/tests/**
+!devtools/shared/tests/unit/test_csslexer.js
 devtools/shared/touch/**
 devtools/shared/transport/**
 devtools/shared/webconsole/test/**
 devtools/shared/worker/**
 
 # Ignore devtools pre-processed files
 devtools/client/framework/toolbox-process-window.js
 devtools/client/performance/system.js
--- a/devtools/client/inspector/shared/utils.js
+++ b/devtools/client/inspector/shared/utils.js
@@ -5,19 +5,17 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
 
 const {Ci} = require("chrome");
 const {parseDeclarations} =
       require("devtools/client/shared/css-parsing-utils");
 const promise = require("promise");
-
-loader.lazyServiceGetter(this, "domUtils",
-  "@mozilla.org/inspector/dom-utils;1", "inIDOMUtils");
+const {getCSSLexer} = require("devtools/shared/css-lexer");
 
 const HTML_NS = "http://www.w3.org/1999/xhtml";
 
 /**
  * Create a child element with a set of attributes.
  *
  * @param {Element} parent
  *        The parent node.
@@ -80,17 +78,17 @@ function advanceValidate(keyCode, value,
     return false;
   }
 
   // Insert the character provisionally and see what happens.  If we
   // end up with a ";" symbol token, then the semicolon terminates the
   // value.  Otherwise it's been inserted in some spot where it has a
   // valid meaning, like a comment or string.
   value = value.slice(0, insertionPoint) + ";" + value.slice(insertionPoint);
-  let lexer = domUtils.getCSSLexer(value);
+  let lexer = getCSSLexer(value);
   while (true) {
     let token = lexer.nextToken();
     if (token.endOffset > insertionPoint) {
       if (token.tokenType === "symbol" && token.text === ";") {
         // The ";" is a terminator.
         return true;
       }
       // The ";" is not a terminator in this context.
--- a/devtools/client/shared/css-color.js
+++ b/devtools/client/shared/css-color.js
@@ -2,16 +2,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
 
 const {Cc, Ci} = require("chrome");
 const Services = require("Services");
 
+const {getCSSLexer} = require("devtools/shared/css-lexer");
 const {cssColors} = require("devtools/client/shared/css-color-db");
 
 const COLOR_UNIT_PREF = "devtools.defaultColorUnit";
 
 const SPECIALVALUES = new Set([
   "currentcolor",
   "initial",
   "inherit",
@@ -718,17 +719,17 @@ function colorToRGBA(name) {
     let result = cssColors[name];
     return {r: result[0], g: result[1], b: result[2], a: result[3]};
   } else if (name === "transparent") {
     return {r: 0, g: 0, b: 0, a: 0};
   } else if (name === "currentcolor") {
     return {r: 0, g: 0, b: 0, a: 1};
   }
 
-  let lexer = DOMUtils.getCSSLexer(name);
+  let lexer = getCSSLexer(name);
 
   let func = getToken(lexer);
   if (!func) {
     return null;
   }
 
   if (func.tokenType === "id" || func.tokenType === "hash") {
     if (getToken(lexer) !== null) {
--- a/devtools/client/shared/css-parsing-utils.js
+++ b/devtools/client/shared/css-parsing-utils.js
@@ -12,16 +12,17 @@
 //     pseudo-classes
 // parseSingleValue - parse a single CSS property value
 
 "use strict";
 
 const {Cc, Ci, Cu} = require("chrome");
 loader.lazyRequireGetter(this, "CSS", "CSS");
 const promise = require("promise");
+const {getCSSLexer} = require("devtools/shared/css-lexer");
 Cu.import("resource://gre/modules/Task.jsm", this);
 loader.lazyGetter(this, "DOMUtils", () => {
   return Cc["@mozilla.org/inspector/dom-utils;1"].getService(Ci.inIDOMUtils);
 });
 
 const SELECTOR_ATTRIBUTE = exports.SELECTOR_ATTRIBUTE = 1;
 const SELECTOR_ELEMENT = exports.SELECTOR_ELEMENT = 2;
 const SELECTOR_PSEUDO_CLASS = exports.SELECTOR_PSEUDO_CLASS = 3;
@@ -46,17 +47,17 @@ const COMMENT_PARSING_HEURISTIC_BYPASS_C
  * A generator function that lexes a CSS source string, yielding the
  * CSS tokens.  Comment tokens are dropped.
  *
  * @param {String} CSS source string
  * @yield {CSSToken} The next CSSToken that is lexed
  * @see CSSToken for details about the returned tokens
  */
 function* cssTokenizer(string) {
-  let lexer = DOMUtils.getCSSLexer(string);
+  let lexer = getCSSLexer(string);
   while (true) {
     let token = lexer.nextToken();
     if (!token) {
       break;
     }
     // None of the existing consumers want comments.
     if (token.tokenType !== "comment") {
       yield token;
@@ -72,24 +73,24 @@ function* cssTokenizer(string) {
  * has an additional "loc" attribute.  This attribute is an object
  * of the form {line: L, column: C}.  Lines and columns are both zero
  * based.
  *
  * It's best not to add new uses of this function.  In general it is
  * simpler and better to use the CSSToken offsets, rather than line
  * and column.  Also, this function lexes the entire input string at
  * once, rather than lazily yielding a token stream.  Use
- * |cssTokenizer| or |DOMUtils.getCSSLexer| instead.
+ * |cssTokenizer| or |getCSSLexer| instead.
  *
  * @param{String} string The input string.
  * @return {Array} An array of tokens (@see CSSToken) that have
  *        line and column information.
  */
 function cssTokenizerWithLineColumn(string) {
-  let lexer = DOMUtils.getCSSLexer(string);
+  let lexer = getCSSLexer(string);
   let result = [];
   let prevToken = undefined;
   while (true) {
     let token = lexer.nextToken();
     let lineNumber = lexer.lineNumber;
     let columnNumber = lexer.columnNumber;
 
     if (prevToken) {
@@ -287,17 +288,17 @@ function getEmptyDeclaration() {
  *        heuristic.
  */
 function parseDeclarationsInternal(inputString, parseComments,
                                    inComment, commentOverride) {
   if (inputString === null || inputString === undefined) {
     throw new Error("empty input string");
   }
 
-  let lexer = DOMUtils.getCSSLexer(inputString);
+  let lexer = getCSSLexer(inputString);
 
   let declarations = [getEmptyDeclaration()];
   let lastProp = declarations[0];
 
   let current = "", hasBang = false;
   while (true) {
     let token = lexer.nextToken();
     if (!token) {
@@ -569,17 +570,17 @@ RuleRewriter.prototype = {
    * @param {String} text The input text.  This should include the trailing ";".
    * @return {Array} An array of the form [anySanitized, text], where
    *                 |anySanitized| is a boolean that indicates
    *                  whether anything substantive has changed; and
    *                  where |text| is the text that has been rewritten
    *                  to be "lexically safe".
    */
   sanitizePropertyValue: function (text) {
-    let lexer = DOMUtils.getCSSLexer(text);
+    let lexer = getCSSLexer(text);
 
     let result = "";
     let previousOffset = 0;
     let braceDepth = 0;
     let anySanitized = false;
     while (true) {
       let token = lexer.nextToken();
       if (!token) {
--- a/devtools/client/shared/output-parser.js
+++ b/devtools/client/shared/output-parser.js
@@ -2,16 +2,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
 
 const {Cc, Ci, Cu} = require("chrome");
 const {angleUtils} = require("devtools/shared/css-angle");
 const {colorUtils} = require("devtools/client/shared/css-color");
+const {getCSSLexer} = require("devtools/shared/css-lexer");
 const Services = require("Services");
 const EventEmitter = require("devtools/shared/event-emitter");
 
 const HTML_NS = "http://www.w3.org/1999/xhtml";
 
 const BEZIER_KEYWORDS = ["linear", "ease-in-out", "ease-in", "ease-out",
                          "ease"];
 
@@ -158,17 +159,17 @@ OutputParser.prototype = {
    *         _mergeOptions().
    * @return {DocumentFragment}
    *         A document fragment.
    */
   _parse: function(text, options = {}) {
     text = text.trim();
     this.parsed.length = 0;
 
-    let tokenStream = DOMUtils.getCSSLexer(text);
+    let tokenStream = getCSSLexer(text);
     let parenDepth = 0;
     let outerMostFunctionTakesColor = false;
 
     let colorOK = function() {
       return options.supportsColor ||
         (options.expectFilter && parenDepth === 1 &&
          outerMostFunctionTakesColor);
     };
@@ -484,17 +485,17 @@ OutputParser.prototype = {
     swatch.emit("unit-change", val);
   },
 
   /**
    * A helper function that sanitizes a possibly-unterminated URL.
    */
   _sanitizeURL: function(url) {
     // Re-lex the URL and add any needed termination characters.
-    let urlTokenizer = DOMUtils.getCSSLexer(url);
+    let urlTokenizer = getCSSLexer(url);
     // Just read until EOF; there will only be a single token.
     while (urlTokenizer.nextToken()) {
       // Nothing.
     }
 
     return urlTokenizer.performEOFFixup(url, true);
   },
 
--- a/devtools/client/shared/widgets/CubicBezierWidget.js
+++ b/devtools/client/shared/widgets/CubicBezierWidget.js
@@ -26,20 +26,18 @@
 "use strict";
 
 const EventEmitter = require("devtools/shared/event-emitter");
 const {
   PREDEFINED,
   PRESETS,
   DEFAULT_PRESET_CATEGORY
 } = require("devtools/client/shared/widgets/CubicBezierPresets");
+const {getCSSLexer} = require("devtools/shared/css-lexer");
 const {Cc, Ci} = require('chrome');
-loader.lazyGetter(this, "DOMUtils", () => {
-  return Cc["@mozilla.org/inspector/dom-utils;1"].getService(Ci.inIDOMUtils);
-});
 
 /**
  * CubicBezier data structure helper
  * Accepts an array of coordinates and exposes a few useful getters
  * @param {Array} coordinates i.e. [.42, 0, .58, 1]
  */
 function CubicBezier(coordinates) {
   if (!coordinates) {
@@ -823,17 +821,17 @@ function distance(x1, y1, x2, y2) {
  * @param {String} value
  * @return {Array} of coordinates, or undefined
  */
 function parseTimingFunction(value) {
   if (value in PREDEFINED) {
     return PREDEFINED[value];
   }
 
-  let tokenStream = DOMUtils.getCSSLexer(value);
+  let tokenStream = getCSSLexer(value);
   let getNextToken = () => {
     while (true) {
       let token = tokenStream.nextToken();
       if (!token || (token.tokenType !== "whitespace" &&
                      token.tokenType !== "comment")) {
         return token;
       }
     }
--- a/devtools/client/shared/widgets/MdnDocsWidget.js
+++ b/devtools/client/shared/widgets/MdnDocsWidget.js
@@ -22,18 +22,17 @@
  * the content in a tooltip, use this in conjunction with Tooltip.js.
  */
 
 "use strict";
 
 const {Cc, Cu, Ci} = require("chrome");
 const Services = require("Services");
 const Promise = require("promise");
-const DOMUtils = Cc["@mozilla.org/inspector/dom-utils;1"]
-                 .getService(Ci.inIDOMUtils);
+const {getCSSLexer} = require("devtools/shared/css-lexer");
 
 // Parameters for the XHR request
 // see https://developer.mozilla.org/en-US/docs/MDN/Kuma/API#Document_parameters
 const XHR_PARAMS = "?raw&macros";
 // URL for the XHR request
 var XHR_CSS_URL = "https://developer.mozilla.org/en-US/docs/Web/CSS/";
 
 // Parameters for the link to MDN in the tooltip, so
@@ -78,17 +77,17 @@ const COMMENT_COLOR = "theme-comment";
  *
  * @param {DOM node} syntaxSection
  * This is the parent for the output nodes. Generated nodes
  * are appended to this as children.
  */
 function appendSyntaxHighlightedCSS(cssText, parentElement) {
   let doc = parentElement.ownerDocument;
   let identClass = PROPERTY_NAME_COLOR;
-  let lexer = DOMUtils.getCSSLexer(cssText);
+  let lexer = getCSSLexer(cssText);
 
   /**
    * Create a SPAN node with the given text content and class.
    */
   function createStyledNode(textContent, className) {
     let newNode = doc.createElement("span");
     newNode.classList.add(className);
     newNode.textContent = textContent;
new file mode 100644
--- /dev/null
+++ b/devtools/shared/css-lexer.js
@@ -0,0 +1,1240 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// A CSS Lexer.  This file is a bit unusual -- it is a more or less
+// direct translation of layout/style/nsCSSScanner.cpp and
+// layout/style/CSSLexer.cpp into JS.  This implements the
+// CSSLexer.webidl interface, and the intent is to try to keep it in
+// sync with changes to the platform CSS lexer.  Due to this goal,
+// this file violates some naming conventions and consequently locally
+// disables some eslint rules.
+
+/* eslint-disable camelcase, no-inline-comments, mozilla/no-aArgs */
+/* eslint-disable no-else-return */
+
+"use strict";
+
+// White space of any kind.  No value fields are used.  Note that
+// comments do *not* count as white space; comments separate tokens
+// but are not themselves tokens.
+const eCSSToken_Whitespace = "whitespace";     //
+// A comment.
+const eCSSToken_Comment = "comment";        // /*...*/
+
+// Identifier-like tokens.  mIdent is the text of the identifier.
+// The difference between ID and Hash is: if the text after the #
+// would have been a valid Ident if the # hadn't been there, the
+// scanner produces an ID token.  Otherwise it produces a Hash token.
+// (This distinction is required by css3-selectors.)
+const eCSSToken_Ident = "ident";          // word
+const eCSSToken_Function = "function";       // word(
+const eCSSToken_AtKeyword = "at";      // @word
+const eCSSToken_ID = "id";             // #word
+const eCSSToken_Hash = "hash";           // #0word
+
+// Numeric tokens.  mNumber is the floating-point value of the
+// number, and mHasSign indicates whether there was an explicit sign
+// (+ or -) in front of the number.  If mIntegerValid is true, the
+// number had the lexical form of an integer, and mInteger is its
+// integer value.  Lexically integer values outside the range of a
+// 32-bit signed number are clamped to the maximum values; mNumber
+// will indicate a 'truer' value in that case.  Percentage tokens
+// are always considered not to be integers, even if their numeric
+// value is integral (100% => mNumber = 1.0).  For Dimension
+// tokens, mIdent holds the text of the unit.
+const eCSSToken_Number = "number";         // 1 -5 +2e3 3.14159 7.297352e-3
+const eCSSToken_Dimension = "dimension";      // 24px 8.5in
+const eCSSToken_Percentage = "percentage";     // 85% 1280.4%
+
+// String-like tokens.  In all cases, mIdent holds the text
+// belonging to the string, and mSymbol holds the delimiter
+// character, which may be ', ", or zero (only for unquoted URLs).
+// Bad_String and Bad_URL tokens are emitted when the closing
+// delimiter or parenthesis was missing.
+const eCSSToken_String = "string";         // 'foo bar' "foo bar"
+const eCSSToken_Bad_String = "bad_string";     // 'foo bar
+const eCSSToken_URL = "url";            // url(foobar) url("foo bar")
+const eCSSToken_Bad_URL = "bad_url";        // url(foo
+
+// Any one-character symbol.  mSymbol holds the character.
+const eCSSToken_Symbol = "symbol";         // . ; { } ! *
+
+// Match operators.  These are single tokens rather than pairs of
+// Symbol tokens because css3-selectors forbids the presence of
+// comments between the two characters.  No value fields are used;
+// the token type indicates which operator.
+const eCSSToken_Includes = "includes";       // ~=
+const eCSSToken_Dashmatch = "dashmatch";      // |=
+const eCSSToken_Beginsmatch = "beginsmatch";    // ^=
+const eCSSToken_Endsmatch = "endsmatch";      // $=
+const eCSSToken_Containsmatch = "containsmatch";  // *=
+
+// Unicode-range token: currently used only in @font-face.
+// The lexical rule for this token includes several forms that are
+// semantically invalid.  Therefore, mIdent always holds the
+// complete original text of the token (so we can print it
+// accurately in diagnostics), and mIntegerValid is true iff the
+// token is semantically valid.  In that case, mInteger holds the
+// lowest value included in the range, and mInteger2 holds the
+// highest value included in the range.
+const eCSSToken_URange = "urange";         // U+007e U+01?? U+2000-206F
+
+// HTML comment delimiters, ignored as a unit when they appear at
+// the top level of a style sheet, for compatibility with websites
+// written for compatibility with pre-CSS browsers.  This token type
+// subsumes the css2.1 CDO and CDC tokens, which are always treated
+// the same by the parser.  mIdent holds the text of the token, for
+// diagnostics.
+const eCSSToken_HTMLComment = "htmlcomment";    // <!-- -->
+
+const eEOFCharacters_None = 0x0000;
+
+// to handle \<EOF> inside strings
+const eEOFCharacters_DropBackslash = 0x0001;
+
+// to handle \<EOF> outside strings
+const eEOFCharacters_ReplacementChar = 0x0002;
+
+// to close comments
+const eEOFCharacters_Asterisk = 0x0004;
+const eEOFCharacters_Slash = 0x0008;
+
+// to close double-quoted strings
+const eEOFCharacters_DoubleQuote = 0x0010;
+
+// to close single-quoted strings
+const eEOFCharacters_SingleQuote = 0x0020;
+
+// to close URLs
+const eEOFCharacters_CloseParen = 0x0040;
+
+// Bridge the char/string divide.
+const APOSTROPHE = "'".charCodeAt(0);
+const ASTERISK = "*".charCodeAt(0);
+const CARRIAGE_RETURN = "\r".charCodeAt(0);
+const CIRCUMFLEX_ACCENT = "^".charCodeAt(0);
+const COMMERCIAL_AT = "@".charCodeAt(0);
+const DIGIT_NINE = "9".charCodeAt(0);
+const DIGIT_ZERO = "0".charCodeAt(0);
+const DOLLAR_SIGN = "$".charCodeAt(0);
+const EQUALS_SIGN = "=".charCodeAt(0);
+const EXCLAMATION_MARK = "!".charCodeAt(0);
+const FULL_STOP = ".".charCodeAt(0);
+const GREATER_THAN_SIGN = ">".charCodeAt(0);
+const HYPHEN_MINUS = "-".charCodeAt(0);
+const LATIN_CAPITAL_LETTER_E = "E".charCodeAt(0);
+const LATIN_CAPITAL_LETTER_U = "U".charCodeAt(0);
+const LATIN_SMALL_LETTER_E = "e".charCodeAt(0);
+const LATIN_SMALL_LETTER_U = "u".charCodeAt(0);
+const LEFT_PARENTHESIS = "(".charCodeAt(0);
+const LESS_THAN_SIGN = "<".charCodeAt(0);
+const LINE_FEED = "\n".charCodeAt(0);
+const NUMBER_SIGN = "#".charCodeAt(0);
+const PERCENT_SIGN = "%".charCodeAt(0);
+const PLUS_SIGN = "+".charCodeAt(0);
+const QUESTION_MARK = "?".charCodeAt(0);
+const QUOTATION_MARK = "\"".charCodeAt(0);
+const REVERSE_SOLIDUS = "\\".charCodeAt(0);
+const RIGHT_PARENTHESIS = ")".charCodeAt(0);
+const SOLIDUS = "/".charCodeAt(0);
+const TILDE = "~".charCodeAt(0);
+const VERTICAL_LINE = "|".charCodeAt(0);
+
+const UCS2_REPLACEMENT_CHAR = 0xFFFD;
+
+const kImpliedEOFCharacters = [
+  UCS2_REPLACEMENT_CHAR,
+  ASTERISK,
+  SOLIDUS,
+  QUOTATION_MARK,
+  APOSTROPHE,
+  RIGHT_PARENTHESIS,
+  0
+];
+
+/**
+ * Ensure that the character is valid.  If it is valid, return it;
+ * otherwise, return the replacement character.
+ *
+ * @param {Number} c the character to check
+ * @return {Number} the character or its replacement
+ */
+function ensureValidChar(c) {
+  if (c >= 0x00110000 || (c & 0xFFF800) == 0xD800) {
+    // Out of range or a surrogate.
+    return UCS2_REPLACEMENT_CHAR;
+  }
+  return c;
+}
+
+/**
+ * Turn a string into an array of character codes.
+ *
+ * @param {String} str the input string
+ * @return {Array} an array of character codes, one per character in
+ *         the input string.
+ */
+function stringToCodes(str) {
+  return Array.prototype.map.call(str, (c) => c.charCodeAt(0));
+}
+
+const IS_HEX_DIGIT = 0x01;
+const IS_IDSTART = 0x02;
+const IS_IDCHAR = 0x04;
+const IS_URL_CHAR = 0x08;
+const IS_HSPACE = 0x10;
+const IS_VSPACE = 0x20;
+const IS_SPACE = IS_HSPACE | IS_VSPACE;
+const IS_STRING = 0x40;
+
+const H = IS_HSPACE;
+const V = IS_VSPACE;
+const I = IS_IDCHAR;
+const J = IS_IDSTART;
+const U = IS_URL_CHAR;
+const S = IS_STRING;
+const X = IS_HEX_DIGIT;
+
+const SH = S | H;
+const SU = S | U;
+const SUI = S | U | I;
+const SUIJ = S | U | I | J;
+const SUIX = S | U | I | X;
+const SUIJX = S | U | I | J | X;
+
+/* eslint-disable indent, no-multi-spaces, comma-spacing, spaced-comment */
+const gLexTable = [
+// 00    01    02    03    04    05    06    07
+    0,    S,    S,    S,    S,    S,    S,    S,
+// 08   TAB    LF    0B    FF    CR    0E    0F
+    S,   SH,    V,    S,    V,    V,    S,    S,
+// 10    11    12    13    14    15    16    17
+    S,    S,    S,    S,    S,    S,    S,    S,
+// 18    19    1A    1B    1C    1D    1E    1F
+    S,    S,    S,    S,    S,    S,    S,    S,
+//SPC     !     "     #     $     %     &     '
+   SH,   SU,    0,   SU,   SU,   SU,   SU,    0,
+//  (     )     *     +     ,     -     .     /
+    S,    S,   SU,   SU,   SU,  SUI,   SU,   SU,
+//  0     1     2     3     4     5     6     7
+ SUIX, SUIX, SUIX, SUIX, SUIX, SUIX, SUIX, SUIX,
+//  8     9     :     ;     <     =     >     ?
+ SUIX, SUIX,   SU,   SU,   SU,   SU,   SU,   SU,
+//  @     A     B     C     D     E     F     G
+   SU,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX, SUIJ,
+//  H     I     J     K     L     M     N     O
+ SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
+//  P     Q     R     S     T     U     V     W
+ SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
+//  X     Y     Z     [     \     ]     ^     _
+ SUIJ, SUIJ, SUIJ,   SU,    J,   SU,   SU, SUIJ,
+//  `     a     b     c     d     e     f     g
+   SU,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX,SUIJX, SUIJ,
+//  h     i     j     k     l     m     n     o
+ SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
+//  p     q     r     s     t     u     v     w
+ SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ, SUIJ,
+//  x     y     z     {     |     }     ~    7F
+ SUIJ, SUIJ, SUIJ,   SU,   SU,   SU,   SU,    S,
+];
+/* eslint-enable indent, no-multi-spaces, comma-spacing, spaced-comment */
+
+/**
+ * True if 'ch' is in character class 'cls', which should be one of
+ * the constants above or some combination of them.  All characters
+ * above U+007F are considered to be in 'cls'.  EOF is never in 'cls'.
+ */
+function IsOpenCharClass(ch, cls) {
+  return ch >= 0 && (ch >= 128 || (gLexTable[ch] & cls) != 0);
+}
+
+/**
+ * True if 'ch' is in character class 'cls', which should be one of
+ * the constants above or some combination of them.  No characters
+ * above U+007F are considered to be in 'cls'. EOF is never in 'cls'.
+ */
+function IsClosedCharClass(ch, cls) {
+  return ch >= 0 && ch < 128 && (gLexTable[ch] & cls) != 0;
+}
+
+/**
+ * True if 'ch' is CSS whitespace, i.e. any of the ASCII characters
+ * TAB, LF, FF, CR, or SPC.
+ */
+function IsWhitespace(ch) {
+  return IsClosedCharClass(ch, IS_SPACE);
+}
+
+/**
+ * True if 'ch' is horizontal whitespace, i.e. TAB or SPC.
+ */
+function IsHorzSpace(ch) {
+  return IsClosedCharClass(ch, IS_HSPACE);
+}
+
+/**
+ * True if 'ch' is vertical whitespace, i.e. LF, FF, or CR.  Vertical
+ * whitespace requires special handling when consumed, see AdvanceLine.
+ */
+function IsVertSpace(ch) {
+  return IsClosedCharClass(ch, IS_VSPACE);
+}
+
+/**
+ * True if 'ch' is a character that can appear in the middle of an identifier.
+ * This includes U+0000 since it is handled as U+FFFD, but for purposes of
+ * GatherText it should not be included in IsOpenCharClass.
+ */
+function IsIdentChar(ch) {
+  return IsOpenCharClass(ch, IS_IDCHAR) || ch == 0;
+}
+
+/**
+ * True if 'ch' is a character that by itself begins an identifier.
+ * This includes U+0000 since it is handled as U+FFFD, but for purposes of
+ * GatherText it should not be included in IsOpenCharClass.
+ * (This is a subset of IsIdentChar.)
+ */
+function IsIdentStart(ch) {
+  return IsOpenCharClass(ch, IS_IDSTART) || ch == 0;
+}
+
+/**
+ * True if the two-character sequence aFirstChar+aSecondChar begins an
+ * identifier.
+ */
+function StartsIdent(aFirstChar, aSecondChar) {
+  return IsIdentStart(aFirstChar) ||
+    (aFirstChar == HYPHEN_MINUS && (aSecondChar == HYPHEN_MINUS ||
+                                    IsIdentStart(aSecondChar)));
+}
+
+/**
+ * True if 'ch' is a decimal digit.
+ */
+function IsDigit(ch) {
+  return (ch >= DIGIT_ZERO) && (ch <= DIGIT_NINE);
+}
+
+/**
+ * True if 'ch' is a hexadecimal digit.
+ */
+function IsHexDigit(ch) {
+  return IsClosedCharClass(ch, IS_HEX_DIGIT);
+}
+
+/**
+ * Assuming that 'ch' is a decimal digit, return its numeric value.
+ */
+function DecimalDigitValue(ch) {
+  return ch - DIGIT_ZERO;
+}
+
+/**
+ * Assuming that 'ch' is a hexadecimal digit, return its numeric value.
+ */
+function HexDigitValue(ch) {
+  if (IsDigit(ch)) {
+    return DecimalDigitValue(ch);
+  } else {
+    // Note: c&7 just keeps the low three bits which causes
+    // upper and lower case alphabetics to both yield their
+    // "relative to 10" value for computing the hex value.
+    return (ch & 0x7) + 9;
+  }
+}
+
+/**
+ * If 'ch' can be the first character of a two-character match operator
+ * token, return the token type code for that token, otherwise return
+ * eCSSToken_Symbol to indicate that it can't.
+ */
+function MatchOperatorType(ch) {
+  switch (ch) {
+    case TILDE: return eCSSToken_Includes;
+    case VERTICAL_LINE: return eCSSToken_Dashmatch;
+    case CIRCUMFLEX_ACCENT: return eCSSToken_Beginsmatch;
+    case DOLLAR_SIGN: return eCSSToken_Endsmatch;
+    case ASTERISK: return eCSSToken_Containsmatch;
+    default: return eCSSToken_Symbol;
+  }
+}
+
+function Scanner(buffer) {
+  this.mBuffer = buffer || "";
+  this.mOffset = 0;
+  this.mCount = this.mBuffer.length;
+  this.mLineNumber = 1;
+  this.mLineOffset = 0;
+  this.mTokenLineOffset = 0;
+  this.mTokenOffset = 0;
+  this.mTokenLineNumber = 1;
+  this.mEOFCharacters = eEOFCharacters_None;
+}
+
+Scanner.prototype = {
+  /**
+   * @see CSSLexer.lineNumber
+   */
+  get lineNumber() {
+    return this.mTokenLineNumber - 1;
+  },
+
+  /**
+   * @see CSSLexer.columnNumber
+   */
+  get columnNumber() {
+    return this.mTokenOffset - this.mTokenLineOffset;
+  },
+
+  /**
+   * @see CSSLexer.performEOFFixup
+   */
+  performEOFFixup: function (aInputString, aPreserveBackslash) {
+    let result = aInputString;
+
+    let eofChars = this.mEOFCharacters;
+
+    if (aPreserveBackslash &&
+        (eofChars & (eEOFCharacters_DropBackslash |
+                     eEOFCharacters_ReplacementChar)) != 0) {
+      eofChars &= ~(eEOFCharacters_DropBackslash |
+                    eEOFCharacters_ReplacementChar);
+      result += "\\";
+    }
+
+    if ((eofChars & eEOFCharacters_DropBackslash) != 0 &&
+        result.length > 0 && result.endsWith("\\")) {
+      result = result.slice(0, -1);
+    }
+
+    let extra = [];
+    this.AppendImpliedEOFCharacters(eofChars, extra);
+    let asString = String.fromCharCode.apply(null, extra);
+
+    return result + asString;
+  },
+
+  /**
+   * @see CSSLexer.nextToken
+   */
+  nextToken: function () {
+    let token = {};
+    if (!this.Next(token)) {
+      return null;
+    }
+
+    let resultToken = {};
+    resultToken.tokenType = token.mType;
+    resultToken.startOffset = this.mTokenOffset;
+    resultToken.endOffset = this.mOffset;
+
+    let constructText = () => {
+      return String.fromCharCode.apply(null, token.mIdent);
+    };
+
+    switch (token.mType) {
+      case eCSSToken_Whitespace:
+        break;
+
+      case eCSSToken_Ident:
+      case eCSSToken_Function:
+      case eCSSToken_AtKeyword:
+      case eCSSToken_ID:
+      case eCSSToken_Hash:
+        resultToken.text = constructText();
+        break;
+
+      case eCSSToken_Dimension:
+        resultToken.text = constructText();
+        /* Fall through.  */
+      case eCSSToken_Number:
+      case eCSSToken_Percentage:
+        resultToken.number = token.mNumber;
+        resultToken.hasSign = token.mHasSign;
+        resultToken.isInteger = token.mIntegerValid;
+        break;
+
+      case eCSSToken_String:
+      case eCSSToken_Bad_String:
+      case eCSSToken_URL:
+      case eCSSToken_Bad_URL:
+        resultToken.text = constructText();
+        /* Don't bother emitting the delimiter, as it is readily extracted
+           from the source string when needed.  */
+        break;
+
+      case eCSSToken_Symbol:
+        resultToken.text = String.fromCharCode(token.mSymbol);
+        break;
+
+      case eCSSToken_Includes:
+      case eCSSToken_Dashmatch:
+      case eCSSToken_Beginsmatch:
+      case eCSSToken_Endsmatch:
+      case eCSSToken_Containsmatch:
+      case eCSSToken_URange:
+        break;
+
+      case eCSSToken_Comment:
+      case eCSSToken_HTMLComment:
+        /* The comment text is easily extracted from the source string,
+           and is rarely useful.  */
+        break;
+    }
+
+    return resultToken;
+  },
+
+  /**
+   * Return the raw UTF-16 code unit at position |this.mOffset + n| within
+   * the read buffer.  If that is beyond the end of the buffer, returns
+   * -1 to indicate end of input.
+   */
+  Peek: function (n = 0) {
+    if (this.mOffset + n >= this.mCount) {
+      return -1;
+    }
+    return this.mBuffer.charCodeAt(this.mOffset + n);
+  },
+
+  /**
+   * Advance |this.mOffset| over |n| code units.  Advance(0) is a no-op.
+   * If |n| is greater than the distance to end of input, will silently
+   * stop at the end.  May not be used to advance over a line boundary;
+   * AdvanceLine() must be used instead.
+   */
+  Advance: function (n = 1) {
+    if (this.mOffset + n >= this.mCount || this.mOffset + n < this.mOffset) {
+      this.mOffset = this.mCount;
+    } else {
+      this.mOffset += n;
+    }
+  },
+
+  /**
+   * Advance |this.mOffset| over a line boundary.
+   */
+  AdvanceLine: function () {
+    // Advance over \r\n as a unit.
+    if (this.mBuffer.charCodeAt(this.mOffset) == CARRIAGE_RETURN &&
+        this.mOffset + 1 < this.mCount &&
+        this.mBuffer.charCodeAt(this.mOffset + 1) == LINE_FEED) {
+      this.mOffset += 2;
+    } else {
+      this.mOffset += 1;
+    }
+    // 0 is a magical line number meaning that we don't know (i.e., script)
+    if (this.mLineNumber != 0) {
+      this.mLineNumber++;
+    }
+    this.mLineOffset = this.mOffset;
+  },
+
+  /**
+   * Skip over a sequence of whitespace characters (vertical or
+   * horizontal) starting at the current read position.
+   */
+  SkipWhitespace: function () {
+    for (;;) {
+      let ch = this.Peek();
+      if (!IsWhitespace(ch)) { // EOF counts as non-whitespace
+        break;
+      }
+      if (IsVertSpace(ch)) {
+        this.AdvanceLine();
+      } else {
+        this.Advance();
+      }
+    }
+  },
+
+  /**
+   * Skip over one CSS comment starting at the current read position.
+   */
+  SkipComment: function () {
+    this.Advance(2);
+    for (;;) {
+      let ch = this.Peek();
+      if (ch < 0) {
+        this.SetEOFCharacters(eEOFCharacters_Asterisk | eEOFCharacters_Slash);
+        return;
+      }
+      if (ch == ASTERISK) {
+        this.Advance();
+        ch = this.Peek();
+        if (ch < 0) {
+          this.SetEOFCharacters(eEOFCharacters_Slash);
+          return;
+        }
+        if (ch == SOLIDUS) {
+          this.Advance();
+          return;
+        }
+      } else if (IsVertSpace(ch)) {
+        this.AdvanceLine();
+      } else {
+        this.Advance();
+      }
+    }
+  },
+
+  /**
+   * If there is a valid escape sequence starting at the current read
+   * position, consume it, decode it, append the result to |aOutput|,
+   * and return true.  Otherwise, consume nothing, leave |aOutput|
+   * unmodified, and return false.  If |aInString| is true, accept the
+   * additional form of escape sequence allowed within string-like tokens.
+   */
+  GatherEscape: function (aOutput, aInString) {
+    let ch = this.Peek(1);
+    if (ch < 0) {
+      // If we are in a string (or a url() containing a string), we want to drop
+      // the backslash on the floor.  Otherwise, we want to treat it as a U+FFFD
+      // character.
+      this.Advance();
+      if (aInString) {
+        this.SetEOFCharacters(eEOFCharacters_DropBackslash);
+      } else {
+        aOutput.push(UCS2_REPLACEMENT_CHAR);
+        this.SetEOFCharacters(eEOFCharacters_ReplacementChar);
+      }
+      return true;
+    }
+    if (IsVertSpace(ch)) {
+      if (aInString) {
+        // In strings (and in url() containing a string), escaped
+        // newlines are completely removed, to allow splitting over
+        // multiple lines.
+        this.Advance();
+        this.AdvanceLine();
+        return true;
+      }
+      // Outside of strings, backslash followed by a newline is not an escape.
+      return false;
+    }
+
+    if (!IsHexDigit(ch)) {
+      // "Any character (except a hexadecimal digit, linefeed, carriage
+      // return, or form feed) can be escaped with a backslash to remove
+      // its special meaning." -- CSS2.1 section 4.1.3
+      this.Advance(2);
+      if (ch == 0) {
+        aOutput.push(UCS2_REPLACEMENT_CHAR);
+      } else {
+        aOutput.push(ch);
+      }
+      return true;
+    }
+
+    // "[at most six hexadecimal digits following a backslash] stand
+    // for the ISO 10646 character with that number, which must not be
+    // zero. (It is undefined in CSS 2.1 what happens if a style sheet
+    // does contain a character with Unicode codepoint zero.)"
+    //   -- CSS2.1 section 4.1.3
+
+    // At this point we know we have \ followed by at least one
+    // hexadecimal digit, therefore the escape sequence is valid and we
+    // can go ahead and consume the backslash.
+    this.Advance();
+    let val = 0;
+    let i = 0;
+    do {
+      val = val * 16 + HexDigitValue(ch);
+      i++;
+      this.Advance();
+      ch = this.Peek();
+    } while (i < 6 && IsHexDigit(ch));
+
+    // "Interpret the hex digits as a hexadecimal number. If this
+    // number is zero, or is greater than the maximum allowed
+    // codepoint, return U+FFFD REPLACEMENT CHARACTER" -- CSS Syntax
+    // Level 3
+    if (val == 0) {
+      aOutput.push(UCS2_REPLACEMENT_CHAR);
+    } else {
+      aOutput.push(ensureValidChar(val));
+    }
+
+    // Consume exactly one whitespace character after a
+    // hexadecimal escape sequence.
+    if (IsVertSpace(ch)) {
+      this.AdvanceLine();
+    } else if (IsHorzSpace(ch)) {
+      this.Advance();
+    }
+    return true;
+  },
+
+  /**
+   * Consume a run of "text" beginning with the current read position,
+   * consisting of characters in the class |aClass| (which must be a
+   * suitable argument to IsOpenCharClass) plus escape sequences.
+   * Append the text to |aText|, after decoding escape sequences.
+   *
+   * Returns true if at least one character was appended to |aText|,
+   * false otherwise.
+   */
+  GatherText: function (aClass, aText) {
+    let start = this.mOffset;
+    let inString = aClass == IS_STRING;
+
+    for (;;) {
+      // Consume runs of unescaped characters in one go.
+      let n = this.mOffset;
+      while (n < this.mCount && IsOpenCharClass(this.mBuffer.charCodeAt(n),
+                                                aClass)) {
+        n++;
+      }
+      if (n > this.mOffset) {
+        let substr = this.mBuffer.slice(this.mOffset, n);
+        Array.prototype.push.apply(aText, stringToCodes(substr));
+        this.mOffset = n;
+      }
+      if (n == this.mCount) {
+        break;
+      }
+
+      let ch = this.Peek();
+      if (ch == 0) {
+        this.Advance();
+        aText.push(UCS2_REPLACEMENT_CHAR);
+        continue;
+      }
+
+      if (ch != REVERSE_SOLIDUS) {
+        break;
+      }
+      if (!this.GatherEscape(aText, inString)) {
+        break;
+      }
+    }
+
+    return this.mOffset > start;
+  },
+
+  /**
+   * Scan an Ident token.  This also handles Function and URL tokens,
+   * both of which begin indistinguishably from an identifier.  It can
+   * produce a Symbol token when an apparent identifier actually led
+   * into an invalid escape sequence.
+   */
+  ScanIdent: function (aToken) {
+    if (!this.GatherText(IS_IDCHAR, aToken.mIdent)) {
+      aToken.mSymbol = this.Peek();
+      this.Advance();
+      return true;
+    }
+
+    if (this.Peek() != LEFT_PARENTHESIS) {
+      aToken.mType = eCSSToken_Ident;
+      return true;
+    }
+
+    this.Advance();
+    aToken.mType = eCSSToken_Function;
+
+    let asString = String.fromCharCode.apply(null, aToken.mIdent);
+    if (asString.toLowerCase() === "url") {
+      this.NextURL(aToken);
+    }
+    return true;
+  },
+
+  /**
+   * Scan an AtKeyword token.  Also handles production of Symbol when
+   * an '@' is not followed by an identifier.
+   */
+  ScanAtKeyword: function (aToken) {
+    // Fall back for when '@' isn't followed by an identifier.
+    aToken.mSymbol = COMMERCIAL_AT;
+    this.Advance();
+
+    let ch = this.Peek();
+    if (StartsIdent(ch, this.Peek(1))) {
+      if (this.GatherText(IS_IDCHAR, aToken.mIdent)) {
+        aToken.mType = eCSSToken_AtKeyword;
+      }
+    }
+    return true;
+  },
+
+  /**
+   * Scan a Hash token.  Handles the distinction between eCSSToken_ID
+   * and eCSSToken_Hash, and handles production of Symbol when a '#'
+   * is not followed by identifier characters.
+   */
+  ScanHash: function (aToken) {
+    // Fall back for when '#' isn't followed by identifier characters.
+    aToken.mSymbol = NUMBER_SIGN;
+    this.Advance();
+
+    let ch = this.Peek();
+    if (IsIdentChar(ch) || ch == REVERSE_SOLIDUS) {
+      let type =
+          StartsIdent(ch, this.Peek(1)) ? eCSSToken_ID : eCSSToken_Hash;
+      aToken.mIdent.length = 0;
+      if (this.GatherText(IS_IDCHAR, aToken.mIdent)) {
+        aToken.mType = type;
+      }
+    }
+
+    return true;
+  },
+
+  /**
+   * Scan a Number, Percentage, or Dimension token (all of which begin
+   * like a Number).  Can produce a Symbol when a '.' is not followed by
+   * digits, or when '+' or '-' are not followed by either a digit or a
+   * '.' and then a digit.  Can also produce a HTMLComment when it
+   * encounters '-->'.
+   */
+  ScanNumber: function (aToken) {
+    let c = this.Peek();
+
+    // Sign of the mantissa (-1 or 1).
+    let sign = c == HYPHEN_MINUS ? -1 : 1;
+    // Absolute value of the integer part of the mantissa.  This is a double so
+    // we don't run into overflow issues for consumers that only care about our
+    // floating-point value while still being able to express the full int32_t
+    // range for consumers who want integers.
+    let intPart = 0;
+    // Fractional part of the mantissa.  This is a double so that when
+    // we convert to float at the end we'll end up rounding to nearest
+    // float instead of truncating down (as we would if fracPart were
+    // a float and we just effectively lost the last several digits).
+    let fracPart = 0;
+    // Absolute value of the power of 10 that we should multiply by
+    // (only relevant for numbers in scientific notation).  Has to be
+    // a signed integer, because multiplication of signed by unsigned
+    // converts the unsigned to signed, so if we plan to actually
+    // multiply by expSign...
+    let exponent = 0;
+    // Sign of the exponent.
+    let expSign = 1;
+
+    aToken.mHasSign = (c == PLUS_SIGN || c == HYPHEN_MINUS);
+    if (aToken.mHasSign) {
+      this.Advance();
+      c = this.Peek();
+    }
+
+    let gotDot = (c == FULL_STOP);
+
+    if (!gotDot) {
+      // Scan the integer part of the mantissa.
+      do {
+        intPart = 10 * intPart + DecimalDigitValue(c);
+        this.Advance();
+        c = this.Peek();
+      } while (IsDigit(c));
+
+      gotDot = (c == FULL_STOP) && IsDigit(this.Peek(1));
+    }
+
+    if (gotDot) {
+      // Scan the fractional part of the mantissa.
+      this.Advance();
+      c = this.Peek();
+      // Power of ten by which we need to divide our next digit
+      let divisor = 10;
+      do {
+        fracPart += DecimalDigitValue(c) / divisor;
+        divisor *= 10;
+        this.Advance();
+        c = this.Peek();
+      } while (IsDigit(c));
+    }
+
+    let gotE = false;
+    if (c == LATIN_SMALL_LETTER_E || c == LATIN_CAPITAL_LETTER_E) {
+      let expSignChar = this.Peek(1);
+      let nextChar = this.Peek(2);
+      if (IsDigit(expSignChar) ||
+          ((expSignChar == HYPHEN_MINUS || expSignChar == PLUS_SIGN) &&
+           IsDigit(nextChar))) {
+        gotE = true;
+        if (expSignChar == HYPHEN_MINUS) {
+          expSign = -1;
+        }
+        this.Advance(); // consumes the E
+        if (expSignChar == HYPHEN_MINUS || expSignChar == PLUS_SIGN) {
+          this.Advance();
+          c = nextChar;
+        } else {
+          c = expSignChar;
+        }
+        do {
+          exponent = 10 * exponent + DecimalDigitValue(c);
+          this.Advance();
+          c = this.Peek();
+        } while (IsDigit(c));
+      }
+    }
+
+    let type = eCSSToken_Number;
+
+    // Set mIntegerValid for all cases (except %, below) because we need
+    // it for the "2n" in :nth-child(2n).
+    aToken.mIntegerValid = false;
+
+    // Time to reassemble our number.
+    // Do all the math in double precision so it's truncated only once.
+    let value = sign * (intPart + fracPart);
+    if (gotE) {
+      // Explicitly cast expSign*exponent to double to avoid issues with
+      // overloaded pow() on Windows.
+      value *= Math.pow(10.0, expSign * exponent);
+    } else if (!gotDot) {
+      // Clamp values outside of integer range.
+      if (sign > 0) {
+        aToken.mInteger = Math.min(intPart, Number.MAX_SAFE_INTEGER);
+      } else {
+        aToken.mInteger = Math.max(-intPart, Number.MIN_SAFE_INTEGER);
+      }
+      aToken.mIntegerValid = true;
+    }
+
+    let ident = aToken.mIdent;
+
+    // Check for Dimension and Percentage tokens.
+    if (c >= 0) {
+      if (StartsIdent(c, this.Peek(1))) {
+        if (this.GatherText(IS_IDCHAR, ident)) {
+          type = eCSSToken_Dimension;
+        }
+      } else if (c == PERCENT_SIGN) {
+        this.Advance();
+        type = eCSSToken_Percentage;
+        value = value / 100.0;
+        aToken.mIntegerValid = false;
+      }
+    }
+    aToken.mNumber = value;
+    aToken.mType = type;
+    return true;
+  },
+
+  /**
+   * Scan a string constant ('foo' or "foo").  Will always produce
+   * either a String or a Bad_String token; the latter occurs when the
+   * close quote is missing.  Always returns true (for convenience in Next()).
+   */
+  ScanString: function (aToken) {
+    let aStop = this.Peek();
+    aToken.mType = eCSSToken_String;
+    aToken.mSymbol = aStop; // Remember how it's quoted.
+    this.Advance();
+
+    for (;;) {
+      this.GatherText(IS_STRING, aToken.mIdent);
+
+      let ch = this.Peek();
+      if (ch == -1) {
+        this.AddEOFCharacters(aStop == QUOTATION_MARK ?
+                              eEOFCharacters_DoubleQuote :
+                              eEOFCharacters_SingleQuote);
+        break; // EOF ends a string token with no error.
+      }
+      if (ch == aStop) {
+        this.Advance();
+        break;
+      }
+      // Both " and ' are excluded from IS_STRING.
+      if (ch == QUOTATION_MARK || ch == APOSTROPHE) {
+        aToken.mIdent.push(ch);
+        this.Advance();
+        continue;
+      }
+
+      aToken.mType = eCSSToken_Bad_String;
+      break;
+    }
+    return true;
+  },
+
+  /**
+   * Scan a unicode-range token.  These match the regular expression
+   *
+   *     u\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?
+   *
+   * However, some such tokens are "invalid".  There are three valid forms:
+   *
+   *     u+[0-9a-f]{x}              1 <= x <= 6
+   *     u+[0-9a-f]{x}\?{y}         1 <= x+y <= 6
+   *     u+[0-9a-f]{x}-[0-9a-f]{y}  1 <= x <= 6, 1 <= y <= 6
+   *
+   * All unicode-range tokens have their text recorded in mIdent; valid ones
+   * are also decoded into mInteger and mInteger2, and mIntegerValid is set.
+   * Note that this does not validate the numeric range, only the syntactic
+   * form.
+   */
+  ScanURange: function (aResult) {
+    let intro1 = this.Peek();
+    let intro2 = this.Peek(1);
+    let ch = this.Peek(2);
+
+    aResult.mIdent.push(intro1);
+    aResult.mIdent.push(intro2);
+    this.Advance(2);
+
+    let valid = true;
+    let haveQues = false;
+    let low = 0;
+    let high = 0;
+    let i = 0;
+
+    do {
+      aResult.mIdent.push(ch);
+      if (IsHexDigit(ch)) {
+        if (haveQues) {
+          valid = false; // All question marks should be at the end.
+        }
+        low = low * 16 + HexDigitValue(ch);
+        high = high * 16 + HexDigitValue(ch);
+      } else {
+        haveQues = true;
+        low = low * 16 + 0x0;
+        high = high * 16 + 0xF;
+      }
+
+      i++;
+      this.Advance();
+      ch = this.Peek();
+    } while (i < 6 && (IsHexDigit(ch) || ch == QUESTION_MARK));
+
+    if (ch == HYPHEN_MINUS && IsHexDigit(this.Peek(1))) {
+      if (haveQues) {
+        valid = false;
+      }
+
+      aResult.mIdent.push(ch);
+      this.Advance();
+      ch = this.Peek();
+      high = 0;
+      i = 0;
+      do {
+        aResult.mIdent.push(ch);
+        high = high * 16 + HexDigitValue(ch);
+
+        i++;
+        this.Advance();
+        ch = this.Peek();
+      } while (i < 6 && IsHexDigit(ch));
+    }
+
+    aResult.mInteger = low;
+    aResult.mInteger2 = high;
+    aResult.mIntegerValid = valid;
+    aResult.mType = eCSSToken_URange;
+    return true;
+  },
+
+  SetEOFCharacters: function (aEOFCharacters) {
+    this.mEOFCharacters = aEOFCharacters;
+  },
+
+  AddEOFCharacters: function (aEOFCharacters) {
+    this.mEOFCharacters = this.mEOFCharacters | aEOFCharacters;
+  },
+
+  AppendImpliedEOFCharacters: function (aEOFCharacters, aResult) {
+    // First, ignore eEOFCharacters_DropBackslash.
+    let c = aEOFCharacters >> 1;
+
+    // All of the remaining EOFCharacters bits represent appended characters,
+    // and the bits are in the order that they need appending.
+    for (let p of kImpliedEOFCharacters) {
+      if (c & 1) {
+        aResult.push(p);
+      }
+      c >>= 1;
+    }
+  },
+
+  /**
+   * Consume the part of an URL token after the initial 'url('.  Caller
+   * is assumed to have consumed 'url(' already.  Will always produce
+   * either an URL or a Bad_URL token.
+   *
+   * Exposed for use by nsCSSParser::ParseMozDocumentRule, which applies
+   * the special lexical rules for URL tokens in a nonstandard context.
+   */
+  NextURL: function (aToken) {
+    this.SkipWhitespace();
+
+    // aToken.mIdent may be "url" at this point; clear that out
+    aToken.mIdent.length = 0;
+
+    let ch = this.Peek();
+    // Do we have a string?
+    if (ch == QUOTATION_MARK || ch == APOSTROPHE) {
+      this.ScanString(aToken);
+      if (aToken.mType == eCSSToken_Bad_String) {
+        aToken.mType = eCSSToken_Bad_URL;
+        return;
+      }
+    } else {
+      // Otherwise, this is the start of a non-quoted url (which may be empty).
+      aToken.mSymbol = 0;
+      this.GatherText(IS_URL_CHAR, aToken.mIdent);
+    }
+
+    // Consume trailing whitespace and then look for a close parenthesis.
+    this.SkipWhitespace();
+    ch = this.Peek();
+    // ch can be less than zero indicating EOF
+    if (ch < 0 || ch == RIGHT_PARENTHESIS) {
+      this.Advance();
+      aToken.mType = eCSSToken_URL;
+      if (ch < 0) {
+        this.AddEOFCharacters(eEOFCharacters_CloseParen);
+      }
+    } else {
+      aToken.mType = eCSSToken_Bad_URL;
+    }
+  },
+
+  /**
+   * Primary scanner entry point.  Consume one token and fill in
+   * |aToken| accordingly.  Will skip over any number of comments first,
+   * and will also skip over rather than return whitespace and comment
+   * tokens, depending on the value of |aSkip|.
+   *
+   * Returns true if it successfully consumed a token, false if EOF has
+   * been reached.  Will always advance the current read position by at
+   * least one character unless called when already at EOF.
+   */
+  Next: function (aToken, aSkip) {
+    let ch;
+
+    // do this here so we don't have to do it in dozens of other places
+    aToken.mIdent = [];
+    aToken.mType = eCSSToken_Symbol;
+
+    this.mTokenOffset = this.mOffset;
+    this.mTokenLineOffset = this.mLineOffset;
+    this.mTokenLineNumber = this.mLineNumber;
+
+    ch = this.Peek();
+    if (IsWhitespace(ch)) {
+      this.SkipWhitespace();
+      aToken.mType = eCSSToken_Whitespace;
+      return true;
+    }
+    if (ch == SOLIDUS && // !IsSVGMode() &&
+        this.Peek(1) == ASTERISK) {
+      this.SkipComment();
+      aToken.mType = eCSSToken_Comment;
+      return true;
+    }
+
+    // EOF
+    if (ch < 0) {
+      return false;
+    }
+
+    // 'u' could be UNICODE-RANGE or an identifier-family token
+    if (ch == LATIN_SMALL_LETTER_U || ch == LATIN_CAPITAL_LETTER_U) {
+      let c2 = this.Peek(1);
+      let c3 = this.Peek(2);
+      if (c2 == PLUS_SIGN && (IsHexDigit(c3) || c3 == QUESTION_MARK)) {
+        return this.ScanURange(aToken);
+      }
+      return this.ScanIdent(aToken);
+    }
+
+    // identifier family
+    if (IsIdentStart(ch)) {
+      return this.ScanIdent(aToken);
+    }
+
+    // number family
+    if (IsDigit(ch)) {
+      return this.ScanNumber(aToken);
+    }
+
+    if (ch == FULL_STOP && IsDigit(this.Peek(1))) {
+      return this.ScanNumber(aToken);
+    }
+
+    if (ch == PLUS_SIGN) {
+      let c2 = this.Peek(1);
+      if (IsDigit(c2) || (c2 == FULL_STOP && IsDigit(this.Peek(2)))) {
+        return this.ScanNumber(aToken);
+      }
+    }
+
+    // HYPHEN_MINUS can start an identifier-family token, a number-family token,
+    // or an HTML-comment
+    if (ch == HYPHEN_MINUS) {
+      let c2 = this.Peek(1);
+      let c3 = this.Peek(2);
+      if (IsIdentStart(c2) || (c2 == HYPHEN_MINUS && c3 != GREATER_THAN_SIGN)) {
+        return this.ScanIdent(aToken);
+      }
+      if (IsDigit(c2) || (c2 == FULL_STOP && IsDigit(c3))) {
+        return this.ScanNumber(aToken);
+      }
+      if (c2 == HYPHEN_MINUS && c3 == GREATER_THAN_SIGN) {
+        this.Advance(3);
+        aToken.mType = eCSSToken_HTMLComment;
+        aToken.mIdent = stringToCodes("-->");
+        return true;
+      }
+    }
+
+    // the other HTML-comment token
+    if (ch == LESS_THAN_SIGN &&
+        this.Peek(1) == EXCLAMATION_MARK &&
+        this.Peek(2) == HYPHEN_MINUS &&
+        this.Peek(3) == HYPHEN_MINUS) {
+      this.Advance(4);
+      aToken.mType = eCSSToken_HTMLComment;
+      aToken.mIdent = stringToCodes("<!--");
+      return true;
+    }
+
+    // AT_KEYWORD
+    if (ch == COMMERCIAL_AT) {
+      return this.ScanAtKeyword(aToken);
+    }
+
+    // HASH
+    if (ch == NUMBER_SIGN) {
+      return this.ScanHash(aToken);
+    }
+
+    // STRING
+    if (ch == QUOTATION_MARK || ch == APOSTROPHE) {
+      return this.ScanString(aToken);
+    }
+
+    // Match operators: ~= |= ^= $= *=
+    let opType = MatchOperatorType(ch);
+    if (opType != eCSSToken_Symbol && this.Peek(1) == EQUALS_SIGN) {
+      aToken.mType = opType;
+      this.Advance(2);
+      return true;
+    }
+
+    // Otherwise, a symbol (DELIM).
+    aToken.mSymbol = ch;
+    this.Advance();
+    return true;
+  },
+};
+
+/**
+ * Create and return a new CSS lexer, conforming to the @see CSSLexer
+ * webidl interface.
+ *
+ * @param {String} input the CSS text to lex
+ * @return {CSSLexer} the new lexer
+ */
+function getCSSLexer(input) {
+  return new Scanner(input);
+}
+
+exports.getCSSLexer = getCSSLexer;
--- a/devtools/shared/inspector/css-logic.js
+++ b/devtools/shared/inspector/css-logic.js
@@ -44,16 +44,18 @@ const { Cc, Ci, Cu } = require("chrome")
 const Services = require("Services");
 const DevToolsUtils = require("devtools/shared/DevToolsUtils");
 const { getRootBindingParent } = require("devtools/shared/layout/utils");
 
 // This should be ok because none of the functions that use this should be used
 // on the worker thread, where Cu is not available.
 loader.lazyRequireGetter(this, "CSS", "CSS");
 
+loader.lazyRequireGetter(this, "CSSLexer", "devtools/shared/css-lexer");
+
 function CssLogic() {
   // The cache of examined CSS properties.
   this._propertyInfos = {};
 }
 
 exports.CssLogic = CssLogic;
 
 /**
@@ -987,17 +989,17 @@ CssLogic.prettifyCSS = function(text, ru
   // * A "}" symbol ensures there is a preceding newline, and
   //   decreases the indentation level.
   // * Ensure there is whitespace before a "{".
   //
   // This approach can be confused sometimes, but should do ok on a
   // minified file.
   let indent = "";
   let indentLevel = 0;
-  let tokens = domUtils.getCSSLexer(text);
+  let tokens = CSSLexer.getCSSLexer(text);
   let result = "";
   let pushbackToken = undefined;
 
   // A helper function that reads tokens, looking for the next
   // non-comment, non-whitespace token.  Comment and whitespace tokens
   // are appended to |result|.  If this encounters EOF, it returns
   // null.  Otherwise it returns the last whitespace token that was
   // seen.  This function also updates |pushbackToken|.
--- a/devtools/shared/moz.build
+++ b/devtools/shared/moz.build
@@ -36,16 +36,17 @@ XPCSHELL_TESTS_MANIFESTS += ['tests/unit
 
 JAR_MANIFESTS += ['jar.mn']
 
 DevToolsModules(
     'async-storage.js',
     'async-utils.js',
     'content-observer.js',
     'css-angle.js',
+    'css-lexer.js',
     'deprecated-sync-thenables.js',
     'DevToolsUtils.js',
     'event-emitter.js',
     'event-parsers.js',
     'indentation.js',
     'Loader.jsm',
     'Parser.jsm',
     'path.js',
new file mode 100644
--- /dev/null
+++ b/devtools/shared/tests/unit/test_csslexer.js
@@ -0,0 +1,242 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+// This file is a copy of layout/style/test/test_csslexer.js, modified
+// to use both our pure-JS lexer and the DOMUtils lexer for
+// cross-checking.
+
+"use strict";
+
+const jsLexer = require("devtools/shared/css-lexer");
+const domutils = Components.classes["@mozilla.org/inspector/dom-utils;1"]
+                           .getService(Components.interfaces.inIDOMUtils);
+
+// An object that acts like a CSSLexer but verifies that the DOM lexer
+// and the JS lexer do the same thing.
+function DoubleLexer(input) {
+  do_print("DoubleLexer input: " + input);
+  this.domLexer = domutils.getCSSLexer(input);
+  this.jsLexer = jsLexer.getCSSLexer(input);
+}
+
+DoubleLexer.prototype = {
+  checkState: function () {
+    equal(this.domLexer.lineNumber, this.jsLexer.lineNumber,
+          "check line number");
+    equal(this.domLexer.columnNumber, this.jsLexer.columnNumber,
+         "check column number");
+  },
+
+  get lineNumber() {
+    return this.domLexer.lineNumber;
+  },
+
+  get columnNumber() {
+    return this.domLexer.columnNumber;
+  },
+
+  performEOFFixup: function (inputString, preserveBackslash) {
+    let d = this.domLexer.performEOFFixup(inputString, preserveBackslash);
+    let j = this.jsLexer.performEOFFixup(inputString, preserveBackslash);
+
+    equal(d, j);
+    return d;
+  },
+
+  mungeNumber: function (token) {
+    if (token && (token.tokenType === "number" ||
+                  token.tokenType === "percentage") &&
+        !token.isInteger) {
+      // The JS lexer does its computations in double, but the
+      // platform lexer does its computations in float.  Account for
+      // this discrepancy in a way that's sufficient for this test.
+      // See https://bugzilla.mozilla.org/show_bug.cgi?id=1163047
+      token.number = parseFloat(token.number.toPrecision(8));
+    }
+  },
+
+  nextToken: function () {
+    // Check state both before and after.
+    this.checkState();
+
+    let d = this.domLexer.nextToken();
+    let j = this.jsLexer.nextToken();
+
+    this.mungeNumber(d);
+    this.mungeNumber(j);
+
+    deepEqual(d, j);
+
+    this.checkState();
+
+    return d;
+  }
+};
+
+function test_lexer(cssText, tokenTypes) {
+  let lexer = new DoubleLexer(cssText);
+  let reconstructed = "";
+  let lastTokenEnd = 0;
+  let i = 0;
+  while (true) {
+    let token = lexer.nextToken();
+    if (!token) {
+      break;
+    }
+    let combined = token.tokenType;
+    if (token.text) {
+      combined += ":" + token.text;
+    }
+    equal(combined, tokenTypes[i]);
+    ok(token.endOffset > token.startOffset);
+    equal(token.startOffset, lastTokenEnd);
+    lastTokenEnd = token.endOffset;
+    reconstructed += cssText.substring(token.startOffset, token.endOffset);
+    ++i;
+  }
+  // Ensure that we saw the correct number of tokens.
+  equal(i, tokenTypes.length);
+  // Ensure that the reported offsets cover all the text.
+  equal(reconstructed, cssText);
+}
+
+var LEX_TESTS = [
+  ["simple", ["ident:simple"]],
+  ["simple: { hi; }",
+             ["ident:simple", "symbol::",
+              "whitespace", "symbol:{",
+              "whitespace", "ident:hi",
+              "symbol:;", "whitespace",
+              "symbol:}"]],
+  ["/* whatever */", ["comment"]],
+  ["'string'", ["string:string"]],
+  ['"string"', ["string:string"]],
+  ["rgb(1,2,3)", ["function:rgb", "number",
+                                      "symbol:,", "number",
+                                      "symbol:,", "number",
+                                      "symbol:)"]],
+  ["@media", ["at:media"]],
+  ["#hibob", ["id:hibob"]],
+  ["#123", ["hash:123"]],
+  ["23px", ["dimension:px"]],
+  ["23%", ["percentage"]],
+  ["url(http://example.com)", ["url:http://example.com"]],
+  ["url('http://example.com')", ["url:http://example.com"]],
+  ["url(  'http://example.com'  )",
+             ["url:http://example.com"]],
+  // In CSS Level 3, this is an ordinary URL, not a BAD_URL.
+  ["url(http://example.com", ["url:http://example.com"]],
+  // See bug 1153981 to understand why this gets a SYMBOL token.
+  ["url(http://example.com @", ["bad_url:http://example.com", "symbol:@"]],
+  ["quo\\ting", ["ident:quoting"]],
+  ["'bad string\n", ["bad_string:bad string", "whitespace"]],
+  ["~=", ["includes"]],
+  ["|=", ["dashmatch"]],
+  ["^=", ["beginsmatch"]],
+  ["$=", ["endsmatch"]],
+  ["*=", ["containsmatch"]],
+
+  // URANGE may be on the way out, and it isn't used by devutils, so
+  // let's skip it.
+
+  ["<!-- html comment -->", ["htmlcomment", "whitespace", "ident:html",
+                             "whitespace", "ident:comment", "whitespace",
+                             "htmlcomment"]],
+
+  // earlier versions of CSS had "bad comment" tokens, but in level 3,
+  // unterminated comments are just comments.
+  ["/* bad comment", ["comment"]]
+];
+
+function test_lexer_linecol(cssText, locations) {
+  let lexer = new DoubleLexer(cssText);
+  let i = 0;
+  while (true) {
+    let token = lexer.nextToken();
+    let startLine = lexer.lineNumber;
+    let startColumn = lexer.columnNumber;
+
+    // We do this in a bit of a funny way so that we can also test the
+    // location of the EOF.
+    let combined = ":" + startLine + ":" + startColumn;
+    if (token) {
+      combined = token.tokenType + combined;
+    }
+
+    equal(combined, locations[i]);
+    ++i;
+
+    if (!token) {
+      break;
+    }
+  }
+  // Ensure that we saw the correct number of tokens.
+  equal(i, locations.length);
+}
+
+function test_lexer_eofchar(cssText, argText, expectedAppend,
+                            expectedNoAppend) {
+  let lexer = new DoubleLexer(cssText);
+  while (lexer.nextToken()) {
+    // Nothing.
+  }
+
+  do_print("EOF char test, input = " + cssText);
+
+  let result = lexer.performEOFFixup(argText, true);
+  equal(result, expectedAppend);
+
+  result = lexer.performEOFFixup(argText, false);
+  equal(result, expectedNoAppend);
+}
+
+var LINECOL_TESTS = [
+  ["simple", ["ident:0:0", ":0:6"]],
+  ["\n    stuff", ["whitespace:0:0", "ident:1:4", ":1:9"]],
+  ['"string with \\\nnewline"    \r\n', ["string:0:0", "whitespace:1:8",
+                                         ":2:0"]]
+];
+
+var EOFCHAR_TESTS = [
+  ["hello", "hello"],
+  ["hello \\", "hello \\\\", "hello \\\uFFFD"],
+  ["'hello", "'hello'"],
+  ["\"hello", "\"hello\""],
+  ["'hello\\", "'hello\\\\'", "'hello'"],
+  ["\"hello\\", "\"hello\\\\\"", "\"hello\""],
+  ["/*hello", "/*hello*/"],
+  ["/*hello*", "/*hello*/"],
+  ["/*hello\\", "/*hello\\*/"],
+  ["url(hello", "url(hello)"],
+  ["url('hello", "url('hello')"],
+  ["url(\"hello", "url(\"hello\")"],
+  ["url(hello\\", "url(hello\\\\)", "url(hello\\\uFFFD)"],
+  ["url('hello\\", "url('hello\\\\')", "url('hello')"],
+  ["url(\"hello\\", "url(\"hello\\\\\")", "url(\"hello\")"],
+];
+
+function run_test() {
+  let text, result;
+  for ([text, result] of LEX_TESTS) {
+    test_lexer(text, result);
+  }
+
+  for ([text, result] of LINECOL_TESTS) {
+    test_lexer_linecol(text, result);
+  }
+
+  let expectedAppend, expectedNoAppend;
+  for ([text, expectedAppend, expectedNoAppend] of EOFCHAR_TESTS) {
+    if (!expectedNoAppend) {
+      expectedNoAppend = expectedAppend;
+    }
+    test_lexer_eofchar(text, text, expectedAppend, expectedNoAppend);
+  }
+
+  // Ensure that passing a different inputString to performEOFFixup
+  // doesn't cause an assertion trying to strip a backslash from the
+  // end of an empty string.
+  test_lexer_eofchar("'\\", "", "\\'", "'");
+}
--- a/devtools/shared/tests/unit/xpcshell.ini
+++ b/devtools/shared/tests/unit/xpcshell.ini
@@ -3,16 +3,17 @@ tags = devtools
 head = head_devtools.js
 tail =
 firefox-appdir = browser
 skip-if = toolkit == 'android' || toolkit == 'gonk'
 support-files =
   exposeLoader.js
 
 [test_assert.js]
+[test_csslexer.js]
 [test_fetch-chrome.js]
 [test_fetch-file.js]
 [test_fetch-http.js]
 [test_fetch-resource.js]
 [test_flatten.js]
 [test_indentation.js]
 [test_independent_loaders.js]
 [test_invisible_loader.js]