Backed out changeset 2fb18533192e (bug 1545916) for test_tokenizer.js failures CLOSED TREE
authorBogdan Tara <btara@mozilla.com>
Thu, 25 Apr 2019 19:57:27 +0300
changeset 530154 e2ee2fb416dc1d46529cf63269dcdef7a45c27ae
parent 530153 765b4140cde753ee6ac95d3cbc0c84003a007a1c
child 530155 09176cea4448959c4f1651b0addd0a5d5745ced1
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1545916
milestone68.0a1
backs out2fb18533192ec35ca71374439938757a5bc008cd
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 2fb18533192e (bug 1545916) for test_tokenizer.js failures CLOSED TREE
browser/components/urlbar/UrlbarTokenizer.jsm
browser/components/urlbar/UrlbarUtils.jsm
browser/components/urlbar/tests/browser/browser_view_resultDisplay.js
browser/components/urlbar/tests/unit/test_UrlbarUtils_getTokenMatches.js
browser/docs/AddressBar.rst
--- a/browser/components/urlbar/UrlbarTokenizer.jsm
+++ b/browser/components/urlbar/UrlbarTokenizer.jsm
@@ -267,17 +267,16 @@ function splitString(searchString) {
  */
 function filterTokens(tokens) {
   let filtered = [];
   let restrictions = [];
   for (let i = 0; i < tokens.length; ++i) {
     let token = tokens[i];
     let tokenObj = {
       value: token,
-      lowerCaseValue: token.toLocaleLowerCase(),
       type: UrlbarTokenizer.TYPE.TEXT,
     };
     let restrictionType = CHAR_TO_TYPE_MAP.get(token);
     if (restrictionType) {
       restrictions.push({index: i, type: restrictionType});
     } else if (UrlbarTokenizer.looksLikeOrigin(token)) {
       tokenObj.type = UrlbarTokenizer.TYPE.POSSIBLE_ORIGIN;
     } else if (UrlbarTokenizer.looksLikeUrl(token, {requirePath: true})) {
--- a/browser/components/urlbar/UrlbarUtils.jsm
+++ b/browser/components/urlbar/UrlbarUtils.jsm
@@ -223,41 +223,39 @@ var UrlbarUtils = {
     let mimeStream = Cc["@mozilla.org/network/mime-input-stream;1"]
                        .createInstance(Ci.nsIMIMEInputStream);
     mimeStream.addHeader("Content-Type", type);
     mimeStream.setData(dataStream);
     return mimeStream.QueryInterface(Ci.nsIInputStream);
   },
 
   /**
-   * Returns a list of all the token substring matches in a string.  Matching is
-   * case insensitive.  Each match in the returned list is a tuple: [matchIndex,
-   * matchLength].  matchIndex is the index in the string of the match, and
-   * matchLength is the length of the match.
+   * Returns a list of all the token substring matches in a string.  Each match
+   * in the list is a tuple: [matchIndex, matchLength].  matchIndex is the index
+   * in the string of the match, and matchLength is the length of the match.
    *
    * @param {array} tokens The tokens to search for.
    * @param {string} str The string to match against.
    * @returns {array} An array: [
    *            [matchIndex_0, matchLength_0],
    *            [matchIndex_1, matchLength_1],
    *            ...
    *            [matchIndex_n, matchLength_n]
    *          ].
    *          The array is sorted by match indexes ascending.
    */
   getTokenMatches(tokens, str) {
-    str = str.toLocaleLowerCase();
     // To generate non-overlapping ranges, we start from a 0-filled array with
     // the same length of the string, and use it as a collision marker, setting
     // 1 where a token matches.
     let hits = new Array(str.length).fill(0);
-    for (let { lowerCaseValue } of tokens) {
+    for (let token of tokens) {
       // Ideally we should never hit the empty token case, but just in case
-      // the `needle` check protects us from an infinite loop.
-      for (let index = 0, needle = lowerCaseValue; index >= 0 && needle;) {
+      // the value check protects us from an infinite loop.
+      for (let index = 0, needle = token.value; index >= 0 && needle;) {
         index = str.indexOf(needle, index);
         if (index >= 0) {
           hits.fill(1, index, index + needle.length);
           index += needle.length;
         }
       }
     }
     // Starting from the collision array, generate [start, len] tuples
--- a/browser/components/urlbar/tests/browser/browser_view_resultDisplay.js
+++ b/browser/components/urlbar/tests/browser/browser_view_resultDisplay.js
@@ -15,23 +15,22 @@ add_task(async function setup() {
     await PlacesUtils.history.clear();
     Services.prefs.clearUserPref("browser.urlbar.trimURLs");
   });
 });
 
 async function testResult(input, expected) {
   const ESCAPED_URL = encodeURI(input.url);
 
-  await PlacesUtils.history.clear();
   await PlacesTestUtils.addVisits({
     uri: input.url,
     title: input.title,
   });
 
-  await promiseAutocompleteResultPopup(input.query);
+  await promiseAutocompleteResultPopup("\u6e2C\u8a66");
 
   let result = await UrlbarTestUtils.getDetailsOfResultAt(window, 1);
   Assert.equal(result.url, ESCAPED_URL,
     "Should have the correct url to load");
   Assert.equal(result.displayed.url, expected.displayedUrl,
     "Should have the correct displayed url");
   Assert.equal(result.displayed.title, input.title,
     "Should have the expected title");
@@ -96,150 +95,8 @@ add_task(async function test_url_result_
       ["http://example.com/", false],
       ["\u6e2C\u8a66", true],
       ["test", false],
     ],
   });
 
   Services.prefs.clearUserPref("browser.urlbar.trimURLs");
 });
-
-add_task(async function test_case_insensitive_highlights_1() {
-  await testResult({
-    query: "exam",
-    title: "The examPLE URL EXAMple",
-    url: "http://example.com/ExAm",
-  }, {
-    displayedUrl: "example.com/ExAm",
-    highlightedTitle: [
-      ["The ", false],
-      ["exam", true],
-      ["PLE URL ", false],
-      ["EXAM", true],
-      ["ple", false],
-    ],
-    highlightedUrl: [
-      ["exam", true],
-      ["ple.com/", false],
-      ["ExAm", true],
-    ],
-  });
-});
-
-add_task(async function test_case_insensitive_highlights_2() {
-  await testResult({
-    query: "EXAM",
-    title: "The examPLE URL EXAMple",
-    url: "http://example.com/ExAm",
-  }, {
-    displayedUrl: "example.com/ExAm",
-    highlightedTitle: [
-      ["The ", false],
-      ["exam", true],
-      ["PLE URL ", false],
-      ["EXAM", true],
-      ["ple", false],
-    ],
-    highlightedUrl: [
-      ["exam", true],
-      ["ple.com/", false],
-      ["ExAm", true],
-    ],
-  });
-});
-
-add_task(async function test_case_insensitive_highlights_3() {
-  await testResult({
-    query: "eXaM",
-    title: "The examPLE URL EXAMple",
-    url: "http://example.com/ExAm",
-  }, {
-    displayedUrl: "example.com/ExAm",
-    highlightedTitle: [
-      ["The ", false],
-      ["exam", true],
-      ["PLE URL ", false],
-      ["EXAM", true],
-      ["ple", false],
-    ],
-    highlightedUrl: [
-      ["exam", true],
-      ["ple.com/", false],
-      ["ExAm", true],
-    ],
-  });
-});
-
-add_task(async function test_case_insensitive_highlights_4() {
-  await testResult({
-    query: "ExAm",
-    title: "The examPLE URL EXAMple",
-    url: "http://example.com/ExAm",
-  }, {
-    displayedUrl: "example.com/ExAm",
-    highlightedTitle: [
-      ["The ", false],
-      ["exam", true],
-      ["PLE URL ", false],
-      ["EXAM", true],
-      ["ple", false],
-    ],
-    highlightedUrl: [
-      ["exam", true],
-      ["ple.com/", false],
-      ["ExAm", true],
-    ],
-  });
-});
-
-add_task(async function test_case_insensitive_highlights_5() {
-  await testResult({
-    query: "exam foo",
-    title: "The examPLE URL foo EXAMple FOO",
-    url: "http://example.com/ExAm/fOo",
-  }, {
-    displayedUrl: "example.com/ExAm/fOo",
-    highlightedTitle: [
-      ["The ", false],
-      ["exam", true],
-      ["PLE URL ", false],
-      ["foo", true],
-      [" ", false],
-      ["EXAM", true],
-      ["ple ", false],
-      ["FOO", true],
-    ],
-    highlightedUrl: [
-      ["exam", true],
-      ["ple.com/", false],
-      ["ExAm", true],
-      ["/", false],
-      ["fOo", true],
-    ],
-  });
-});
-
-add_task(async function test_case_insensitive_highlights_6() {
-  await testResult({
-    query: "EXAM FOO",
-    title: "The examPLE URL foo EXAMple FOO",
-    url: "http://example.com/ExAm/fOo",
-  }, {
-    displayedUrl: "example.com/ExAm/fOo",
-    highlightedTitle: [
-      ["The ", false],
-      ["exam", true],
-      ["PLE URL ", false],
-      ["foo", true],
-      [" ", false],
-      ["EXAM", true],
-      ["ple ", false],
-      ["FOO", true],
-    ],
-    highlightedUrl: [
-      ["exam", true],
-      ["ple.com/", false],
-      ["ExAm", true],
-      ["/", false],
-      ["fOo", true],
-    ],
-  });
-});
--- a/browser/components/urlbar/tests/unit/test_UrlbarUtils_getTokenMatches.js
+++ b/browser/components/urlbar/tests/unit/test_UrlbarUtils_getTokenMatches.js
@@ -10,153 +10,54 @@
 add_task(function test() {
   const tests = [
     {
       tokens: ["mozilla", "is", "i"],
       phrase: "mozilla is for the Open Web",
       expected: [[0, 7], [8, 2]],
     },
     {
-      tokens: ["mozilla", "is", "i"],
-      phrase: "MOZILLA IS for the Open Web",
-      expected: [[0, 7], [8, 2]],
-    },
-    {
-      tokens: ["mozilla", "is", "i"],
-      phrase: "MoZiLlA Is for the Open Web",
-      expected: [[0, 7], [8, 2]],
-    },
-    {
-      tokens: ["MOZILLA", "IS", "I"],
-      phrase: "mozilla is for the Open Web",
-      expected: [[0, 7], [8, 2]],
-    },
-    {
-      tokens: ["MoZiLlA", "Is", "I"],
-      phrase: "mozilla is for the Open Web",
-      expected: [[0, 7], [8, 2]],
-    },
-    {
       tokens: ["mo", "b"],
       phrase: "mozilla is for the Open Web",
       expected: [[0, 2], [26, 1]],
     },
     {
-      tokens: ["mo", "b"],
-      phrase: "MOZILLA is for the OPEN WEB",
-      expected: [[0, 2], [26, 1]],
-    },
-    {
-      tokens: ["MO", "B"],
-      phrase: "mozilla is for the Open Web",
-      expected: [[0, 2], [26, 1]],
-    },
-    {
       tokens: ["mo", ""],
       phrase: "mozilla is for the Open Web",
       expected: [[0, 2]],
     },
     {
       tokens: ["mozilla"],
       phrase: "mozilla",
       expected: [[0, 7]],
     },
     {
-      tokens: ["mozilla"],
-      phrase: "MOZILLA",
-      expected: [[0, 7]],
-    },
-    {
-      tokens: ["mozilla"],
-      phrase: "MoZiLlA",
-      expected: [[0, 7]],
-    },
-    {
-      tokens: ["mozilla"],
-      phrase: "mOzIlLa",
-      expected: [[0, 7]],
-    },
-    {
-      tokens: ["MOZILLA"],
-      phrase: "mozilla",
-      expected: [[0, 7]],
-    },
-    {
-      tokens: ["MoZiLlA"],
-      phrase: "mozilla",
-      expected: [[0, 7]],
-    },
-    {
-      tokens: ["mOzIlLa"],
-      phrase: "mozilla",
-      expected: [[0, 7]],
-    },
-    {
       tokens: ["\u9996"],
       phrase: "Test \u9996\u9875 Test",
       expected: [[5, 1]],
     },
     {
       tokens: ["mo", "zilla"],
       phrase: "mozilla",
       expected: [[0, 7]],
     },
     {
-      tokens: ["mo", "zilla"],
-      phrase: "MOZILLA",
-      expected: [[0, 7]],
-    },
-    {
-      tokens: ["mo", "zilla"],
-      phrase: "MoZiLlA",
-      expected: [[0, 7]],
-    },
-    {
-      tokens: ["mo", "zilla"],
-      phrase: "mOzIlLa",
-      expected: [[0, 7]],
-    },
-    {
-      tokens: ["MO", "ZILLA"],
-      phrase: "mozilla",
-      expected: [[0, 7]],
-    },
-    {
-      tokens: ["Mo", "Zilla"],
-      phrase: "mozilla",
-      expected: [[0, 7]],
-    },
-    {
       tokens: ["moz", "zilla"],
       phrase: "mozilla",
       expected: [[0, 7]],
     },
     {
       tokens: [""], // Should never happen in practice.
       phrase: "mozilla",
       expected: [],
     },
     {
       tokens: ["mo", "om"],
       phrase: "mozilla mozzarella momo",
       expected: [[0, 2], [8, 2], [19, 4]],
     },
-    {
-      tokens: ["mo", "om"],
-      phrase: "MOZILLA MOZZARELLA MOMO",
-      expected: [[0, 2], [8, 2], [19, 4]],
-    },
-    {
-      tokens: ["MO", "OM"],
-      phrase: "mozilla mozzarella momo",
-      expected: [[0, 2], [8, 2], [19, 4]],
-    },
   ];
   for (let {tokens, phrase, expected} of tests) {
-    let mockContext = {
-      searchString: tokens.join(" "),
-    };
-    UrlbarTokenizer.tokenize(mockContext);
-    tokens = mockContext.tokens;
+    tokens = tokens.map(t => ({value: t}));
     Assert.deepEqual(UrlbarUtils.getTokenMatches(tokens, phrase), expected,
                      `Match "${tokens.map(t => t.value).join(", ")}" on "${phrase}"`);
   }
 });
--- a/browser/docs/AddressBar.rst
+++ b/browser/docs/AddressBar.rst
@@ -73,17 +73,17 @@ It is augmented as it progresses through
                // registered through the UrlbarProvidersManager.
     sources; // {array} If provided is the list of sources, as defined by
              // RESULT_SOURCE.*, that can be returned by the model.
 
     // Properties added by the Model.
     preselected; // {boolean} whether the first result should be preselected.
     results; // {array} list of UrlbarResult objects.
     tokens; // {array} tokens extracted from the searchString, each token is an
-            // object in the form {type, value, lowerCaseValue}.
+            // object in the form {type, value}.
   }
 
 
 The Model
 =========
 
 The *Model* is the component responsible for retrieving search results based on
 the user's input, and sorting them accordingly to their importance.