Bug 1486819 - support mozParams in webext search engines r=aswan,adw,mkaply
authorShane Caraveo <scaraveo@mozilla.com>
Wed, 03 Oct 2018 20:23:16 +0000
changeset 487889 1f268b2058c9e498547c2a902127fa45a93673f8
parent 487888 da259f95580506589a87462b31d7585578e735a4
child 487890 47f1126313c624f27bf6b6622162727fa596f9d9
push id246
push userfmarier@mozilla.com
push dateSat, 13 Oct 2018 00:15:40 +0000
reviewersaswan, adw, mkaply
bugs1486819, 1485508, 1488517
milestone64.0a1
Bug 1486819 - support mozParams in webext search engines r=aswan,adw,mkaply mkaply for overall search engine api changes adw for searchservice changes. note that a small part of it will be removed in favor of the fix from bug 1485508 aswan for webextension side. note that I want to do better with the distribution signal, that can be in bug 1488517 Differential Revision: https://phabricator.services.mozilla.com/D4473
browser/components/extensions/parent/ext-chrome-settings-overrides.js
browser/components/extensions/schemas/chrome_settings_overrides.json
browser/components/extensions/test/xpcshell/test_ext_settings_overrides_search.js
browser/components/extensions/test/xpcshell/test_ext_settings_overrides_search_mozParam.js
browser/components/extensions/test/xpcshell/xpcshell-common.ini
toolkit/components/search/nsSearchService.js
--- a/browser/components/extensions/parent/ext-chrome-settings-overrides.js
+++ b/browser/components/extensions/parent/ext-chrome-settings-overrides.js
@@ -1,21 +1,26 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
 
+ChromeUtils.import("resource://gre/modules/ExtensionParent.jsm");
 ChromeUtils.defineModuleGetter(this, "ExtensionPreferencesManager",
                                "resource://gre/modules/ExtensionPreferencesManager.jsm");
 ChromeUtils.defineModuleGetter(this, "ExtensionSettingsStore",
                                "resource://gre/modules/ExtensionSettingsStore.jsm");
 ChromeUtils.defineModuleGetter(this, "ExtensionControlledPopup",
                                "resource:///modules/ExtensionControlledPopup.jsm");
 
+var {
+  IconDetails,
+} = ExtensionParent;
+
 const DEFAULT_SEARCH_STORE_TYPE = "default_search";
 const DEFAULT_SEARCH_SETTING_NAME = "defaultSearch";
 const ENGINE_ADDED_SETTING_NAME = "engineAdded";
 
 const HOMEPAGE_PREF = "browser.startup.homepage";
 const HOMEPAGE_CONFIRMED_TYPE = "homepageNotification";
 const HOMEPAGE_SETTING_TYPE = "prefs";
 const HOMEPAGE_SETTING_NAME = "homepage_override";
@@ -273,29 +278,41 @@ this.chrome_settings_overrides = class e
       if (engines.length > 0) {
         // There can be only one engine right now
         isCurrent = Services.search.currentEngine == engines[0];
         // Get position of engine and store it
         index = Services.search.getEngines().indexOf(engines[0]);
         Services.search.removeEngine(engines[0]);
       }
     }
+
+    let icons = extension.manifest.icons;
+    let iconURL = searchProvider.favicon_url ||
+                  (icons && extension.baseURI.resolve(IconDetails.getPreferredIcon(icons).icon));
+    let iconList = [];
+    if (icons) {
+      iconList = Object.entries(icons).map(icon => {
+        return {width: icon[0], height: icon[0],
+                url: extension.baseURI.resolve(icon[1])};
+      });
+    }
     try {
       let params = {
         template: searchProvider.search_url,
-        iconURL: searchProvider.favicon_url,
+        searchPostParams: searchProvider.search_url_post_params,
+        iconURL,
+        icons: iconList,
         alias: searchProvider.keyword,
         extensionID: extension.id,
+        isBuiltIn: extension.isPrivileged,
         suggestURL: searchProvider.suggest_url,
+        suggestPostParams: searchProvider.suggest_url_post_params,
         queryCharset: "UTF-8",
+        mozParams: searchProvider.params,
       };
-      if (searchProvider.search_url_post_params) {
-        params.method = "POST";
-        params.postData = searchProvider.search_url_post_params;
-      }
       Services.search.addEngineWithDetails(searchProvider.name.trim(), params);
       await ExtensionSettingsStore.addSetting(
         extension.id, DEFAULT_SEARCH_STORE_TYPE, ENGINE_ADDED_SETTING_NAME,
         searchProvider.name.trim());
       if (extension.startupReason === "ADDON_UPGRADE") {
         let engine = Services.search.getEngineByName(searchProvider.name.trim());
         if (isCurrent) {
           Services.search.currentEngine = engine;
--- a/browser/components/extensions/schemas/chrome_settings_overrides.json
+++ b/browser/components/extensions/schemas/chrome_settings_overrides.json
@@ -63,16 +63,22 @@
                     "deprecated": "Unsupported on Firefox at this time."
                   },
                   "search_url_post_params": {
                     "type": "string",
                     "optional": true,
                     "preprocess": "localize",
                     "description": "POST parameters to the search_url as a query string."
                   },
+                  "suggest_url_post_params": {
+                    "type": "string",
+                    "optional": true,
+                    "preprocess": "localize",
+                    "description": "POST parameters to the suggest_url as a query string."
+                  },
                   "instant_url_post_params": {
                     "type": "string",
                     "optional": true,
                     "preprocess": "localize",
                     "deprecated": "Unsupported on Firefox at this time."
                   },
                   "image_url_post_params": {
                     "type": "string",
@@ -94,16 +100,52 @@
                     "type": "integer",
                     "optional": true,
                     "deprecated": "Unsupported on Firefox."
                   },
                   "is_default": {
                     "type": "boolean",
                     "optional": true,
                     "description": "Sets the default engine to a built-in engine only."
+                  },
+                  "params": {
+                    "optional": true,
+                    "type": "array",
+                    "items": {
+                      "type": "object",
+                      "properties": {
+                        "name": {
+                          "type": "string",
+                          "description": "A url parameter name"
+                        },
+                        "condition": {
+                          "type": "string",
+                          "optional": true,
+                          "enum": ["purpose", "pref"],
+                          "description": "The type of param can be either \"purpose\" or \"pref\"."
+                        },
+                        "pref": {
+                          "type": "string",
+                          "optional": true,
+                          "description": "The preference to retreive the value from."
+                        },
+                        "purpose": {
+                          "type": "string",
+                          "optional": true,
+                          "enum": ["contextmenu", "searchbar", "homepage", "keyword", "newtab"],
+                          "description": "The context that initiates a search, required if condition is \"purpose\"."
+                        },
+                        "value": {
+                          "type": "string",
+                          "optional": true,
+                          "description": "A url parameter value."
+                        }
+                      }
+                    },
+                    "description": "A list of optional search url parameters. This allows the additon of search url parameters based on how the search is performed in Firefox."
                   }
                 }
               }
             }
           }
         }
       }
     ]
--- a/browser/components/extensions/test/xpcshell/test_ext_settings_overrides_search.js
+++ b/browser/components/extensions/test/xpcshell/test_ext_settings_overrides_search.js
@@ -20,16 +20,20 @@ AddonTestUtils.createAppInfo("xpcshell@t
 add_task(async function setup() {
   await AddonTestUtils.promiseStartupManager();
   Services.search.init();
 });
 
 add_task(async function test_extension_adding_engine() {
   let ext1 = ExtensionTestUtils.loadExtension({
     manifest: {
+      "icons": {
+        "16": "foo.ico",
+        "32": "foo32.ico",
+      },
       "chrome_settings_overrides": {
         "search_provider": {
           "name": "MozSearch",
           "keyword": "MozSearch",
           "search_url": kSearchEngineURL,
           "suggest_url": kSearchSuggestURL,
         },
       },
@@ -37,16 +41,23 @@ add_task(async function test_extension_a
     useAddonManager: "temporary",
   });
 
   await ext1.startup();
 
   let engine = Services.search.getEngineByName("MozSearch");
   ok(engine, "Engine should exist.");
 
+  let {baseURI} = ext1.extension;
+  equal(engine.iconURI.spec, baseURI.resolve("foo.ico"), "icon path matches");
+  let icons = engine.getIcons();
+  equal(icons.length, 2, "both icons avialable");
+  equal(icons[0].url, baseURI.resolve("foo.ico"), "icon path matches");
+  equal(icons[1].url, baseURI.resolve("foo32.ico"), "icon path matches");
+
   let expectedSuggestURL = kSearchSuggestURL.replace("{searchTerms}", kSearchTerm);
   let submissionSuggest = engine.getSubmission(kSearchTerm, URLTYPE_SUGGEST_JSON);
   let encodedSubmissionURL = engine.getSubmission(kSearchTermIntl).uri.spec;
   let testSubmissionURL = kSearchEngineURL.replace("{searchTerms}", encodeURIComponent(kSearchTermIntl));
   equal(encodedSubmissionURL, testSubmissionURL, "Encoded UTF-8 URLs should match");
 
   equal(submissionSuggest.uri.spec, expectedSuggestURL, "Suggest URLs should match");
 
@@ -134,8 +145,47 @@ add_task(async function test_upgrade_def
   equal(Services.search.getEngines().indexOf(engine), 1, "Engine is in position 1");
 
   await ext1.unload();
   await delay();
 
   engine = Services.search.getEngineByName("MozSearch");
   ok(!engine, "Engine should not exist");
 });
+
+add_task(async function test_extension_post_params() {
+  let ext1 = ExtensionTestUtils.loadExtension({
+    manifest: {
+      "chrome_settings_overrides": {
+        "search_provider": {
+          "name": "MozSearch",
+          "keyword": "MozSearch",
+          "search_url": kSearchEngineURL,
+          "search_url_post_params": "foo=bar&bar=foo",
+          "suggest_url": kSearchSuggestURL,
+          "suggest_url_post_params": "foo=bar&bar=foo",
+        },
+      },
+    },
+    useAddonManager: "temporary",
+  });
+
+  await ext1.startup();
+
+  let engine = Services.search.getEngineByName("MozSearch");
+  ok(engine, "Engine should exist.");
+
+  let url = engine.wrappedJSObject._getURLOfType("text/html");
+  equal(url.method, "POST", "Search URLs method is POST");
+
+  let expectedURL = kSearchEngineURL.replace("{searchTerms}", kSearchTerm);
+  let submission = engine.getSubmission(kSearchTerm);
+  equal(submission.uri.spec, expectedURL, "Search URLs should match");
+  // postData is a nsIMIMEInputStream which contains a nsIStringInputStream.
+  equal(submission.postData.data.data, "foo=bar&bar=foo", "Search postData should match");
+
+  let expectedSuggestURL = kSearchSuggestURL.replace("{searchTerms}", kSearchTerm);
+  let submissionSuggest = engine.getSubmission(kSearchTerm, URLTYPE_SUGGEST_JSON);
+  equal(submissionSuggest.uri.spec, expectedSuggestURL, "Suggest URLs should match");
+  equal(submissionSuggest.postData.data.data, "foo=bar&bar=foo", "Suggest postData should match");
+
+  await ext1.unload();
+});
new file mode 100644
--- /dev/null
+++ b/browser/components/extensions/test/xpcshell/test_ext_settings_overrides_search_mozParam.js
@@ -0,0 +1,121 @@
+/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set sts=2 sw=2 et tw=80: */
+
+"use strict";
+
+ChromeUtils.import("resource://testing-common/AddonTestUtils.jsm");
+
+AddonTestUtils.init(this);
+AddonTestUtils.overrideCertDB();
+AddonTestUtils.createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "42", "42");
+
+let {
+  promiseShutdownManager,
+  promiseStartupManager,
+} = AddonTestUtils;
+
+add_task(async function setup() {
+  await promiseStartupManager();
+  Services.search.init();
+  registerCleanupFunction(async () => {
+    await promiseShutdownManager();
+  });
+});
+
+/* This tests setting moz params. */
+add_task(async function test_extension_setting_moz_params() {
+  let defaultBranch = Services.prefs.getDefaultBranch("browser.search.");
+  defaultBranch.setCharPref("param.code", "good");
+
+  // These params are conditional based on how search is initiated.
+  let mozParams = [
+    {name: "test-0", condition: "purpose", purpose: "contextmenu", value: "0"},
+    {name: "test-1", condition: "purpose", purpose: "searchbar", value: "1"},
+    {name: "test-2", condition: "purpose", purpose: "homepage", value: "2"},
+    {name: "test-3", condition: "purpose", purpose: "keyword", value: "3"},
+    {name: "test-4", condition: "purpose", purpose: "newtab", value: "4"},
+  ];
+  // These params are always included.
+  let params = [
+    {name: "simple", value: "5"},
+    {name: "term", value: "{searchTerms}"},
+    {name: "lang", value: "{language}"},
+    {name: "locale", value: "{moz:locale}"},
+    {name: "prefval", condition: "pref", pref: "code"},
+  ];
+
+  let extension = ExtensionTestUtils.loadExtension({
+    manifest: {
+      "applications": {
+        "gecko": {"id": "test@mochitest"},
+      },
+      "chrome_settings_overrides": {
+        "search_provider": {
+          "name": "MozParamsTest",
+          "search_url": "https://example.com/?q={searchTerms}",
+          "params": [...mozParams, ...params],
+        },
+      },
+    },
+    useAddonManager: "permanent",
+  });
+  await extension.startup();
+  equal(extension.extension.isPrivileged, true, "extension is priviledged");
+
+  let engine = Services.search.getEngineByName("MozParamsTest");
+
+  let extraParams = [];
+  for (let p of params) {
+    if (p.condition == "pref") {
+      extraParams.push(`${p.name}=good`);
+    } else if (p.value == "{searchTerms}") {
+      extraParams.push(`${p.name}=test`);
+    } else if (p.value == "{language}") {
+      extraParams.push(`${p.name}=${Services.locale.requestedLocale || "*"}`);
+    } else if (p.value == "{moz:locale}") {
+      extraParams.push(`${p.name}=${Services.locale.requestedLocale}`);
+    } else {
+      extraParams.push(`${p.name}=${p.value}`);
+    }
+  }
+  let paramStr = extraParams.join("&");
+
+  for (let p of mozParams) {
+    let expectedURL = engine.getSubmission("test", null, p.condition == "purpose" ? p.purpose : null).uri.spec;
+    equal(expectedURL, `https://example.com/?q=test&${p.name}=${p.value}&${paramStr}`, "search url is expected");
+  }
+
+  await extension.unload();
+});
+
+add_task(async function test_extension_setting_moz_params_fail() {
+  // Ensure that the test infra does not automatically make
+  // this privileged.
+  AddonTestUtils.usePrivilegedSignatures = false;
+
+  let extension = ExtensionTestUtils.loadExtension({
+    manifest: {
+      "applications": {
+        "gecko": {"id": "test@mochitest"},
+      },
+      "chrome_settings_overrides": {
+        "search_provider": {
+          "name": "MozParamsTest",
+          "search_url": "https://example.com/",
+          "params": [
+            {name: "testParam", condition: "purpose", purpose: "contextmenu", value: "0"},
+            {name: "prefval", condition: "pref", pref: "code"},
+            {name: "q", value: "{searchTerms}"},
+          ],
+        },
+      },
+    },
+    useAddonManager: "permanent",
+  });
+  await extension.startup();
+  equal(extension.extension.isPrivileged, false, "extension is not priviledged");
+  let engine = Services.search.getEngineByName("MozParamsTest");
+  let expectedURL = engine.getSubmission("test", null, "contextmenu").uri.spec;
+  equal(expectedURL, "https://example.com/?q=test", "engine cannot have conditional or pref params");
+  await extension.unload();
+});
--- a/browser/components/extensions/test/xpcshell/xpcshell-common.ini
+++ b/browser/components/extensions/test/xpcshell/xpcshell-common.ini
@@ -4,11 +4,12 @@
 [test_ext_browsingData_downloads.js]
 [test_ext_browsingData_passwords.js]
 [test_ext_browsingData_settings.js]
 [test_ext_chrome_settings_overrides_update.js]
 [test_ext_distribution_popup.js]
 [test_ext_geckoProfiler_control.js]
 [test_ext_history.js]
 [test_ext_settings_overrides_search.js]
+[test_ext_settings_overrides_search_mozParam.js]
 [test_ext_url_overrides_newtab.js]
 [test_ext_url_overrides_newtab_update.js]
 
--- a/toolkit/components/search/nsSearchService.js
+++ b/toolkit/components/search/nsSearchService.js
@@ -60,16 +60,17 @@ const PERMS_FILE    = 0o644;
 
 // Directory service keys
 const NS_APP_DISTRIBUTION_SEARCH_DIR_LIST = "SrchPluginsDistDL";
 const NS_APP_USER_PROFILE_50_DIR = "ProfD";
 
 // We load plugins from APP_SEARCH_PREFIX, where a list.json
 // file needs to exist to list available engines.
 const APP_SEARCH_PREFIX = "resource://search-plugins/";
+const EXT_SEARCH_PREFIX = "resource://search-extensions/";
 
 // See documentation in nsIBrowserSearchService.idl.
 const SEARCH_ENGINE_TOPIC        = "browser-search-engine-modified";
 const TOPIC_LOCALES_CHANGE       = "intl:app-locales-changed";
 const QUIT_APPLICATION_TOPIC     = "quit-application";
 
 const SEARCH_ENGINE_REMOVED      = "engine-removed";
 const SEARCH_ENGINE_ADDED        = "engine-added";
@@ -1074,19 +1075,23 @@ EngineURL.prototype = {
       dataArray.push(param.name + "=" + value);
     }
     let dataString = dataArray.join("&");
 
     var postData = null;
     if (this.method == "GET") {
       // GET method requests have no post data, and append the encoded
       // query string to the url...
-      if (!url.includes("?") && dataString)
-        url += "?";
-      url += dataString;
+      if (dataString) {
+        if (url.includes("?")) {
+          url = `${url}&${dataString}`;
+        } else {
+          url = `${url}?${dataString}`;
+        }
+      }
     } else if (this.method == "POST") {
       // POST method requests must wrap the encoded text in a MIME
       // stream and supply that as POSTDATA.
       var stringStream = Cc["@mozilla.org/io/string-input-stream;1"].
                          createInstance(Ci.nsIStringInputStream);
       stringStream.data = dataString;
 
       postData = Cc["@mozilla.org/network/mime-input-stream;1"].
@@ -1277,16 +1282,19 @@ Engine.prototype = {
   // The number of days between update checks for new versions
   _updateInterval: null,
   // The url to check at for a new update
   _updateURL: null,
   // The url to check for a new icon
   _iconUpdateURL: null,
   /* The extension ID if added by an extension. */
   _extensionID: null,
+  // If the extension is builtin we treat it as a builtin search engine as well.
+  // Both System and Distribution extensions are considered builtin for search engines.
+  _isBuiltinExtension: false,
 
   /**
    * Retrieves the data from the engine's file.
    * The document element is placed in the engine's data field.
    */
   _initFromFile: function SRCH_ENG_initFromFile(file) {
     if (!file || !file.exists())
       FAIL("File must exist before calling initFromFile!", Cr.NS_ERROR_UNEXPECTED);
@@ -1672,16 +1680,17 @@ Engine.prototype = {
     switch (uri.scheme) {
       case "resource":
       case "chrome":
         // We only allow chrome and resource icon URLs for built-in search engines
         if (!this._isDefault) {
           return;
         }
         // Fall through to the data case
+      case "moz-extension":
       case "data":
         if (!this._hasPreferredIcon || aIsPreferred) {
           this._iconURI = uri;
           notifyAction(this, SEARCH_ENGINE_CHANGED);
           this._hasPreferredIcon = aIsPreferred;
         }
 
         if (aWidth && aHeight) {
@@ -1767,43 +1776,95 @@ Engine.prototype = {
       FAIL(this._location + " is not a valid search plugin.", Cr.NS_ERROR_FILE_CORRUPTED);
     }
     // No need to keep a ref to our data (which in some cases can be a document
     // element) past this point
     this._data = null;
   },
 
   /**
+   * Initialize an EngineURL object from metadata.
+   */
+  _initEngineURLFromMetaData(aType, aParams) {
+    let url = new EngineURL(aType, aParams.method || "GET", aParams.template);
+
+    if (aParams.postParams) {
+      let queries = new URLSearchParams(aParams.postParams);
+      for (let [name, value] of queries) {
+        url.addParam(name, value);
+      }
+    }
+
+    if (aParams.mozParams) {
+      for (let p of aParams.mozParams) {
+        if ((p.condition || p.purpose) && !this._isDefault) {
+          continue;
+        }
+        if (p.condition == "pref") {
+          let value = getMozParamPref(p.pref);
+          url.addParam(p.name, value);
+          url._addMozParam(p);
+        } else {
+          url.addParam(p.name, p.value, p.purpose || undefined);
+        }
+      }
+    }
+
+    this._urls.push(url);
+    return url;
+  },
+
+  /**
    * Initialize this Engine object from a collection of metadata.
    */
   _initFromMetadata: function SRCH_ENG_initMetaData(aName, aParams) {
     ENSURE_WARN(!this._readOnly,
                 "Can't call _initFromMetaData on a readonly engine!",
                 Cr.NS_ERROR_FAILURE);
 
-    let method = aParams.method || "GET";
-    this._urls.push(new EngineURL(URLTYPE_SEARCH_HTML, method, aParams.template));
+    this._extensionID = aParams.extensionID;
+    this._isBuiltinExtension = !!aParams.isBuiltIn;
+
+    this._initEngineURLFromMetaData(URLTYPE_SEARCH_HTML, {
+      method: (aParams.searchPostParams && "POST") || aParams.method || "GET",
+      template: aParams.template,
+      postParams: aParams.searchPostParams,
+      mozParams: aParams.mozParams,
+    });
+
     if (aParams.suggestURL) {
-      this._urls.push(new EngineURL(URLTYPE_SUGGEST_JSON, "GET", aParams.suggestURL));
+      this._initEngineURLFromMetaData(URLTYPE_SUGGEST_JSON, {
+        method: (aParams.suggestPostParams && "POST") || aParams.method || "GET",
+        template: aParams.suggestURL,
+        postParams: aParams.suggestPostParams,
+      });
     }
+
     if (aParams.queryCharset) {
       this._queryCharset = aParams.queryCharset;
     }
     if (aParams.postData) {
       let queries = new URLSearchParams(aParams.postData);
       for (let [name, value] of queries) {
         this.addParam(name, value);
       }
     }
 
     this._name = aName;
     this.alias = aParams.alias;
     this._description = aParams.description;
-    this._setIcon(aParams.iconURL, true);
-    this._extensionID = aParams.extensionID;
+    if (aParams.iconURL) {
+      this._setIcon(aParams.iconURL, true);
+    }
+    // Other sizes
+    if (aParams.icons) {
+      for (let icon of aParams.icons) {
+        this._addIconToMap(icon.size, icon.size, icon.url);
+      }
+    }
   },
 
   /**
    * Extracts data from an OpenSearch URL element and creates an EngineURL
    * object which is then added to the engine's list of URLs.
    *
    * @throws NS_ERROR_FAILURE if a URL object could not be created.
    *
@@ -2209,16 +2270,20 @@ Engine.prototype = {
     // avoid leaking user identifiable data.
     if (!id)
       id = "[other]/" + file.leafName;
 
     return prefix + id + suffix;
   },
 
   get _isDefault() {
+    if (this._extensionID) {
+      return this._isBuiltinExtension;
+    }
+
     // If we don't have a shortName, the engine is being parsed from a
     // downloaded file, so this can't be a default engine.
     if (!this._shortName)
       return false;
 
     // An engine is a default one if we initially loaded it from the application
     // or distribution directory.
     if (/^(?:jar:)?(?:\[app\]|\[distribution\])/.test(this._loadPath))