Bug 1656220 - Implement recording attributions for search engines. r=dao
authorMark Banner <standard8@mozilla.com>
Thu, 20 Aug 2020 12:58:23 +0000
changeset 610141 5c795477d83a25be5a6ade887d52d5261c319885
parent 610140 3b983f83c30730c14be4f9c55949c6a823c6f857
child 610142 d92766e56d92847d4639466c0d5aea89252371f8
push id13553
push userffxbld-merge
push dateMon, 24 Aug 2020 12:51:36 +0000
treeherdermozilla-beta@a54f8b5d0977 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdao
bugs1656220
milestone81.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1656220 - Implement recording attributions for search engines. r=dao Differential Revision: https://phabricator.services.mozilla.com/D87501
browser/actors/ContentSearchParent.jsm
browser/app/profile/firefox.js
browser/base/content/browser.js
browser/components/extensions/parent/ext-search.js
browser/components/search/content/searchbar.js
browser/components/urlbar/UrlbarInput.jsm
browser/modules/BrowserUsageTelemetry.jsm
browser/modules/PartnerLinkAttribution.jsm
browser/modules/test/browser/browser.ini
browser/modules/test/browser/browser_PartnerLinkAttribution.js
browser/modules/test/browser/search-engines/basic/manifest.json
browser/modules/test/browser/search-engines/engines.json
browser/modules/test/browser/search-engines/simple/manifest.json
toolkit/components/search/SearchEngine.jsm
toolkit/components/search/nsISearchService.idl
toolkit/components/search/tests/SearchTestUtils.jsm
--- a/browser/actors/ContentSearchParent.jsm
+++ b/browser/actors/ContentSearchParent.jsm
@@ -248,16 +248,17 @@ let ContentSearch = {
         inBackground: Services.prefs.getBoolPref(
           "browser.tabs.loadInBackground"
         ),
       };
       win.openTrustedLinkIn(submission.uri.spec, where, params);
     }
     win.BrowserSearch.recordSearchInTelemetry(engine, data.healthReportKey, {
       selection: data.selection,
+      url: submission.uri,
     });
   },
 
   async getSuggestions(engineName, searchString, browser) {
     let engine = Services.search.getEngineByName(engineName);
     if (!engine) {
       throw new Error("Unknown engine name: " + engineName);
     }
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1308,17 +1308,18 @@ pref("browser.menu.showCharacterEncoding
 // Allow using tab-modal prompts when possible.
 pref("prompts.tab_modal.enabled", true);
 
 // Whether prompts should be content modal (1) tab modal (2) or window modal(3) by default
 // This is a fallback value for when prompt callers do not specify a modalType.
 pref("prompts.defaultModalType", 3);
 
 pref("browser.topsites.useRemoteSetting", false);
-pref("browser.topsites.attributionURL", "");
+
+pref("browser.partnerlink.attributionURL", "https://topsites.mozilla.io/cid/amzn_2020_a1");
 
 // Whether to show tab level system prompts opened via nsIPrompt(Service) as
 // SubDialogs in the TabDialogBox (true) or as TabModalPrompt in the
 // TabModalPromptBox (false).
 #ifdef NIGHTLY_BUILD
   pref("prompts.tabChromePromptSubDialog", true);
 #else
   pref("prompts.tabChromePromptSubDialog", false);
--- a/browser/base/content/browser.js
+++ b/browser/base/content/browser.js
@@ -4341,57 +4341,57 @@ const BrowserSearch = {
       private: usePrivate && !PrivateBrowsingUtils.isWindowPrivate(window),
       postData: submission.postData,
       inBackground,
       relatedToCurrent: true,
       triggeringPrincipal,
       csp,
     });
 
-    return engine;
+    return { engine, url: submission.uri };
   },
 
   /**
    * Perform a search initiated from the context menu.
    *
    * This should only be called from the context menu. See
    * BrowserSearch.loadSearch for the preferred API.
    */
   async loadSearchFromContext(terms, usePrivate, triggeringPrincipal, csp) {
-    let engine = await BrowserSearch._loadSearch(
+    let { engine, url } = await BrowserSearch._loadSearch(
       terms,
       usePrivate && !PrivateBrowsingUtils.isWindowPrivate(window)
         ? "window"
         : "tab",
       usePrivate,
       "contextmenu",
       Services.scriptSecurityManager.createNullPrincipal(
         triggeringPrincipal.originAttributes
       ),
       csp
     );
     if (engine) {
-      BrowserSearch.recordSearchInTelemetry(engine, "contextmenu");
+      BrowserSearch.recordSearchInTelemetry(engine, "contextmenu", { url });
     }
   },
 
   /**
    * Perform a search initiated from the command line.
    */
   async loadSearchFromCommandLine(terms, usePrivate, triggeringPrincipal, csp) {
-    let engine = await BrowserSearch._loadSearch(
+    let { engine, url } = await BrowserSearch._loadSearch(
       terms,
       "current",
       usePrivate,
       "system",
       triggeringPrincipal,
       csp
     );
     if (engine) {
-      BrowserSearch.recordSearchInTelemetry(engine, "system");
+      BrowserSearch.recordSearchInTelemetry(engine, "system", { url });
     }
   },
 
   pasteAndSearch(event) {
     BrowserSearch.searchBar.select();
     goDoCommand("cmd_paste");
     BrowserSearch.searchBar.handleSearchCommand(event);
   },
--- a/browser/components/extensions/parent/ext-search.js
+++ b/browser/components/extensions/parent/ext-search.js
@@ -112,15 +112,16 @@ this.search = class extends ExtensionAPI
           } else {
             let tab = tabTracker.getTab(searchProperties.tabId);
             tab.linkedBrowser.loadURI(submission.uri.spec, options);
             tabbrowser = tab.linkedBrowser.getTabBrowser();
           }
           BrowserUsageTelemetry.recordSearch(
             tabbrowser,
             engine,
-            "webextension"
+            "webextension",
+            { url: submission.uri }
           );
         },
       },
     };
   }
 };
--- a/browser/components/search/content/searchbar.js
+++ b/browser/components/search/content/searchbar.js
@@ -416,16 +416,17 @@
       if (telemetrySearchDetails && telemetrySearchDetails.index == -1) {
         telemetrySearchDetails = null;
       }
       // If we hit here, we come either from a one-off, a plain search or a suggestion.
       const details = {
         isOneOff: aOneOff,
         isSuggestion: !aOneOff && telemetrySearchDetails,
         selection: telemetrySearchDetails,
+        url: submission.uri,
       };
       BrowserSearch.recordSearchInTelemetry(engine, "searchbar", details);
       // null parameter below specifies HTML response for search
       let params = {
         postData: submission.postData,
       };
       if (aParams) {
         for (let key in aParams) {
--- a/browser/components/urlbar/UrlbarInput.jsm
+++ b/browser/components/urlbar/UrlbarInput.jsm
@@ -469,17 +469,17 @@ class UrlbarInput {
       result = this._resultForCurrentValue;
       let searchString =
         (result && (result.payload.suggestion || result.payload.query)) ||
         this._lastSearchString;
       [url, openParams.postData] = UrlbarUtils.getSearchQueryUrl(
         selectedOneOff.engine,
         searchString
       );
-      this._recordSearch(selectedOneOff.engine, event);
+      this._recordSearch(selectedOneOff.engine, event, { url });
 
       UrlbarUtils.addToFormHistory(
         this,
         searchString,
         selectedOneOff.engine.name
       ).catch(Cu.reportError);
     } else {
       // Use the current value if we don't have a UrlbarResult e.g. because the
@@ -771,16 +771,17 @@ class UrlbarInput {
           where = "window";
           openParams.private = true;
         }
 
         const actionDetails = {
           isSuggestion: !!result.payload.suggestion,
           isFormHistory: result.source == UrlbarUtils.RESULT_SOURCE.HISTORY,
           alias: result.payload.keyword,
+          url,
         };
         const engine = Services.search.getEngineByName(result.payload.engine);
         this._recordSearch(engine, event, actionDetails);
 
         if (!result.payload.inPrivateWindow) {
           UrlbarUtils.addToFormHistory(
             this,
             result.payload.suggestion || result.payload.query,
@@ -1820,16 +1821,18 @@ class UrlbarInput {
    * @param {object} searchActionDetails
    *   The details associated with this search query.
    * @param {boolean} searchActionDetails.isSuggestion
    *   True if this query was initiated from a suggestion from the search engine.
    * @param {boolean} searchActionDetails.alias
    *   True if this query was initiated via a search alias.
    * @param {boolean} searchActionDetails.isFormHistory
    *   True if this query was initiated from a form history result.
+   * @param {string} searchActionDetails.url
+   *   The url this query was triggered with.
    */
   _recordSearch(engine, event, searchActionDetails = {}) {
     const isOneOff = this.view.oneOffSearchButtons.maybeRecordTelemetry(event);
     // Infer the type of the event which triggered the search.
     let eventType = "unknown";
     if (event instanceof KeyboardEvent) {
       eventType = "key";
     } else if (event instanceof MouseEvent) {
--- a/browser/modules/BrowserUsageTelemetry.jsm
+++ b/browser/modules/BrowserUsageTelemetry.jsm
@@ -18,16 +18,17 @@ const { XPCOMUtils } = ChromeUtils.impor
 
 XPCOMUtils.defineLazyModuleGetters(this, {
   AppConstants: "resource://gre/modules/AppConstants.jsm",
   ClientID: "resource://gre/modules/ClientID.jsm",
   BrowserUtils: "resource://gre/modules/BrowserUtils.jsm",
   CustomizableUI: "resource:///modules/CustomizableUI.jsm",
   OS: "resource://gre/modules/osfile.jsm",
   PageActions: "resource:///modules/PageActions.jsm",
+  PartnerLinkAttribution: "resource:///modules/PartnerLinkAttribution.jsm",
   PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.jsm",
   SearchTelemetry: "resource:///modules/SearchTelemetry.jsm",
   Services: "resource://gre/modules/Services.jsm",
   setTimeout: "resource://gre/modules/Timer.jsm",
   clearTimeout: "resource://gre/modules/Timer.jsm",
 });
 
 // This pref is in seconds!
@@ -599,17 +600,21 @@ let BrowserUsageTelemetry = {
         histogram.add(countIdSource);
       }
     }
 
     // Dispatch the search signal to other handlers.
     this._handleSearchAction(engine, source, details);
   },
 
-  _recordSearch(engine, source, action = null) {
+  _recordSearch(engine, url, source, action = null) {
+    PartnerLinkAttribution.makeSearchEngineRequest(engine, url).catch(
+      Cu.reportError
+    );
+
     let scalarKey = action ? "search_" + action : "search";
     Services.telemetry.keyedScalarAdd(
       "browser.engagement.navigation." + source,
       scalarKey,
       1
     );
     Services.telemetry.recordEvent("navigation", "search", source, action, {
       engine: engine.telemetryId,
@@ -621,25 +626,25 @@ let BrowserUsageTelemetry = {
       case "urlbar":
       case "oneoff-urlbar":
       case "searchbar":
       case "oneoff-searchbar":
       case "unknown": // Edge case: this is the searchbar (see bug 1195733 comment 7).
         this._handleSearchAndUrlbar(engine, source, details);
         break;
       case "abouthome":
-        this._recordSearch(engine, "about_home", "enter");
+        this._recordSearch(engine, details.url, "about_home", "enter");
         break;
       case "newtab":
-        this._recordSearch(engine, "about_newtab", "enter");
+        this._recordSearch(engine, details.url, "about_newtab", "enter");
         break;
       case "contextmenu":
       case "system":
       case "webextension":
-        this._recordSearch(engine, source);
+        this._recordSearch(engine, details.url, source);
         break;
     }
   },
 
   /**
    * This function handles the "urlbar", "urlbar-oneoff", "searchbar" and
    * "searchbar-oneoff" sources.
    */
@@ -660,37 +665,37 @@ let BrowserUsageTelemetry = {
       // Moreover, we skip the "unknown" source that comes from the searchbar
       // when performing searches from the default search engine. See bug 1195733
       // comment 7 for context.
       if (["urlbar", "searchbar", "unknown"].includes(source)) {
         return;
       }
 
       // If that's a legit one-off search signal, record it using the relative key.
-      this._recordSearch(engine, sourceName, "oneoff");
+      this._recordSearch(engine, details.url, sourceName, "oneoff");
       return;
     }
 
     // The search was not a one-off. It was a search with the default search engine.
     if (details.isFormHistory) {
       // It came from a form history result.
-      this._recordSearch(engine, sourceName, "formhistory");
+      this._recordSearch(engine, details.url, sourceName, "formhistory");
       return;
     } else if (details.isSuggestion) {
       // It came from a suggested search, so count it as such.
-      this._recordSearch(engine, sourceName, "suggestion");
+      this._recordSearch(engine, details.url, sourceName, "suggestion");
       return;
     } else if (details.alias) {
       // This one came from a search that used an alias.
-      this._recordSearch(engine, sourceName, "alias");
+      this._recordSearch(engine, details.url, sourceName, "alias");
       return;
     }
 
     // The search signal was generated by typing something and pressing enter.
-    this._recordSearch(engine, sourceName, "enter");
+    this._recordSearch(engine, details.url, sourceName, "enter");
   },
 
   /**
    * Records the method by which the user selected a result from the urlbar.
    *
    * @param {Event} event
    *        The event that triggered the selection.
    * @param {number} index
--- a/browser/modules/PartnerLinkAttribution.jsm
+++ b/browser/modules/PartnerLinkAttribution.jsm
@@ -3,59 +3,108 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
 
 Cu.importGlobalProperties(["fetch"]);
 
 var EXPORTED_SYMBOLS = ["PartnerLinkAttribution"];
 
-ChromeUtils.defineModuleGetter(
-  this,
-  "Services",
-  "resource://gre/modules/Services.jsm"
+const { XPCOMUtils } = ChromeUtils.import(
+  "resource://gre/modules/XPCOMUtils.jsm"
 );
 
-ChromeUtils.defineModuleGetter(
-  this,
-  "Region",
-  "resource://gre/modules/Region.jsm"
-);
+XPCOMUtils.defineLazyModuleGetters(this, {
+  Services: "resource://gre/modules/Services.jsm",
+  Region: "resource://gre/modules/Region.jsm",
+});
 
 var PartnerLinkAttribution = {
   async makeRequest({ targetURL, source }) {
     let partner = targetURL.match(/^https?:\/\/(?:www.)?([^.]*)/)[1];
 
     function record(objectString, value = "") {
       recordTelemetryEvent("interaction", objectString, value, {
         partner,
         source,
       });
     }
     record("click");
 
     const attributionUrl = Services.prefs.getStringPref(
       Services.prefs.getBoolPref("browser.topsites.useRemoteSetting")
-        ? "browser.topsites.attributionURL"
+        ? "browser.partnerlink.attributionURL"
         : `browser.newtabpage.searchTileOverride.${partner}.attributionURL`,
       ""
     );
     if (!attributionUrl) {
       record("attribution", "abort");
       return;
     }
-    const request = new Request(attributionUrl);
-    request.headers.set("X-Region", Region.home);
-    request.headers.set("X-Source", source);
-    request.headers.set("X-Target-URL", targetURL);
-    const response = await fetch(request);
-    record("attribution", response.ok ? "success" : "failure");
+    let result = await sendRequest(attributionUrl, source, targetURL);
+    record("attribution", result ? "success" : "failure");
+  },
+
+  /**
+   * Makes a request to the attribution URL for a search engine search.
+   *
+   * @param {nsISearchEngine} engine
+   *   The search engine to save the attribution for.
+   * @param {nsIURI} targetUrl
+   *   The target URL to filter and include in the attribution.
+   */
+  async makeSearchEngineRequest(engine, targetUrl) {
+    if (!engine.sendAttributionRequest) {
+      return;
+    }
+
+    let searchUrlQueryParamName = engine.searchUrlQueryParamName;
+    if (!searchUrlQueryParamName) {
+      Cu.reportError("makeSearchEngineRequest can't find search terms key");
+      return;
+    }
+
+    let url = targetUrl;
+    if (typeof url == "string") {
+      url = Services.io.newURI(url);
+    }
+
+    let targetParams = new URLSearchParams(url.query);
+    if (!targetParams.has(searchUrlQueryParamName)) {
+      Cu.reportError(
+        "makeSearchEngineRequest can't remove target search terms"
+      );
+      return;
+    }
+
+    const attributionUrl = Services.prefs.getStringPref(
+      "browser.partnerlink.attributionURL",
+      ""
+    );
+
+    targetParams.delete(searchUrlQueryParamName);
+    let strippedTargetUrl = `${url.prePath}${url.filePath}`;
+    let newParams = targetParams.toString();
+    if (newParams) {
+      strippedTargetUrl += "?" + newParams;
+    }
+
+    await sendRequest(attributionUrl, "searchurl", strippedTargetUrl);
   },
 };
 
+async function sendRequest(attributionUrl, source, targetURL) {
+  const request = new Request(attributionUrl);
+  request.headers.set("X-Region", Region.home);
+  request.headers.set("X-Source", source);
+  request.headers.set("X-Target-URL", targetURL);
+  const response = await fetch(request);
+  return response.ok;
+}
+
 function recordTelemetryEvent(method, objectString, value, extra) {
   Services.telemetry.setEventRecordingEnabled("partner_link", true);
   Services.telemetry.recordEvent(
     "partner_link",
     method,
     objectString,
     value,
     extra
--- a/browser/modules/test/browser/browser.ini
+++ b/browser/modules/test/browser/browser.ini
@@ -10,20 +10,25 @@ prefs =
 support-files =
   contentSearchBadImage.xml
   contentSearchSuggestions.sjs
   contentSearchSuggestions.xml
   !/browser/components/search/test/browser/head.js
   !/browser/components/search/test/browser/testEngine.xml
   !/browser/components/search/test/browser/testEngine_diacritics.xml
   testEngine_chromeicon.xml
-skip-if = (debug && os == "linux" && bits == 64 && os_version == "18.04") # Bug 1649755 
+skip-if = (debug && os == "linux" && bits == 64 && os_version == "18.04") # Bug 1649755
 [browser_EveryWindow.js]
 [browser_LiveBookmarkMigrator.js]
 [browser_PageActions.js]
+[browser_PartnerLinkAttribution.js]
+support-files =
+  search-engines/basic/manifest.json
+  search-engines/simple/manifest.json
+  search-engines/engines.json
 [browser_PermissionUI.js]
 [browser_PermissionUI_prompts.js]
 [browser_preloading_tab_moving.js]
 [browser_ProcessHangNotifications.js]
 skip-if = !e10s
 [browser_SitePermissions.js]
 [browser_SitePermissions_combinations.js]
 [browser_SitePermissions_expiry.js]
new file mode 100644
--- /dev/null
+++ b/browser/modules/test/browser/browser_PartnerLinkAttribution.js
@@ -0,0 +1,286 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+/**
+ * This file tests urlbar telemetry with search related actions.
+ */
+
+"use strict";
+
+const SCALAR_URLBAR = "browser.engagement.navigation.urlbar";
+
+// The preference to enable suggestions in the urlbar.
+const SUGGEST_URLBAR_PREF = "browser.urlbar.suggest.searches";
+// The name of the search engine used to generate suggestions.
+const SUGGESTION_ENGINE_NAME =
+  "browser_UsageTelemetry usageTelemetrySearchSuggestions.xml";
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  CustomizableUITestUtils:
+    "resource://testing-common/CustomizableUITestUtils.jsm",
+  Region: "resource://gre/modules/Region.jsm",
+  SearchTelemetry: "resource:///modules/SearchTelemetry.jsm",
+  SearchTestUtils: "resource://testing-common/SearchTestUtils.jsm",
+  UrlbarTestUtils: "resource://testing-common/UrlbarTestUtils.jsm",
+  HttpServer: "resource://testing-common/httpd.js",
+});
+
+let gCUITestUtils = new CustomizableUITestUtils(window);
+SearchTestUtils.init(Assert, registerCleanupFunction);
+
+var gHttpServer = null;
+var gRequests = [];
+
+function submitHandler(request, response) {
+  gRequests.push(request);
+  response.setStatusLine(request.httpVersion, 200, "Ok");
+}
+
+add_task(async function setup() {
+  // Ensure the initial init is complete.
+  await Services.search.init();
+
+  // Clear history so that history added by previous tests doesn't mess up this
+  // test when it selects results in the urlbar.
+  await PlacesUtils.history.clear();
+
+  let searchExtensions = getChromeDir(getResolvedURI(gTestPath));
+  searchExtensions.append("search-engines");
+
+  await SearchTestUtils.useMochitestEngines(searchExtensions);
+
+  SearchTestUtils.useMockIdleService();
+  let response = await fetch(`resource://search-extensions/engines.json`);
+  let json = await response.json();
+  await SearchTestUtils.updateRemoteSettingsConfig(json.data);
+
+  gHttpServer = new HttpServer();
+  gHttpServer.registerPathHandler("/", submitHandler);
+  gHttpServer.start(-1);
+
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      // Enable search suggestions in the urlbar.
+      [SUGGEST_URLBAR_PREF, true],
+      // Clear historical search suggestions to avoid interference from previous
+      // tests.
+      ["browser.urlbar.maxHistoricalSearchSuggestions", 0],
+      // Use the default matching bucket configuration.
+      ["browser.urlbar.matchBuckets", "general:5,suggestion:4"],
+      //
+      [
+        "browser.partnerlink.attributionURL",
+        `http://localhost:${gHttpServer.identity.primaryPort}/`,
+      ],
+    ],
+  });
+
+  await gCUITestUtils.addSearchBar();
+
+  // Make sure to restore the engine once we're done.
+  registerCleanupFunction(async function() {
+    await SearchTestUtils.updateRemoteSettingsConfig();
+    await gHttpServer.stop();
+    gHttpServer = null;
+    await PlacesUtils.history.clear();
+    gCUITestUtils.removeSearchBar();
+  });
+});
+
+function searchInAwesomebar(value, win = window) {
+  return UrlbarTestUtils.promiseAutocompleteResultPopup({
+    window: win,
+    waitForFocus,
+    value,
+    fireInputEvent: true,
+  });
+}
+
+add_task(async function test_simpleQuery_no_attribution() {
+  await Services.search.setDefault(
+    Services.search.getEngineByName("Simple Engine")
+  );
+
+  let tab = await BrowserTestUtils.openNewForegroundTab(
+    gBrowser,
+    "about:blank"
+  );
+
+  info("Simulate entering a simple search.");
+  let promiseLoad = BrowserTestUtils.waitForDocLoadAndStopIt(
+    "https://example.com/?sourceId=Mozilla-search&search=simple+query",
+    tab
+  );
+  await searchInAwesomebar("simple query");
+  EventUtils.synthesizeKey("KEY_Enter");
+  await promiseLoad;
+
+  await new Promise(resolve => Services.tm.dispatchToMainThread(resolve));
+
+  Assert.equal(gRequests.length, 0, "Should not have submitted an attribution");
+
+  BrowserTestUtils.removeTab(tab);
+
+  await Services.search.setDefault(Services.search.getEngineByName("basic"));
+});
+
+async function checkAttributionRecorded(actionFn, cleanupFn) {
+  let tab = await BrowserTestUtils.openNewForegroundTab(
+    gBrowser,
+    "data:text/plain;charset=utf8,simple%20query"
+  );
+
+  let promiseLoad = BrowserTestUtils.waitForDocLoadAndStopIt(
+    "https://mochi.test:8888/browser/browser/components/search/test/browser/?search=simple+query&foo=1",
+    tab
+  );
+  await actionFn(tab);
+  await promiseLoad;
+
+  await BrowserTestUtils.waitForCondition(
+    () => gRequests.length == 1,
+    "Should have received an attribution submission"
+  );
+  Assert.equal(
+    gRequests[0].getHeader("x-region"),
+    Region.home,
+    "Should have set the region correctly"
+  );
+  Assert.equal(
+    gRequests[0].getHeader("X-Source"),
+    "searchurl",
+    "Should have set the source correctly"
+  );
+  Assert.equal(
+    gRequests[0].getHeader("X-Target-url"),
+    "https://mochi.test:8888/browser/browser/components/search/test/browser/?foo=1",
+    "Should have set the target url correctly and stripped the search terms"
+  );
+  if (cleanupFn) {
+    await cleanupFn();
+  }
+  BrowserTestUtils.removeTab(tab);
+  gRequests = [];
+}
+
+add_task(async function test_urlbar() {
+  await checkAttributionRecorded(async tab => {
+    await searchInAwesomebar("simple query");
+    EventUtils.synthesizeKey("KEY_Enter");
+  });
+});
+
+add_task(async function test_searchbar() {
+  await checkAttributionRecorded(async tab => {
+    let sb = BrowserSearch.searchBar;
+    // Write the search query in the searchbar.
+    sb.focus();
+    sb.value = "simple query";
+    sb.textbox.controller.startSearch("simple query");
+    // Wait for the popup to show.
+    await BrowserTestUtils.waitForEvent(sb.textbox.popup, "popupshown");
+    // And then for the search to complete.
+    await BrowserTestUtils.waitForCondition(
+      () =>
+        sb.textbox.controller.searchStatus >=
+        Ci.nsIAutoCompleteController.STATUS_COMPLETE_NO_MATCH,
+      "The search in the searchbar must complete."
+    );
+    EventUtils.synthesizeKey("KEY_Enter");
+  });
+});
+
+add_task(async function test_context_menu() {
+  let contextMenu;
+  await checkAttributionRecorded(
+    async tab => {
+      info("Select all the text in the page.");
+      await SpecialPowers.spawn(tab.linkedBrowser, [""], async function() {
+        return new Promise(resolve => {
+          content.document.addEventListener(
+            "selectionchange",
+            () => resolve(),
+            {
+              once: true,
+            }
+          );
+          content.document
+            .getSelection()
+            .selectAllChildren(content.document.body);
+        });
+      });
+
+      info("Open the context menu.");
+      contextMenu = document.getElementById("contentAreaContextMenu");
+      let popupPromise = BrowserTestUtils.waitForEvent(
+        contextMenu,
+        "popupshown"
+      );
+      BrowserTestUtils.synthesizeMouseAtCenter(
+        "body",
+        { type: "contextmenu", button: 2 },
+        gBrowser.selectedBrowser
+      );
+      await popupPromise;
+
+      info("Click on search.");
+      let searchItem = contextMenu.getElementsByAttribute(
+        "id",
+        "context-searchselect"
+      )[0];
+      searchItem.click();
+    },
+    () => {
+      contextMenu.hidePopup();
+      BrowserTestUtils.removeTab(gBrowser.selectedTab);
+    }
+  );
+});
+
+add_task(async function test_about_newtab() {
+  let tab = await BrowserTestUtils.openNewForegroundTab(
+    gBrowser,
+    "about:newtab",
+    false
+  );
+  await SpecialPowers.spawn(tab.linkedBrowser, [], async function() {
+    await ContentTaskUtils.waitForCondition(() => !content.document.hidden);
+  });
+
+  info("Trigger a simple serch, just text + enter.");
+  let promiseLoad = BrowserTestUtils.waitForDocLoadAndStopIt(
+    "https://mochi.test:8888/browser/browser/components/search/test/browser/?search=simple+query&foo=1",
+    tab
+  );
+  await typeInSearchField(
+    tab.linkedBrowser,
+    "simple query",
+    "newtab-search-text"
+  );
+  await BrowserTestUtils.synthesizeKey("VK_RETURN", {}, tab.linkedBrowser);
+  await promiseLoad;
+
+  await BrowserTestUtils.waitForCondition(
+    () => gRequests.length == 1,
+    "Should have received an attribution submission"
+  );
+  Assert.equal(
+    gRequests[0].getHeader("x-region"),
+    Region.home,
+    "Should have set the region correctly"
+  );
+  Assert.equal(
+    gRequests[0].getHeader("X-Source"),
+    "searchurl",
+    "Should have set the source correctly"
+  );
+  Assert.equal(
+    gRequests[0].getHeader("X-Target-url"),
+    "https://mochi.test:8888/browser/browser/components/search/test/browser/?foo=1",
+    "Should have set the target url correctly and stripped the search terms"
+  );
+
+  BrowserTestUtils.removeTab(tab);
+  gRequests = [];
+});
new file mode 100644
--- /dev/null
+++ b/browser/modules/test/browser/search-engines/basic/manifest.json
@@ -0,0 +1,19 @@
+{
+  "name": "basic",
+  "manifest_version": 2,
+  "version": "1.0",
+  "description": "basic",
+  "applications": {
+    "gecko": {
+      "id": "basic@search.mozilla.org"
+    }
+  },
+  "hidden": true,
+  "chrome_settings_overrides": {
+    "search_provider": {
+      "name": "basic",
+      "search_url": "https://mochi.test:8888/browser/browser/components/search/test/browser/?search={searchTerms}&foo=1",
+      "suggest_url": "https://mochi.test:8888/browser/browser/modules/test/browser/usageTelemetrySearchSuggestions.sjs?{searchTerms}"
+    }
+  }
+}
new file mode 100644
--- /dev/null
+++ b/browser/modules/test/browser/search-engines/engines.json
@@ -0,0 +1,24 @@
+{
+  "data": [
+    {
+      "webExtension": {
+        "id":"basic@search.mozilla.org"
+      },
+      "telemetryId": "telemetry",
+      "appliesTo": [{
+        "included": { "everywhere": true },
+        "default": "yes",
+        "sendAttributionRequest": true
+      }]
+    },
+    {
+      "webExtension": {
+        "id":"simple@search.mozilla.org"
+      },
+      "appliesTo": [{
+        "included": { "everywhere": true },
+        "default": "yes"
+      }]
+    }
+  ]
+}
new file mode 100644
--- /dev/null
+++ b/browser/modules/test/browser/search-engines/simple/manifest.json
@@ -0,0 +1,29 @@
+{
+  "name": "Simple Engine",
+  "manifest_version": 2,
+  "version": "1.0",
+  "description": "Simple engine with a different name from the WebExtension id prefix",
+  "applications": {
+    "gecko": {
+      "id": "simple@search.mozilla.org"
+    }
+  },
+  "hidden": true,
+  "chrome_settings_overrides": {
+    "search_provider": {
+      "name": "Simple Engine",
+      "search_url": "https://example.com",
+      "params": [
+        {
+          "name": "sourceId",
+          "value": "Mozilla-search"
+        },
+        {
+          "name": "search",
+          "value": "{searchTerms}"
+        }
+      ],
+      "suggest_url": "https://example.com?search={searchTerms}"
+    }
+  }
+}
--- a/toolkit/components/search/SearchEngine.jsm
+++ b/toolkit/components/search/SearchEngine.jsm
@@ -617,16 +617,19 @@ class SearchEngine {
   // notification sent. This allows to skip sending notifications during
   // initialization.
   _engineAddedToStore = false;
   // The aliases coming from the engine definition (via webextension
   // keyword field for example).
   _definedAliases = [];
   // The urls associated with this engine.
   _urls = [];
+  // The query parameter name of the search url, cached in memory to avoid
+  // repeated look-ups.
+  _searchUrlQueryParamName = null;
 
   /**
    * Constructor.
    *
    * @param {object} options
    *   The options for this search engine. At least one of options.name,
    *   or options.shortName are required.
    * @param {string} [options.name]
@@ -1501,16 +1504,42 @@ class SearchEngine {
       submissionData = Services.textToSubURI.ConvertAndEscape(
         SearchUtils.DEFAULT_QUERY_CHARSET,
         data
       );
     }
     return url.getSubmission(submissionData, this, purpose);
   }
 
+  get searchUrlQueryParamName() {
+    if (this._searchUrlQueryParamName != null) {
+      return this._searchUrlQueryParamName;
+    }
+
+    let submission = this.getSubmission(
+      "{searchTerms}",
+      SearchUtils.URL_TYPE.SEARCH
+    );
+
+    if (submission.postData) {
+      Cu.reportError("searchUrlQueryParamName can't handle POST urls.");
+      return (this._searchUrlQueryParamName = "");
+    }
+
+    let queryParams = new URLSearchParams(submission.uri.query);
+    let searchUrlQueryParamName = "";
+    for (let [key, value] of queryParams) {
+      if (value == "{searchTerms}") {
+        searchUrlQueryParamName = key;
+      }
+    }
+
+    return (this._searchUrlQueryParamName = searchUrlQueryParamName);
+  }
+
   // from nsISearchEngine
   supportsResponseType(type) {
     return this._getURLOfType(type) != null;
   }
 
   // from nsISearchEngine
   getResultDomain(responseType) {
     if (!responseType) {
--- a/toolkit/components/search/nsISearchService.idl
+++ b/toolkit/components/search/nsISearchService.idl
@@ -46,16 +46,25 @@ interface nsISearchEngine : nsISupports
    *          to send to the search engine.  If no submission can be
    *          obtained for the given responseType, returns null.
    */
   nsISearchSubmission getSubmission(in AString data,
                                     [optional] in AString responseType,
                                     [optional] in AString purpose);
 
   /**
+   * Returns the name of the parameter used for the search terms for a submission
+   * URL of type `SearchUtils.URL_TYPE.SEARCH`.
+   *
+   * @returns A string which is the name of the parameter, or empty string
+   *          if no parameter cannot be found or is not supported (e.g. POST).
+   */
+  readonly attribute AString searchUrlQueryParamName;
+
+  /**
    * Determines whether the engine can return responses in the given
    * MIME type.  Returns true if the engine spec has a URL with the
    * given responseType, false otherwise.
    *
    * @param responseType
    *        The MIME type to check for
    */
   boolean supportsResponseType(in AString responseType);
--- a/toolkit/components/search/tests/SearchTestUtils.jsm
+++ b/toolkit/components/search/tests/SearchTestUtils.jsm
@@ -103,16 +103,32 @@ var SearchTestUtils = Object.freeze({
       sinon.stub(settings, "get").returns(config);
     } else {
       let response = await fetch(`resource://search-extensions/engines.json`);
       let json = await response.json();
       sinon.stub(settings, "get").returns(json.data);
     }
   },
 
+  async useMochitestEngines(testDir) {
+    // Replace the path we load search engines from with
+    // the path to our test data.
+    let resProt = Services.io
+      .getProtocolHandler("resource")
+      .QueryInterface(Ci.nsIResProtocolHandler);
+    let originalSubstitution = resProt.getSubstitution("search-extensions");
+    resProt.setSubstitution(
+      "search-extensions",
+      Services.io.newURI("file://" + testDir.path)
+    );
+    gTestGlobals.registerCleanupFunction(() => {
+      resProt.setSubstitution("search-extensions", originalSubstitution);
+    });
+  },
+
   /**
    * Convert a list of engine configurations into engine objects.
    *
    * @param {Array} engineConfigurations
    **/
   async searchConfigToEngines(engineConfigurations) {
     let engines = [];
     for (let config of engineConfigurations) {
@@ -279,20 +295,24 @@ var SearchTestUtils = Object.freeze({
     gTestGlobals.registerCleanupFunction(() => {
       MockRegistrar.unregister(fakeIdleService);
     });
   },
 
   /**
    * Simulates an update to the RemoteSettings configuration.
    *
-   * @param {object} config
+   * @param {object} [config]
    *  The new configuration.
    */
   async updateRemoteSettingsConfig(config) {
+    if (!config) {
+      let settings = RemoteSettings(SearchUtils.SETTINGS_KEY);
+      config = await settings.get();
+    }
     const reloadObserved = SearchTestUtils.promiseSearchNotification(
       "engines-reloaded"
     );
     await RemoteSettings(SearchUtils.SETTINGS_KEY).emit("sync", {
       data: { current: config },
     });
 
     this.idleService._fireObservers("idle");