Bug 1484251 - Part 4 - Test updates for new content blocking telemetry. r=Gijs
authorJohann Hofmann <jhofmann@mozilla.com>
Thu, 11 Oct 2018 11:25:57 +0000
changeset 499154 29d9b2d2192d80733b6be89a31816d97ffda9163
parent 499153 8794df889e3fc61be3f1fc17e65f59d18f5d2d83
child 499155 8d8c2a0e01b424c7719f76f553ff2923f1ddce69
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersGijs
bugs1484251
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1484251 - Part 4 - Test updates for new content blocking telemetry. r=Gijs Differential Revision: https://phabricator.services.mozilla.com/D6896
browser/base/content/test/siteIdentity/browser_identityPopup_clearSiteData.js
browser/base/content/test/trackingUI/browser_trackingUI_open_preferences.js
browser/base/content/test/trackingUI/browser_trackingUI_report_breakage.js
browser/base/content/test/trackingUI/browser_trackingUI_telemetry.js
--- a/browser/base/content/test/siteIdentity/browser_identityPopup_clearSiteData.js
+++ b/browser/base/content/test/siteIdentity/browser_identityPopup_clearSiteData.js
@@ -3,16 +3,25 @@
 
 const TEST_ORIGIN = "https://example.com";
 const TEST_SUB_ORIGIN = "https://test1.example.com";
 const REMOVE_DIALOG_URL = "chrome://browser/content/preferences/siteDataRemoveSelected.xul";
 
 ChromeUtils.defineModuleGetter(this, "SiteDataTestUtils",
                                "resource://testing-common/SiteDataTestUtils.jsm");
 
+add_task(async function setup() {
+  let oldCanRecord = Services.telemetry.canRecordExtended;
+  Services.telemetry.canRecordExtended = true;
+
+  registerCleanupFunction(() => {
+    Services.telemetry.canRecordExtended = oldCanRecord;
+  });
+});
+
 async function testClearing(testQuota, testCookies) {
   // Add some test quota storage.
   if (testQuota) {
     await SiteDataTestUtils.addToIndexedDB(TEST_ORIGIN);
     await SiteDataTestUtils.addToIndexedDB(TEST_SUB_ORIGIN);
   }
 
   // Add some test cookies.
@@ -48,24 +57,31 @@ async function testClearing(testQuota, t
     if (testCookies) {
       cookiesCleared = Promise.all([
         TestUtils.topicObserved("cookie-changed", (subj, data) => data == "deleted" && subj.name == "test1"),
         TestUtils.topicObserved("cookie-changed", (subj, data) => data == "deleted" && subj.name == "test2"),
         TestUtils.topicObserved("cookie-changed", (subj, data) => data == "deleted" && subj.name == "test3"),
       ]);
     }
 
+    Services.telemetry.clearEvents();
+
     // Click the "Clear data" button.
     siteDataUpdated = TestUtils.topicObserved("sitedatamanager:sites-updated");
     let hideEvent = BrowserTestUtils.waitForEvent(gIdentityHandler._identityPopup, "popuphidden");
     let removeDialogPromise = BrowserTestUtils.promiseAlertDialogOpen("accept", REMOVE_DIALOG_URL);
     clearButton.click();
     await hideEvent;
     await removeDialogPromise;
 
+    let events = Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN).parent;
+    let buttonEvents = events.filter(
+      e => e[1] == "security.ui.identitypopup" && e[2] == "click" && e[3] == "clear_sitedata");
+    is(buttonEvents.length, 1, "recorded telemetry for the button click");
+
     await siteDataUpdated;
 
     // Check that cookies were deleted.
     if (testCookies) {
       await cookiesCleared;
       let uri = Services.io.newURI(TEST_ORIGIN);
       is(Services.cookies.countCookiesFromHost(uri.host), 0, "Cookies from the base domain should be cleared");
       uri = Services.io.newURI(TEST_SUB_ORIGIN);
--- a/browser/base/content/test/trackingUI/browser_trackingUI_open_preferences.js
+++ b/browser/base/content/test/trackingUI/browser_trackingUI_open_preferences.js
@@ -26,16 +26,24 @@ async function waitAndAssertPreferencesS
       "The trackingprotection section is spotlighted.");
   });
 
   BrowserTestUtils.removeTab(gBrowser.selectedTab);
 }
 
 add_task(async function setup() {
   await UrlClassifierTestUtils.addTestTrackers();
+  let oldCanRecord = Services.telemetry.canRecordExtended;
+  Services.telemetry.canRecordExtended = true;
+
+  registerCleanupFunction(() => {
+    Services.telemetry.canRecordExtended = oldCanRecord;
+  });
+
+  Services.telemetry.clearEvents();
 });
 
 // Tests that pressing the preferences icon in the identity popup
 // links to about:preferences
 add_task(async function testOpenPreferencesFromPrefsButton() {
   await BrowserTestUtils.withNewTab("https://example.com", async function() {
     let promisePanelOpen = BrowserTestUtils.waitForEvent(gIdentityHandler._identityPopup, "popupshown");
     gIdentityHandler._identityBox.click();
@@ -43,16 +51,21 @@ add_task(async function testOpenPreferen
 
     let preferencesButton = document.getElementById("tracking-protection-preferences-button");
 
     ok(!BrowserTestUtils.is_hidden(preferencesButton), "The enable tracking protection button is shown.");
 
     let shown = waitAndAssertPreferencesShown();
     preferencesButton.click();
     await shown;
+
+    let events = Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true).parent;
+    let clickEvents = events.filter(
+      e => e[1] == "security.ui.identitypopup" && e[2] == "click" && e[3] == "cb_prefs_button");
+    is(clickEvents.length, 1, "recorded telemetry for the click");
   });
 });
 
 // Tests that clicking the contentblocking category items "add blocking" labels
 // links to about:preferences
 add_task(async function testOpenPreferencesFromAddBlockingButtons() {
   SpecialPowers.pushPrefEnv({set: [
     [CB_PREF, true],
@@ -71,16 +84,21 @@ add_task(async function testOpenPreferen
       let promisePanelOpen = BrowserTestUtils.waitForEvent(gIdentityHandler._identityPopup, "popupshown");
       gIdentityHandler._identityBox.click();
       await promisePanelOpen;
 
       ok(BrowserTestUtils.is_visible(button), "Button is shown.");
       let shown = waitAndAssertPreferencesShown();
       button.click();
       await shown;
+
+      let events = Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true).parent;
+      let clickEvents = events.filter(
+        e => e[1] == "security.ui.identitypopup" && e[2] == "click" && e[3].endsWith("_add_blocking"));
+      is(clickEvents.length, 1, "recorded telemetry for the click");
     }
   });
 });
 
 
 add_task(async function cleanup() {
   UrlClassifierTestUtils.cleanupTestTrackers();
 });
--- a/browser/base/content/test/trackingUI/browser_trackingUI_report_breakage.js
+++ b/browser/base/content/test/trackingUI/browser_trackingUI_report_breakage.js
@@ -17,16 +17,23 @@ let {HttpServer} = ChromeUtils.import("r
 let {CommonUtils} = ChromeUtils.import("resource://services-common/utils.js", {});
 let {Preferences} = ChromeUtils.import("resource://gre/modules/Preferences.jsm", {});
 
 add_task(async function setup() {
   await SpecialPowers.pushPrefEnv({set: [
     [CB_UI_PREF, true],
   ]});
   await UrlClassifierTestUtils.addTestTrackers();
+
+  let oldCanRecord = Services.telemetry.canRecordExtended;
+  Services.telemetry.canRecordExtended = true;
+
+  registerCleanupFunction(() => {
+    Services.telemetry.canRecordExtended = oldCanRecord;
+  });
 });
 
 function openIdentityPopup() {
   let mainView = document.getElementById("identity-popup-mainView");
   let viewShown = BrowserTestUtils.waitForEvent(mainView, "ViewShown");
   gIdentityHandler._identityBox.click();
   return viewShown;
 }
@@ -121,23 +128,30 @@ add_task(async function testReportBreaka
 add_task(async function testReportBreakageCancel() {
   Services.prefs.setBoolPref(TP_PREF, true);
   Services.prefs.setBoolPref(CB_PREF, true);
   Services.prefs.setBoolPref(PREF_REPORT_BREAKAGE_ENABLED, true);
 
   await BrowserTestUtils.withNewTab(TRACKING_PAGE, async function() {
     await openIdentityPopup();
 
+    Services.telemetry.clearEvents();
+
     let reportBreakageButton = document.getElementById("identity-popup-content-blocking-report-breakage");
     ok(BrowserTestUtils.is_visible(reportBreakageButton), "report breakage button is visible");
     let reportBreakageView = document.getElementById("identity-popup-breakageReportView");
     let viewShown = BrowserTestUtils.waitForEvent(reportBreakageView, "ViewShown");
     reportBreakageButton.click();
     await viewShown;
 
+    let events = Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN).parent;
+    let clickEvents = events.filter(
+      e => e[1] == "security.ui.identitypopup" && e[2] == "click" && e[3] == "report_breakage");
+    is(clickEvents.length, 1, "recorded telemetry for the click");
+
     ok(true, "Report breakage view was shown");
 
     let mainView = document.getElementById("identity-popup-mainView");
     viewShown = BrowserTestUtils.waitForEvent(mainView, "ViewShown");
     let cancelButton = document.getElementById("identity-popup-breakageReportView-cancel");
     cancelButton.click();
     await viewShown;
 
--- a/browser/base/content/test/trackingUI/browser_trackingUI_telemetry.js
+++ b/browser/base/content/test/trackingUI/browser_trackingUI_telemetry.js
@@ -6,138 +6,131 @@ const PREF = "privacy.trackingprotection
 const BENIGN_PAGE = "http://tracking.example.org/browser/browser/base/content/test/trackingUI/benignPage.html";
 const TRACKING_PAGE = "http://tracking.example.org/browser/browser/base/content/test/trackingUI/trackingPage.html";
 
 /**
  * Enable local telemetry recording for the duration of the tests.
  */
 var oldCanRecord = Services.telemetry.canRecordExtended;
 Services.telemetry.canRecordExtended = true;
-Services.prefs.setBoolPref(PREF, false);
-Services.telemetry.getHistogramById("TRACKING_PROTECTION_ENABLED").clear();
 registerCleanupFunction(function() {
   UrlClassifierTestUtils.cleanupTestTrackers();
   Services.telemetry.canRecordExtended = oldCanRecord;
   Services.prefs.clearUserPref(PREF);
 });
 
 function getShieldHistogram() {
   return Services.telemetry.getHistogramById("TRACKING_PROTECTION_SHIELD");
 }
 
-function getEnabledHistogram() {
-  return Services.telemetry.getHistogramById("TRACKING_PROTECTION_ENABLED");
-}
-
-function getEventsHistogram() {
-  return Services.telemetry.getHistogramById("TRACKING_PROTECTION_EVENTS");
-}
-
 function getShieldCounts() {
   return getShieldHistogram().snapshot().counts;
 }
 
-function getEnabledCounts() {
-  return getEnabledHistogram().snapshot().counts;
-}
-
-function getEventCounts() {
-  return getEventsHistogram().snapshot().counts;
-}
-
 add_task(async function setup() {
   await UrlClassifierTestUtils.addTestTrackers();
 
   let TrackingProtection = gBrowser.ownerGlobal.TrackingProtection;
   ok(TrackingProtection, "TP is attached to the browser window");
   ok(!TrackingProtection.enabled, "TP is not enabled");
 
-  // Open a window with TP disabled to make sure 'enabled' is logged correctly.
-  let newWin = await BrowserTestUtils.openNewBrowserWindow({});
-  newWin.close();
+  let enabledCounts =
+    Services.telemetry.getHistogramById("TRACKING_PROTECTION_ENABLED").snapshot().counts;
+  is(enabledCounts[0], 1, "TP was not enabled on start up");
 
-  is(getEnabledCounts()[0], 1, "TP was disabled once on start up");
-  is(getEnabledCounts()[1], 0, "TP was not enabled on start up");
+  let scalars = Services.telemetry.snapshotScalars(
+    Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTOUT, false).parent;
 
-  // Enable TP so the next browser to open will log 'enabled'
-  Services.prefs.setBoolPref(PREF, true);
+  is(scalars["contentblocking.enabled"], true, "CB was enabled at startup");
+  is(scalars["contentblocking.fastblock_enabled"], true, "FB was enabled at startup");
+  is(scalars["contentblocking.exceptions"], 0, "no CB exceptions at startup");
 });
 
 
-add_task(async function testNewWindow() {
-  let newWin = await BrowserTestUtils.openNewBrowserWindow({});
-  let tab = await BrowserTestUtils.openNewForegroundTab(newWin.gBrowser);
-  let TrackingProtection = newWin.TrackingProtection;
-  ok(TrackingProtection, "TP is attached to the browser window");
-
-  is(getEnabledCounts()[0], 1, "TP was disabled once on start up");
-  is(getEnabledCounts()[1], 1, "TP was enabled once on start up");
+add_task(async function testShieldHistogram() {
+  Services.prefs.setBoolPref(PREF, true);
+  let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser);
 
   // Reset these to make counting easier
-  getEventsHistogram().clear();
   getShieldHistogram().clear();
 
   await promiseTabLoadEvent(tab, BENIGN_PAGE);
-  is(getEventCounts()[0], 1, "Total page loads");
-  is(getEventCounts()[1], 0, "Disable actions");
-  is(getEventCounts()[2], 0, "Enable actions");
   is(getShieldCounts()[0], 1, "Page loads without tracking");
 
   await promiseTabLoadEvent(tab, TRACKING_PAGE);
-  // Note that right now the events and shield histogram is not measuring what
+  // Note that right now the shield histogram is not measuring what
   // you might think.  Since onSecurityChange fires twice for a tracking page,
   // the total page loads count is double counting, and the shield count
   // (which is meant to measure times when the shield wasn't shown) fires even
   // when tracking elements exist on the page.
-  todo_is(getEventCounts()[0], 2, "FIXME: TOTAL PAGE LOADS IS DOUBLE COUNTING");
-  is(getEventCounts()[1], 0, "Disable actions");
-  is(getEventCounts()[2], 0, "Enable actions");
   todo_is(getShieldCounts()[0], 1, "FIXME: TOTAL PAGE LOADS WITHOUT TRACKING IS DOUBLE COUNTING");
 
   info("Disable TP for the page (which reloads the page)");
   let tabReloadPromise = promiseTabLoadEvent(tab);
-  newWin.document.querySelector("#tracking-action-unblock").doCommand();
+  document.querySelector("#tracking-action-unblock").doCommand();
   await tabReloadPromise;
-  todo_is(getEventCounts()[0], 3, "FIXME: TOTAL PAGE LOADS IS DOUBLE COUNTING");
-  is(getEventCounts()[1], 1, "Disable actions");
-  is(getEventCounts()[2], 0, "Enable actions");
   todo_is(getShieldCounts()[0], 1, "FIXME: TOTAL PAGE LOADS WITHOUT TRACKING IS DOUBLE COUNTING");
 
   info("Re-enable TP for the page (which reloads the page)");
   tabReloadPromise = promiseTabLoadEvent(tab);
-  newWin.document.querySelector("#tracking-action-block").doCommand();
+  document.querySelector("#tracking-action-block").doCommand();
   await tabReloadPromise;
-  todo_is(getEventCounts()[0], 4, "FIXME: TOTAL PAGE LOADS IS DOUBLE COUNTING");
-  is(getEventCounts()[1], 1, "Disable actions");
-  is(getEventCounts()[2], 1, "Enable actions");
   todo_is(getShieldCounts()[0], 1, "FIXME: TOTAL PAGE LOADS WITHOUT TRACKING IS DOUBLE COUNTING");
 
-  newWin.close();
+  gBrowser.removeCurrentTab();
 
   // Reset these to make counting easier for the next test
-  getEventsHistogram().clear();
   getShieldHistogram().clear();
-  getEnabledHistogram().clear();
 });
 
-add_task(async function testPrivateBrowsing() {
-  let privateWin = await BrowserTestUtils.openNewBrowserWindow({private: true});
-  let tab = await BrowserTestUtils.openNewForegroundTab(privateWin.gBrowser);
-  let TrackingProtection = privateWin.TrackingProtection;
-  ok(TrackingProtection, "TP is attached to the browser window");
+add_task(async function testIdentityPopupEvents() {
+  Services.prefs.setBoolPref(PREF, true);
+  let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser);
+
+  await promiseTabLoadEvent(tab, BENIGN_PAGE);
+
+  Services.telemetry.clearEvents();
+
+  let { gIdentityHandler } = gBrowser.ownerGlobal;
+  let promisePanelOpen = BrowserTestUtils.waitForEvent(gIdentityHandler._identityPopup, "popupshown");
+  gIdentityHandler._identityBox.click();
+  await promisePanelOpen;
 
-  // Do a bunch of actions and make sure that no telemetry data is gathered
-  await promiseTabLoadEvent(tab, BENIGN_PAGE);
+  let events = Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true).parent;
+  let openEvents = events.filter(
+    e => e[1] == "security.ui.identitypopup" && e[2] == "open" && e[3] == "identity_popup");
+  is(openEvents.length, 1, "recorded telemetry for opening the identity popup");
+  is(openEvents[0][4], "shield-hidden", "recorded the shield as hidden");
+
   await promiseTabLoadEvent(tab, TRACKING_PAGE);
+
+  promisePanelOpen = BrowserTestUtils.waitForEvent(gIdentityHandler._identityPopup, "popupshown");
+  gIdentityHandler._identityBox.click();
+  await promisePanelOpen;
+
+  events = Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true).parent;
+  openEvents = events.filter(
+    e => e[1] == "security.ui.identitypopup" && e[2] == "open" && e[3] == "identity_popup");
+  is(openEvents.length, 1, "recorded telemetry for opening the identity popup");
+  is(openEvents[0][4], "shield-showing", "recorded the shield as showing");
+
+  info("Disable TP for the page (which reloads the page)");
   let tabReloadPromise = promiseTabLoadEvent(tab);
-  privateWin.document.querySelector("#tracking-action-unblock").doCommand();
-  await tabReloadPromise;
-  tabReloadPromise = promiseTabLoadEvent(tab);
-  privateWin.document.querySelector("#tracking-action-block").doCommand();
+  document.querySelector("#tracking-action-unblock").doCommand();
   await tabReloadPromise;
 
-  // Sum up all the counts to make sure that nothing got logged
-  is(getEnabledCounts().reduce((p, c) => p + c), 0, "Telemetry logging off in PB mode");
-  is(getEventCounts().reduce((p, c) => p + c), 0, "Telemetry logging off in PB mode");
-  is(getShieldCounts().reduce((p, c) => p + c), 0, "Telemetry logging off in PB mode");
+  events = Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true).parent;
+  let clickEvents = events.filter(
+    e => e[1] == "security.ui.identitypopup" && e[2] == "click" && e[3] == "unblock");
+  is(clickEvents.length, 1, "recorded telemetry for the click");
 
-  privateWin.close();
+  info("Re-enable TP for the page (which reloads the page)");
+  tabReloadPromise = promiseTabLoadEvent(tab);
+  document.querySelector("#tracking-action-block").doCommand();
+  await tabReloadPromise;
+
+  events = Services.telemetry.snapshotEvents(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true).parent;
+  clickEvents = events.filter(
+    e => e[1] == "security.ui.identitypopup" && e[2] == "click" && e[3] == "block");
+  is(clickEvents.length, 1, "recorded telemetry for the click");
+
+  gBrowser.removeCurrentTab();
 });