Bug 1461690 Part 1: Write uninstall ping. r=chutten,Gijs
authorAdam Gashlin <agashlin@mozilla.com>
Tue, 20 Oct 2020 23:16:02 +0000
changeset 553724 c842251ed63538a76db865459650d795d20d5262
parent 553723 34b7425fedc3f754efcf5d55481e5dad681552b8
child 553725 fe4ce626ccc07c16d4acb122fdc0c2a407d8ebb4
push id37881
push usersmolnar@mozilla.com
push dateWed, 21 Oct 2020 09:51:28 +0000
treeherdermozilla-central@d8861d51b01e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschutten, Gijs
bugs1461690
milestone84.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1461690 Part 1: Write uninstall ping. r=chutten,Gijs This covers the in-Firefox portions of the install ping, except for the otherInstalls count, which is added in part 2. Later parts will cover the uninstaller. Differential Revision: https://phabricator.services.mozilla.com/D92522
toolkit/components/telemetry/app/TelemetryControllerParent.jsm
toolkit/components/telemetry/app/TelemetryStorage.jsm
toolkit/components/telemetry/docs/data/uninstall-ping.rst
toolkit/components/telemetry/tests/unit/head.js
toolkit/components/telemetry/tests/unit/test_UninstallPing.js
toolkit/components/telemetry/tests/unit/xpcshell.ini
--- a/toolkit/components/telemetry/app/TelemetryControllerParent.jsm
+++ b/toolkit/components/telemetry/app/TelemetryControllerParent.jsm
@@ -37,16 +37,17 @@ const TELEMETRY_TEST_DELAY = 1;
 
 // How long to wait (ms) before sending the new profile ping on the first
 // run of a new profile.
 const NEWPROFILE_PING_DEFAULT_DELAY = 30 * 60 * 1000;
 
 // Ping types.
 const PING_TYPE_MAIN = "main";
 const PING_TYPE_DELETION_REQUEST = "deletion-request";
+const PING_TYPE_UNINSTALL = "uninstall";
 
 // Session ping reasons.
 const REASON_GATHER_PAYLOAD = "gather-payload";
 const REASON_GATHER_SUBSESSION_PAYLOAD = "gather-subsession-payload";
 
 XPCOMUtils.defineLazyServiceGetter(
   this,
   "Telemetry",
@@ -249,16 +250,29 @@ var TelemetryController = Object.freeze(
    *
    * @return {Promise} Promise that is resolved when the ping was removed.
    */
   removeAbortedSessionPing() {
     return Impl.removeAbortedSessionPing();
   },
 
   /**
+   * Create an uninstall ping and write it to disk, replacing any already present.
+   * This is stored independently from other pings, and only read by
+   * the Windows uninstaller.
+   *
+   * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
+   *
+   * @return {Promise} Resolved when the ping has been saved.
+   */
+  saveUninstallPing() {
+    return Impl.saveUninstallPing();
+  },
+
+  /**
    * Allows waiting for TelemetryControllers delayed initialization to complete.
    * The returned promise is guaranteed to resolve before TelemetryController is shutting down.
    * @return {Promise} Resolved when delayed TelemetryController initialization completed.
    */
   promiseInitialized() {
     return Impl.promiseInitialized();
   },
 });
@@ -684,16 +698,40 @@ var Impl = {
     const pingData = this.assemblePing(PING_TYPE_MAIN, aPayload, options);
     return TelemetryStorage.saveAbortedSessionPing(pingData);
   },
 
   removeAbortedSessionPing() {
     return TelemetryStorage.removeAbortedSessionPing();
   },
 
+  _countOtherInstalls() {
+    // TODO
+    throw new Error("_countOtherInstalls - not implemented");
+  },
+
+  async saveUninstallPing() {
+    if (AppConstants.platform != "win") {
+      return undefined;
+    }
+
+    this._log.trace("saveUninstallPing");
+
+    let payload = {};
+    try {
+      payload.otherInstalls = this._countOtherInstalls();
+    } catch (e) {
+      this._log.warn("saveUninstallPing - _countOtherInstalls failed", e);
+    }
+    const options = { addClientId: true, addEnvironment: true };
+    const pingData = this.assemblePing(PING_TYPE_UNINSTALL, payload, options);
+
+    return TelemetryStorage.saveUninstallPing(pingData);
+  },
+
   /**
    * This triggers basic telemetry initialization and schedules a full initialized for later
    * for performance reasons.
    *
    * This delayed initialization means TelemetryController init can be in the following states:
    * 1) setupTelemetry was never called
    * or it was called and
    *   2) _delayedInitTask was scheduled, but didn't run yet.
@@ -831,16 +869,26 @@ var Impl = {
               TelemetryUntrustedModulesPing.start();
             }
           }
 
           TelemetryEventPing.startup();
           EcosystemTelemetry.startup();
           TelemetryPrioPing.startup();
 
+          if (uploadEnabled) {
+            await this.saveUninstallPing().catch(e =>
+              this._log.warn("_delayedInitTask - saveUninstallPing failed", e)
+            );
+          } else {
+            await TelemetryStorage.removeUninstallPings().catch(e =>
+              this._log.warn("_delayedInitTask - saveUninstallPing", e)
+            );
+          }
+
           this._delayedInitTaskDeferred.resolve();
         } catch (e) {
           this._delayedInitTaskDeferred.reject(e);
         } finally {
           this._delayedInitTask = null;
         }
       },
       this._testMode ? TELEMETRY_TEST_DELAY : TELEMETRY_DELAY,
@@ -999,16 +1047,20 @@ var Impl = {
       this._clientID = null;
 
       // Generate a new client ID and make sure this module uses the new version
       let p = (async () => {
         await ClientID.removeClientIDs();
         let id = await ClientID.getClientID();
         this._clientID = id;
         Telemetry.scalarSet("telemetry.data_upload_optin", true);
+
+        await this.saveUninstallPing().catch(e =>
+          this._log.warn("_onUploadPrefChange - saveUninstallPing failed", e)
+        );
       })();
 
       this._shutdownBarrier.client.addBlocker(
         "TelemetryController: resetting client ID after data upload was enabled",
         p
       );
 
       return;
@@ -1018,16 +1070,17 @@ var Impl = {
       try {
         // 1. Cancel the current pings.
         // 2. Clear unpersisted pings
         await TelemetrySend.clearCurrentPings();
 
         // 3. Remove all pending pings
         await TelemetryStorage.removeAppDataPings();
         await TelemetryStorage.runRemovePendingPingsTask();
+        await TelemetryStorage.removeUninstallPings();
       } catch (e) {
         this._log.error(
           "_onUploadPrefChange - error clearing pending pings",
           e
         );
       } finally {
         // 4. Reset session and subsession counter
         TelemetrySession.resetSubsessionCounter();
--- a/toolkit/components/telemetry/app/TelemetryStorage.jsm
+++ b/toolkit/components/telemetry/app/TelemetryStorage.jsm
@@ -105,16 +105,36 @@ PingParseError.prototype.constructor = P
  */
 var Policy = {
   now: () => new Date(),
   getArchiveQuota: () => ARCHIVE_QUOTA_BYTES,
   getPendingPingsQuota: () =>
     AppConstants.platform == "android"
       ? PENDING_PINGS_QUOTA_BYTES_MOBILE
       : PENDING_PINGS_QUOTA_BYTES_DESKTOP,
+  /**
+   * @param {string} id The ID of the ping that will be written into the file. Can be "*" to
+   *                    make a pattern to find all pings for this installation.
+   * @return
+   *         {
+   *           directory: <nsIFile>, // Directory to save pings
+   *           file: <string>, // File name for this ping (or pattern for all pings)
+   *         }
+   */
+  getUninstallPingPath: id => {
+    // UpdRootD is e.g. C:\ProgramData\Mozilla\updates\<PATH HASH>
+    const updateDirectory = Services.dirsvc.get("UpdRootD", Ci.nsIFile);
+    const installPathHash = updateDirectory.leafName;
+
+    return {
+      // e.g. C:\ProgramData\Mozilla
+      directory: updateDirectory.parent.parent.clone(),
+      file: `uninstall_ping_${installPathHash}_${id}.json`,
+    };
+  },
 };
 
 /**
  * Wait for all promises in iterable to resolve or reject. This function
  * always resolves its promise with undefined, and never rejects.
  */
 function waitForAll(it) {
   let dummy = () => {};
@@ -342,16 +362,41 @@ var TelemetryStorage = {
    *
    * @return {promise} Promise that is resolved once the ping is removed.
    */
   removeAbortedSessionPing() {
     return TelemetryStorageImpl.removeAbortedSessionPing();
   },
 
   /**
+   * Save an uninstall ping to disk, removing any old ones from this
+   * installation first.
+   * This is stored independently from other pings, and only read by
+   * the Windows uninstaller.
+   *
+   * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
+   *
+   * @return {promise} Promise that is resolved when the ping has been saved.
+   */
+  saveUninstallPing(ping) {
+    return TelemetryStorageImpl.saveUninstallPing(ping);
+  },
+
+  /**
+   * Remove all uninstall pings from this installation.
+   *
+   * WINDOWS ONLY, does nothing and resolves immediately on other platforms.
+   *
+   * @return {promise} Promise that is resolved when the pings have been removed.
+   */
+  removeUninstallPings() {
+    return TelemetryStorageImpl.removeUninstallPings();
+  },
+
+  /**
    * Save a single ping to a file.
    *
    * @param {object} ping The content of the ping to save.
    * @param {string} file The destination file.
    * @param {bool} overwrite If |true|, the file will be overwritten if it exists,
    * if |false| the file will not be overwritten and no error will be reported if
    * the file exists.
    * @returns {promise}
@@ -1970,16 +2015,59 @@ var TelemetryStorageImpl = {
           this._log.trace("removeAbortedSessionPing - no such file");
         } else {
           this._log.error("removeAbortedSessionPing - error removing ping", ex);
         }
       }
     });
   },
 
+  async saveUninstallPing(ping) {
+    if (AppConstants.platform != "win") {
+      return;
+    }
+
+    // Remove any old pings from this install first.
+    await this.removeUninstallPings();
+
+    let { directory: pingFile, file } = Policy.getUninstallPingPath(ping.id);
+    pingFile.append(file);
+
+    await this.savePingToFile(ping, pingFile.path, /* overwrite */ true);
+  },
+
+  async removeUninstallPings() {
+    if (AppConstants.platform != "win") {
+      return;
+    }
+
+    const { directory, file } = Policy.getUninstallPingPath("*");
+
+    const iteratorOptions = { winPattern: file };
+    const iterator = new OS.File.DirectoryIterator(
+      directory.path,
+      iteratorOptions
+    );
+
+    await iterator.forEach(async entry => {
+      this._log.trace("removeUninstallPings - removing", entry.path);
+      try {
+        await OS.File.remove(entry.path);
+        this._log.trace("removeUninstallPings - success");
+      } catch (ex) {
+        if (ex.becauseNoSuchFile) {
+          this._log.trace("removeUninstallPings - no such file");
+        } else {
+          this._log.error("removeUninstallPings - error removing ping", ex);
+        }
+      }
+    });
+    iterator.close();
+  },
+
   /**
    * Remove FHR database files. This is temporary and will be dropped in
    * the future.
    * @return {Promise} Resolved when the database files are deleted.
    */
   async removeFHRDatabase() {
     this._log.trace("removeFHRDatabase");
 
new file mode 100644
--- /dev/null
+++ b/toolkit/components/telemetry/docs/data/uninstall-ping.rst
@@ -0,0 +1,36 @@
+
+"uninstall" ping
+================
+
+This opt-out ping is sent from the Windows uninstaller when the uninstall finishes. Notably it includes ``clientId`` and the :doc:`Telemetry Environment <environment>`. It follows the :doc:`common ping format <common-ping>`.
+
+Structure:
+
+.. code-block:: js
+
+    {
+      type: "uninstall",
+      ... common ping data
+      clientId: <UUID>,
+      environment: { ... },
+      payload: {
+        otherInstalls: <integer>, // Optional, number of other installs on the system, max 11.
+      }
+    }
+
+See also the `JSON schema <https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/master/templates/telemetry/uninstall/uninstall.4.schema.json>`_. These pings are recorded in the ``telemetry.uninstall`` table in Redash, using the default "Telemetry (BigQuery)" data source.
+
+payload.otherInstalls
+---------------------
+This is a count of how many other installs of Firefox were present on the system at the time the ping was written. It is the number of values in the ``Software\Mozilla\Firefox\TaskBarIDs`` registry key, for both 32-bit and 64-bit architectures, for both HKCU and HKLM, excluding duplicates, and excluding a value for this install (if present). For example, if this is the only install on the system, the value will be 0. It may be missing in case of an error.
+
+This count is capped at 11. This avoids introducing a high-resolution identifier in case of a system with a large, unique number of installs.
+
+Uninstall Ping storage and lifetime
+-----------------------------------
+
+On delayed Telemetry init (about 1 minute into each run of Firefox), if opt-out telemetry is enabled, this ping is written to disk. There is a single ping for each install, any uninstall pings from the same install are removed before the new ping is written.
+
+The ping is removed if Firefox notices that opt-out telemetry has been disabled, either when the ``datareporting.healthreport.uploadEnabled`` pref goes false or when it is false on delayed init. Conversely, when opt-out telemetry is re-enabled, the ping is written as Telemetry is setting itself up again.
+
+The ping is sent by the uninstaller some arbitrary time after it is written to disk by Firefox, so it could be significantly out of date when it is submitted. There should be little impact from stale data, since analysis is likely to focus on clients that uninstalled soon after running Firefox, and this ping mostly changes when Firefox itself is updated.
--- a/toolkit/components/telemetry/tests/unit/head.js
+++ b/toolkit/components/telemetry/tests/unit/head.js
@@ -434,16 +434,30 @@ function fakeIntlReady() {
     "resource://gre/modules/TelemetryEnvironment.jsm",
     null
   );
   m.Policy._intlLoaded = true;
   // Dispatch the observer event in case the promise has been registered already.
   Services.obs.notifyObservers(null, "browser-delayed-startup-finished");
 }
 
+// Override the uninstall ping file names
+function fakeUninstallPingPath(aPathFcn) {
+  const m = ChromeUtils.import(
+    "resource://gre/modules/TelemetryStorage.jsm",
+    null
+  );
+  m.Policy.getUninstallPingPath =
+    aPathFcn ||
+    (id => ({
+      directory: new FileUtils.File(OS.Constants.Path.profileDir),
+      file: `uninstall_ping_0123456789ABCDEF_${id}.json`,
+    }));
+}
+
 // Return a date that is |offset| ms in the future from |date|.
 function futureDate(date, offset) {
   return new Date(date.getTime() + offset);
 }
 
 function truncateToDays(aMsec) {
   return Math.floor(aMsec / MILLISECONDS_PER_DAY);
 }
@@ -565,8 +579,11 @@ TelemetryController.testInitLogging();
 
 // Avoid timers interrupting test behavior.
 fakeSchedulerTimer(
   () => {},
   () => {}
 );
 // Make pind sending predictable.
 fakeMidnightPingFuzzingDelay(0);
+
+// Avoid using the directory service, which is not registered in some tests.
+fakeUninstallPingPath();
new file mode 100644
--- /dev/null
+++ b/toolkit/components/telemetry/tests/unit/test_UninstallPing.js
@@ -0,0 +1,127 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+"use strict";
+
+const { TelemetryStorage } = ChromeUtils.import(
+  "resource://gre/modules/TelemetryStorage.jsm"
+);
+const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm");
+const { FileUtils } = ChromeUtils.import(
+  "resource://gre/modules/FileUtils.jsm"
+);
+
+const gFakeInstallPathHash = "0123456789ABCDEF";
+let gFakeVendorDirectory;
+let gFakeGetUninstallPingPath;
+
+add_task(async function setup() {
+  do_get_profile();
+
+  let fakeVendorDirectoryNSFile = new FileUtils.File(
+    OS.Path.join(OS.Constants.Path.profileDir, "uninstall-ping-test")
+  );
+  fakeVendorDirectoryNSFile.createUnique(
+    Ci.nsIFile.DIRECTORY_TYPE,
+    FileUtils.PERMS_DIRECTORY
+  );
+  gFakeVendorDirectory = fakeVendorDirectoryNSFile.path;
+
+  gFakeGetUninstallPingPath = id => ({
+    directory: fakeVendorDirectoryNSFile.clone(),
+    file: `uninstall_ping_${gFakeInstallPathHash}_${id}.json`,
+  });
+
+  fakeUninstallPingPath(gFakeGetUninstallPingPath);
+
+  registerCleanupFunction(() => {
+    OS.File.removeDir(gFakeVendorDirectory);
+  });
+});
+
+function ping_path(ping) {
+  let { directory: pingFile, file } = gFakeGetUninstallPingPath(ping.id);
+  pingFile.append(file);
+  return pingFile.path;
+}
+
+add_task(async function test_store_ping() {
+  // Remove shouldn't throw on an empty dir.
+  await TelemetryStorage.removeUninstallPings();
+
+  // Write ping
+  const ping1 = {
+    id: "58b63aac-999e-4efb-9d5a-20f368670721",
+    payload: { some: "thing" },
+  };
+  const ping1Path = ping_path(ping1);
+  await TelemetryStorage.saveUninstallPing(ping1);
+
+  // Check the ping
+  Assert.ok(await OS.File.exists(ping1Path));
+  const readPing1 = JSON.parse(
+    await OS.File.read(ping1Path, { encoding: "utf-8" })
+  );
+  Assert.deepEqual(ping1, readPing1);
+
+  // Write another file that shouldn't match the pattern
+  const otherFilePath = OS.Path.join(gFakeVendorDirectory, "other_file.json");
+  await OS.File.writeAtomic(otherFilePath, "");
+  Assert.ok(await OS.File.exists(otherFilePath));
+
+  // Write another ping, should remove the earlier one
+  const ping2 = {
+    id: "7202c564-8f23-41b4-8a50-1744e9549260",
+    payload: { another: "thing" },
+  };
+  const ping2Path = ping_path(ping2);
+  await TelemetryStorage.saveUninstallPing(ping2);
+
+  Assert.ok(!(await OS.File.exists(ping1Path)));
+  Assert.ok(await OS.File.exists(ping2Path));
+  Assert.ok(await OS.File.exists(otherFilePath));
+
+  // Write an additional file manually so there are multiple matching pings to remove
+  const ping3 = { id: "yada-yada" };
+  const ping3Path = ping_path(ping3);
+
+  await OS.File.writeAtomic(ping3Path, "");
+  Assert.ok(await OS.File.exists(ping3Path));
+
+  // Remove pings
+  await TelemetryStorage.removeUninstallPings();
+
+  // Check our pings are removed but other file isn't
+  Assert.ok(!(await OS.File.exists(ping1Path)));
+  Assert.ok(!(await OS.File.exists(ping2Path)));
+  Assert.ok(!(await OS.File.exists(ping3Path)));
+  Assert.ok(await OS.File.exists(otherFilePath));
+
+  // Remove again, confirming that the remove doesn't cause an error if nothing to remove
+  await TelemetryStorage.removeUninstallPings();
+
+  const ping4 = {
+    id: "1f113673-753c-4fbe-9143-fe197f936036",
+    payload: { any: "thing" },
+  };
+  const ping4Path = ping_path(ping4);
+  await TelemetryStorage.saveUninstallPing(ping4);
+
+  // Open the ping without FILE_SHARE_DELETE, so a delete should fail.
+  const ping4File = await OS.File.open(
+    ping4Path,
+    { read: true, existing: true },
+    { winShare: OS.Constants.Win.FILE_SHARE_READ }
+  );
+
+  // Check that there is no error if the file can't be removed.
+  await TelemetryStorage.removeUninstallPings();
+
+  // And file should still exist.
+  Assert.ok(await OS.File.exists(ping4Path));
+
+  // Close the file, it should be possible to remove now.
+  ping4File.close();
+  await TelemetryStorage.removeUninstallPings();
+  Assert.ok(!(await OS.File.exists(ping4Path)));
+});
--- a/toolkit/components/telemetry/tests/unit/xpcshell.ini
+++ b/toolkit/components/telemetry/tests/unit/xpcshell.ini
@@ -90,8 +90,10 @@ run-if = nightly_build && (os == 'win')
 [test_EcosystemTelemetry.js]
 skip-if = appname == "thunderbird"
 [test_EventPing.js]
 [test_EventPing_disabled.js]
 tags = coverage
 [test_CoveragePing.js]
 [test_PrioPing.js]
 [test_bug1555798.js]
+[test_UninstallPing.js]
+run-if = os == 'win'