Bug 1313573 - Validate the sync ping during TPS test runs r=markh
authorThom Chiovoloni <tchiovoloni@mozilla.com>
Wed, 02 Nov 2016 14:25:07 -0400
changeset 321586 c05a3ad32d21c1707f37f5dfd4e98ec6fb148d84
parent 321585 ca0017c90ad0fb11dbf4ecd3409f9bf196059869
child 321587 b8386ff9818fd5682b32a4ea8e0a516c9d78d586
push id83647
push userkwierso@gmail.com
push dateTue, 08 Nov 2016 22:08:41 +0000
treeherdermozilla-inbound@1d0b02250149 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmarkh
bugs1313573
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1313573 - Validate the sync ping during TPS test runs r=markh MozReview-Commit-ID: Jy7VpAvhbPf
services/sync/tests/unit/sync_ping_schema.json
services/sync/tps/extensions/tps/resource/tps.jsm
testing/tps/tps/testrunner.py
--- a/services/sync/tests/unit/sync_ping_schema.json
+++ b/services/sync/tests/unit/sync_ping_schema.json
@@ -23,17 +23,17 @@
         "didLogin": { "type": "boolean" },
         "when": { "type": "integer" },
         "uid": {
           "type": "string",
           "pattern": "^[0-9a-f]{32}$"
         },
         "devices": {
           "type": "array",
-          "items": { "$ref": "#/definitions/engine" }
+          "items": { "$ref": "#/definitions/device" }
         },
         "deviceID": {
           "type": "string",
           "pattern": "^[0-9a-f]{64}$"
         },
         "status": {
           "type": "object",
           "anyOf": [
--- a/services/sync/tps/extensions/tps/resource/tps.jsm
+++ b/services/sync/tps/extensions/tps/resource/tps.jsm
@@ -14,16 +14,17 @@ const {classes: Cc, interfaces: Ci, util
 var module = this;
 
 // Global modules
 Cu.import("resource://gre/modules/Log.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/AppConstants.jsm");
 Cu.import("resource://gre/modules/PlacesUtils.jsm");
+Cu.import("resource://gre/modules/FileUtils.jsm");
 Cu.import("resource://services-common/async.js");
 Cu.import("resource://services-sync/constants.js");
 Cu.import("resource://services-sync/main.js");
 Cu.import("resource://services-sync/util.js");
 Cu.import("resource://services-sync/telemetry.js");
 Cu.import("resource://services-sync/bookmark_validator.js");
 Cu.import("resource://services-sync/engines/passwords.js");
 Cu.import("resource://services-sync/engines/forms.js");
@@ -44,16 +45,21 @@ Cu.import("resource://tps/modules/window
 var hh = Cc["@mozilla.org/network/protocol;1?name=http"]
          .getService(Ci.nsIHttpProtocolHandler);
 var prefs = Cc["@mozilla.org/preferences-service;1"]
             .getService(Ci.nsIPrefBranch);
 
 var mozmillInit = {};
 Cu.import('resource://mozmill/driver/mozmill.js', mozmillInit);
 
+XPCOMUtils.defineLazyGetter(this, "fileProtocolHandler", () => {
+  let fileHandler = Services.io.getProtocolHandler("file");
+  return fileHandler.QueryInterface(Ci.nsIFileProtocolHandler);
+});
+
 // Options for wiping data during a sync
 const SYNC_RESET_CLIENT = "resetClient";
 const SYNC_WIPE_CLIENT  = "wipeClient";
 const SYNC_WIPE_REMOTE  = "wipeRemote";
 
 // Actions a test can perform
 const ACTION_ADD                = "add";
 const ACTION_DELETE             = "delete";
@@ -739,24 +745,28 @@ var TPS = {
           this.ValidatePasswords();
         }
         if (this.shouldValidateForms) {
           this.ValidateForms();
         }
         if (this.shouldValidateAddons) {
           this.ValidateAddons();
         }
+        // Force this early so that we run the validation and detect missing pings
+        // *before* we start shutting down, since if we do it after, the python
+        // code won't notice the failure.
+        SyncTelemetry.shutdown();
         // we're all done
         Logger.logInfo("test phase " + this._currentPhase + ": " +
                        (this._errors ? "FAIL" : "PASS"));
         this._phaseFinished = true;
         this.quit();
         return;
       }
-
+      this.seconds_since_epoch = prefs.getIntPref("tps.seconds_since_epoch", 0);
       if (this.seconds_since_epoch)
         this._usSinceEpoch = this.seconds_since_epoch * 1000 * 1000;
       else {
         this.DumpError("seconds-since-epoch not set");
         return;
       }
 
       let phase = this._phaselist[this._currentPhase];
@@ -780,16 +790,60 @@ var TPS = {
       } else {
         this.DumpError("RunNextTestAction failed", e);
       }
       return;
     }
     this.RunNextTestAction();
   },
 
+  _getFileRelativeToSourceRoot(testFileURL, relativePath) {
+    let file = fileProtocolHandler.getFileFromURLSpec(testFileURL);
+    let root = file // <root>/services/sync/tests/tps/test_foo.js
+      .parent // <root>/services/sync/tests/tps
+      .parent // <root>/services/sync/tests
+      .parent // <root>/services/sync
+      .parent // <root>/services
+      .parent // <root>
+      ;
+    root.appendRelativePath(relativePath);
+    return root;
+  },
+
+  // Attempt to load the sync_ping_schema.json and initialize `this.pingValidator`
+  // based on the source of the tps file. Assumes that it's at "../unit/sync_ping_schema.json"
+  // relative to the directory the tps test file (testFile) is contained in.
+  _tryLoadPingSchema(testFile) {
+    try {
+      let schemaFile = this._getFileRelativeToSourceRoot(testFile,
+        "services/sync/tests/unit/sync_ping_schema.json");
+
+      let stream = Cc["@mozilla.org/network/file-input-stream;1"]
+                   .createInstance(Ci.nsIFileInputStream);
+
+      let jsonReader = Cc["@mozilla.org/dom/json;1"]
+                       .createInstance(Components.interfaces.nsIJSON);
+
+      stream.init(schemaFile, FileUtils.MODE_RDONLY, FileUtils.PERMS_FILE, 0);
+      let schema = jsonReader.decodeFromStream(stream, stream.available());
+      Logger.logInfo("Successfully loaded schema")
+
+      // Importing resource://testing-common/* isn't possible from within TPS,
+      // so we load Ajv manually.
+      let ajvFile = this._getFileRelativeToSourceRoot(testFile, "testing/modules/ajv-4.1.1.js");
+      let ajvURL = fileProtocolHandler.getURLSpecFromFile(ajvFile);
+      let ns = {};
+      Cu.import(ajvURL, ns);
+      let ajv = new ns.Ajv({ async: "co*" });
+      this.pingValidator = ajv.compile(schema);
+    } catch (e) {
+      this.DumpError(`Failed to load ping schema and AJV relative to "${testFile}".`, e);
+    }
+  },
+
   /**
    * Runs a single test phase.
    *
    * This is the main entry point for each phase of a test. The TPS command
    * line driver loads this module and calls into the function with the
    * arguments from the command line.
    *
    * When a phase is executed, the file is loaded as JavaScript into the
@@ -848,25 +902,29 @@ var TPS = {
 
   /**
    * Executes a single test phase.
    *
    * This is called by RunTestPhase() after the environment is validated.
    */
   _executeTestPhase: function _executeTestPhase(file, phase, settings) {
     try {
+      this.config = JSON.parse(prefs.getCharPref('tps.config'));
       // parse the test file
       Services.scriptloader.loadSubScript(file, this);
       this._currentPhase = phase;
       if (this._currentPhase.startsWith("cleanup-")) {
         let profileToClean = Cc["@mozilla.org/toolkit/profile-service;1"]
                              .getService(Ci.nsIToolkitProfileService)
                              .selectedProfile.name;
         this.phases[this._currentPhase] = profileToClean;
         this.Phase(this._currentPhase, [[this.Cleanup]]);
+      } else {
+        // Don't bother doing this for cleanup phases.
+        this._tryLoadPingSchema(file);
       }
       let this_phase = this._phaselist[this._currentPhase];
 
       if (this_phase == undefined) {
         this.DumpError("invalid phase " + this._currentPhase);
         return;
       }
 
@@ -890,34 +948,16 @@ var TPS = {
           }
         }
       }
       Logger.logInfo("Starting phase " + this._currentPhase);
 
       Logger.logInfo("setting client.name to " + this.phases[this._currentPhase]);
       Weave.Svc.Prefs.set("client.name", this.phases[this._currentPhase]);
 
-      // If a custom server was specified, set it now
-      if (this.config["serverURL"]) {
-        Weave.Service.serverURL = this.config.serverURL;
-        prefs.setCharPref('tps.serverURL', this.config.serverURL);
-      }
-
-      // Store account details as prefs so they're accessible to the Mozmill
-      // framework.
-      if (this.fxaccounts_enabled) {
-        prefs.setCharPref('tps.account.username', this.config.fx_account.username);
-        prefs.setCharPref('tps.account.password', this.config.fx_account.password);
-      }
-      else {
-        prefs.setCharPref('tps.account.username', this.config.sync_account.username);
-        prefs.setCharPref('tps.account.password', this.config.sync_account.password);
-        prefs.setCharPref('tps.account.passphrase', this.config.sync_account.passphrase);
-      }
-
       this._interceptSyncTelemetry();
 
       // start processing the test actions
       this._currentAction = 0;
     }
     catch(e) {
       this.DumpError("_executeTestPhase failed", e);
       return;
@@ -937,25 +977,37 @@ var TPS = {
       } catch (e) {
         self.DumpError("Error when generating sync telemetry", e);
       }
     };
     SyncTelemetry.submit = record => {
       Logger.logInfo("Intercepted sync telemetry submission: " + JSON.stringify(record));
       this._syncsReportedViaTelemetry += record.syncs.length + (record.discarded || 0);
       if (record.discarded) {
-        Logger.AssertTrue(record.syncs.length == SyncTelemetry.maxPayloadCount,
-                          "Syncs discarded from ping before maximum payload count reached");
+        if (record.syncs.length != SyncTelemetry.maxPayloadCount) {
+          this.DumpError("Syncs discarded from ping before maximum payload count reached");
+        }
       }
       // If this is the shutdown ping, check and see that the telemetry saw all the syncs.
       if (record.why === "shutdown") {
         // If we happen to sync outside of tps manually causing it, its not an
         // error in the telemetry, so we only complain if we didn't see all of them.
-        Logger.AssertTrue(this._syncsReportedViaTelemetry >= this._syncCount,
-                          `Telemetry missed syncs: Saw ${this._syncsReportedViaTelemetry}, should have >= ${this._syncCount}.`);
+        if (this._syncsReportedViaTelemetry < this._syncCount) {
+          this.DumpError(`Telemetry missed syncs: Saw ${this._syncsReportedViaTelemetry}, should have >= ${this._syncCount}.`);
+        }
+      }
+      if (!record.syncs.length) {
+        // Note: we're overwriting submit, so this is called even for pings that
+        // may have no data (which wouldn't be submitted to telemetry and would
+        // fail validation).
+        return;
+      }
+      if (!this.pingValidator(record)) {
+        // Note that we already logged the record.
+        this.DumpError("Sync ping validation failed with errors: " + JSON.stringify(this.pingValidator.errors));
       }
     };
   },
 
   /**
    * Register a single phase with the test harness.
    *
    * This is called when loading individual test files.
--- a/testing/tps/tps/testrunner.py
+++ b/testing/tps/tps/testrunner.py
@@ -233,22 +233,17 @@ class TPSTestRunner(object):
         f = open(testpath, 'r')
         testcontent = f.read()
         f.close()
         try:
             test = json.loads(testcontent)
         except:
             test = json.loads(testcontent[testcontent.find('{'):testcontent.find('}') + 1])
 
-        testcontent += 'var config = %s;\n' % json.dumps(self.config, indent=2)
-        testcontent += 'var seconds_since_epoch = %d;\n' % int(time.time())
-
-        tmpfile = TempFile(prefix='tps_test_')
-        tmpfile.write(testcontent)
-        tmpfile.close()
+        self.preferences['tps.seconds_since_epoch'] = int(time.time())
 
         # generate the profiles defined in the test, and a list of test phases
         profiles = {}
         phaselist = []
         for phase in test:
             profilename = test[phase]
 
             # create the profile if necessary
@@ -256,17 +251,17 @@ class TPSTestRunner(object):
                 profiles[profilename] = Profile(preferences = self.preferences,
                                                 addons = self.extensions)
 
             # create the test phase
             phaselist.append(TPSTestPhase(
                 phase,
                 profiles[profilename],
                 testname,
-                tmpfile.filename,
+                testpath,
                 self.logfile,
                 self.env,
                 self.firefoxRunner,
                 self.log,
                 ignore_unused_engines=self.ignore_unused_engines))
 
         # sort the phase list by name
         phaselist = sorted(phaselist, key=lambda phase: phase.phase)
@@ -278,17 +273,17 @@ class TPSTestRunner(object):
             if phase.status != 'PASS':
                 failed = True
                 break;
 
         for profilename in profiles:
             cleanup_phase = TPSTestPhase(
                 'cleanup-' + profilename,
                 profiles[profilename], testname,
-                tmpfile.filename,
+                testpath,
                 self.logfile,
                 self.env,
                 self.firefoxRunner,
                 self.log)
 
             cleanup_phase.run()
             if cleanup_phase.status != 'PASS':
                 failed = True
@@ -369,16 +364,18 @@ class TPSTestRunner(object):
             self.preferences.update({'services.sync.username': "dummy"})
 
         if self.debug:
             self.preferences.update(self.debug_preferences)
 
         if 'preferences' in self.config:
             self.preferences.update(self.config['preferences'])
 
+        self.preferences['tps.config'] = json.dumps(self.config)
+
     def run_tests(self):
         # delete the logfile if it already exists
         if os.access(self.logfile, os.F_OK):
             os.remove(self.logfile)
 
         # Copy the system env variables, and update them for custom settings
         self.env = os.environ.copy()
         self.env.update(self.extra_env)