Bug 1002414 - Part 1: Add retry logic to PushServer user agent. r=standard8
authorPaul Kerr [:pkerr] <pkerr@mozilla.com>
Thu, 17 Jul 2014 15:28:38 -0700
changeset 216745 a561499de7a4a22429d3e82a0f4a259a4abce734
parent 216744 568b73c5bd31bfe3b597e67ae07feb4fa4c8d1f2
child 216746 89b195298ca341b4797682c4a3659ab60dc844da
push id515
push userraliiev@mozilla.com
push dateMon, 06 Oct 2014 12:51:51 +0000
treeherdermozilla-release@267c7a481bef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersstandard8
bugs1002414
milestone33.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1002414 - Part 1: Add retry logic to PushServer user agent. r=standard8
browser/app/profile/firefox.js
browser/components/loop/MozLoopPushHandler.jsm
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1520,16 +1520,18 @@ pref("loop.enabled", true);
 #else
 pref("loop.enabled", false);
 #endif
 
 pref("loop.server", "https://loop.services.mozilla.com");
 pref("loop.seenToS", "unseen");
 pref("loop.do_not_disturb", false);
 pref("loop.ringtone", "chrome://browser/content/loop/shared/sounds/Firefox-Long.ogg");
+pref("loop.retry_delay.start", 60000);
+pref("loop.retry_delay.limit", 300000);
 
 // serverURL to be assigned by services team
 pref("services.push.serverURL", "wss://push.services.mozilla.com/");
 
 // Default social providers
 pref("social.manifest.facebook", "{\"origin\":\"https://www.facebook.com\",\"name\":\"Facebook Share\",\"shareURL\":\"https://www.facebook.com/sharer/sharer.php?u=%{url}\",\"iconURL\":\"data:image/x-icon;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8%2F9hAAAAX0lEQVQ4jWP4%2F%2F8%2FAyUYTFhHzjgDxP9JxGeQDSBVMxgTbUBCxer%2Fr999%2BQ8DJBuArJksA9A10s8AXIBoA0B%2BR%2FY%2FjD%2BEwoBoA1yT5v3PbdmCE8MAshhID%2FUMoDgzUYIBj0Cgi7ar4coAAAAASUVORK5CYII%3D\",\"icon32URL\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAACXBIWXMAAAsTAAALEwEAmpwYAAADbklEQVRYCc1Xv08UQRj99tctexAuCEFjRE0kGBEtLDSGqIWNxkYKbTAxNlY2JhaGWltNtNFeKgsKKxITK43/gCYW+IsoRhA4D47bH7fn9+bcvdm5JR7sefolC3Ozu9978+bNN7PayUv3HN3umdY0Y6IWBtSJ0HSTarXqTOiuTep6Lj+tdxAcA8RAgSmwdd2aCDs0clldYALb/FvgYVhjmfliVA2XpjEgWo0Attn42Z6WH1RFor5ehwo9XQIUZMoVn4qlCoVMSo62EvD8Kh0b3U2Xz43R2PBO6mUCGDlAf65V6MadZzT/rUimoccc2kYA4BfPHqJb105RzjJigKhRq9kEJUBIjgYVuXeL7SAI6eD+Abp5dTwVHOmEHxT50d8WBYJqSOdPj5BjW8gZR8UNqFR2xagx/65XFYaMH+BGWwiYpi4UkBPPLxTp9v1Z+lHc4DWvCQXWmIy6EjITgKowVd5Jjv7N3Hd6y5esigoOwpkJIAmMpZpLJGdiaaC4F0UmAj6bD84GCEwmB/qxMmRilmnwb/mpjAocHh4UEoNAt5NLZB7oy9OJo0PxqkAtePdhiSqunyC1LQUwWMPQaOr6GRre258Ajn4cP7KHcEXhsxpXbj+lT19X2TMNGTLVAcjcalS8gDwsQ2UOMhH4k8FkcrEn5E5ub2sKohxLK2VR77Hl9RUcsrgeRIEiVOT6z+tDbIeLy+vk+kGTCbXxycet6xhl//3f6bJEkdHYhA+mLtDIvoH4ieev5+juoxdk5+pjhALYEdXIpEB5w+NlSKSzqVQ/+H7IO6BLtl3fngGMiqhGJgIwlM6qpyUGFjySdk8m0Zg0ubeD7X9OIDEFajltRQgUJaUKx69tdgaQa0FMADuahZPMFtcEwNPm2hA7ZI5sK4aoE2NvYI+o8hkCIe7CwTv68zS0q9Dk5vpbm/8FXxitSzmMFHpsGj0wyLUheTwD2Y9fVgh1Ae0EPUgD9241ZEnld+v5kgnVZ/8fE0brVh5BK+1oCqKKF72Dk7HwBsssB/pklU1dfChy3S659H5+uelgIb+8WRv1/uGTV9Sdb5wJFlfW6fPCalMhwhSU1j2xKwKbP838GcOwJja4TqO0bjdmXxYTy1EYjFdCWoCEYZhseH/GDL3yJPHnuW6YmT7P1SlIA4768Hke4vOcsX8BE346lLHhDUQAAAAASUVORK5CYII=\", \"icon64URL\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAAadEVYdFNvZnR3YXJlAFBhaW50Lk5FVCB2My41LjEwMPRyoQAACNNJREFUeNrtm3tw1NUVxz/399hHHkgCaCBGEFEEREVFYFQcSoOKdkZay4z+4dDpYIsjHWx1WoTMhFi1gzBSpVgVGbU4U1sHfPESKODwEEnRYDFAICEIeZIQshs2u/v73ds/drMsyW7YLEkl2Z6Z32yy+9v7u+fc7znne8+5KzgvAjDunzlv0M13PjDZ6c4cARj0WhEoaZ1tOn3yq9XLf/tNU0O1D5Ad7wq/OpxpaXOL1j5uZAwuaGlVgwNBhULRm0XXBG6HZrlNa9uRrzfM+3DlgjIgGMsA7rl/XDdHOnNf9vosTfVuvTsaQhdkZ4iykh2rHtqydvkxwI58BhjTfv7MmP55E9/1nLNdfU15ACkVvoAaMCRvRPa+re9+DgTaPjMAx+DrJv3M67Mz+6LybWLb4NfTHhxzx31DDhZvOtqGAgNwWbjGICV9XQJB0e/KobcOP1i8qTzaAEYgaDtNU/V5A9hSaUFLuQEt2gVQSgml+j4CUAIppYgK/m0GkCjZ9xGAUNAu0LUhgJRAAAIVzwBSqVRQH4hlAClRKZAFhOgEASoFECBR8QwgUyQGdJT/B8HzCEiBNKhUJzEgBYIgQsTJAkohe9oFZHgHKvQoHtZ9K3tewfiixXABLdoFeuSSEmkF+PH4QTz7+M3o+ENptzvGtS36uSwmjMpAYF10XCllHCYoe84FlLS555Zs5jx6J6ahY+iCl98pJiDNS1hwSZop+cm91zJmxEBefGsPlu1AxKC67V3gf5oGlZSMuz6Dp2fdhWnoAEwaN5T5hsYLb+4hKB1dcgelFDpB8ifk8thDt3DO5+fZxRvxBV0IjQR0EB3KfD1GhJS0GZnnYuGcKTgdF9ZWx4/No/BJjUUrdtJqm4iL+K5SCmSAiWMHMevhcQzNzaa6ron5SzfQ7HeiaSKx+au4m6HupcJKSYZdZVI4dypuV2yo3zoql0VP3cOiFV/Q4jdiGkGhQFqMGpbJL346kbE3DEYIQWNTC39Ysp4Gr4HQtZDyiRhA0NlmSHZbRM7pr1H0m6lckeHqdGXG3jCYoqfupeC17bT49fNRXIFSNrkDTGbNGM9dtw1D10M1DI/Xx3NLP6OqETRdDy1eglPT4rqA7K56gCIrXfHCvHwGZqUnBMtR113FS/N+xHPLtuJpDa1mVobg0emjmX7vqEjsUErhaw1Q8Mo6yk4F0A1HeOW7kIlFx/u7jworRabLpmhuPjmD+iG7YNDrrhnIS09P5cW/buOeO67lkftvJt3tDE06PE7Qsnh++QYOHPOim86wcVUS0+whJug0ghTMmcK1V2eH8m2UHP++nrwhAyIwjiXDcrN5vXAGhqGhFBeMYUvJ0re2sPfgGTTDGUZrEogVopMgmGQWUCgMEWTBE5MZPSLngnGqas/w9j92s31fJfmThvPM7HyMMKRj+qgmOiBHSsnr73/B5r1V6A53KD4k3bFS8dNgckFQoWPxu1kTGDc6N7JqzR4ff/+smE+2H8FSLkx3FluLawhaG3n2iXwcppGoV/Hemt18tK0c3UwLIfUS2nVafBeQJNUXkAF+/dht3H37cKSUBC2bTTv+w98++YazPg1dT0NoIUhruoMd+2sJrtjI7381rQM3iCVrN33N++tL0c30xFNdp0GQeFRYQhcRIO0gv5xxE/fdfSO2bfNVSTmrPtxLZW0A3XSh6VporaLG1XQHu0pOU/TaOhY8+QAuZ3w6vHnnQd74536EkZ50wOsSE0zcugolbWZOG8GM/LGUVdSw6sM97D/UgGa60QxXzMJDZAq6yb7SJgr//CkLn5pOmsvR4Z5dxUdZ9t6XoKfFjNyXkqZjuoBUCpGgCyhp8eDdQ5k++UaWrdrMlr2VSFxoZlpE8YtNWGgGXx9ppnDZpxTMfTCS8gAOlJ5g8ds7kCItTIi6j6FqMVxAC2sV2RB1ekmLCTcNpH+myeyFH7BxT1Voopoe4RKJXQqhmxw45mXh0o/xeH0opThcXs2iv2wmoFyhAnbC4yX+3PgISMDShrA5XHGa3d9UITRniIeTfHASmsGhEz7mL/mI2TPv4sU3t+KzHAnu7JKpCosOkcSIICAB5hZE0OiRCM0Iwb0b6LPQdI5W+Zn/yucoYYayRk+16eK1xqRMDAHtA0r3lep0lNAjO8kfpCpMqpTF4xZEUqA7rIlOCiKpgADVWXc4FQwgEfHPB5AiByTixIDUCYJx+wJoqdIcJV5VOAWygEZcF7BToT2upFDKtuz2BrAtf8v3mju972cBJX2exso6ok6N64BhOtM11xXXPBz6v6340PcuO+DZfaJkzWqgqY3L64Bqaaz0ZV45Mkc308dG2kd97FLSaq4v317gazr5HeCLRoACFTxbfeBw+oDhWYYj4/rw+30H+rb/VMPxXQsbKnbuABqJOi4vogyRiRB5/XNvvz3zytFTDEf61eF9b0dCKTS36c4afymTsgLeQ9Ly13X/aYnzE1Uy6PV7679trNy1xe+tKwPqAH/0Vla0qw65gH7AFeG/Y3Uy9P45o0bm3PTIaplM6lTK9jWf/OBUyQcrpdXaTIyfsXQb9QcLaAn7vJd2vxY5XxBpo8pwDmgFGsLKx1oeh8OVmUUSLXUlrWZPzbdLag9v+BjUqfDzepKAyDDcZbznGHG+1NmqSKHpVlfbadJqLW+o2LHobNX+PUB1WPkfnHwYyTmX6lI7Lehr3F576NM/+T3V3wH17f2w1xkg2ggXuSvga6p8p+bgmpVKWpXAmVh+2AsNEKogdYYAJa0GT03J4obyf60HTgKe6PTTqw0QOpcQ3wXs4LlDZyq2FXrrS4uBmjDxuCw3G5eIgA46yeC5ho11pWsWW35PWTibBC4Xf+9eBLRPg0q2+s5UvHG6bMNqJYPHw7nXutxZYvIIiMoCSgbrPVX/fv7syS+3AKfC5MOmF4iRpP6RjrId8O5vrNhS1NpUWQLUholUr6muXEoatP3emrWNR9e/avk9R8P+HuxNypPkrk93pGdnK0VtXemaN6UdOHo55vdE5b/0NKx+K4AxtAAAAABJRU5ErkJggg==\", \"description\":\"Easily share the web to your Facebook friends.\",\"author\":\"Facebook\",\"homepageURL\":\"https://www.facebook.com\",\"builtin\":\"true\",\"version\":1}");
 
 pref("social.sidebar.unload_timeout_ms", 10000);
--- a/browser/components/loop/MozLoopPushHandler.jsm
+++ b/browser/components/loop/MozLoopPushHandler.jsm
@@ -3,36 +3,56 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 "use strict";
 
 const { classes: Cc, interfaces: Ci, utils: Cu } = Components;
 
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Timer.jsm");
 
 this.EXPORTED_SYMBOLS = ["MozLoopPushHandler"];
 
 XPCOMUtils.defineLazyModuleGetter(this, "console",
                                   "resource://gre/modules/devtools/Console.jsm");
 
 /**
  * We don't have push notifications on desktop currently, so this is a
  * workaround to get them going for us.
- *
- * XXX Handle auto-reconnections if connection fails for whatever reason
- * (bug 1013248).
  */
 let MozLoopPushHandler = {
   // This is the uri of the push server.
   pushServerUri: Services.prefs.getCharPref("services.push.serverURL"),
   // This is the channel id we're using for notifications
   channelID: "8b1081ce-9b35-42b5-b8f5-3ff8cb813a50",
+  // This is the UserAgent UUID assigned by the PushServer
+  uaID: undefined,
   // Stores the push url if we're registered and we have one.
   pushUrl: undefined,
+  // Set to true once the channelID has been registered with the PushServer.
+  registered: false,
+
+  _minRetryDelay_ms: (() => {
+    try {
+      return Services.prefs.getIntPref("loop.retry_delay.start")
+    }
+    catch (e) {
+      return 60000 // 1 minute
+    }
+  })(),
+
+  _maxRetryDelay_ms: (() => {
+    try {
+      return Services.prefs.getIntPref("loop.retry_delay.limit")
+    }
+    catch (e) {
+      return 300000 // 5 minutes
+    }
+  })(),
 
    /**
     * Starts a connection to the push socket server. On
     * connection, it will automatically say hello and register the channel
     * id with the server.
     *
     * Register callback parameters:
     * - {String|null} err: Encountered error, if any
@@ -46,109 +66,196 @@ let MozLoopPushHandler = {
     *                     registered.
     * @param {Function} notificationCallback Callback to be called when a
     *                     push notification is received (may be called multiple
     *                     times).
     * @param {Object} mockPushHandler Optional, test-only object, to allow
     *                                 the websocket to be mocked for tests.
     */
   initialize: function(registerCallback, notificationCallback, mockPushHandler) {
-    if (Services.io.offline) {
-      registerCallback("offline");
-      return;
+    if (mockPushHandler) {
+      this._mockPushHandler = mockPushHandler;
     }
 
     this._registerCallback = registerCallback;
     this._notificationCallback = notificationCallback;
-
-    if (mockPushHandler) {
-      // For tests, use the mock instance.
-      this._websocket = mockPushHandler;
-    } else {
-      this._websocket = Cc["@mozilla.org/network/protocol;1?name=wss"]
-        .createInstance(Ci.nsIWebSocketChannel);
-    }
-    this._websocket.protocol = "push-notification";
-
-    let pushURI = Services.io.newURI(this.pushServerUri, null, null);
-    this._websocket.asyncOpen(pushURI, this.pushServerUri, this, null);
+    this._openSocket();
   },
 
   /**
    * Listener method, handles the start of the websocket stream.
    * Sends a hello message to the server.
    *
    * @param {nsISupports} aContext Not used
    */
   onStart: function() {
-    let helloMsg = { messageType: "hello", uaid: "", channelIDs: [] };
-    this._websocket.sendMsg(JSON.stringify(helloMsg));
+    this._retryEnd();
+    // If a uaID has already been assigned, assume this is a re-connect
+    // and send the uaID in order to re-synch with the
+    // PushServer. If a registration has been completed, send the channelID.
+    let helloMsg = { messageType: "hello",
+		     uaid: this.uaID,
+		     channelIDs: this.registered ? [this.channelID] :[] };
+    this._retryOperation(() => this.onStart(), this._maxRetryDelay_ms);
+    try { // in case websocket has closed before this handler is run
+      this._websocket.sendMsg(JSON.stringify(helloMsg));
+    }
+    catch (e) {console.warn("MozLoopPushHandler::onStart websocket.sendMsg() failure");}
   },
 
   /**
    * Listener method, called when the websocket is closed.
    *
    * @param {nsISupports} aContext Not used
    * @param {nsresult} aStatusCode Reason for stopping (NS_OK = successful)
    */
   onStop: function(aContext, aStatusCode) {
-    // XXX We really should be handling auto-reconnect here, this will be
-    // implemented in bug 994151. For now, just log a warning, so that a
-    // developer can find out it has happened and not get too confused.
     Cu.reportError("Loop Push server web socket closed! Code: " + aStatusCode);
-    this.pushUrl = undefined;
+    this._retryOperation(() => this._openSocket());
   },
 
   /**
    * Listener method, called when the websocket is closed by the server.
    * If there are errors, onStop may be called without ever calling this
    * method.
    *
    * @param {nsISupports} aContext Not used
    * @param {integer} aCode the websocket closing handshake close code
    * @param {String} aReason the websocket closing handshake close reason
    */
   onServerClose: function(aContext, aCode) {
-    // XXX We really should be handling auto-reconnect here, this will be
-    // implemented in bug 994151. For now, just log a warning, so that a
-    // developer can find out it has happened and not get too confused.
     Cu.reportError("Loop Push server web socket closed (server)! Code: " + aCode);
-    this.pushUrl = undefined;
+    this._retryOperation(() => this._openSocket());
   },
 
   /**
    * Listener method, called when the websocket receives a message.
    *
    * @param {nsISupports} aContext Not used
    * @param {String} aMsg The message data
    */
   onMessageAvailable: function(aContext, aMsg) {
     let msg = JSON.parse(aMsg);
 
     switch(msg.messageType) {
       case "hello":
-        this._registerChannel();
+        this._retryEnd();
+	if (this.uaID !== msg.uaid) {
+	  this.uaID = msg.uaid;
+          this._registerChannel();
+	}
         break;
+
       case "register":
-        this.pushUrl = msg.pushEndpoint;
-        this._registerCallback(null, this.pushUrl);
+        this._onRegister(msg);
         break;
+
       case "notification":
-        msg.updates.forEach(function(update) {
+        msg.updates.forEach((update) => {
           if (update.channelID === this.channelID) {
             this._notificationCallback(update.version);
           }
-        }.bind(this));
+        });
         break;
     }
   },
 
   /**
+   * Handles the PushServer registration response.
+   *
+   * @param {} msg PushServer to UserAgent registration response (parsed from JSON).
+   */
+  _onRegister: function(msg) {
+    switch (msg.status) {
+      case 200:
+        this._retryEnd(); // reset retry mechanism
+	this.registered = true;
+        if (this.pushUrl !== msg.pushEndpoint) {
+          this.pushUrl = msg.pushEndpoint;
+          this._registerCallback(null, this.pushUrl);
+        }
+        break;
+
+      case 500:
+        // retry the registration request after a suitable delay
+        this._retryOperation(() => this._registerChannel());
+        break;
+
+      case 409:
+        this._registerCallback("error: PushServer ChannelID already in use");
+	break;
+
+      default:
+        this._registerCallback("error: PushServer registration failure, status = " + msg.status);
+	break;
+    }
+  },
+
+  /**
+   * Attempts to open a websocket.
+   *
+   * A new websocket interface is used each time. If an onStop callback
+   * was received, calling asyncOpen() on the same interface will
+   * trigger a "alreay open socket" exception even though the channel
+   * is logically closed.
+   */
+  _openSocket: function() {
+    if (this._mockPushHandler) {
+      // For tests, use the mock instance.
+      this._websocket = this._mockPushHandler;
+    } else if (!Services.io.offline) {
+      this._websocket = Cc["@mozilla.org/network/protocol;1?name=wss"]
+                        .createInstance(Ci.nsIWebSocketChannel);
+    } else {
+      this._registerCallback("offline");
+      console.warn("MozLoopPushHandler - IO offline");
+      return;
+    }
+
+    this._websocket.protocol = "push-notification";
+    let uri = Services.io.newURI(this.pushServerUri, null, null);
+    this._websocket.asyncOpen(uri, this.pushServerUri, this, null);
+  },
+
+  /**
    * Handles registering a service
    */
   _registerChannel: function() {
-    this._websocket.sendMsg(JSON.stringify({
-      messageType: "register",
-      channelID: this.channelID
-    }));
+    this.registered = false;
+    try { // in case websocket has closed
+      this._websocket.sendMsg(JSON.stringify({messageType: "register",
+                                              channelID: this.channelID}));
+    }
+    catch (e) {console.warn("MozLoopPushHandler::_registerChannel websocket.sendMsg() failure");}
+  },
+
+  /**
+   * Method to handle retrying UserAgent to PushServer request following
+   * a retry back-off scheme managed by this function.
+   *
+   * @param {function} delayedOp Function to call after current delay is satisfied
+   *
+   * @param {number} [optional] retryDelay This parameter will be used as the initial delay
+   */
+  _retryOperation: function(delayedOp, retryDelay) {
+    if (!this._retryCount) {
+      this._retryDelay = retryDelay || this._minRetryDelay_ms;
+      this._retryCount = 1;
+    } else {
+      let nextDelay = this._retryDelay * 2;
+      this._retryDelay = nextDelay > this._maxRetryDelay_ms ? this._maxRetryDelay_ms : nextDelay;
+      this._retryCount += 1;
+    }
+    this._timeoutID = setTimeout(delayedOp, this._retryDelay);
+  },
+
+  /**
+   * Method used to reset the retry delay back-off logic.
+   *
+   */
+  _retryEnd: function() {
+    if (this._retryCount) {
+      clearTimeout(this._timeoutID);
+      this._retryCount = 0;
+    }
   }
 };