Bug 984508 - Marionette should monitor listeners to ensure they're still alive; r=mdas

testing/marionette/client/marionette/errors.py

@@ -31,6 +31,7 @@ class ErrorCodes(object):
     INVALID_RESPONSE = 53
     FRAME_SEND_NOT_INITIALIZED_ERROR = 54
     FRAME_SEND_FAILURE_ERROR = 55
+    FRAME_NOT_RESPONDING = 56
     UNSUPPORTED_OPERATION = 405
     MARIONETTE_ERROR = 500
 

testing/marionette/client/marionette/marionette.py

@@ -1584,3 +1584,13 @@ class Marionette(object):
         """
 
         return self._send_message("maximizeWindow", "ok")
+
+    def set_frame_timeout(self, timeout):
+        """ Set the OOP frame timeout value in ms. When focus is on a
+        remote frame, if the heartbeat pong is not received within this
+        specified value, the frame will timeout.
+
+        :param timeout: The frame timeout value in ms.
+        """
+
+        return self._send_message("setFrameTimeout", "ok", ms=timeout)

testing/marionette/client/marionette/tests/unit/test_set_frame_timeout.py

@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from errors import MarionetteException
+from marionette_test import MarionetteTestCase
+
+class TestSetFrameTimeout(MarionetteTestCase):
+
+    def test_set_valid_frame_timeout(self):
+        self.marionette.set_frame_timeout(10000)
+
+    def test_set_invalid_frame_timeout(self):
+        with self.assertRaisesRegexp(MarionetteException, "Not a number"):
+            self.marionette.set_frame_timeout("timeout")

testing/marionette/client/marionette/tests/unit/unit-tests.ini

@@ -132,5 +132,6 @@ browser = false
 b2g = false
 [test_set_window_size.py]
 b2g = false
+[test_set_frame_timeout.py]
 skip-if = os == "linux" # Bug 1085717
 [test_with_using_context.py]

testing/marionette/marionette-frame-manager.js

@@ -108,6 +108,16 @@ FrameManager.prototype = {
     let oopFrame = frameWindow.document.getElementsByTagName("iframe")[message.json.frame]; //find the OOP frame
     let mm = oopFrame.QueryInterface(Ci.nsIFrameLoaderOwner).frameLoader.messageManager; //get the OOP frame's mm
 
+    // Grab the app name
+    let appName = null;
+    try {
+      appName = oopFrame.getAttribute("mozapp");
+    }
+    catch(e) {
+      appName = "mozapp name unavailable";
+      logger.info("Error getting mozapp: " + e.result)
+    }
+
     // See if this frame already has our frame script loaded in it; if so,
     // just wake it up.
     for (let i = 0; i < remoteFrames.length; i++) {
@@ -133,7 +143,7 @@ FrameManager.prototype = {
         }
 
         mm.sendAsyncMessage("Marionette:restart", {});
-        return oopFrame.id;
+        return [oopFrame.id, appName];
       }
     }
 
@@ -150,7 +160,7 @@ FrameManager.prototype = {
 
     aFrame.specialPowersObserver = new specialpowers.SpecialPowersObserver();
     aFrame.specialPowersObserver.init(mm);
-    return oopFrame.id;
+    return [oopFrame.id, appName];
   },
 
   /*
@@ -166,6 +176,22 @@ FrameManager.prototype = {
     this.handledModal = false;
   },
 
+  /*
+   *  Remove specified frame from the remote frames list
+   */
+  removeRemoteFrame: function FM_removeRemoteFrame(frameId) {
+    logger.info("Deleting frame from remote frames list: " + frameId);
+    startLen = remoteFrames.length;
+    for (let i = 0; i < remoteFrames.length; i++) {
+      if (remoteFrames[i].frameId == frameId) {
+       remoteFrames.splice(i, 1);
+      }
+    }
+    if (remoteFrames.length == startLen) {
+     logger.info("Frame not found in remote frames list");
+    }
+  },
+
   /**
    * This function removes any SpecialPowersObservers from OOP frames.
    */
@@ -205,9 +231,11 @@ FrameManager.prototype = {
     messageManager.addWeakMessageListener("Marionette:addCookie", this.server);
     messageManager.addWeakMessageListener("Marionette:getVisibleCookies", this.server);
     messageManager.addWeakMessageListener("Marionette:deleteCookie", this.server);
+    messageManager.addWeakMessageListener("Marionette:pong", this.server);
     messageManager.addWeakMessageListener("MarionetteFrame:handleModal", this);
     messageManager.addWeakMessageListener("MarionetteFrame:getCurrentFrameId", this);
     messageManager.addWeakMessageListener("MarionetteFrame:getInterruptedState", this);
+    messageManager.addWeakMessageListener("Marionette:startHeartbeat", this.server);
   },
 
   /**
@@ -236,8 +264,10 @@ FrameManager.prototype = {
     messageManager.removeWeakMessageListener("Marionette:addCookie", this.server);
     messageManager.removeWeakMessageListener("Marionette:getVisibleCookies", this.server);
     messageManager.removeWeakMessageListener("Marionette:deleteCookie", this.server);
+    messageManager.removeWeakMessageListener("Marionette:pong", this.server);
     messageManager.removeWeakMessageListener("MarionetteFrame:handleModal", this);
     messageManager.removeWeakMessageListener("MarionetteFrame:getCurrentFrameId", this);
+    messageManager.removeWeakMessageListener("Marionette:startHeartbeat", this.server);
   },
 
 };

testing/marionette/marionette-listener.js

@@ -188,6 +188,7 @@ function startListeners() {
   addMessageListenerId("Marionette:getCookies", getCookies);
   addMessageListenerId("Marionette:deleteAllCookies", deleteAllCookies);
   addMessageListenerId("Marionette:deleteCookie", deleteCookie);
+  addMessageListenerId("Marionette:ping", ping);
 }
 
 /**
@@ -290,6 +291,7 @@ function deleteSession(msg) {
   removeMessageListenerId("Marionette:getCookies", getCookies);
   removeMessageListenerId("Marionette:deleteAllCookies", deleteAllCookies);
   removeMessageListenerId("Marionette:deleteCookie", deleteCookie);
+  removeMessageListenerId("Marionette:ping", ping);
   if (isB2G) {
     content.removeEventListener("mozbrowsershowmodalprompt", modalHandler, false);
   }
@@ -1287,6 +1289,8 @@ function get(msg) {
       if (curFrame.document.readyState == "complete") {
         removeEventListener("DOMContentLoaded", onDOMContentLoaded, false);
         sendOk(command_id);
+        // Restart the OOP frame heartbeat now that the URL is loaded
+        sendToServer("Marionette:startHeartbeat");
       }
       else if (curFrame.document.readyState == "interactive" &&
                errorRegex.exec(curFrame.document.baseURI)) {
@@ -1918,6 +1922,13 @@ function getAppCacheStatus(msg) {
                msg.json.command_id);
 }
 
+/**
+ * Received heartbeat ping
+ */
+function ping(msg) {
+  sendToServer("Marionette:pong", {}, msg.json.command_id);
+}
+
 // emulator callbacks
 let _emu_cb_id = 0;
 let _emu_cbs = {};

testing/marionette/marionette-server.js

@@ -98,18 +98,14 @@ function FrameSendNotInitializedError(frame) {
   this.code = 54;
   this.frame = frame;
   this.message = "Error sending message to frame (NS_ERROR_NOT_INITIALIZED)";
-  this.toString = function() {
-    return this.message + " " + this.frame + "; frame has closed.";
-  }
+  this.errMsg = this.message + " " + this.frame + "; frame has closed.";
 }
 
 function FrameSendFailureError(frame) {
   this.code = 55;
   this.frame = frame;
   this.message = "Error sending message to frame (NS_ERROR_FAILURE)";
-  this.toString = function() {
-    return this.message + " " + this.frame + "; frame not responding.";
-  }
+  this.errMsg = this.message + " " + this.frame + "; frame not responding.";
 }
 
 /**
@@ -155,6 +151,11 @@ function MarionetteServerConnection(aPrefix, aTransport, aServer)
   this.currentFrameElement = null;
   this.testName = null;
   this.mozBrowserClose = null;
+  this.frameHeartbeatTimer = null;
+  this.frameHeartbeatLastPong = null;
+  this.frameHeartbeatLastApp = null;
+  this.frameHeartbeatExceptionPending = false;
+  this.frameTimeout = 5000; // default, set with setFrameTimeout
   this.oopFrameId = null; // frame ID of current remote frame, used for mozbrowserclose events
   this.sessionCapabilities = {
     // Mandated capabilities
@@ -243,11 +244,21 @@ MarionetteServerConnection.prototype = {
    * @param object values
    *        Object to send to the listener
    */
-  sendAsync: function MDA_sendAsync(name, values, commandId, ignoreFailure) {
+  sendAsync: function MDA_sendAsync(name, values, commandId, ignoreFailure, throwError) {
     let success = true;
     if (commandId) {
       values.command_id = commandId;
     }
+    if (typeof(throwError) !== "boolean") {
+      throwError = false;
+    }
+    if (this.frameHeartbeatExceptionPending) {
+      // Previous frame was not responding; send exception indicating have switched to system frame
+      this.frameHeartbeatExceptionPending = false;
+      let errorTxt = "Frame not responding (" + this.frameHeartbeatLastApp + "), switching to root frame";
+      this.sendError(errorTxt, 56, null, this.command_id);
+      return false;
+    }
     if (this.curBrowser.frameManager.currentRemoteFrame !== null) {
       try {
         this.messageManager.sendAsyncMessage(
@@ -268,7 +279,12 @@ MarionetteServerConnection.prototype = {
               break;
           }
           let code = error.hasOwnProperty('code') ? e.code : 500;
-          this.sendError(error.toString(), code, error.stack, commandId);
+          if (throwError == false) {
+            this.sendError(error.toString(), code, error.stack, commandId);
+          }
+          else {
+            throw {message:"sendAsync failed: " + error.hasOwnProperty('type'), code, stack:null};
+          }
         }
       }
     }
@@ -724,6 +740,10 @@ MarionetteServerConnection.prototype = {
     else {
       this.context = context;
       this.sendOk(this.command_id);
+      // Stop the OOP frame heartbeat if switched into chrome
+      if (context == "chrome") {
+        this.stopHeartbeat();
+      }
     }
   },
 
@@ -1179,6 +1199,8 @@ MarionetteServerConnection.prototype = {
     if (this.context != "chrome") {
       aRequest.command_id = command_id;
       aRequest.parameters.pageTimeout = this.pageTimeout;
+      // stop OOP frame heartbeat if it's running, so it won't timeout during URL load
+      this.stopHeartbeat();
       this.sendAsync("get", aRequest.parameters, command_id);
       return;
     }
@@ -1471,7 +1493,6 @@ MarionetteServerConnection.prototype = {
         this.sendError("Error loading page", 13, null, command_id);
         return;
       }
-
       checkTimer.initWithCallback(checkLoad.bind(this), 100, Ci.nsITimer.TYPE_ONE_SHOT);
     }
     if (this.context == "chrome") {
@@ -2327,6 +2348,7 @@ MarionetteServerConnection.prototype = {
    */
   deleteSession: function MDA_deleteSession() {
     let command_id = this.command_id = this.getCommandId();
+    this.stopHeartbeat();
     try {
       this.sessionTearDown();
     }
@@ -2641,6 +2663,22 @@ MarionetteServerConnection.prototype = {
     this.sendOk(this.command_id);
   },
 
+  /**
+   * Sets the OOP frame timeout value (ms)
+   */
+  setFrameTimeout: function MDA_setFrameTimeout(aRequest) {
+    this.command_id = this.getCommandId();
+    let timeout = parseInt(aRequest.parameters.ms);
+    if (isNaN(timeout)) {
+      this.sendError("Not a number", 500, null, this.command_id);
+      return;
+    }
+    else {
+      this.frameTimeout = timeout;
+    }
+    this.sendOk(this.command_id);
+  },
+
   /**
    * Helper function to convert an outerWindowID into a UID that Marionette
    * tracks.
@@ -2650,6 +2688,54 @@ MarionetteServerConnection.prototype = {
     return uid;
   },
 
+  /**
+   * Start the OOP frame heartbeat
+   */
+  startHeartbeat: function MDA_startHeartbeat() {
+    this.frameHeartbeatLastPong = new Date().getTime();
+    function pulse() {
+      let noResponse = false;
+      let now = new Date().getTime();
+      let elapsed = now - this.frameHeartbeatLastPong;
+      try {
+        if (elapsed > this.frameTimeout) {
+          throw {message:null, code:56, stack:null};
+        }
+        let result = this.sendAsync("ping", {}, this.command_id, false, true);
+        if (result == false) {
+          throw {message:null, code:56, stack:null};
+        }
+      }
+      catch (e) {
+        let lastApp = this.frameHeartbeatLastApp ? this.frameHeartbeatLastApp : "undefined";
+        this.stopHeartbeat();
+        this.curBrowser.frameManager.removeRemoteFrame(this.curBrowser.frameManager.currentRemoteFrame.frameId);
+        this.switchToGlobalMessageManager();
+        // If there is an active request, send back an exception now, otherwise wait until next request
+        if (this.command_id) {
+          let errorTxt = "Frame not responding (" + lastApp + "), switching to root frame";
+          this.sendError(errorTxt, e.code, e.stack, this.command_id);
+        }
+        else {
+          this.frameHeartbeatExceptionPending = true;
+        }
+        return;
+      }
+    }
+    this.frameHeartbeatTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
+    this.frameHeartbeatTimer.initWithCallback(pulse.bind(this), 500, Ci.nsITimer.TYPE_REPEATING_PRECISE_CAN_SKIP);
+  },
+
+  /**
+   * Stop the OOP frame heartbeat
+   */
+  stopHeartbeat: function MDA_stopHeartbeat() {
+    if (this.frameHeartbeatTimer !== null) {
+      this.frameHeartbeatTimer.cancel();
+      this.frameHeartbeatTimer = null;
+    }
+  },
+
   /**
    * Receives all messages from content messageManager
    */
@@ -2686,7 +2772,7 @@ MarionetteServerConnection.prototype = {
         this.sendToClient(message.json, -1);
         break;
       case "Marionette:switchToFrame":
-        this.oopFrameId = this.curBrowser.frameManager.switchToFrame(message);
+        [this.oopFrameId, this.frameHeartbeatLastApp] = this.curBrowser.frameManager.switchToFrame(message);
         this.messageManager = this.curBrowser.frameManager.currentRemoteFrame.messageManager.get();
         break;
       case "Marionette:switchToModalOrigin":
@@ -2707,6 +2793,7 @@ MarionetteServerConnection.prototype = {
           }
           this.currentFrameElement = message.json.frameValue;
         }
+        this.stopHeartbeat();
         break;
       case "Marionette:getVisibleCookies":
         let [currentPath, host] = message.json.value;
@@ -2770,6 +2857,7 @@ MarionetteServerConnection.prototype = {
           // is from a remote frame.
           this.curBrowser.frameManager.currentRemoteFrame.targetFrameId = this.generateFrameId(message.json.value);
           this.sendOk(this.command_id);
+          this.startHeartbeat();
         }
 
         let browserType;
@@ -2820,6 +2908,12 @@ MarionetteServerConnection.prototype = {
         globalMessageManager.broadcastAsyncMessage(
           "MarionetteMainListener:emitTouchEvent", message.json);
         return;
+      case "Marionette:pong":
+        this.frameHeartbeatLastPong = new Date().getTime();
+        break;
+      case "Marionette:startHeartbeat":
+        this.startHeartbeat();
+        break;
     }
   }
 };
@@ -2903,7 +2997,8 @@ MarionetteServerConnection.prototype.requestTypes = {
   "setScreenOrientation": MarionetteServerConnection.prototype.setScreenOrientation,
   "getWindowSize": MarionetteServerConnection.prototype.getWindowSize,
   "setWindowSize": MarionetteServerConnection.prototype.setWindowSize,
-  "maximizeWindow": MarionetteServerConnection.prototype.maximizeWindow
+  "maximizeWindow": MarionetteServerConnection.prototype.maximizeWindow,
+  "setFrameTimeout": MarionetteServerConnection.prototype.setFrameTimeout
 };
 
 /**