Bug 841101 - Add support for multi touch action chains in marionette, r=mdas
authorYiming Yang <yiyang@mozilla.com>
Mon, 18 Mar 2013 13:42:46 -0700
changeset 125282 284df6d78bc55ec816f7c162d858cb8361239c0f
parent 125281 75b300cd1c3dbb0017420606d5157c707ff7163b
child 125283 8da24aaaaa26e1243a130af6921e875cb8a12207
push id24851
push usermdas@mozilla.com
push dateMon, 18 Mar 2013 20:54:33 +0000
treeherdermozilla-inbound@284df6d78bc5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmdas
bugs841101
milestone22.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 841101 - Add support for multi touch action chains in marionette, r=mdas
testing/marionette/client/marionette/__init__.py
testing/marionette/client/marionette/marionette.py
testing/marionette/client/marionette/tests/unit/test_multi_finger.py
testing/marionette/client/marionette/tests/unit/unit-tests.ini
testing/marionette/client/marionette/www/testAction.html
testing/marionette/client/setup.py
testing/marionette/marionette-actors.js
testing/marionette/marionette-listener.js
--- a/testing/marionette/client/marionette/__init__.py
+++ b/testing/marionette/client/marionette/__init__.py
@@ -1,10 +1,10 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from marionette import Marionette, HTMLElement, Actions
+from marionette import Marionette, HTMLElement, Actions, MultiActions
 from marionette_test import MarionetteTestCase, CommonTestCase
 from marionette_touch import MarionetteTouchMixin
 from emulator import Emulator
 from runtests import MarionetteTestRunner
 
--- a/testing/marionette/client/marionette/marionette.py
+++ b/testing/marionette/client/marionette/marionette.py
@@ -129,16 +129,31 @@ class Actions(object):
 
     def wait(self, time=None):
         self.action_chain.append(['wait', time])
         return self
 
     def perform(self):
         return self.marionette._send_message('actionChain', 'ok', value=self.action_chain)
 
+class MultiActions(object):
+    def __init__(self, marionette):
+        self.multi_actions = []
+        self.max_length = 0
+        self.marionette = marionette
+
+    def add(self, action):
+        self.multi_actions.append(action.action_chain)
+        if len(action.action_chain) > self.max_length:
+          self.max_length = len(action.action_chain)
+        return self
+
+    def perform(self):
+        return self.marionette._send_message('multiAction', 'ok', value=self.multi_actions, max_length=self.max_length)
+
 class Marionette(object):
 
     CONTEXT_CHROME = 'chrome'
     CONTEXT_CONTENT = 'content'
     TIMEOUT_SEARCH = 'implicit'
     TIMEOUT_SCRIPT = 'script'
     TIMEOUT_PAGE = 'page load'
 
new file mode 100644
--- /dev/null
+++ b/testing/marionette/client/marionette/tests/unit/test_multi_finger.py
@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import time
+from marionette_test import MarionetteTestCase
+from marionette import MultiActions, Actions
+
+class testSingleFinger(MarionetteTestCase):
+    def test_move_element(self):
+      testTouch = self.marionette.absolute_url("testAction.html")
+      self.marionette.navigate(testTouch)
+      start = self.marionette.find_element("id", "mozLink")
+      drop = self.marionette.find_element("id", "mozLinkPos")
+      ele = self.marionette.find_element("id", "mozLinkCopy")
+      multi_action = MultiActions(self.marionette)
+      action1 = Actions(self.marionette)
+      action2 = Actions(self.marionette)
+      action1.press(start).move(drop).wait(3).release()
+      action2.press(ele).wait().release()
+      multi_action.add(action1).add(action2).perform()
+      time.sleep(15)
+      self.assertEqual("Move", self.marionette.execute_script("return document.getElementById('mozLink').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkPos').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkCopy').innerHTML;"))
+
+    def test_move_offset_element(self):
+      testTouch = self.marionette.absolute_url("testAction.html")
+      self.marionette.navigate(testTouch)
+      start = self.marionette.find_element("id", "mozLink")
+      ele = self.marionette.find_element("id", "mozLinkCopy")
+      multi_action = MultiActions(self.marionette)
+      action1 = Actions(self.marionette)
+      action2 = Actions(self.marionette)
+      action1.press(start).move_by_offset(0,300).wait().release()
+      action2.press(ele).wait(5).release()
+      multi_action.add(action1).add(action2).perform()
+      time.sleep(15)
+      self.assertEqual("Move", self.marionette.execute_script("return document.getElementById('mozLink').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkPos').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkCopy').innerHTML;"))
+
+    def test_three_fingers(self):
+      testTouch = self.marionette.absolute_url("testAction.html")
+      self.marionette.navigate(testTouch)
+      start_one = self.marionette.find_element("id", "mozLink")
+      start_two = self.marionette.find_element("id", "mozLinkStart")
+      drop_two = self.marionette.find_element("id", "mozLinkEnd")
+      ele = self.marionette.find_element("id", "mozLinkCopy2")
+      multi_action = MultiActions(self.marionette)
+      action1 = Actions(self.marionette)
+      action2 = Actions(self.marionette)
+      action3 = Actions(self.marionette)
+      action1.press(start_one).move_by_offset(0,300).release()
+      action2.press(ele).wait().wait(5).release()
+      action3.press(start_two).move(drop_two).wait(2).release()
+      multi_action.add(action1).add(action2).add(action3).perform()
+      time.sleep(15)
+      self.assertEqual("Move", self.marionette.execute_script("return document.getElementById('mozLink').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkPos').innerHTML;"))
+      self.assertTrue(self.marionette.execute_script("return document.getElementById('mozLinkCopy2').innerHTML >= 5000;"))
+      self.assertTrue(self.marionette.execute_script("return document.getElementById('mozLinkEnd').innerHTML >= 5000;"))
--- a/testing/marionette/client/marionette/tests/unit/unit-tests.ini
+++ b/testing/marionette/client/marionette/tests/unit/unit-tests.ini
@@ -51,16 +51,20 @@ browser = false
 [test_press_release.py]
 b2g = true
 browser = false
 
 [test_single_finger.py]
 b2g = true
 browser = false
 
+[test_multi_finger.py]
+b2g = true
+browser = false
+
 [test_simpletest_pass.js]
 [test_simpletest_sanity.py]
 [test_simpletest_chrome.js]
 [test_simpletest_timeout.js]
 [test_specialpowers.py]
 [test_switch_frame.py]
 b2g = false
 
--- a/testing/marionette/client/marionette/www/testAction.html
+++ b/testing/marionette/client/marionette/www/testAction.html
@@ -10,61 +10,110 @@
 </head>
 <body>
   <h1 id="testh1">Test Page</h1>
   <!-- "mozLink" and "mozLinkPos" work together to perform touchdown on mozLink, vertical move and then touchup on mozLinkPos-->
   <button id="mozLink" style="position:absolute;left:0px;top:55px;" type="button" allowevents=true>Button1</button>
   <button id="mozLinkPos" style="position:absolute;left:0px;top:355px;" type="button" allowevents=true>Button2</button>
   <!-- "mozLinkCopy" listens for a touchdown and touchup -->
   <button id="mozLinkCopy" style="position:absolute;left:0px;top:455px;" type="button" allowevents=true>Button3</button>
+  <!-- "mozLinkStart" and "mozLinkEnd" work together to perform touchdown on mozLinkStart, horizontal move and then touchup on mozLinkEnd -->  
+  <button id="mozLinkStart" style="position:absolute;left:10px;top:200px;" type="button" allowevents=true>Press</button>
+  <button id="mozLinkEnd" style="position:absolute;left:140px;top:200px;" type="button" allowevents=true>Release</button>
+  <!-- "mozLinkCopy2" listens for a touchdown and touchup. It shows the time when it's fired-->
+  <button id="mozLinkCopy2" style="position:absolute;left:80px;top:455px;" type="button" allowevents=true>Button4</button>
   <script type="text/javascript">
     window.ready = true;
     var press = document.getElementById("mozLink");
     var second = document.getElementById("mozLinkCopy");
+    var third = document.getElementById("mozLinkStart");
+    var fourth = document.getElementById("mozLinkCopy2");
     // touchmove and touchend must be performed on the same element as touchstart
     // here is press for vertical move
-    press.addEventListener("touchstart", changePressText, false);
+    press.addEventListener("touchstart", function(){changePressText("mozLink")}, false);
     press.addEventListener("touchmove", changeMoveText, false);
     press.addEventListener("touchend", changeReleaseText, false);
     // here is second for a tap
-    second.addEventListener("touchstart", changeCopyText, false);
+    second.addEventListener("touchstart", function(){changePressText("mozLinkCopy")}, false);
     second.addEventListener("touchend", changeClickText, false);
-    function changePressText() {
-      var press = document.getElementById("mozLink");
+    // here is third for horizontal move
+    third.addEventListener("touchstart", function(){changePressText("mozLinkStart")}, false);
+    third.addEventListener("touchmove", changeHorizontalMove, false);
+    third.addEventListener("touchend", changeHorizontalRelease, false);
+    // here is fourth for touch up and down with time shown
+    fourth.addEventListener("touchstart", changeTimePress, false);
+    fourth.addEventListener("touchend", changeTimeRelease, false);
+    function changePressText(strId) {
+      var press = document.getElementById(strId);
       press.innerHTML = "Start";
     }
 
     function changeMoveText() {
       var move = document.getElementById("mozLink");
       move.innerHTML = "Move";
     }
 
-    function changeReleaseText(event) {
+    function checkPosition(event, ele) {
       var touches = event.changedTouches;
       var clientX = touches[0].clientX;
       var clientY = touches[0].clientY;
-      var release = document.getElementById("mozLinkPos");
+      var release = document.getElementById(ele);
       var boxr = release.getBoundingClientRect();
-      if (clientY >= boxr.top &&
-          clientY <= boxr.bottom &&
-          clientX >= boxr.left &&
-          clientX <= boxr.right) {
-        release.innerHTML ="End";
+      return (clientY >= boxr.top &&
+              clientY <= boxr.bottom &&
+              clientX >= boxr.left &&
+              clientX <= boxr.right);
+    }
+
+    function changeReleaseText(event) {
+      if (checkPosition(event, "mozLinkPos")) {
+        document.getElementById("mozLinkPos").innerHTML = "End";
       }
     }
 
-    function changeCopyText() {
-      var second = document.getElementById("mozLinkCopy");
-      second.innerHTML = "Start";
+    function changeHorizontalMove() {
+      var press = document.getElementById("mozLinkStart");
+      if (press.innerHTML == "Start") {
+        var d = new Date();
+        press.innerHTML = d.getTime();
+      }
+    }
+
+    function changeHorizontalRelease(event) {
+      if (checkPosition(event, "mozLinkEnd")) {
+        var press = document.getElementById("mozLinkStart");
+        var d = new Date();
+        var timeDiff = d.getTime() - press.innerHTML;
+        document.getElementById("mozLinkEnd").innerHTML = timeDiff;
+        
+      }
     }
 
     function changeClickText() {
       var second = document.getElementById("mozLinkCopy");
       if (second.innerHTML == "Start") {
         second.innerHTML = "End";
       }
       else {
         second.innerHTML = "Error";
       }
     }
+
+    function changeTimePress() {
+      var fourth = document.getElementById("mozLinkCopy2");
+      var d = new Date();
+      fourth.innerHTML = d.getTime();
+    }
+
+    function changeTimeRelease() {
+      var fourth = document.getElementById("mozLinkCopy2");
+      if (fourth.innerHTML != "Button4") {
+        var d = new Date();
+        var timeDiff = d.getTime() - fourth.innerHTML;
+        fourth.innerHTML = timeDiff;
+      }
+      else {
+        fourth.innerHTML = "Error";
+      }
+    }
   </script>
 </body>
 </html>
--- a/testing/marionette/client/setup.py
+++ b/testing/marionette/client/setup.py
@@ -1,12 +1,12 @@
 import os
 from setuptools import setup, find_packages
 
-version = '0.5.20'
+version = '0.5.21'
 
 # get documentation from the README
 try:
     here = os.path.dirname(os.path.abspath(__file__))
     description = file(os.path.join(here, 'README.md')).read()
 except (OSError, IOError):
     description = ''
 
--- a/testing/marionette/marionette-actors.js
+++ b/testing/marionette/marionette-actors.js
@@ -1378,16 +1378,37 @@ MarionetteDriverActor.prototype = {
     }
     else {
       this.sendAsync("actionChain", {value: aRequest.value,
                                      command_id: this.command_id});
     }
   },
 
   /**
+   * multiAction
+   *
+   * @param object aRequest
+   *        'value' represents a nested array: inner array represents each event;
+   *        middle array represents collection of events for each finger
+   *        outer array represents all the fingers
+   */
+
+  multiAction: function MDA_multiAction(aRequest) {
+    this.command_id = this.getCommandId();
+    if (this.context == "chrome") {
+       this.sendError("Not in Chrome", 500, null, this.command_id);
+    }
+    else {
+      this.sendAsync("multiAction", {value: aRequest.value,
+                                     maxlen: aRequest.max_length,
+                                     command_id: this.command_id});
+   }
+ },
+
+  /**
    * Find an element using the indicated search strategy.
    *
    * @param object aRequest
    *        'using' member indicates which search method to use
    *        'value' member is the value the client is looking for
    */
   findElement: function MDA_findElement(aRequest) {
     let command_id = this.command_id = this.getCommandId();
@@ -2122,16 +2143,17 @@ MarionetteDriverActor.prototype.requestT
   "executeScript": MarionetteDriverActor.prototype.execute,
   "setScriptTimeout": MarionetteDriverActor.prototype.setScriptTimeout,
   "timeouts": MarionetteDriverActor.prototype.timeouts,
   "singleTap": MarionetteDriverActor.prototype.singleTap,
   "doubleTap": MarionetteDriverActor.prototype.doubleTap,
   "press": MarionetteDriverActor.prototype.press,
   "release": MarionetteDriverActor.prototype.release,
   "actionChain": MarionetteDriverActor.prototype.actionChain,
+  "multiAction": MarionetteDriverActor.prototype.multiAction,
   "executeAsyncScript": MarionetteDriverActor.prototype.executeWithCallback,
   "executeJSScript": MarionetteDriverActor.prototype.executeJSScript,
   "setSearchTimeout": MarionetteDriverActor.prototype.setSearchTimeout,
   "findElement": MarionetteDriverActor.prototype.findElement,
   "findElements": MarionetteDriverActor.prototype.findElements,
   "clickElement": MarionetteDriverActor.prototype.clickElement,
   "getElementAttribute": MarionetteDriverActor.prototype.getElementAttribute,
   "getElementText": MarionetteDriverActor.prototype.getElementText,
--- a/testing/marionette/marionette-listener.js
+++ b/testing/marionette/marionette-listener.js
@@ -59,16 +59,18 @@ let originalOnError;
 let checkTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
 // Send move events about this often
 let EVENT_INTERVAL = 30; // milliseconds
 // The current array of all pending touches
 let touches = [];
 // For assigning unique ids to all touches
 let nextTouchId = 1000;
 let touchIds = {};
+// last touch for each fingerId
+let multiLast = {};
 // last touch for single finger
 let lastTouch = null;
 /**
  * Called when listener is first started up. 
  * The listener sends its unique window ID and its current URI to the actor.
  * If the actor returns an ID, we start the listeners. Otherwise, nothing happens.
  */
 function registerSelf() {
@@ -104,16 +106,17 @@ function startListeners() {
   addMessageListenerId("Marionette:executeScript", executeScript);
   addMessageListenerId("Marionette:executeAsyncScript", executeAsyncScript);
   addMessageListenerId("Marionette:executeJSScript", executeJSScript);
   addMessageListenerId("Marionette:singleTap", singleTap);
   addMessageListenerId("Marionette:doubleTap", doubleTap);
   addMessageListenerId("Marionette:press", press);
   addMessageListenerId("Marionette:release", release);
   addMessageListenerId("Marionette:actionChain", actionChain);
+  addMessageListenerId("Marionette:multiAction", multiAction);
   addMessageListenerId("Marionette:setSearchTimeout", setSearchTimeout);
   addMessageListenerId("Marionette:goUrl", goUrl);
   addMessageListenerId("Marionette:getUrl", getUrl);
   addMessageListenerId("Marionette:getTitle", getTitle);
   addMessageListenerId("Marionette:getPageSource", getPageSource);
   addMessageListenerId("Marionette:goBack", goBack);
   addMessageListenerId("Marionette:goForward", goForward);
   addMessageListenerId("Marionette:refresh", refresh);
@@ -196,16 +199,17 @@ function deleteSession(msg) {
   removeMessageListenerId("Marionette:executeScript", executeScript);
   removeMessageListenerId("Marionette:executeAsyncScript", executeAsyncScript);
   removeMessageListenerId("Marionette:executeJSScript", executeJSScript);
   removeMessageListenerId("Marionette:singleTap", singleTap);
   removeMessageListenerId("Marionette:doubleTap", doubleTap);
   removeMessageListenerId("Marionette:press", press);
   removeMessageListenerId("Marionette:release", release);
   removeMessageListenerId("Marionette:actionChain", actionChain);
+  removeMessageListenerId("Marionette:multiAction", multiAction);
   removeMessageListenerId("Marionette:setSearchTimeout", setSearchTimeout);
   removeMessageListenerId("Marionette:goUrl", goUrl);
   removeMessageListenerId("Marionette:getTitle", getTitle);
   removeMessageListenerId("Marionette:getPageSource", getPageSource);
   removeMessageListenerId("Marionette:getUrl", getUrl);
   removeMessageListenerId("Marionette:goBack", goBack);
   removeMessageListenerId("Marionette:goForward", goForward);
   removeMessageListenerId("Marionette:refresh", refresh);
@@ -989,16 +993,195 @@ function actionChain(msg) {
     actions(commandArray, touchId, command_id);
   }
   catch (e) {
     sendError(e.message, e.code, e.stack, msg.json.command_id);
   }
 }
 
 /**
+ * Function to emit touch events which allow multi touch on the screen
+ * @param type represents the type of event, touch represents the current touch,touches are all pending touches
+ */
+function emitMultiEvents(type, touch, touches) {
+  let target = touch.target;
+  let doc = target.ownerDocument;
+  let win = doc.defaultView;
+  // touches that are in the same document
+  let documentTouches = doc.createTouchList(touches.filter(function(t) {
+    return t.target.ownerDocument === doc;
+  }));
+  // touches on the same target
+  let targetTouches = doc.createTouchList(touches.filter(function(t) {
+    return t.target === target;
+  }));
+  // Create changed touches
+  let changedTouches = doc.createTouchList(touch);
+  // Create the event object
+  let event = curWindow.document.createEvent('TouchEvent');
+  event.initTouchEvent(type,
+                       true,
+                       true,
+                       win,
+                       0,
+                       false, false, false, false,
+                       documentTouches,
+                       targetTouches,
+                       changedTouches);
+  target.dispatchEvent(event);
+}
+
+/**
+ * Function to dispatch one set of actions
+ * @param touches represents all pending touches, batchIndex represents the batch we are dispatching right now
+ */
+function setDispatch(batches, touches, command_id, batchIndex) {
+  if (typeof batchIndex === "undefined") {
+    batchIndex = 0;
+  }
+  // check if all the sets have been fired
+  if (batchIndex >= batches.length) {
+    multiLast = {};
+    sendOk(command_id);
+    return;
+  }
+  // a set of actions need to be done
+  let batch = batches[batchIndex];
+  // each action for some finger
+  let pack;
+  // the touch id for the finger (pack)
+  let touchId;
+  // command for the finger
+  let command;
+  // touch that will be created for the finger
+  let el;
+  let corx;
+  let cory;
+  let touch;
+  let lastTouch;
+  let touchIndex;
+  let waitTime = 0;
+  let maxTime = 0;
+  batchIndex++;
+  // loop through the batch
+  for (let i = 0; i < batch.length; i++) {
+    pack = batch[i];
+    touchId = pack[0];
+    command = pack[1];
+    switch (command) {
+      case 'press':
+        el = elementManager.getKnownElement(pack[2], curWindow);
+        // after this block, the element will be scrolled into view
+        if (!checkVisible(el, command_id)) {
+           sendError("Element is not currently visible and may not be manipulated", 11, null, command_id);
+           return;
+        }
+        corx = pack[3];
+        cory = pack[4];
+        touch = createATouch(el, corx, cory, touchId);
+        multiLast[touchId] = touch;
+        touches.push(touch);
+        emitMultiEvents('touchstart', touch, touches);
+        break;
+      case 'release':
+        touch = multiLast[touchId];
+        // the index of the previous touch for the finger may change in the touches array
+        touchIndex = touches.indexOf(touch);
+        touches.splice(touchIndex, 1);
+        emitMultiEvents('touchend', touch, touches);
+        break;
+      case 'move':
+        el = elementManager.getKnownElement(pack[2], curWindow);
+        lastTouch = multiLast[touchId];
+        let boxTarget = el.getBoundingClientRect();
+        let startTarget = lastTouch.target;
+        let boxStart = startTarget.getBoundingClientRect();
+        // note here corx and cory are relative to the target, not the viewport
+        // we always want to touch the center of the element if the element is specified
+        corx = boxTarget.left - boxStart.left + boxTarget.width * 0.5;
+        cory = boxTarget.top - boxStart.top + boxTarget.height * 0.5;
+        touch = createATouch(startTarget, corx, cory, touchId);
+        touchIndex = touches.indexOf(lastTouch);
+        touches[touchIndex] = touch;
+        multiLast[touchId] = touch;
+        emitMultiEvents('touchmove', touch, touches);
+        break;
+      case 'moveByOffset':
+        el = multiLast[touchId].target;
+        lastTouch = multiLast[touchId];
+        touchIndex = touches.indexOf(lastTouch);
+        let doc = el.ownerDocument;
+        let win = doc.defaultView;
+        // since x and y are relative to the last touch, therefore, it's relative to the position of the last touch
+        let clientX = lastTouch.clientX + pack[2],
+            clientY = lastTouch.clientY + pack[3];
+        let pageX = clientX + win.pageXOffset,
+            pageY = clientY + win.pageYOffset;
+        let screenX = clientX + win.mozInnerScreenX,
+            screenY = clientY + win.mozInnerScreenY;
+        touch = doc.createTouch(win, el, touchId, pageX, pageY, screenX, screenY, clientX, clientY);
+        touches[touchIndex] = touch;
+        multiLast[touchId] = touch;
+        emitMultiEvents('touchmove', touch, touches);
+        break;
+      case 'wait':
+        if (pack[2] != undefined ) {
+          waitTime = pack[2]*1000;
+          if (waitTime > maxTime) {
+            maxTime = waitTime;
+          }
+        }
+        break;
+    }//end of switch block
+  }//end of for loop
+  if (maxTime != 0) {
+    checkTimer.initWithCallback(function(){setDispatch(batches, touches, command_id, batchIndex);}, maxTime, Ci.nsITimer.TYPE_ONE_SHOT);
+  }
+  else {
+    setDispatch(batches, touches, command_id, batchIndex);
+  }
+}
+
+/**
+ * Function to start multi-action
+ */
+function multiAction(msg) {
+  let command_id = msg.json.command_id;
+  let args = msg.json.value;
+  // maxlen is the longest action chain for one finger
+  let maxlen = msg.json.maxlen;
+  try {
+    // unwrap the original nested array
+    let commandArray = elementManager.convertWrappedArguments(args, curWindow);
+    let concurrentEvent = [];
+    let temp;
+    for (let i = 0; i < maxlen; i++) {
+      let row = [];
+      for (let j = 0; j < commandArray.length; j++) {
+        if (commandArray[j][i] != undefined) {
+          // add finger id to the front of each action, i.e. [finger_id, action, element]
+          temp = commandArray[j][i];
+          temp.unshift(j);
+          row.push(temp);
+        }
+      }
+      concurrentEvent.push(row);
+    }
+    // now concurrent event is made of sets where each set contain a list of actions that need to be fired.
+    // note: each action belongs to a different finger
+    // pendingTouches keeps track of current touches that's on the screen
+    let pendingTouches = [];
+    setDispatch(concurrentEvent, pendingTouches, command_id);
+  }
+  catch (e) {
+    sendError(e.message, e.code, e.stack, msg.json.command_id);
+  }
+}
+
+/**
  * Function to set the timeout period for element searching 
  */
 function setSearchTimeout(msg) {
   try {
     elementManager.setSearchTimeout(msg.json.value);
   }
   catch (e) {
     sendError(e.message, e.code, e.stack, msg.json.command_id);