Bug 766779 - Introduce Explore by touch
authorEitan Isaacson <eitan@monotonous.org>
Mon, 25 Jun 2012 10:34:52 -0700
changeset 97583 b4fff5e5afb3051970f811a01bff24f0d574634d
parent 97582 678d9ad1e2447259167ef48ecbb4914808e2b861
child 97584 ac2bddd99d8f5c98307bcaba964cf2939c42f22d
push id11116
push usereisaacson@mozilla.com
push dateMon, 25 Jun 2012 17:35:03 +0000
treeherdermozilla-inbound@b4fff5e5afb3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs766779
milestone16.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 766779 - Introduce Explore by touch
accessible/src/jsat/AccessFu.jsm
accessible/src/jsat/Presenters.jsm
accessible/src/jsat/Utils.jsm
accessible/src/jsat/VirtualCursorController.jsm
--- a/accessible/src/jsat/AccessFu.jsm
+++ b/accessible/src/jsat/AccessFu.jsm
@@ -227,18 +227,19 @@ var AccessFu = {
             QueryInterface(Ci.nsIAccessibleCursorable).virtualCursor;
           let event = aEvent.
             QueryInterface(Ci.nsIAccessibleVirtualCursorChangeEvent);
           let position = pivot.position;
           let doc = aEvent.DOMNode;
 
           let presenterContext =
             new PresenterContext(position, event.oldAccessible);
+          let reason = event.reason;
           this.presenters.forEach(
-            function(p) { p.pivotChanged(presenterContext); });
+            function(p) { p.pivotChanged(presenterContext, reason); });
 
           break;
         }
       case Ci.nsIAccessibleEvent.EVENT_STATE_CHANGE:
         {
           let event = aEvent.QueryInterface(Ci.nsIAccessibleStateChangeEvent);
           if (event.state == Ci.nsIAccessibleStates.STATE_CHECKED &&
               !(event.isExtraState())) {
--- a/accessible/src/jsat/Presenters.jsm
+++ b/accessible/src/jsat/Presenters.jsm
@@ -34,18 +34,20 @@ Presenter.prototype = {
    * Detach function.
    */
   detach: function detach() {},
 
   /**
    * The virtual cursor's position changed.
    * @param {PresenterContext} aContext the context object for the new pivot
    *   position.
+   * @param {int} aReason the reason for the pivot change.
+   *   See nsIAccessiblePivot.
    */
-  pivotChanged: function pivotChanged(aContext) {},
+  pivotChanged: function pivotChanged(aContext, aReason) {},
 
   /**
    * An object's action has been invoked.
    * @param {nsIAccessible} aObject the object that has been invoked.
    * @param {string} aActionName the name of the action.
    */
   actionInvoked: function actionInvoked(aObject, aActionName) {},
 
@@ -135,17 +137,17 @@ VisualPresenter.prototype = {
     this.highlightBox = this.stylesheet = null;
   },
 
   viewportChanged: function VisualPresenter_viewportChanged() {
     if (this._currentObject)
       this._highlight(this._currentObject);
   },
 
-  pivotChanged: function VisualPresenter_pivotChanged(aContext) {
+  pivotChanged: function VisualPresenter_pivotChanged(aContext, aReason) {
     this._currentObject = aContext.accessible;
 
     if (!aContext.accessible) {
       this._hide();
       return;
     }
 
     try {
@@ -154,17 +156,17 @@ VisualPresenter.prototype = {
       this._highlight(aContext.accessible);
     } catch (e) {
       Logger.error('Failed to get bounds: ' + e);
       return;
     }
   },
 
   tabSelected: function VisualPresenter_tabSelected(aDocContext, aVCContext) {
-    this.pivotChanged(aVCContext);
+    this.pivotChanged(aVCContext, Ci.nsIAccessiblePivot.REASON_NONE);
   },
 
   tabStateChanged: function VisualPresenter_tabStateChanged(aDocObj,
                                                             aPageState) {
     if (aPageState == 'newdoc')
       this._hide();
   },
 
@@ -224,46 +226,79 @@ AndroidPresenter.prototype = {
 
   // Android AccessibilityEvent type constants.
   ANDROID_VIEW_CLICKED: 0x01,
   ANDROID_VIEW_LONG_CLICKED: 0x02,
   ANDROID_VIEW_SELECTED: 0x04,
   ANDROID_VIEW_FOCUSED: 0x08,
   ANDROID_VIEW_TEXT_CHANGED: 0x10,
   ANDROID_WINDOW_STATE_CHANGED: 0x20,
+  ANDROID_VIEW_HOVER_ENTER: 0x80,
+  ANDROID_VIEW_HOVER_EXIT: 0x100,
   ANDROID_VIEW_SCROLLED: 0x1000,
 
   attach: function AndroidPresenter_attach(aWindow) {
     this.chromeWin = aWindow;
   },
 
-  pivotChanged: function AndroidPresenter_pivotChanged(aContext) {
+  pivotChanged: function AndroidPresenter_pivotChanged(aContext, aReason) {
     if (!aContext.accessible)
       return;
 
+    let isExploreByTouch = (aReason == Ci.nsIAccessiblePivot.REASON_POINT &&
+                            Utils.AndroidSdkVersion >= 14);
+
+    if (isExploreByTouch) {
+      // This isn't really used by TalkBack so this is a half-hearted attempt
+      // for now.
+      this.sendMessageToJava({
+         gecko: {
+           type: 'Accessibility:Event',
+           eventType: this.ANDROID_VIEW_HOVER_EXIT,
+           text: []
+         }
+      });
+    }
+
     let output = [];
-    aContext.newAncestry.forEach(
-      function(acc) {
-        output.push.apply(output, UtteranceGenerator.genForObject(acc));
+
+    if (isExploreByTouch) {
+      // Just provide the parent for some context, no need to utter the entire
+      // ancestry change since it doesn't make sense in spatial navigation.
+      for (var i = aContext.newAncestry.length - 1; i >= 0; i--) {
+        let utter = UtteranceGenerator.genForObject(aContext.newAncestry[i]);
+        if (utter.length) {
+          output.push.apply(output, utter);
+          break;
+        }
       }
-    );
+    } else {
+      // Utter the entire context change in linear navigation.
+      aContext.newAncestry.forEach(
+        function(acc) {
+          output.push.apply(output, UtteranceGenerator.genForObject(acc));
+        }
+      );
+    }
 
     output.push.apply(output,
                       UtteranceGenerator.genForObject(aContext.accessible));
 
     aContext.subtreePreorder.forEach(
       function(acc) {
         output.push.apply(output, UtteranceGenerator.genForObject(acc));
       }
     );
 
     this.sendMessageToJava({
       gecko: {
         type: 'Accessibility:Event',
-        eventType: this.ANDROID_VIEW_FOCUSED,
+        eventType: isExploreByTouch ?
+          this.ANDROID_VIEW_HOVER_ENTER :
+          this.ANDROID_VIEW_FOCUSED,
         text: output
       }
     });
   },
 
   actionInvoked: function AndroidPresenter_actionInvoked(aObject, aActionName) {
     this.sendMessageToJava({
       gecko: {
@@ -271,17 +306,17 @@ AndroidPresenter.prototype = {
         eventType: this.ANDROID_VIEW_CLICKED,
         text: UtteranceGenerator.genForAction(aObject, aActionName)
       }
     });
   },
 
   tabSelected: function AndroidPresenter_tabSelected(aDocContext, aVCContext) {
     // Send a pivot change message with the full context utterance for this doc.
-    this.pivotChanged(aVCContext);
+    this.pivotChanged(aVCContext, Ci.nsIAccessiblePivot.REASON_NONE);
   },
 
   tabStateChanged: function AndroidPresenter_tabStateChanged(aDocObj,
                                                              aPageState) {
     let stateUtterance = UtteranceGenerator.
       genForTabStateChange(aDocObj, aPageState);
 
     if (!stateUtterance.length)
--- a/accessible/src/jsat/Utils.jsm
+++ b/accessible/src/jsat/Utils.jsm
@@ -43,16 +43,20 @@ var Utils = {
     switch (this.OS) {
       case 'Android':
         return aWindow.BrowserApp;
       default:
         return aWindow.gBrowser;
     }
   },
 
+  getCurrentContentDoc: function getCurrentContentDoc(aWindow) {
+    return this.getBrowserApp(aWindow).selectedBrowser.contentDocument;
+  },
+
   getViewport: function getViewport(aWindow) {
     switch (this.OS) {
       case 'Android':
         return aWindow.BrowserApp.selectedTab.getViewport();
       default:
         return null;
     }
   }
--- a/accessible/src/jsat/VirtualCursorController.jsm
+++ b/accessible/src/jsat/VirtualCursorController.jsm
@@ -403,30 +403,64 @@ var TraversalRules = {
   }
 };
 
 var VirtualCursorController = {
   NOT_EDITABLE: 0,
   SINGLE_LINE_EDITABLE: 1,
   MULTI_LINE_EDITABLE: 2,
 
-  explorebytouch: false,
+  exploreByTouch: false,
 
   attach: function attach(aWindow) {
     this.chromeWin = aWindow;
     this.chromeWin.document.addEventListener('keypress', this, true);
+    this.chromeWin.document.addEventListener('mousemove', this, true);
   },
 
   detach: function detach() {
     this.chromeWin.document.removeEventListener('keypress', this, true);
+    this.chromeWin.document.removeEventListener('mousemove', this, true);
+  },
+
+  handleEvent: function VirtualCursorController_handleEvent(aEvent) {
+    switch (aEvent.type) {
+      case 'keypress':
+        this._handleKeypress(aEvent);
+        break;
+      case 'mousemove':
+        this._handleMousemove(aEvent);
+        break;
+    }
   },
 
-  handleEvent: function handleEvent(aEvent) {
-    let document = Utils.getBrowserApp(this.chromeWin).
-      selectedBrowser.contentDocument;
+  _handleMousemove: function _handleMousemove(aEvent) {
+    // Explore by touch is disabled.
+    if (!this.exploreByTouch)
+      return;
+
+    // On non-Android we use the shift key to simulate touch.
+    if (Utils.OS != 'Android' && !aEvent.shiftKey)
+      return;
+
+    // We should not be calling moveToPoint more than 10 times a second.
+    // It is granular enough to feel natural, and it does not hammer the CPU.
+    if (!this._handleMousemove._lastEventTime ||
+        aEvent.timeStamp - this._handleMousemove._lastEventTime >= 100) {
+      this.moveToPoint(Utils.getCurrentContentDoc(this.chromeWin),
+                       aEvent.screenX, aEvent.screenY);
+      this._handleMousemove._lastEventTime = aEvent.timeStamp;
+    }
+
+    aEvent.preventDefault();
+    aEvent.stopImmediatePropagation();
+  },
+
+  _handleKeypress: function _handleKeypress(aEvent) {
+    let document = Utils.getCurrentContentDoc(this.chromeWin);
     let target = aEvent.target;
 
     switch (aEvent.keyCode) {
       case 0:
         // an alphanumeric key was pressed, handle it separately.
         // If it was pressed with either alt or ctrl, just pass through.
         // If it was pressed with meta, pass the key on without the meta.
         if (this._isEditableText(target) ||
@@ -494,16 +528,21 @@ var VirtualCursorController = {
       default:
         return;
     }
 
     aEvent.preventDefault();
     aEvent.stopPropagation();
   },
 
+  moveToPoint: function moveToPoint(aDocument, aX, aY) {
+    this.getVirtualCursor(aDocument).moveToPoint(TraversalRules.Simple,
+                                                 aX, aY, true);
+  },
+
   _isEditableText: function _isEditableText(aElement) {
     // XXX: Support contentEditable and design mode
     if (aElement instanceof Ci.nsIDOMHTMLInputElement &&
         aElement.mozIsTextField(false))
       return this.SINGLE_LINE_EDITABLE;
 
     if (aElement instanceof Ci.nsIDOMHTMLTextAreaElement)
       return this.MULTI_LINE_EDITABLE;
@@ -577,18 +616,17 @@ var VirtualCursorController = {
     while (doc) {
       let vc = null;
       try {
         vc = doc.QueryInterface(Ci.nsIAccessibleCursorable).virtualCursor;
       } catch (x) {
         doc = doc.parentDocument;
         continue;
       }
-      if (vc)
-        vc.moveNext(aRule || TraversalRules.Simple, aAccessible, true);
+      vc.moveNext(aRule || TraversalRules.Simple, aAccessible, true);
       break;
     }
   },
 
   keyMap: {
     a: ['moveForward', TraversalRules.Anchor],
     A: ['moveBackward', TraversalRules.Anchor],
     b: ['moveForward', TraversalRules.Button],