Bug 1471951 - Support set selection and clipboard actions (1/2). r=yzen r=jchen
authorEitan Isaacson <eitan@monotonous.org>
Thu, 12 Jul 2018 08:33:13 -0700
changeset 426753 1839e4b613bd71540882780c7968d8c60ca68aa5
parent 426752 896f75ea76345497797778f66b7ad9386052229a
child 426754 d75a549c4e930a50b240b0d28f574dd211d6be28
push id34284
push userbtara@mozilla.com
push dateMon, 16 Jul 2018 21:55:18 +0000
treeherdermozilla-central@da5b3e1dca89 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersyzen, jchen
bugs1471951
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1471951 - Support set selection and clipboard actions (1/2). r=yzen r=jchen
accessible/jsat/AccessFu.jsm
accessible/jsat/ContentControl.jsm
accessible/jsat/EventManager.jsm
accessible/jsat/Utils.jsm
mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/AccessibilityTest.kt
mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
--- a/accessible/jsat/AccessFu.jsm
+++ b/accessible/jsat/AccessFu.jsm
@@ -19,17 +19,19 @@ const GECKOVIEW_MESSAGE = {
   ACTIVATE: "GeckoView:AccessibilityActivate",
   VIEW_FOCUSED: "GeckoView:AccessibilityViewFocused",
   LONG_PRESS: "GeckoView:AccessibilityLongPress",
   BY_GRANULARITY: "GeckoView:AccessibilityByGranularity",
   NEXT: "GeckoView:AccessibilityNext",
   PREVIOUS: "GeckoView:AccessibilityPrevious",
   SCROLL_BACKWARD: "GeckoView:AccessibilityScrollBackward",
   SCROLL_FORWARD: "GeckoView:AccessibilityScrollForward",
-  EXPLORE_BY_TOUCH: "GeckoView:AccessibilityExploreByTouch"
+  EXPLORE_BY_TOUCH: "GeckoView:AccessibilityExploreByTouch",
+  SET_SELECTION: "GeckoView:AccessibilitySetSelection",
+  CLIPBOARD: "GeckoView:AccessibilityClipboard",
 };
 
 var AccessFu = {
   /**
    * A lazy getter for event handler that binds the scope to AccessFu object.
    */
   get handleEvent() {
     delete this.handleEvent;
@@ -262,16 +264,22 @@ var AccessFu = {
         }
         break;
       case GECKOVIEW_MESSAGE.BY_GRANULARITY:
         this.Input.moveByGranularity(data);
         break;
       case GECKOVIEW_MESSAGE.EXPLORE_BY_TOUCH:
         this.Input.moveToPoint("Simple", ...data.coordinates);
         break;
+      case GECKOVIEW_MESSAGE.SET_SELECTION:
+        this.Input.setSelection(data);
+        break;
+      case GECKOVIEW_MESSAGE.CLIPBOARD:
+        this.Input.clipboard(data);
+        break;
     }
   },
 
   observe: function observe(aSubject, aTopic, aData) {
     switch (aTopic) {
       case "remote-browser-shown":
       case "inprocess-browser-shown":
       {
@@ -323,17 +331,17 @@ var AccessFu = {
         break;
       }
       default:
         break;
     }
   },
 
   autoMove: function autoMove(aOptions) {
-    let mm = Utils.getMessageManager();
+    const mm = Utils.getMessageManager();
     mm.sendAsyncMessage("AccessFu:AutoMove", aOptions);
   },
 
   announce: function announce(aAnnouncement) {
     this._output(Presentation.announce(aAnnouncement), Utils.getCurrentBrowser());
   },
 
   // So we don't enable/disable twice
@@ -361,56 +369,66 @@ var AccessFu = {
       bounds = bounds.scale(1 / devicePixelRatio, 1 / devicePixelRatio);
       bounds = bounds.translate(-mozInnerScreenX, -mozInnerScreenY);
       return bounds.expandToIntegers();
     }
 };
 
 var Input = {
   moveToPoint: function moveToPoint(aRule, aX, aY) {
-    let mm = Utils.getMessageManager();
+    const mm = Utils.getMessageManager();
     mm.sendAsyncMessage("AccessFu:MoveToPoint",
       {rule: aRule, x: aX, y: aY, origin: "top"});
   },
 
   moveCursor: function moveCursor(aAction, aRule, aInputType, aAdjustRange) {
-    let mm = Utils.getMessageManager();
+    const mm = Utils.getMessageManager();
     mm.sendAsyncMessage("AccessFu:MoveCursor",
                         { action: aAction, rule: aRule,
                           origin: "top", inputType: aInputType,
                           adjustRange: aAdjustRange });
   },
 
   androidScroll: function androidScroll(aDirection) {
-    let mm = Utils.getMessageManager();
+    const mm = Utils.getMessageManager();
     mm.sendAsyncMessage("AccessFu:AndroidScroll",
                         { direction: aDirection, origin: "top" });
   },
 
   moveByGranularity: function moveByGranularity(aDetails) {
-    let mm = Utils.getMessageManager();
+    const mm = Utils.getMessageManager();
     mm.sendAsyncMessage("AccessFu:MoveByGranularity", aDetails);
   },
 
+  setSelection: function setSelection(aDetails) {
+    const mm = Utils.getMessageManager();
+    mm.sendAsyncMessage("AccessFu:SetSelection", aDetails);
+  },
+
+  clipboard: function clipboard(aDetails) {
+    const mm = Utils.getMessageManager();
+    mm.sendAsyncMessage("AccessFu:Clipboard", aDetails);
+  },
+
   activateCurrent: function activateCurrent(aData, aActivateIfKey = false) {
     let mm = Utils.getMessageManager();
     let offset = 0;
 
     mm.sendAsyncMessage("AccessFu:Activate",
                         {offset, activateIfKey: aActivateIfKey});
   },
 
   // XXX: This is here for backwards compatability with screen reader simulator
   // it should be removed when the extension is updated on amo.
   scroll: function scroll(aPage, aHorizontal) {
     this.sendScrollMessage(aPage, aHorizontal);
   },
 
   sendScrollMessage: function sendScrollMessage(aPage, aHorizontal) {
-    let mm = Utils.getMessageManager();
+    const mm = Utils.getMessageManager();
     mm.sendAsyncMessage("AccessFu:Scroll",
       {page: aPage, horizontal: aHorizontal, origin: "top"});
   },
 
   doScroll: function doScroll(aDetails, aBrowser) {
     let horizontal = aDetails.horizontal;
     let page = aDetails.page;
     let win = aBrowser.ownerGlobal;
--- a/accessible/jsat/ContentControl.jsm
+++ b/accessible/jsat/ContentControl.jsm
@@ -19,29 +19,35 @@ ChromeUtils.defineModuleGetter(this, "Pr
   "resource://gre/modules/accessibility/Presentation.jsm");
 
 var EXPORTED_SYMBOLS = ["ContentControl"];
 
 const MOVEMENT_GRANULARITY_CHARACTER = 1;
 const MOVEMENT_GRANULARITY_WORD = 2;
 const MOVEMENT_GRANULARITY_PARAGRAPH = 8;
 
+const CLIPBOARD_COPY = 0x4000;
+const CLIPBOARD_PASTE = 0x8000;
+const CLIPBOARD_CUT = 0x10000;
+
 function ContentControl(aContentScope) {
   this._contentScope = Cu.getWeakReference(aContentScope);
   this._childMessageSenders = new WeakMap();
 }
 
 this.ContentControl.prototype = {
   messagesOfInterest: ["AccessFu:MoveCursor",
                        "AccessFu:ClearCursor",
                        "AccessFu:MoveToPoint",
                        "AccessFu:AutoMove",
                        "AccessFu:Activate",
                        "AccessFu:MoveByGranularity",
-                       "AccessFu:AndroidScroll"],
+                       "AccessFu:AndroidScroll",
+                       "AccessFu:SetSelection",
+                       "AccessFu:Clipboard"],
 
   start: function cc_start() {
     let cs = this._contentScope.get();
     for (let message of this.messagesOfInterest) {
       cs.addMessageListener(message, this);
     }
   },
 
@@ -322,16 +328,58 @@ this.ContentControl.prototype = {
 
     if (direction === "Previous") {
       this.vc.movePreviousByText(pivotGranularity);
     } else if (direction === "Next") {
       this.vc.moveNextByText(pivotGranularity);
     }
   },
 
+  handleSetSelection: function cc_handleSetSelection(aMessage) {
+    const { start, end } = aMessage.json;
+    const focusedAcc =
+      Utils.AccService.getAccessibleFor(this.document.activeElement);
+    if (focusedAcc) {
+      const accText = focusedAcc.QueryInterface(Ci.nsIAccessibleText);
+      if (start == end) {
+        accText.caretOffset = start;
+      } else {
+        accText.setSelectionBounds(0, start, end);
+      }
+    }
+  },
+
+  handleClipboard: function cc_handleClipboard(aMessage) {
+    const { action } = aMessage.json;
+    const focusedAcc =
+      Utils.AccService.getAccessibleFor(this.document.activeElement);
+    if (focusedAcc) {
+      const [startSel, endSel] = Utils.getTextSelection(focusedAcc);
+      const editText = focusedAcc.QueryInterface(Ci.nsIAccessibleEditableText);
+      switch (action) {
+        case CLIPBOARD_COPY:
+          if (startSel != endSel) {
+            editText.copyText(startSel, endSel);
+          }
+          break;
+        case CLIPBOARD_PASTE:
+          if (startSel != endSel) {
+            editText.deleteText(startSel, endSel);
+          }
+          editText.pasteText(startSel);
+          break;
+        case CLIPBOARD_CUT:
+          if (startSel != endSel) {
+            editText.cutText(startSel, endSel);
+          }
+          break;
+      }
+    }
+  },
+
   presentCaretChange: function cc_presentCaretChange(
     aText, aOldOffset, aNewOffset) {
     if (aOldOffset !== aNewOffset) {
       let msg = Presentation.textSelectionChanged(aText, aNewOffset, aNewOffset,
         aOldOffset, aOldOffset, true);
       this._contentScope.get().sendAsyncMessage("AccessFu:Present", msg);
     }
   },
--- a/accessible/jsat/EventManager.jsm
+++ b/accessible/jsat/EventManager.jsm
@@ -205,18 +205,24 @@ this.EventManager.prototype = {
         let caretOffset = aEvent.
           QueryInterface(Ci.nsIAccessibleCaretMoveEvent).caretOffset;
 
         // We could get a caret move in an accessible that is not focused,
         // it doesn't mean we are not on any editable accessible. just not
         // on this one..
         let state = Utils.getState(acc);
         if (state.contains(States.FOCUSED) && state.contains(States.EDITABLE)) {
-          this.present(Presentation.textSelectionChanged(acc.getText(0, -1),
-            caretOffset, caretOffset, 0, 0, aEvent.isFromUserInput));
+          let fromIndex = caretOffset;
+          if (acc.selectionCount) {
+            const [startSel, endSel] = Utils.getTextSelection(acc);
+            fromIndex = startSel == caretOffset ? endSel : startSel;
+          }
+          this.present(Presentation.textSelectionChanged(
+            acc.getText(0, -1), fromIndex, caretOffset, 0, 0,
+            aEvent.isFromUserInput));
         }
         break;
       }
       case Events.SHOW:
       {
         this._handleShow(aEvent);
         break;
       }
--- a/accessible/jsat/Utils.jsm
+++ b/accessible/jsat/Utils.jsm
@@ -229,16 +229,28 @@ var Utils = { // jshint ignore:line
 
   getBounds: function getBounds(aAccessible) {
     let objX = {}, objY = {}, objW = {}, objH = {};
     aAccessible.getBounds(objX, objY, objW, objH);
 
     return new Rect(objX.value, objY.value, objW.value, objH.value);
   },
 
+  getTextSelection: function getTextSelection(aAccessible) {
+    const accText = aAccessible.QueryInterface(Ci.nsIAccessibleText);
+    const start = {}, end = {};
+    if (accText.selectionCount) {
+      accText.getSelectionBounds(0, start, end);
+    } else {
+      start.value = end.value = accText.caretOffset;
+    }
+
+    return [start.value, end.value];
+  },
+
   getTextBounds: function getTextBounds(aAccessible, aStart, aEnd,
                                         aPreserveContentScale) {
     let accText = aAccessible.QueryInterface(Ci.nsIAccessibleText);
     let objX = {}, objY = {}, objW = {}, objH = {};
     accText.getRangeExtents(aStart, aEnd, objX, objY, objW, objH,
       Ci.nsIAccessibleCoordinateType.COORDTYPE_SCREEN_RELATIVE);
 
     return new Rect(objX.value, objY.value, objW.value, objH.value);
--- a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/AccessibilityTest.kt
+++ b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/AccessibilityTest.kt
@@ -4,16 +4,17 @@
 
 package org.mozilla.geckoview.test
 
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.AssertCalled
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.WithDisplay
 import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.WithDevToolsAPI
 
 import android.os.Build
+import android.os.Bundle
 
 import android.support.test.filters.MediumTest
 import android.support.test.InstrumentationRegistry
 import android.support.test.runner.AndroidJUnit4
 
 import android.view.accessibility.AccessibilityNodeInfo
 import android.view.accessibility.AccessibilityNodeProvider
 import android.view.accessibility.AccessibilityEvent
@@ -163,9 +164,80 @@ class AccessibilityTest : BaseSessionTes
                 if (Build.VERSION.SDK_INT >= 19) {
                     assertThat("Hint has field name",
                             node.extras.getString("AccessibilityNodeInfo.hint"),
                             equalTo("Name"))
                 }
             }
         })
     }
-}
\ No newline at end of file
+
+    private fun waitUntilTextSelectionChanged(fromIndex: Int, toIndex: Int) {
+        var eventFromIndex = 0;
+        var eventToIndex = 0;
+        do {
+            sessionRule.waitUntilCalled(object : EventDelegate {
+                override fun onTextSelectionChanged(event: AccessibilityEvent) {
+                    eventFromIndex = event.fromIndex;
+                    eventToIndex = event.toIndex;
+                }
+            })
+        } while (fromIndex != eventFromIndex || toIndex != eventToIndex)
+    }
+
+    private fun setSelectionArguments(start: Int, end: Int): Bundle {
+        val arguments = Bundle(2)
+        arguments.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, start)
+        arguments.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, end)
+        return arguments
+    }
+
+    @Test fun testClipboard() {
+        var nodeId = AccessibilityNodeProvider.HOST_VIEW_ID;
+        sessionRule.session.loadString("<input value='hello cruel world' id='input'>", "text/html")
+        sessionRule.waitForPageStop()
+
+        mainSession.evaluateJS("$('input').focus()")
+
+        sessionRule.waitUntilCalled(object : EventDelegate {
+            @AssertCalled(count = 1)
+            override fun onAccessibilityFocused(event: AccessibilityEvent) {
+                nodeId = getSourceId(event)
+                val node = provider.createAccessibilityNodeInfo(nodeId)
+                assertThat("Focused EditBox", node.className.toString(),
+                        equalTo("android.widget.EditText"))
+            }
+
+            @AssertCalled(count = 1)
+            override fun onTextSelectionChanged(event: AccessibilityEvent) {
+                assertThat("fromIndex should be at start", event.fromIndex, equalTo(0))
+                assertThat("toIndex should be at start", event.toIndex, equalTo(0))
+            }
+        })
+
+        provider.performAction(nodeId, AccessibilityNodeInfo.ACTION_SET_SELECTION, setSelectionArguments(5, 11))
+        waitUntilTextSelectionChanged(5, 11)
+
+        provider.performAction(nodeId, AccessibilityNodeInfo.ACTION_COPY, null)
+
+        provider.performAction(nodeId, AccessibilityNodeInfo.ACTION_SET_SELECTION, setSelectionArguments(11, 11))
+        waitUntilTextSelectionChanged(11, 11)
+
+        provider.performAction(nodeId, AccessibilityNodeInfo.ACTION_PASTE, null)
+        sessionRule.waitUntilCalled(object : EventDelegate {
+            @AssertCalled(count = 1)
+            override fun onTextChanged(event: AccessibilityEvent) {
+                assertThat("text should be pasted", event.text[0].toString(), equalTo("hello cruel cruel world"))
+            }
+        })
+
+        provider.performAction(nodeId, AccessibilityNodeInfo.ACTION_SET_SELECTION, setSelectionArguments(17, 23))
+        waitUntilTextSelectionChanged(17, 23)
+
+        provider.performAction(nodeId, AccessibilityNodeInfo.ACTION_PASTE, null)
+        sessionRule.waitUntilCalled(object : EventDelegate {
+            @AssertCalled
+            override fun onTextChanged(event: AccessibilityEvent) {
+                assertThat("text should be pasted", event.text[0].toString(), equalTo("hello cruel cruel cruel"))
+            }
+        })
+    }
+}
--- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
@@ -222,16 +222,34 @@ public class SessionAccessibility {
                                 mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityActivate", data);
                             } else if (granularity > 0) {
                                 data = new GeckoBundle(2);
                                 data.putString("direction", action == AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY ? "Next" : "Previous");
                                 data.putInt("granularity", granularity);
                                 mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityByGranularity", data);
                             }
                             return true;
+                        case AccessibilityNodeInfo.ACTION_SET_SELECTION:
+                            if (arguments == null) {
+                                return false;
+                            }
+                            int selectionStart = arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT);
+                            int selectionEnd = arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT);
+                            data = new GeckoBundle(2);
+                            data.putInt("start", selectionStart);
+                            data.putInt("end", selectionEnd);
+                            mSession.getEventDispatcher().dispatch("GeckoView:AccessibilitySetSelection", data);
+                            return true;
+                        case AccessibilityNodeInfo.ACTION_CUT:
+                        case AccessibilityNodeInfo.ACTION_COPY:
+                        case AccessibilityNodeInfo.ACTION_PASTE:
+                            data = new GeckoBundle(1);
+                            data.putInt("action", action);
+                            mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityClipboard", data);
+                            return true;
                         }
 
                         return mView.performAccessibilityAction(action, arguments);
                     }
 
                     private void assertAttachedView(final View view) {
                         if (view != mView) {
                             throw new AssertionError("delegate used with wrong view.");
@@ -355,33 +373,38 @@ public class SessionAccessibility {
 
     private void populateNodeInfoFromJSON(AccessibilityNodeInfo node, final GeckoBundle message) {
         node.setEnabled(message.getBoolean("enabled", true));
         node.setCheckable(message.getBoolean("checkable"));
         node.setChecked(message.getBoolean("checked"));
         node.setPassword(message.getBoolean("password"));
         node.setFocusable(message.getBoolean("focusable"));
         node.setFocused(message.getBoolean("focused"));
-        if (Build.VERSION.SDK_INT >= 18) {
-            node.setEditable(message.getBoolean("editable"));
-        }
 
         node.setClassName(message.getString("className", "android.view.View"));
 
         final String[] textArray = message.getStringArray("text");
         StringBuilder sb = new StringBuilder();
         if (textArray != null && textArray.length > 0) {
             sb.append(textArray[0] != null ? textArray[0] : "");
             for (int i = 1; i < textArray.length; i++) {
                 sb.append(' ').append(textArray[i] != null ? textArray[i] : "");
             }
             node.setText(sb.toString());
         }
         node.setContentDescription(message.getString("description", ""));
 
+        if (Build.VERSION.SDK_INT >= 18 && message.getBoolean("editable")) {
+            node.addAction(AccessibilityNodeInfo.ACTION_SET_SELECTION);
+            node.addAction(AccessibilityNodeInfo.ACTION_CUT);
+            node.addAction(AccessibilityNodeInfo.ACTION_COPY);
+            node.addAction(AccessibilityNodeInfo.ACTION_PASTE);
+            node.setEditable(true);
+        }
+
         if (message.getBoolean("clickable")) {
             node.setClickable(true);
             node.addAction(AccessibilityNodeInfo.ACTION_CLICK);
         }
 
         if (Build.VERSION.SDK_INT >= 19 && message.containsKey("hint")) {
             Bundle bundle = node.getExtras();
             bundle.putCharSequence("AccessibilityNodeInfo.hint", message.getString("hint"));