Bug 1330257 - 5. Add auto-fill accessibility frontend; r=eeejay
authorJim Chen <nchen@mozilla.com>
Tue, 21 Aug 2018 15:23:28 -0400
changeset 487737 6541f81c8e25558cc8a72773aea2247c6cdef5e5
parent 487736 5f44c144b1e52f8f3fa26f837eeea0d7c0f88f60
child 487738 d91afaee03b064e482e1efc63f5651ff1b61ef34
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerseeejay
bugs1330257
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1330257 - 5. Add auto-fill accessibility frontend; r=eeejay Add an auto-fill frontend that listens to events from Gecko. It populates accessibility nodes for input fields and sends accessibility events, in order to support auto-fill clients that use accessibility services to perform auto-fill. Differential Revision: https://phabricator.services.mozilla.com/D3254
mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java
mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
--- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java
@@ -1003,16 +1003,21 @@ public class GeckoSession extends LayerS
 
         onWindowChanged(WINDOW_CLOSE, /* inProgress */ false);
     }
 
     private void onWindowChanged(int change, boolean inProgress) {
         if ((change == WINDOW_OPEN || change == WINDOW_TRANSFER_IN) && !inProgress) {
             mTextInput.onWindowChanged(mWindow);
         }
+        if ((change == WINDOW_CLOSE || change == WINDOW_TRANSFER_OUT) && !inProgress) {
+            if (mAccessibility != null) {
+                mAccessibility.clearAutoFill();
+            }
+        }
     }
 
     /**
      * Get the SessionTextInput instance for this session. May be called on any thread.
      *
      * @return SessionTextInput instance.
      */
     public @NonNull SessionTextInput getTextInput() {
@@ -1320,16 +1325,20 @@ public class GeckoSession extends LayerS
      *                false if the session should lose focus.
      *
      * @see #setActive
      */
     public void setFocused(boolean focused) {
         final GeckoBundle msg = new GeckoBundle(1);
         msg.putBoolean("focused", focused);
         mEventDispatcher.dispatch("GeckoView:SetFocused", msg);
+
+        if (focused && mAccessibility != null) {
+            mAccessibility.onWindowFocus();
+        }
     }
 
     /**
      * Class representing a saved session state.
      */
     public static class SessionState implements Parcelable {
         private String mState;
 
--- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
@@ -12,66 +12,93 @@ import org.mozilla.gecko.util.BundleEven
 import org.mozilla.gecko.util.EventCallback;
 import org.mozilla.gecko.util.GeckoBundle;
 
 import android.content.Context;
 import android.graphics.Matrix;
 import android.graphics.Rect;
 import android.os.Build;
 import android.os.Bundle;
+import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
+import android.text.InputType;
 import android.util.Log;
+import android.util.SparseArray;
 import android.view.InputDevice;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.ViewParent;
 import android.view.accessibility.AccessibilityEvent;
 import android.view.accessibility.AccessibilityManager;
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityNodeProvider;
 
 public class SessionAccessibility {
     private static final String LOGTAG = "GeckoAccessibility";
+    private static final boolean DEBUG = false;
+
     // This is a special ID we use for nodes that are event sources.
     // We expose it as a fragment and not an actual child of the View node.
     private static final int VIRTUAL_CONTENT_ID = -2;
 
     // This is the number BrailleBack uses to start indexing routing keys.
     private static final int BRAILLE_CLICK_BASE_INDEX = -275000000;
 
+    private static final int ACTION_SET_TEXT = 0x200000;
+    private static final String ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE =
+            "ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE";
+
     /* package */ final class NodeProvider extends AccessibilityNodeProvider {
         @Override
         public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualDescendantId) {
-            AccessibilityNodeInfo info = (virtualDescendantId == VIRTUAL_CONTENT_ID && mVirtualContentNode != null) ?
-                                         AccessibilityNodeInfo.obtain(mVirtualContentNode) :
-                                         AccessibilityNodeInfo.obtain(mView, virtualDescendantId);
+            AccessibilityNodeInfo info = getAutoFillNode(virtualDescendantId);
+            if (info != null) {
+                // Try auto-fill nodes first.
+                return info;
+            }
+
+            info = (virtualDescendantId == VIRTUAL_CONTENT_ID && mVirtualContentNode != null)
+                   ? AccessibilityNodeInfo.obtain(mVirtualContentNode)
+                   : AccessibilityNodeInfo.obtain(mView, virtualDescendantId);
 
             switch (virtualDescendantId) {
             case View.NO_ID:
                 // This is the parent View node.
                 // We intentionally don't add VIRTUAL_CONTENT_ID
                 // as a child. It is a source for events,
                 // but not a member of the tree you
                 // can get to by traversing down.
                 if (Build.VERSION.SDK_INT < 17 || mView.getDisplay() != null) {
                     // When running junit tests we don't have a display
                     mView.onInitializeAccessibilityNodeInfo(info);
                 }
+                info.setPackageName(GeckoAppShell.getApplicationContext().getPackageName());
                 info.setClassName("android.webkit.WebView"); // TODO: WTF
+
                 if (Build.VERSION.SDK_INT >= 19) {
                     Bundle bundle = info.getExtras();
                     bundle.putCharSequence(
                         "ACTION_ARGUMENT_HTML_ELEMENT_STRING_VALUES",
                         "ARTICLE,BUTTON,CHECKBOX,COMBOBOX,CONTROL," +
                         "FOCUSABLE,FRAME,GRAPHIC,H1,H2,H3,H4,H5,H6," +
                         "HEADING,LANDMARK,LINK,LIST,LIST_ITEM,MAIN," +
                         "MEDIA,RADIO,SECTION,TABLE,TEXT_FIELD," +
                         "UNVISITED_LINK,VISITED_LINK");
                 }
                 info.addAction(AccessibilityNodeInfo.ACTION_NEXT_HTML_ELEMENT);
-                info.addChild(hostView, VIRTUAL_CONTENT_ID);
+
+                if (mAutoFillRoots != null) {
+                    // Add auto-fill nodes.
+                    if (DEBUG) {
+                        Log.d(LOGTAG, "Adding roots " + mAutoFillRoots);
+                    }
+                    for (int i = 0; i < mAutoFillRoots.size(); i++) {
+                        info.addChild(mView, mAutoFillRoots.keyAt(i));
+                    }
+                }
                 break;
             default:
                 info.setParent(mView);
                 info.setSource(mView, virtualDescendantId);
                 info.setVisibleToUser(mView.isShown());
                 info.setPackageName(GeckoAppShell.getApplicationContext().getPackageName());
                 info.setEnabled(true);
                 info.addAction(AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY);
@@ -88,16 +115,22 @@ public class SessionAccessibility {
             return info;
         }
 
         @Override
         public boolean performAction(int virtualViewId, int action, Bundle arguments) {
             if (virtualViewId == View.NO_ID) {
                 return performRootAction(action, arguments);
             }
+            if (action == AccessibilityNodeInfo.ACTION_SET_TEXT) {
+                final String value = arguments.getString(Build.VERSION.SDK_INT >= 21
+                        ? AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE
+                        : ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE);
+                return performAutoFill(virtualViewId, value);
+            }
             return performContentAction(action, arguments);
         }
 
         private boolean performRootAction(int action, Bundle arguments) {
             switch (action) {
             case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS:
             case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS:
                 final GeckoBundle data = new GeckoBundle(1);
@@ -198,29 +231,46 @@ public class SessionAccessibility {
     // Gecko session we are proxying
     /* package */  final GeckoSession mSession;
     // This is the view that delegates accessibility to us. We also sends event through it.
     private View mView;
     // Have we reached the last item in content?
     private boolean mLastItem;
     // Used to store the JSON message and populate the event later in the code path.
     private AccessibilityNodeInfo mVirtualContentNode;
+    // Auto-fill nodes.
+    private SparseArray<GeckoBundle> mAutoFillNodes;
+    private SparseArray<EventCallback> mAutoFillRoots;
+    private int mAutoFillFocusedId = View.NO_ID;
 
     /* package */ SessionAccessibility(final GeckoSession session) {
         mSession = session;
 
         Settings.updateAccessibilitySettings();
 
         session.getEventDispatcher().registerUiThreadListener(new BundleEventListener() {
-            @Override
-            public void handleMessage(final String event, final GeckoBundle message,
-                                      final EventCallback callback) {
-                sendAccessibilityEvent(message);
-            }
-        }, "GeckoView:AccessibilityEvent", null);
+                @Override
+                public void handleMessage(final String event, final GeckoBundle message,
+                                          final EventCallback callback) {
+                    if ("GeckoView:AccessibilityEvent".equals(event)) {
+                        sendAccessibilityEvent(message);
+                    } else if ("GeckoView:AddAutoFill".equals(event)) {
+                        addAutoFill(message, callback);
+                    } else if ("GeckoView:ClearAutoFill".equals(event)) {
+                        clearAutoFill();
+                    } else if ("GeckoView:OnAutoFillFocus".equals(event)) {
+                        onAutoFillFocus(message);
+                    }
+                }
+            },
+            "GeckoView:AccessibilityEvent",
+            "GeckoView:AddAutoFill",
+            "GeckoView:ClearAutoFill",
+            "GeckoView:OnAutoFillFocus",
+            null);
     }
 
     /**
       * Get the View instance that delegates accessibility to this session.
       *
       * @return View instance.
       */
     public View getView() {
@@ -259,16 +309,17 @@ public class SessionAccessibility {
             }
         });
     }
 
     private static class Settings {
         private static final String FORCE_ACCESSIBILITY_PREF = "accessibility.force_disabled";
 
         private static volatile boolean sEnabled;
+        private static volatile boolean sTouchExplorationEnabled;
         /* package */ static volatile boolean sForceEnabled;
 
         static {
             final Context context = GeckoAppShell.getApplicationContext();
             AccessibilityManager accessibilityManager =
                 (AccessibilityManager) context.getSystemService(Context.ACCESSIBILITY_SERVICE);
 
             accessibilityManager.addAccessibilityStateChangeListener(
@@ -302,21 +353,25 @@ public class SessionAccessibility {
             };
             PrefsHelper.addObserver(new String[]{ FORCE_ACCESSIBILITY_PREF }, prefHandler);
         }
 
         public static boolean isEnabled() {
             return sEnabled || sForceEnabled;
         }
 
+        public static boolean isTouchExplorationEnabled() {
+            return sTouchExplorationEnabled || sForceEnabled;
+        }
+
         public static void updateAccessibilitySettings() {
             final AccessibilityManager accessibilityManager = (AccessibilityManager)
                     GeckoAppShell.getApplicationContext().getSystemService(Context.ACCESSIBILITY_SERVICE);
-            mEnabled = accessibilityManager.isEnabled() && accessibilityManager.isTouchExplorationEnabled();
-
+            sEnabled = accessibilityManager.isEnabled();
+            sTouchExplorationEnabled = sEnabled && accessibilityManager.isTouchExplorationEnabled();
             dispatch();
         }
 
         /* package */ static void dispatch() {
             final GeckoBundle ret = new GeckoBundle(1);
             ret.putBoolean("enabled", isTouchExplorationEnabled());
             // "GeckoView:AccessibilitySettings" is dispatched to the Gecko thread.
             EventDispatcher.getInstance().dispatch("GeckoView:AccessibilitySettings", ret);
@@ -425,17 +480,17 @@ public class SessionAccessibility {
             node.setChecked(message.getBoolean("checked"));
         }
         if (message.containsKey("selected")) {
             node.setSelected(message.getBoolean("selected"));
         }
     }
 
     private void sendAccessibilityEvent(final GeckoBundle message) {
-        if (mView == null || !Settings.isEnabled())
+        if (mView == null || !Settings.isTouchExplorationEnabled())
             return;
 
         final int eventType = message.getInt("eventType", -1);
         if (eventType < 0) {
             Log.e(LOGTAG, "No accessibility event type provided");
             return;
         }
 
@@ -479,17 +534,17 @@ public class SessionAccessibility {
         }
 
         final AccessibilityEvent accessibilityEvent = obtainEvent(eventType, eventSource);
         populateEventFromJSON(accessibilityEvent, message);
         ((ViewParent) mView).requestSendAccessibilityEvent(mView, accessibilityEvent);
     }
 
     public boolean onMotionEvent(final MotionEvent event) {
-        if (!Settings.isEnabled()) {
+        if (!Settings.isTouchExplorationEnabled()) {
             return false;
         }
 
         if (event.getSource() != InputDevice.SOURCE_TOUCHSCREEN) {
             return false;
         }
 
         final int action = event.getActionMasked();
@@ -499,9 +554,253 @@ public class SessionAccessibility {
             return false;
         }
 
         final GeckoBundle data = new GeckoBundle(2);
         data.putDoubleArray("coordinates", new double[] {event.getRawX(), event.getRawY()});
         mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityExploreByTouch", data);
         return true;
     }
+
+    private int getAutoFillRootId(final int id) {
+        int root = View.NO_ID;
+        for (int newId = id; newId != View.NO_ID;) {
+            root = newId;
+            newId = mAutoFillNodes.get(newId).getInt("parent", View.NO_ID);
+        }
+        return root;
+    }
+
+    /* package */ AccessibilityNodeInfo getAutoFillNode(final int id) {
+        if (mView == null || mAutoFillRoots == null) {
+            return null;
+        }
+
+        final GeckoBundle bundle = mAutoFillNodes.get(id);
+        if (bundle == null) {
+            return null;
+        }
+
+        if (DEBUG) {
+            Log.d(LOGTAG, "getAutoFillNode(" + id + ')');
+        }
+
+        final AccessibilityNodeInfo node = AccessibilityNodeInfo.obtain(mView, id);
+        node.setPackageName(GeckoAppShell.getApplicationContext().getPackageName());
+        node.setParent(mView, bundle.getInt("parent", View.NO_ID));
+        node.setEnabled(true);
+
+        if (getAutoFillRootId(mAutoFillFocusedId) == getAutoFillRootId(id)) {
+            // Some auto-fill clients require a dummy rect for the focused View.
+            final Rect rect = new Rect();
+            mSession.getSurfaceBounds(rect);
+            node.setVisibleToUser(!rect.isEmpty());
+            node.setBoundsInParent(rect);
+
+            final int[] offset = new int[2];
+            mView.getLocationOnScreen(offset);
+            rect.offset(offset[0], offset[1]);
+            node.setBoundsInScreen(rect);
+        }
+
+        final GeckoBundle[] children = bundle.getBundleArray("children");
+        if (children != null) {
+            for (final GeckoBundle child : children) {
+                final int childId = child.getInt("id");
+                node.addChild(mView, childId);
+                mAutoFillNodes.append(childId, child);
+            }
+        }
+
+        String tag = bundle.getString("tag", "");
+        final String type = bundle.getString("type", "text");
+        final GeckoBundle attrs = bundle.getBundle("attributes");
+
+        if ("INPUT".equals(tag) && !bundle.getBoolean("editable", false)) {
+            tag = ""; // Don't process non-editable inputs (e.g. type="button").
+        }
+        switch (tag) {
+            case "INPUT":
+            case "TEXTAREA": {
+                final boolean disabled = bundle.getBoolean("disabled");
+                node.setClassName("android.widget.EditText");
+                node.setEnabled(!disabled);
+                node.setFocusable(!disabled);
+                node.setFocused(id == mAutoFillFocusedId);
+
+                if ("password".equals(type)) {
+                    node.setPassword(true);
+                }
+                if (Build.VERSION.SDK_INT >= 18) {
+                    node.setEditable(!disabled);
+                }
+                if (Build.VERSION.SDK_INT >= 19) {
+                    node.setMultiLine("TEXTAREA".equals(tag));
+                }
+                if (Build.VERSION.SDK_INT >= 21) {
+                    try {
+                        node.setMaxTextLength(Integer.parseInt(
+                                String.valueOf(attrs.get("maxlength"))));
+                    } catch (final NumberFormatException ignore) {
+                    }
+                }
+
+                if (!disabled) {
+                    if (Build.VERSION.SDK_INT >= 21) {
+                        node.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_TEXT);
+                    } else {
+                        node.addAction(ACTION_SET_TEXT);
+                    }
+                }
+                break;
+            }
+            default:
+                if (children != null) {
+                    node.setClassName("android.view.ViewGroup");
+                } else {
+                    node.setClassName("android.view.View");
+                }
+                break;
+        }
+
+        if (Build.VERSION.SDK_INT >= 19 && "INPUT".equals(tag)) {
+            switch (type) {
+                case "email":
+                    node.setInputType(InputType.TYPE_CLASS_TEXT |
+                                      InputType.TYPE_TEXT_VARIATION_WEB_EMAIL_ADDRESS);
+                    break;
+                case "number":
+                    node.setInputType(InputType.TYPE_CLASS_NUMBER);
+                    break;
+                case "password":
+                    node.setInputType(InputType.TYPE_CLASS_TEXT |
+                                      InputType.TYPE_TEXT_VARIATION_WEB_PASSWORD);
+                    break;
+                case "tel":
+                    node.setInputType(InputType.TYPE_CLASS_PHONE);
+                    break;
+                case "text":
+                    node.setInputType(InputType.TYPE_CLASS_TEXT |
+                                      InputType.TYPE_TEXT_VARIATION_WEB_EDIT_TEXT);
+                    break;
+                case "url":
+                    node.setInputType(InputType.TYPE_CLASS_TEXT |
+                                      InputType.TYPE_TEXT_VARIATION_URI);
+                    break;
+            }
+        }
+        return node;
+    }
+
+    /* package */ boolean performAutoFill(final int id, final String value) {
+        if (mAutoFillRoots == null) {
+            return false;
+        }
+
+        int rootId = id;
+        for (int currentId = id; currentId != View.NO_ID;) {
+            final GeckoBundle bundle = mAutoFillNodes.get(currentId);
+            if (bundle == null) {
+                return false;
+            }
+            rootId = currentId;
+            currentId = bundle.getInt("parent", View.NO_ID);
+        }
+
+        if (DEBUG) {
+            Log.d(LOGTAG, "performAutoFill(" + id + ", " + value + ')');
+        }
+
+        final EventCallback callback = mAutoFillRoots.get(rootId);
+        if (callback == null) {
+            return false;
+        }
+
+        final GeckoBundle response = new GeckoBundle(1);
+        response.putString(String.valueOf(id), value);
+        callback.sendSuccess(response);
+        return true;
+    }
+
+    private void fireWindowChangedEvent(final int id) {
+        if (mView instanceof ViewParent) {
+            final AccessibilityEvent event = obtainEvent(
+                    AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED, id);
+            if (Build.VERSION.SDK_INT >= 19) {
+                event.setContentChangeTypes(AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE);
+            }
+            ((ViewParent) mView).requestSendAccessibilityEvent(mView, event);
+        }
+    }
+
+    /* package */ void addAutoFill(final GeckoBundle message, final EventCallback callback) {
+        if (!Settings.isEnabled()) {
+            return;
+        }
+
+        if (mAutoFillRoots == null) {
+            mAutoFillRoots = new SparseArray<>();
+            mAutoFillNodes = new SparseArray<>();
+        }
+
+        final int id = message.getInt("id");
+        if (DEBUG) {
+            Log.d(LOGTAG, "addAutoFill(" + id + ')');
+        }
+
+        mAutoFillRoots.append(id, callback);
+        mAutoFillNodes.append(id, message);
+        fireWindowChangedEvent(id);
+    }
+
+    /* package */ void clearAutoFill() {
+        if (mAutoFillRoots != null) {
+            if (DEBUG) {
+                Log.d(LOGTAG, "clearAutoFill()");
+            }
+            mAutoFillRoots = null;
+            mAutoFillNodes = null;
+            mAutoFillFocusedId = View.NO_ID;
+            fireWindowChangedEvent(View.NO_ID);
+        }
+    }
+
+    /* package */ void onAutoFillFocus(final GeckoBundle message) {
+        if (!Settings.isEnabled() || !(mView instanceof ViewParent) || mAutoFillNodes == null) {
+            return;
+        }
+
+        final int id;
+        if (message != null) {
+            id = message.getInt("id");
+            mAutoFillNodes.put(id, message);
+        } else {
+            id = View.NO_ID;
+        }
+
+        if (DEBUG) {
+            Log.d(LOGTAG, "onAutoFillFocus(" + id + ')');
+        }
+        if (mAutoFillFocusedId == id) {
+            return;
+        }
+        mAutoFillFocusedId = id;
+
+        // We already send "TYPE_VIEW_FOCUSED" in touch exploration mode,
+        // so in that case don't send it here.
+        if (!Settings.isTouchExplorationEnabled()) {
+            AccessibilityEvent event = obtainEvent(AccessibilityEvent.TYPE_VIEW_FOCUSED, id);
+            ((ViewParent) mView).requestSendAccessibilityEvent(mView, event);
+        }
+    }
+
+    /* package */ void onWindowFocus() {
+        // Auto-fill clients expect a state change event on focus.
+        if (Settings.isEnabled() && mView instanceof ViewParent) {
+            if (DEBUG) {
+                Log.d(LOGTAG, "onWindowFocus()");
+            }
+            final AccessibilityEvent event = obtainEvent(
+                    AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED, View.NO_ID);
+            ((ViewParent) mView).requestSendAccessibilityEvent(mView, event);
+        }
+    }
 }