Bug 1330257 - 5. Add auto-fill accessibility frontend; r=eeejay
authorJim Chen <nchen@mozilla.com>
Tue, 21 Aug 2018 15:23:28 -0400
changeset 480955 6541f81c8e25558cc8a72773aea2247c6cdef5e5
parent 480954 5f44c144b1e52f8f3fa26f837eeea0d7c0f88f60
child 480956 d91afaee03b064e482e1efc63f5651ff1b61ef34
push id232
push userfmarier@mozilla.com
push dateWed, 05 Sep 2018 20:45:54 +0000
reviewerseeejay
bugs1330257
milestone63.0a1
Bug 1330257 - 5. Add auto-fill accessibility frontend; r=eeejay Add an auto-fill frontend that listens to events from Gecko. It populates accessibility nodes for input fields and sends accessibility events, in order to support auto-fill clients that use accessibility services to perform auto-fill. Differential Revision: https://phabricator.services.mozilla.com/D3254
mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java
mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
--- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java
@@ -1003,16 +1003,21 @@ public class GeckoSession extends LayerS
 
         onWindowChanged(WINDOW_CLOSE, /* inProgress */ false);
     }
 
     private void onWindowChanged(int change, boolean inProgress) {
         if ((change == WINDOW_OPEN || change == WINDOW_TRANSFER_IN) && !inProgress) {
             mTextInput.onWindowChanged(mWindow);
         }
+        if ((change == WINDOW_CLOSE || change == WINDOW_TRANSFER_OUT) && !inProgress) {
+            if (mAccessibility != null) {
+                mAccessibility.clearAutoFill();
+            }
+        }
     }
 
     /**
      * Get the SessionTextInput instance for this session. May be called on any thread.
      *
      * @return SessionTextInput instance.
      */
     public @NonNull SessionTextInput getTextInput() {
@@ -1320,16 +1325,20 @@ public class GeckoSession extends LayerS
      *                false if the session should lose focus.
      *
      * @see #setActive
      */
     public void setFocused(boolean focused) {
         final GeckoBundle msg = new GeckoBundle(1);
         msg.putBoolean("focused", focused);
         mEventDispatcher.dispatch("GeckoView:SetFocused", msg);
+
+        if (focused && mAccessibility != null) {
+            mAccessibility.onWindowFocus();
+        }
     }
 
     /**
      * Class representing a saved session state.
      */
     public static class SessionState implements Parcelable {
         private String mState;
 
--- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/SessionAccessibility.java
@@ -12,66 +12,93 @@ import org.mozilla.gecko.util.BundleEven
 import org.mozilla.gecko.util.EventCallback;
 import org.mozilla.gecko.util.GeckoBundle;
 
 import android.content.Context;
 import android.graphics.Matrix;
 import android.graphics.Rect;
 import android.os.Build;
 import android.os.Bundle;
+import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
+import android.text.InputType;
 import android.util.Log;
+import android.util.SparseArray;
 import android.view.InputDevice;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.ViewParent;
 import android.view.accessibility.AccessibilityEvent;
 import android.view.accessibility.AccessibilityManager;
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityNodeProvider;
 
 public class SessionAccessibility {
     private static final String LOGTAG = "GeckoAccessibility";
+    private static final boolean DEBUG = false;
+
     // This is a special ID we use for nodes that are event sources.
     // We expose it as a fragment and not an actual child of the View node.
     private static final int VIRTUAL_CONTENT_ID = -2;
 
     // This is the number BrailleBack uses to start indexing routing keys.
     private static final int BRAILLE_CLICK_BASE_INDEX = -275000000;
 
+    private static final int ACTION_SET_TEXT = 0x200000;
+    private static final String ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE =
+            "ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE";
+
     /* package */ final class NodeProvider extends AccessibilityNodeProvider {
         @Override
         public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualDescendantId) {
-            AccessibilityNodeInfo info = (virtualDescendantId == VIRTUAL_CONTENT_ID && mVirtualContentNode != null) ?
-                                         AccessibilityNodeInfo.obtain(mVirtualContentNode) :
-                                         AccessibilityNodeInfo.obtain(mView, virtualDescendantId);
+            AccessibilityNodeInfo info = getAutoFillNode(virtualDescendantId);
+            if (info != null) {
+                // Try auto-fill nodes first.
+                return info;
+            }
+
+            info = (virtualDescendantId == VIRTUAL_CONTENT_ID && mVirtualContentNode != null)
+                   ? AccessibilityNodeInfo.obtain(mVirtualContentNode)
+                   : AccessibilityNodeInfo.obtain(mView, virtualDescendantId);
 
             switch (virtualDescendantId) {
             case View.NO_ID:
                 // This is the parent View node.
                 // We intentionally don't add VIRTUAL_CONTENT_ID
                 // as a child. It is a source for events,
                 // but not a member of the tree you
                 // can get to by traversing down.
                 if (Build.VERSION.SDK_INT < 17 || mView.getDisplay() != null) {
                     // When running junit tests we don't have a display
                     mView.onInitializeAccessibilityNodeInfo(info);
                 }
+                info.setPackageName(GeckoAppShell.getApplicationContext().getPackageName());
                 info.setClassName("android.webkit.WebView"); // TODO: WTF
+
                 if (Build.VERSION.SDK_INT >= 19) {
                     Bundle bundle = info.getExtras();
                     bundle.putCharSequence(
                         "ACTION_ARGUMENT_HTML_ELEMENT_STRING_VALUES",
                         "ARTICLE,BUTTON,CHECKBOX,COMBOBOX,CONTROL," +
                         "FOCUSABLE,FRAME,GRAPHIC,H1,H2,H3,H4,H5,H6," +
                         "HEADING,LANDMARK,LINK,LIST,LIST_ITEM,MAIN," +
                         "MEDIA,RADIO,SECTION,TABLE,TEXT_FIELD," +
                         "UNVISITED_LINK,VISITED_LINK");
                 }
                 info.addAction(AccessibilityNodeInfo.ACTION_NEXT_HTML_ELEMENT);
-                info.addChild(hostView, VIRTUAL_CONTENT_ID);
+
+                if (mAutoFillRoots != null) {
+                    // Add auto-fill nodes.
+                    if (DEBUG) {
+                        Log.d(LOGTAG, "Adding roots " + mAutoFillRoots);
+                    }
+                    for (int i = 0; i < mAutoFillRoots.size(); i++) {
+                        info.addChild(mView, mAutoFillRoots.keyAt(i));
+                    }
+                }
                 break;
             default:
                 info.setParent(mView);
                 info.setSource(mView, virtualDescendantId);
                 info.setVisibleToUser(mView.isShown());
                 info.setPackageName(GeckoAppShell.getApplicationContext().getPackageName());
                 info.setEnabled(true);
                 info.addAction(AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY);
@@ -88,16 +115,22 @@ public class SessionAccessibility {
             return info;
         }
 
         @Override
         public boolean performAction(int virtualViewId, int action, Bundle arguments) {
             if (virtualViewId == View.NO_ID) {
                 return performRootAction(action, arguments);
             }
+            if (action == AccessibilityNodeInfo.ACTION_SET_TEXT) {
+                final String value = arguments.getString(Build.VERSION.SDK_INT >= 21
+                        ? AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE
+                        : ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE);
+                return performAutoFill(virtualViewId, value);
+            }
             return performContentAction(action, arguments);
         }
 
         private boolean performRootAction(int action, Bundle arguments) {
             switch (action) {
             case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS:
             case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS:
                 final GeckoBundle data = new GeckoBundle(1);
@@ -198,29 +231,46 @@ public class SessionAccessibility {
     // Gecko session we are proxying
     /* package */  final GeckoSession mSession;
     // This is the view that delegates accessibility to us. We also sends event through it.
     private View mView;
     // Have we reached the last item in content?
     private boolean mLastItem;
     // Used to store the JSON message and populate the event later in the code path.
     private AccessibilityNodeInfo mVirtualContentNode;
+    // Auto-fill nodes.
+    private SparseArray<GeckoBundle> mAutoFillNodes;
+    private SparseArray<EventCallback> mAutoFillRoots;
+    private int mAutoFillFocusedId = View.NO_ID;
 
     /* package */ SessionAccessibility(final GeckoSession session) {
         mSession = session;
 
         Settings.updateAccessibilitySettings();
 
         session.getEventDispatcher().registerUiThreadListener(new BundleEventListener() {
-            @Override
-            public void handleMessage(final String event, final GeckoBundle message,
-                                      final EventCallback callback) {
-                sendAccessibilityEvent(message);
-            }
-        }, "GeckoView:AccessibilityEvent", null);
+                @Override
+                public void handleMessage(final String event, final GeckoBundle message,
+                                          final EventCallback callback) {
+                    if ("GeckoView:AccessibilityEvent".equals(event)) {
+                        sendAccessibilityEvent(message);
+                    } else if ("GeckoView:AddAutoFill".equals(event)) {
+                        addAutoFill(message, callback);
+                    } else if ("GeckoView:ClearAutoFill".equals(event)) {
+                        clearAutoFill();
+                    } else if ("GeckoView:OnAutoFillFocus".equals(event)) {
+                        onAutoFillFocus(message);
+                    }
+                }
+            },
+            "GeckoView:AccessibilityEvent",
+            "GeckoView:AddAutoFill",
+            "GeckoView:ClearAutoFill",
+            "GeckoView:OnAutoFillFocus",
+            null);
     }
 
     /**
       * Get the View instance that delegates accessibility to this session.
       *
       * @return View instance.
       */
     public View getView() {
@@ -259,16 +309,17 @@ public class SessionAccessibility {
             }
         });
     }
 
     private static class Settings {
         private static final String FORCE_ACCESSIBILITY_PREF = "accessibility.force_disabled";
 
         private static volatile boolean sEnabled;
+        private static volatile boolean sTouchExplorationEnabled;
         /* package */ static volatile boolean sForceEnabled;
 
         static {
             final Context context = GeckoAppShell.getApplicationContext();
             AccessibilityManager accessibilityManager =
                 (AccessibilityManager) context.getSystemService(Context.ACCESSIBILITY_SERVICE);
 
             accessibilityManager.addAccessibilityStateChangeListener(
@@ -302,21 +353,25 @@ public class SessionAccessibility {
             };
             PrefsHelper.addObserver(new String[]{ FORCE_ACCESSIBILITY_PREF }, prefHandler);
         }
 
         public static boolean isEnabled() {
             return sEnabled || sForceEnabled;
         }
 
+        public static boolean isTouchExplorationEnabled() {
+            return sTouchExplorationEnabled || sForceEnabled;
+        }
+
         public static void updateAccessibilitySettings() {
             final AccessibilityManager accessibilityManager = (AccessibilityManager)
                     GeckoAppShell.getApplicationContext().getSystemService(Context.ACCESSIBILITY_SERVICE);
-            mEnabled = accessibilityManager.isEnabled() && accessibilityManager.isTouchExplorationEnabled();
-
+            sEnabled = accessibilityManager.isEnabled();
+            sTouchExplorationEnabled = sEnabled && accessibilityManager.isTouchExplorationEnabled();
             dispatch();
         }
 
         /* package */ static void dispatch() {
             final GeckoBundle ret = new GeckoBundle(1);
             ret.putBoolean("enabled", isTouchExplorationEnabled());
             // "GeckoView:AccessibilitySettings" is dispatched to the Gecko thread.
             EventDispatcher.getInstance().dispatch("GeckoView:AccessibilitySettings", ret);
@@ -425,17 +480,17 @@ public class SessionAccessibility {
             node.setChecked(message.getBoolean("checked"));
         }
         if (message.containsKey("selected")) {
             node.setSelected(message.getBoolean("selected"));
         }
     }
 
     private void sendAccessibilityEvent(final GeckoBundle message) {
-        if (mView == null || !Settings.isEnabled())
+        if (mView == null || !Settings.isTouchExplorationEnabled())
             return;
 
         final int eventType = message.getInt("eventType", -1);
         if (eventType < 0) {
             Log.e(LOGTAG, "No accessibility event type provided");
             return;
         }
 
@@ -479,17 +534,17 @@ public class SessionAccessibility {
         }
 
         final AccessibilityEvent accessibilityEvent = obtainEvent(eventType, eventSource);
         populateEventFromJSON(accessibilityEvent, message);
         ((ViewParent) mView).requestSendAccessibilityEvent(mView, accessibilityEvent);
     }
 
     public boolean onMotionEvent(final MotionEvent event) {
-        if (!Settings.isEnabled()) {
+        if (!Settings.isTouchExplorationEnabled()) {
             return false;
         }
 
         if (event.getSource() != InputDevice.SOURCE_TOUCHSCREEN) {
             return false;
         }
 
         final int action = event.getActionMasked();
@@ -499,9 +554,253 @@ public class SessionAccessibility {
             return false;
         }
 
         final GeckoBundle data = new GeckoBundle(2);
         data.putDoubleArray("coordinates", new double[] {event.getRawX(), event.getRawY()});
         mSession.getEventDispatcher().dispatch("GeckoView:AccessibilityExploreByTouch", data);
         return true;
     }
+
+    private int getAutoFillRootId(final int id) {
+        int root = View.NO_ID;
+        for (int newId = id; newId != View.NO_ID;) {
+            root = newId;
+            newId = mAutoFillNodes.get(newId).getInt("parent", View.NO_ID);
+        }
+        return root;
+    }
+
+    /* package */ AccessibilityNodeInfo getAutoFillNode(final int id) {
+        if (mView == null || mAutoFillRoots == null) {
+            return null;
+        }
+
+        final GeckoBundle bundle = mAutoFillNodes.get(id);
+        if (bundle == null) {
+            return null;
+        }
+
+        if (DEBUG) {
+            Log.d(LOGTAG, "getAutoFillNode(" + id + ')');
+        }
+
+        final AccessibilityNodeInfo node = AccessibilityNodeInfo.obtain(mView, id);
+        node.setPackageName(GeckoAppShell.getApplicationContext().getPackageName());
+        node.setParent(mView, bundle.getInt("parent", View.NO_ID));
+        node.setEnabled(true);
+
+        if (getAutoFillRootId(mAutoFillFocusedId) == getAutoFillRootId(id)) {
+            // Some auto-fill clients require a dummy rect for the focused View.
+            final Rect rect = new Rect();
+            mSession.getSurfaceBounds(rect);
+            node.setVisibleToUser(!rect.isEmpty());
+            node.setBoundsInParent(rect);
+
+            final int[] offset = new int[2];
+            mView.getLocationOnScreen(offset);
+            rect.offset(offset[0], offset[1]);
+            node.setBoundsInScreen(rect);
+        }
+
+        final GeckoBundle[] children = bundle.getBundleArray("children");
+        if (children != null) {
+            for (final GeckoBundle child : children) {
+                final int childId = child.getInt("id");
+                node.addChild(mView, childId);
+                mAutoFillNodes.append(childId, child);
+            }
+        }
+
+        String tag = bundle.getString("tag", "");
+        final String type = bundle.getString("type", "text");
+        final GeckoBundle attrs = bundle.getBundle("attributes");
+
+        if ("INPUT".equals(tag) && !bundle.getBoolean("editable", false)) {
+            tag = ""; // Don't process non-editable inputs (e.g. type="button").
+        }
+        switch (tag) {
+            case "INPUT":
+            case "TEXTAREA": {
+                final boolean disabled = bundle.getBoolean("disabled");
+                node.setClassName("android.widget.EditText");
+                node.setEnabled(!disabled);
+                node.setFocusable(!disabled);
+                node.setFocused(id == mAutoFillFocusedId);
+
+                if ("password".equals(type)) {
+                    node.setPassword(true);
+                }
+                if (Build.VERSION.SDK_INT >= 18) {
+                    node.setEditable(!disabled);
+                }
+                if (Build.VERSION.SDK_INT >= 19) {
+                    node.setMultiLine("TEXTAREA".equals(tag));
+                }
+                if (Build.VERSION.SDK_INT >= 21) {
+                    try {
+                        node.setMaxTextLength(Integer.parseInt(
+                                String.valueOf(attrs.get("maxlength"))));
+                    } catch (final NumberFormatException ignore) {
+                    }
+                }
+
+                if (!disabled) {
+                    if (Build.VERSION.SDK_INT >= 21) {
+                        node.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_TEXT);
+                    } else {
+                        node.addAction(ACTION_SET_TEXT);
+                    }
+                }
+                break;
+            }
+            default:
+                if (children != null) {
+                    node.setClassName("android.view.ViewGroup");
+                } else {
+                    node.setClassName("android.view.View");
+                }
+                break;
+        }
+
+        if (Build.VERSION.SDK_INT >= 19 && "INPUT".equals(tag)) {
+            switch (type) {
+                case "email":
+                    node.setInputType(InputType.TYPE_CLASS_TEXT |
+                                      InputType.TYPE_TEXT_VARIATION_WEB_EMAIL_ADDRESS);
+                    break;
+                case "number":
+                    node.setInputType(InputType.TYPE_CLASS_NUMBER);
+                    break;
+                case "password":
+                    node.setInputType(InputType.TYPE_CLASS_TEXT |
+                                      InputType.TYPE_TEXT_VARIATION_WEB_PASSWORD);
+                    break;
+                case "tel":
+                    node.setInputType(InputType.TYPE_CLASS_PHONE);
+                    break;
+                case "text":
+                    node.setInputType(InputType.TYPE_CLASS_TEXT |
+                                      InputType.TYPE_TEXT_VARIATION_WEB_EDIT_TEXT);
+                    break;
+                case "url":
+                    node.setInputType(InputType.TYPE_CLASS_TEXT |
+                                      InputType.TYPE_TEXT_VARIATION_URI);
+                    break;
+            }
+        }
+        return node;
+    }
+
+    /* package */ boolean performAutoFill(final int id, final String value) {
+        if (mAutoFillRoots == null) {
+            return false;
+        }
+
+        int rootId = id;
+        for (int currentId = id; currentId != View.NO_ID;) {
+            final GeckoBundle bundle = mAutoFillNodes.get(currentId);
+            if (bundle == null) {
+                return false;
+            }
+            rootId = currentId;
+            currentId = bundle.getInt("parent", View.NO_ID);
+        }
+
+        if (DEBUG) {
+            Log.d(LOGTAG, "performAutoFill(" + id + ", " + value + ')');
+        }
+
+        final EventCallback callback = mAutoFillRoots.get(rootId);
+        if (callback == null) {
+            return false;
+        }
+
+        final GeckoBundle response = new GeckoBundle(1);
+        response.putString(String.valueOf(id), value);
+        callback.sendSuccess(response);
+        return true;
+    }
+
+    private void fireWindowChangedEvent(final int id) {
+        if (mView instanceof ViewParent) {
+            final AccessibilityEvent event = obtainEvent(
+                    AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED, id);
+            if (Build.VERSION.SDK_INT >= 19) {
+                event.setContentChangeTypes(AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE);
+            }
+            ((ViewParent) mView).requestSendAccessibilityEvent(mView, event);
+        }
+    }
+
+    /* package */ void addAutoFill(final GeckoBundle message, final EventCallback callback) {
+        if (!Settings.isEnabled()) {
+            return;
+        }
+
+        if (mAutoFillRoots == null) {
+            mAutoFillRoots = new SparseArray<>();
+            mAutoFillNodes = new SparseArray<>();
+        }
+
+        final int id = message.getInt("id");
+        if (DEBUG) {
+            Log.d(LOGTAG, "addAutoFill(" + id + ')');
+        }
+
+        mAutoFillRoots.append(id, callback);
+        mAutoFillNodes.append(id, message);
+        fireWindowChangedEvent(id);
+    }
+
+    /* package */ void clearAutoFill() {
+        if (mAutoFillRoots != null) {
+            if (DEBUG) {
+                Log.d(LOGTAG, "clearAutoFill()");
+            }
+            mAutoFillRoots = null;
+            mAutoFillNodes = null;
+            mAutoFillFocusedId = View.NO_ID;
+            fireWindowChangedEvent(View.NO_ID);
+        }
+    }
+
+    /* package */ void onAutoFillFocus(final GeckoBundle message) {
+        if (!Settings.isEnabled() || !(mView instanceof ViewParent) || mAutoFillNodes == null) {
+            return;
+        }
+
+        final int id;
+        if (message != null) {
+            id = message.getInt("id");
+            mAutoFillNodes.put(id, message);
+        } else {
+            id = View.NO_ID;
+        }
+
+        if (DEBUG) {
+            Log.d(LOGTAG, "onAutoFillFocus(" + id + ')');
+        }
+        if (mAutoFillFocusedId == id) {
+            return;
+        }
+        mAutoFillFocusedId = id;
+
+        // We already send "TYPE_VIEW_FOCUSED" in touch exploration mode,
+        // so in that case don't send it here.
+        if (!Settings.isTouchExplorationEnabled()) {
+            AccessibilityEvent event = obtainEvent(AccessibilityEvent.TYPE_VIEW_FOCUSED, id);
+            ((ViewParent) mView).requestSendAccessibilityEvent(mView, event);
+        }
+    }
+
+    /* package */ void onWindowFocus() {
+        // Auto-fill clients expect a state change event on focus.
+        if (Settings.isEnabled() && mView instanceof ViewParent) {
+            if (DEBUG) {
+                Log.d(LOGTAG, "onWindowFocus()");
+            }
+            final AccessibilityEvent event = obtainEvent(
+                    AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED, View.NO_ID);
+            ((ViewParent) mView).requestSendAccessibilityEvent(mView, event);
+        }
+    }
 }