Bug 777560 - Add Jelly Bean accessibility features to Java layer. r=mbrubeck
authorEitan Isaacson <eitan@monotonous.org>
Mon, 20 Aug 2012 18:29:22 -0400
changeset 102923 3e0e57eb237aac0a5f9c59927c46d7d439e59e7b
parent 102922 eb4392d9bc48dec32c8350ff7a1dbf8a3d9229a8
child 102924 a295ff4319fbafd50072a69fd233b95cdb515d2d
push id13721
push usereisaacson@mozilla.com
push dateTue, 21 Aug 2012 13:47:07 +0000
treeherdermozilla-inbound@aaadec548e0b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmbrubeck
bugs777560
milestone17.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 777560 - Add Jelly Bean accessibility features to Java layer. r=mbrubeck
mobile/android/base/GeckoAccessibility.java
mobile/android/base/GeckoApp.java
mobile/android/base/Makefile.in
mobile/android/base/gfx/LayerView.java
new file mode 100644
--- /dev/null
+++ b/mobile/android/base/GeckoAccessibility.java
@@ -0,0 +1,245 @@
+/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko;
+
+import org.mozilla.gecko.GeckoApp;
+import org.mozilla.gecko.gfx.LayerView;
+
+import android.view.accessibility.*;
+import android.view.View;
+import android.util.Log;
+import android.os.Build;
+import android.os.Bundle;
+import android.content.Context;
+import android.graphics.Rect;
+
+import org.json.*;
+
+public class GeckoAccessibility {
+    private static final String LOGTAG = "GeckoAccessibility";
+    private static final int VIRTUAL_CURSOR_PREVIOUS = 1;
+    private static final int VIRTUAL_CURSOR_POSITION = 2;
+    private static final int VIRTUAL_CURSOR_NEXT = 3;
+
+    private static JSONObject mEventMessage = null;
+    private static AccessibilityNodeInfo mVirtualCursorNode = null;
+
+    public static void updateAccessibilitySettings () {
+        GeckoAppShell.getHandler().post(new Runnable() {
+                public void run() {
+                    JSONObject ret = new JSONObject();
+                    AccessibilityManager accessibilityManager =
+                        (AccessibilityManager) GeckoApp.mAppContext.getSystemService(Context.ACCESSIBILITY_SERVICE);
+                    try {
+                        ret.put("enabled", accessibilityManager.isEnabled());
+                        if (Build.VERSION.SDK_INT >= 14) { // Build.VERSION_CODES.ICE_CREAM_SANDWICH
+                            ret.put("exploreByTouch", accessibilityManager.isTouchExplorationEnabled());
+                        } else {
+                            ret.put("exploreByTouch", false);
+                        }
+                    } catch (Exception ex) {
+                        Log.e(LOGTAG, "Error building JSON arguments for Accessibility:Settings:", ex);
+                    }
+                    GeckoAppShell.sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:Settings",
+                                                                                   ret.toString()));
+                }
+            });
+    }
+
+    private static void populateEventFromJSON (AccessibilityEvent event, JSONObject message) {
+        final JSONArray textArray = message.optJSONArray("text");
+        if (textArray != null) {
+            for (int i = 0; i < textArray.length(); i++)
+                event.getText().add(textArray.optString(i));
+        }
+
+        event.setContentDescription(message.optString("description"));
+        event.setEnabled(message.optBoolean("enabled", true));
+        event.setChecked(message.optBoolean("checked"));
+        event.setPassword(message.optBoolean("password"));
+        event.setAddedCount(message.optInt("addedCount", -1));
+        event.setRemovedCount(message.optInt("removedCount", -1));
+        event.setFromIndex(message.optInt("fromIndex", -1));
+        event.setItemCount(message.optInt("itemCount", -1));
+        event.setCurrentItemIndex(message.optInt("currentItemIndex", -1));
+        event.setBeforeText(message.optString("beforeText"));
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
+            event.setToIndex(message.optInt("toIndex", -1));
+            event.setScrollable(message.optBoolean("scrollable"));
+            event.setScrollX(message.optInt("scrollX", -1));
+            event.setScrollY(message.optInt("scrollY", -1));
+        }
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH_MR1) {
+            event.setMaxScrollX(message.optInt("maxScrollX", -1));
+            event.setMaxScrollY(message.optInt("maxScrollY", -1));
+        }
+    }
+
+    private static void sendDirectAccessibilityEvent(int eventType, JSONObject message) {
+        final AccessibilityEvent accEvent = AccessibilityEvent.obtain(eventType);
+        accEvent.setClassName(LayerView.class.getName());
+        accEvent.setPackageName(GeckoApp.mAppContext.getPackageName());
+        populateEventFromJSON(accEvent, message);
+        AccessibilityManager accessibilityManager =
+            (AccessibilityManager) GeckoApp.mAppContext.getSystemService(Context.ACCESSIBILITY_SERVICE);
+        try {
+            accessibilityManager.sendAccessibilityEvent(accEvent);
+        } catch (IllegalStateException e) {
+            // Accessibility is off.
+        }
+    }
+
+    public static void sendAccessibilityEvent (final JSONObject message) {
+        final int eventType = message.optInt("eventType", -1);
+        if (eventType < 0) {
+            Log.e(LOGTAG, "No accessibility event type provided");
+            return;
+        }
+
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN) {
+            // Before Jelly Bean we send events directly from here while spoofing the source by setting
+            // the package and class name manually.
+            GeckoAppShell.getHandler().post(new Runnable() {
+                    public void run() {
+                        sendDirectAccessibilityEvent(eventType, message);
+                }
+            });
+        } else {
+            // In Jelly Bean we populate an AccessibilityNodeInfo with the minimal amount of data to have
+            // it work with TalkBack.
+            final LayerView view = GeckoApp.mAppContext.getLayerView();
+            if (mVirtualCursorNode == null)
+                mVirtualCursorNode = AccessibilityNodeInfo.obtain(view, VIRTUAL_CURSOR_POSITION);
+            mVirtualCursorNode.setEnabled(message.optBoolean("enabled", true));
+            mVirtualCursorNode.setChecked(message.optBoolean("checked"));
+            mVirtualCursorNode.setPassword(message.optBoolean("password"));
+            JSONObject bounds = message.optJSONObject("bounds");
+            if (bounds != null) {
+                Rect relativeBounds = new Rect(bounds.optInt("left"), bounds.optInt("top"),
+                                               bounds.optInt("right"), bounds.optInt("bottom"));
+                mVirtualCursorNode.setBoundsInParent(relativeBounds);
+                int[] locationOnScreen = new int[2];
+                view.getLocationOnScreen(locationOnScreen);
+                Rect screenBounds = new Rect(relativeBounds);
+                screenBounds.offset(locationOnScreen[0], locationOnScreen[1]);
+                mVirtualCursorNode.setBoundsInScreen(screenBounds);
+            }
+
+            // Store the JSON message and use it to populate the event later in the code path.
+            mEventMessage = message;
+            GeckoApp.mAppContext.mMainHandler.post(new Runnable() {
+                    public void run() {
+                        // If this is an accessibility focus, a lot of internal voodoo happens so we perform an
+                        // accessibility focus action on the view, and it in turn sends the right events.
+                        switch (eventType) {
+                        case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED:
+                            view.performAccessibilityAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS, null);
+                            break;
+                        case AccessibilityEvent.TYPE_ANNOUNCEMENT:
+                            sendDirectAccessibilityEvent(eventType, message);
+                            break;
+                        default:
+                            view.sendAccessibilityEvent(eventType);
+                            break;
+                        }
+                    }
+                });
+
+        }
+    }
+
+    public static void setDelegate(LayerView layerview) {
+        // Only use this delegate in Jelly Bean.
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN) {
+            layerview.setAccessibilityDelegate(new GeckoAccessibilityDelegate());
+            layerview.setImportantForAccessibility(View.IMPORTANT_FOR_ACCESSIBILITY_YES);
+        }
+    }
+
+    public static class GeckoAccessibilityDelegate extends View.AccessibilityDelegate {
+        AccessibilityNodeProvider mAccessibilityNodeProvider;
+
+        @Override
+        public void onPopulateAccessibilityEvent (View host, AccessibilityEvent event) {
+            super.onPopulateAccessibilityEvent(host, event);
+            if (mEventMessage != null)
+                populateEventFromJSON(event, mEventMessage);
+            mEventMessage = null;
+            // No matter where the a11y focus is requested, we always force it back to the current vc position.
+            event.setSource(host, VIRTUAL_CURSOR_POSITION);
+        }
+
+        @Override
+        public AccessibilityNodeProvider getAccessibilityNodeProvider(final View host) {
+            if (mAccessibilityNodeProvider == null)
+                // The accessibility node structure for web content consists of 3 LayerView child nodes:
+                // 1. VIRTUAL_CURSOR_PREVIOUS: Represents the virtual cursor position that is previous to the
+                // current one.
+                // 2. VIRTUAL_CURSOR_POSITION: Represents the current position of the virtual cursor.
+                // 3. VIRTUAL_CURSOR_NEXT: Represents the next virtual cursor position.
+                mAccessibilityNodeProvider = new AccessibilityNodeProvider() {
+                        @Override
+                        public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualDescendantId) {
+                            AccessibilityNodeInfo info = (virtualDescendantId == VIRTUAL_CURSOR_POSITION && mVirtualCursorNode != null) ?
+                                AccessibilityNodeInfo.obtain(mVirtualCursorNode) :
+                                AccessibilityNodeInfo.obtain(host, virtualDescendantId);
+
+
+                            switch (virtualDescendantId) {
+                            case View.NO_ID:
+                                // This is the parent LayerView node, populate it with children.
+                                onInitializeAccessibilityNodeInfo(host, info);
+                                info.addChild(host, VIRTUAL_CURSOR_PREVIOUS);
+                                info.addChild(host, VIRTUAL_CURSOR_POSITION);
+                                info.addChild(host, VIRTUAL_CURSOR_NEXT);
+                                break;
+                            default:
+                                info.setParent(host);
+                                info.setSource(host, virtualDescendantId);
+                                info.setVisibleToUser(true);
+                                info.setPackageName(GeckoApp.mAppContext.getPackageName());
+                                info.setClassName(host.getClass().getName());
+                                info.addAction(AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS);
+                                info.addAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS);
+                                info.addAction(AccessibilityNodeInfo.ACTION_CLICK);
+                                break;
+                            }
+
+                            return info;
+                        }
+
+                        @Override
+                        public boolean performAction (int virtualViewId, int action, Bundle arguments) {
+                            if (action == AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS) {
+                                // The accessibility focus is permanently on the middle node, VIRTUAL_CURSOR_POSITION.
+                                // When accessibility focus is requested on one of its siblings we move the virtual cursor
+                                // either forward or backward depending on which sibling was selected.
+
+                                switch (virtualViewId) {
+                                case VIRTUAL_CURSOR_PREVIOUS:
+                                    GeckoAppShell.
+                                        sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:PreviousObject", null));
+                                    return true;
+                                case VIRTUAL_CURSOR_POSITION:
+                                    GeckoAppShell.
+                                        sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:CurrentObject", null));
+                                    return true;
+                                case VIRTUAL_CURSOR_NEXT:
+                                    GeckoAppShell.
+                                        sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:NextObject", null));
+                                    return true;
+                                default:
+                                    break;
+                                }
+                            }
+                            return host.performAccessibilityAction(action, arguments);
+                        }
+                    };
+
+            return mAccessibilityNodeProvider;
+        }
+    }
+}
--- a/mobile/android/base/GeckoApp.java
+++ b/mobile/android/base/GeckoApp.java
@@ -10,16 +10,17 @@ import org.mozilla.gecko.gfx.Layer;
 import org.mozilla.gecko.gfx.LayerView;
 import org.mozilla.gecko.gfx.PluginLayer;
 import org.mozilla.gecko.gfx.PointUtils;
 import org.mozilla.gecko.ui.PanZoomController;
 import org.mozilla.gecko.util.GeckoAsyncTask;
 import org.mozilla.gecko.util.GeckoBackgroundThread;
 import org.mozilla.gecko.util.GeckoEventListener;
 import org.mozilla.gecko.util.GeckoEventResponder;
+import org.mozilla.gecko.GeckoAccessibility;
 
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
 import android.app.Activity;
 import android.app.AlertDialog;
 import android.content.Context;
@@ -67,16 +68,17 @@ import android.view.MenuItem;
 import android.view.MotionEvent;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 import android.view.View;
 import android.view.ViewConfiguration;
 import android.view.ViewGroup;
 import android.view.Window;
 import android.view.WindowManager;
+
 import android.view.accessibility.AccessibilityEvent;
 import android.view.accessibility.AccessibilityManager;
 import android.widget.AbsoluteLayout;
 import android.widget.FrameLayout;
 import android.widget.LinearLayout;
 import android.widget.ListView;
 import android.widget.RelativeLayout;
 import android.widget.ScrollView;
@@ -1029,58 +1031,19 @@ abstract public class GeckoApp
                         GeckoAppShell.getHandler().post(new Runnable() {
                             public void run() {
                                 BrowserDB.addBookmark(GeckoApp.mAppContext.getContentResolver(), title, url);
                             }
                         });
                     }
                 });
             } else if (event.equals("Accessibility:Event")) {
-                final AccessibilityEvent accEvent = AccessibilityEvent.obtain(message.getInt("eventType"));
-                accEvent.setClassName(LayerView.class.getName());
-                accEvent.setPackageName(mAppContext.getPackageName());
-
-                final JSONArray text = message.getJSONArray("text");
-                for (int i = 0; i < text.length(); i++)
-                    accEvent.getText().add(text.getString(i));
-
-                accEvent.setContentDescription(message.optString("description"));
-                accEvent.setEnabled(message.optBoolean("enabled", true));
-                accEvent.setChecked(message.optBoolean("checked"));
-                accEvent.setPassword(message.optBoolean("password"));
-                accEvent.setAddedCount(message.optInt("addedCount", -1));
-                accEvent.setRemovedCount(message.optInt("removedCount", -1));
-                accEvent.setFromIndex(message.optInt("fromIndex", -1));
-                accEvent.setItemCount(message.optInt("itemCount", -1));
-                accEvent.setCurrentItemIndex(message.optInt("currentItemIndex", -1));
-                accEvent.setBeforeText(message.optString("beforeText"));
-                if (Build.VERSION.SDK_INT >= 14) { // Build.VERSION_CODES.ICE_CREAM_SANDWICH
-                    accEvent.setToIndex(message.optInt("toIndex", -1));
-                    accEvent.setScrollable(message.optBoolean("scrollable"));
-                    accEvent.setScrollX(message.optInt("scrollX", -1));
-                    accEvent.setScrollY(message.optInt("scrollY", -1));
-                }
-                if (Build.VERSION.SDK_INT >= 15) { // Build.VERSION_CODES.ICE_CREAM_SANDWICH_MR1
-                    AccessibilityCompat.setMaxScrollX(accEvent, message.optInt("maxScrollX", -1));
-                    AccessibilityCompat.setMaxScrollY(accEvent, message.optInt("maxScrollY", -1));
-                }
-
-                mMainHandler.post(new Runnable() {
-                    public void run() {
-                        AccessibilityManager accessibilityManager =
-                            (AccessibilityManager) mAppContext.getSystemService(Context.ACCESSIBILITY_SERVICE);
-                        try {
-                            accessibilityManager.sendAccessibilityEvent(accEvent);
-                        } catch (IllegalStateException e) {
-                            // Accessibility is off.
-                        }
-                    }
-                });
+                GeckoAccessibility.sendAccessibilityEvent(message);
             } else if (event.equals("Accessibility:Ready")) {
-                updateAccessibilitySettings();
+                GeckoAccessibility.updateAccessibilitySettings();
             } else if (event.equals("Shortcut:Remove")) {
                 final String url = message.getString("url");
                 final String origin = message.getString("origin");
                 final String title = message.getString("title");
                 final String type = message.getString("shortcutType");
                 GeckoAppShell.removeShortcut(title, url, origin, type);
             } else if (event.equals("WebApps:Open")) {
                 String url = message.getString("uri");
@@ -1407,38 +1370,16 @@ abstract public class GeckoApp
 
                 PluginLayer layer = (PluginLayer) tab.removePluginLayer(view);
                 if (layer != null) {
                     layer.destroy();
                 }
             }
         });
     }
-
-    public void updateAccessibilitySettings () {
-        mMainHandler.post(new Runnable() {
-                public void run() {
-                    JSONObject ret = new JSONObject();
-                    AccessibilityManager accessibilityManager =
-                        (AccessibilityManager) mAppContext.getSystemService(Context.ACCESSIBILITY_SERVICE);
-                    try {
-                        ret.put("enabled", accessibilityManager.isEnabled());
-                        if (Build.VERSION.SDK_INT >= 14) { // Build.VERSION_CODES.ICE_CREAM_SANDWICH
-                            ret.put("exploreByTouch", accessibilityManager.isTouchExplorationEnabled());
-                        } else {
-                            ret.put("exploreByTouch", false);
-                        }
-                    } catch (Exception ex) {
-                        Log.e(LOGTAG, "Error building JSON arguments for Accessibility:Settings:", ex);
-                    }
-                    GeckoAppShell.sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:Settings",
-                                                                                   ret.toString()));
-                }
-            });
-    }
     
     private void hidePluginLayer(Layer layer) {
         LayerView layerView = mLayerView;
         layerView.removeLayer(layer);
         layerView.requestRender();
     }
 
     private void showPluginLayer(Layer layer) {
@@ -2009,17 +1950,17 @@ abstract public class GeckoApp
         int newOrientation = getResources().getConfiguration().orientation;
 
         if (mOrientation != newOrientation) {
             mOrientation = newOrientation;
             refreshChrome();
         }
 
         // User may have enabled/disabled accessibility.
-        updateAccessibilitySettings();
+        GeckoAccessibility.updateAccessibilitySettings();
 
         GeckoBackgroundThread.getHandler().post(new Runnable() {
             public void run() {
                 SharedPreferences prefs =
                     GeckoApp.mAppContext.getSharedPreferences(GeckoApp.PREFS_NAME, 0);
                 SharedPreferences.Editor editor = prefs.edit();
                 editor.putBoolean(GeckoApp.PREFS_WAS_STOPPED, false);
                 editor.commit();
@@ -2871,50 +2812,9 @@ abstract public class GeckoApp
 
         if (currentThreadId != expectedThreadId) {
             throw new IllegalThreadStateException("Expected thread " + expectedThreadId + " (\""
                                                   + expectedThread.getName()
                                                   + "\"), but running on thread " + currentThreadId
                                                   + " (\"" + currentThread.getName() + ")");
         }
     }
-
-    // SDK version 15 accessibility methods retrieved through reflection.
-    private static class AccessibilityCompat {
-        private static boolean mInitialized = false;
-        private static Method mAccessibilityEvent_setMaxScrollX = null;
-        private static Method mAccessibilityEvent_setMaxScrollY = null;
-
-        private static void initialize () {
-            try {
-                mAccessibilityEvent_setMaxScrollX =
-                    AccessibilityEvent.class.getMethod("setMaxScrollX", int.class);
-                mAccessibilityEvent_setMaxScrollY =
-                    AccessibilityEvent.class.getMethod("setMaxScrollY", int.class);
-            } catch (NoSuchMethodException e) {
-                Log.e(LOGTAG, "Error initializing AccessibilityCompat", e);
-            }
-            mInitialized = true;
-        }
-
-        public static void setMaxScrollX (AccessibilityEvent event, int maxScrollX) {
-            if (!mInitialized)
-                initialize();
-            try {
-                if (mAccessibilityEvent_setMaxScrollX != null)
-                    mAccessibilityEvent_setMaxScrollX.invoke(event, maxScrollX);
-            } catch (Exception e) {
-                Log.e(LOGTAG, "Error invoking AccessibilityEvent.setMaxScrollX", e);
-            }
-        }
-
-        public static void setMaxScrollY (AccessibilityEvent event, int maxScrollY) {
-            if (!mInitialized)
-                initialize();
-            try {
-                if (mAccessibilityEvent_setMaxScrollY != null)
-                    mAccessibilityEvent_setMaxScrollY.invoke(event, maxScrollY);
-            } catch (Exception e) {
-                Log.e(LOGTAG, "Error invoking AccessibilityEvent.setMaxScrollY", e);
-            }
-        }
-    }
 }
--- a/mobile/android/base/Makefile.in
+++ b/mobile/android/base/Makefile.in
@@ -60,16 +60,17 @@ FENNEC_JAVA_FILES = \
   DoorHangerPopup.java \
   Favicons.java \
   FilePickerResultHandler.java \
   FilePickerResultHandlerSync.java \
   FindInPageBar.java \
   FlowLayout.java \
   FontSizePreference.java \
   FormAssistPopup.java \
+  GeckoAccessibility.java \
   GeckoApplication.java \
   GeckoApp.java \
   GeckoAppShell.java \
   GeckoBatteryManager.java \
   GeckoConnectivityReceiver.java \
   GeckoEvent.java \
   GeckoHalDefines.java \
   GeckoInputConnection.java \
--- a/mobile/android/base/gfx/LayerView.java
+++ b/mobile/android/base/gfx/LayerView.java
@@ -4,16 +4,17 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.gfx;
 
 import org.mozilla.gecko.GeckoApp;
 import org.mozilla.gecko.R;
 import org.mozilla.gecko.ZoomConstraints;
 import org.mozilla.gecko.util.EventDispatcher;
+import org.mozilla.gecko.GeckoAccessibility;
 
 import android.content.Context;
 import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
 import android.graphics.Color;
 import android.graphics.PixelFormat;
 import android.graphics.PointF;
 import android.graphics.SurfaceTexture;
@@ -108,16 +109,18 @@ public class LayerView extends FrameLayo
         mLayerClient = new GeckoLayerClient(getContext(), this, eventDispatcher);
 
         mTouchEventHandler = new TouchEventHandler(getContext(), this, mLayerClient);
         mRenderer = new LayerRenderer(this);
         mInputConnectionHandler = null;
 
         setFocusable(true);
         setFocusableInTouchMode(true);
+
+        GeckoAccessibility.setDelegate(this);
     }
 
     public void destroy() {
         if (mLayerClient != null) {
             mLayerClient.destroy();
         }
     }