Bug 1577003 - Handle Android autofill service in GeckoView class r=geckoview-reviewers,esawin
authorJames Willcox <snorp@snorp.net>
Fri, 04 Oct 2019 17:54:39 +0000
changeset 496373 58284841398d105f6b5742d5919a46848818e0d7
parent 496372 8be3fae908128e6aac63b72a795ffb19836b741c
child 496374 9964d7bfa61992d80f78215631fbb1df1a22df81
push id97206
push userjwillcox@mozilla.com
push dateFri, 04 Oct 2019 17:57:55 +0000
treeherderautoland@204658865fb7 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgeckoview-reviewers, esawin
bugs1577003
milestone71.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1577003 - Handle Android autofill service in GeckoView class r=geckoview-reviewers,esawin This also adds `GeckoView.setAutofillEnabled()`, which is enabled by default. Apps can disable this if they don't want to use the Android autofill service. This will allow them to set their own `AutofillDelegate` on the `GeckoSession` and provide their own autofill implementation. Differential Revision: https://phabricator.services.mozilla.com/D47483
mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/SessionLifecycleTest.kt
mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoView.java
--- a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/SessionLifecycleTest.kt
+++ b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/SessionLifecycleTest.kt
@@ -281,16 +281,18 @@ class SessionLifecycleTest : BaseSession
             }
         })
     }
 
     private fun testRestoreInstanceState(fromSession: GeckoSession?,
                                          ontoSession: GeckoSession?) =
             GeckoView(InstrumentationRegistry.getTargetContext()).apply {
                 id = 0
+                autofillEnabled = false
+
                 if (fromSession != null) {
                     setSession(fromSession)
                 }
 
                 val state = SparseArray<Parcelable>()
                 saveHierarchyState(state)
 
                 if (ontoSession !== fromSession) {
--- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoView.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoView.java
@@ -15,38 +15,42 @@ import org.mozilla.gecko.util.ThreadUtil
 import android.annotation.SuppressLint;
 import android.annotation.TargetApi;
 import android.app.Activity;
 import android.content.Context;
 import android.content.res.Configuration;
 import android.graphics.Bitmap;
 import android.graphics.Canvas;
 import android.graphics.Color;
+import android.graphics.Matrix;
 import android.graphics.Rect;
+import android.graphics.RectF;
 import android.graphics.Region;
 import android.os.Build;
 import android.os.Handler;
 import android.os.Parcel;
 import android.os.Parcelable;
 import android.support.annotation.AnyThread;
 import android.support.annotation.NonNull;
 import android.support.annotation.Nullable;
 import android.support.annotation.UiThread;
 import android.support.v4.view.ViewCompat;
+import android.text.InputType;
 import android.util.AttributeSet;
 import android.util.DisplayMetrics;
 import android.util.SparseArray;
 import android.util.TypedValue;
 import android.view.KeyEvent;
 import android.view.MotionEvent;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 import android.view.View;
 import android.view.ViewGroup;
 import android.view.ViewStructure;
+import android.view.autofill.AutofillManager;
 import android.view.autofill.AutofillValue;
 import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputConnection;
 import android.view.inputmethod.InputMethodManager;
 import android.widget.FrameLayout;
 
 @UiThread
 public class GeckoView extends FrameLayout {
@@ -56,17 +60,20 @@ public class GeckoView extends FrameLayo
     protected final @NonNull Display mDisplay = new Display();
     protected @Nullable GeckoSession mSession;
     private boolean mStateSaved;
 
     protected @Nullable SurfaceView mSurfaceView;
 
     private boolean mIsResettingFocus;
 
+    private boolean mAutofillEnabled = true;
+
     private GeckoSession.SelectionActionDelegate mSelectionActionDelegate;
+    private GeckoSession.AutofillDelegate mAutofillDelegate;
 
     private static class SavedState extends BaseSavedState {
         public final GeckoSession session;
 
         public SavedState(final Parcelable superState, final GeckoSession session) {
             super(superState);
             this.session = session;
         }
@@ -230,16 +237,18 @@ public class GeckoView extends FrameLayo
                                            ViewGroup.LayoutParams.MATCH_PARENT));
 
         mSurfaceView.getHolder().addCallback(mDisplay);
 
         final Activity activity = ActivityUtils.getActivityFromContext(getContext());
         if (activity != null) {
             mSelectionActionDelegate = new BasicSelectionActionDelegate(activity);
         }
+
+        mAutofillDelegate = new AndroidAutofillDelegate();
     }
 
     /**
      * Set a color to cover the display surface while a document is being shown. The color
      * is automatically cleared once the new document starts painting. Set to
      * Color.TRANSPARENT to undo the cover.
      *
      * @param color Cover color.
@@ -321,16 +330,20 @@ public class GeckoView extends FrameLayo
         if (mSession.getTextInput().getView() == this) {
             mSession.getTextInput().setView(null);
         }
 
         if (mSession.getSelectionActionDelegate() == mSelectionActionDelegate) {
             mSession.setSelectionActionDelegate(null);
         }
 
+        if (mSession.getAutofillDelegate() == mAutofillDelegate) {
+            mSession.setAutofillDelegate(null);
+        }
+
         if (isFocused()) {
             mSession.setFocused(false);
         }
         mSession = null;
         return session;
     }
 
     /**
@@ -393,16 +406,20 @@ public class GeckoView extends FrameLayo
         if (session.getAccessibility().getView() == null) {
             session.getAccessibility().setView(this);
         }
 
         if (session.getSelectionActionDelegate() == null && mSelectionActionDelegate != null) {
             session.setSelectionActionDelegate(mSelectionActionDelegate);
         }
 
+        if (mAutofillEnabled) {
+            session.setAutofillDelegate(mAutofillDelegate);
+        }
+
         if (isFocused()) {
             session.setFocused(true);
         }
     }
 
     @AnyThread
     public @Nullable GeckoSession getSession() {
         return mSession;
@@ -708,18 +725,109 @@ public class GeckoView extends FrameLayo
         return mSession.getPanZoomController().onMotionEvent(event);
     }
 
     @Override
     public void onProvideAutofillVirtualStructure(final ViewStructure structure,
                                                   final int flags) {
         super.onProvideAutofillVirtualStructure(structure, flags);
 
-        if (mSession != null) {
-            mSession.provideAutofillVirtualStructure(this, structure, flags);
+        if (mSession == null) {
+            return;
+        }
+
+        final AutofillElement root = mSession.getAutofillElements();
+        fillViewStructure(root, structure, flags);
+    }
+
+    @TargetApi(23)
+    private void fillViewStructure(final AutofillElement element, final ViewStructure structure, final int flags) {
+        if (Build.VERSION.SDK_INT >= 26) {
+            structure.setAutofillId(getAutofillId(), element.id);
+            structure.setWebDomain(element.domain);
+        }
+
+        structure.setId(element.id, null, null, null);
+        structure.setDimens(0, 0, 0, 0, element.dimensions.width(), element.dimensions.height());
+
+        if (Build.VERSION.SDK_INT >= 26) {
+            final ViewStructure.HtmlInfo.Builder htmlBuilder = structure.newHtmlInfoBuilder(element.tag);
+            for (final String key : element.attributes.keySet()) {
+                htmlBuilder.addAttribute(key, String.valueOf(element.attributes.get(key)));
+            }
+
+            structure.setHtmlInfo(htmlBuilder.build());
+        }
+
+        int childCount = 0;
+        for (final AutofillElement child : element.children) {
+            final ViewStructure childStructure = structure.newChild(childCount);
+            fillViewStructure(child, childStructure, flags);
+            childCount++;
+        }
+
+        structure.setChildCount(childCount);
+
+        switch (element.tag) {
+            case "input":
+            case "textarea":
+                structure.setClassName("android.widget.EditText");
+                structure.setEnabled(element.enabled);
+                structure.setFocusable(element.focusable);
+                structure.setFocused(element.focused);
+                structure.setVisibility(View.VISIBLE);
+
+                if (Build.VERSION.SDK_INT >= 26) {
+                    structure.setAutofillType(View.AUTOFILL_TYPE_TEXT);
+                }
+                break;
+            default:
+                if (childCount > 0) {
+                    structure.setClassName("android.view.ViewGroup");
+                } else {
+                    structure.setClassName("android.view.View");
+                }
+                break;
+        }
+
+        if (Build.VERSION.SDK_INT >= 26 && "input".equals(element.tag)) {
+            // LastPass will fill password to the field that setAutofillHints is unset and setInputType is set.
+            switch (element.hint) {
+                case AutofillElement.HINT_EMAIL_ADDRESS:
+                    structure.setAutofillHints(new String[] { View.AUTOFILL_HINT_EMAIL_ADDRESS });
+                    structure.setInputType(InputType.TYPE_CLASS_TEXT |
+                            InputType.TYPE_TEXT_VARIATION_EMAIL_ADDRESS);
+                    break;
+                case AutofillElement.HINT_PASSWORD:
+                    structure.setAutofillHints(new String[] { View.AUTOFILL_HINT_PASSWORD });
+                    structure.setInputType(InputType.TYPE_CLASS_TEXT |
+                            InputType.TYPE_TEXT_VARIATION_WEB_PASSWORD);
+                    break;
+                case AutofillElement.HINT_URL:
+                    structure.setInputType(InputType.TYPE_CLASS_TEXT |
+                            InputType.TYPE_TEXT_VARIATION_URI);
+                    break;
+                case AutofillElement.HINT_USERNAME:
+                    structure.setAutofillHints(new String[] { View.AUTOFILL_HINT_USERNAME });
+                    structure.setInputType(InputType.TYPE_CLASS_TEXT |
+                            InputType.TYPE_TEXT_VARIATION_WEB_EDIT_TEXT);
+                    break;
+            }
+
+            switch (element.inputType) {
+                case AutofillElement.INPUT_TYPE_NUMBER:
+                    structure.setInputType(InputType.TYPE_CLASS_NUMBER);
+                    break;
+                case AutofillElement.INPUT_TYPE_PHONE:
+                    structure.setAutofillHints(new String[] { View.AUTOFILL_HINT_PHONE });
+                    structure.setInputType(InputType.TYPE_CLASS_PHONE);
+                    break;
+                default:
+                    break;
+            }
         }
     }
 
     @Override
     @TargetApi(26)
     public void autofill(@NonNull final SparseArray<AutofillValue> values) {
         super.autofill(values);
 
@@ -745,9 +853,111 @@ public class GeckoView extends FrameLayo
      *
      * @return A {@link GeckoResult} that completes with a {@link Bitmap} containing
      * the pixels and size information of the currently visible rendered web page.
      */
     @UiThread
     public @NonNull GeckoResult<Bitmap> capturePixels() {
         return mDisplay.capturePixels();
     }
+
+    /**
+     * Sets whether or not this View participates in Android autofill.
+     *
+     * When enabled, this will set an {@link GeckoSession.AutofillDelegate} on the
+     * {@link GeckoSession} for this instance.
+     *
+     * @param enabled Whether or not Android autofill is enabled for this view.
+     */
+    @TargetApi(26)
+    public void setAutofillEnabled(final boolean enabled) {
+        mAutofillEnabled = enabled;
+
+        if (mSession != null) {
+            if (!enabled && mSession.getAutofillDelegate() == mAutofillDelegate) {
+                mSession.setAutofillDelegate(null);
+            } else if (enabled) {
+                mSession.setAutofillDelegate(mAutofillDelegate);
+            }
+        }
+    }
+
+    /**
+     * @return Whether or not Android autofill is enabled for this view.
+     */
+    @TargetApi(26)
+    public boolean getAutofillEnabled() {
+        return mAutofillEnabled;
+    }
+
+    private class AndroidAutofillDelegate implements GeckoSession.AutofillDelegate {
+
+        private AutofillElement findElementWithId(final AutofillElement root, final int id) {
+            if (root.id == id) {
+                return root;
+            }
+
+            for (AutofillElement child : root.children) {
+                final AutofillElement found = findElementWithId(child, id);
+                if (found != null) {
+                    return found;
+                }
+            }
+
+            return null;
+        }
+
+        private Rect displayRectForId(@NonNull final GeckoSession session,
+                                      @NonNull final int virtualId,
+                                      @Nullable final View view) {
+            final AutofillElement structure = session.getAutofillElements();
+            final AutofillElement element = findElementWithId(structure, virtualId);
+
+            if (element == null) {
+                return new Rect(0, 0, 0, 0);
+            }
+
+            final Matrix matrix = new Matrix();
+            final RectF rectF = new RectF(element.dimensions);
+            session.getPageToScreenMatrix(matrix);
+            matrix.mapRect(rectF);
+
+            final Rect screenRect = new Rect();
+            rectF.roundOut(screenRect);
+            return screenRect;
+        }
+
+        @Override
+        public void onAutofill(@NonNull final GeckoSession session,
+                               @GeckoSession.AutofillNotification final int notification,
+                               final int virtualId) {
+            ThreadUtils.assertOnUiThread();
+            if (Build.VERSION.SDK_INT < 26) {
+                return;
+            }
+
+            final AutofillManager manager =
+                    GeckoView.this.getContext().getSystemService(AutofillManager.class);
+            if (manager == null) {
+                return;
+            }
+
+            switch (notification) {
+                case AUTO_FILL_NOTIFY_STARTED:
+                    // This line seems necessary for auto-fill to work on the initial page.
+                    manager.cancel();
+                    break;
+                case AUTO_FILL_NOTIFY_COMMITTED:
+                    manager.commit();
+                    break;
+                case AUTO_FILL_NOTIFY_CANCELED:
+                    manager.cancel();
+                    break;
+                case AUTO_FILL_NOTIFY_VIEW_ENTERED:
+                    manager.notifyViewEntered(GeckoView.this, virtualId, displayRectForId(session, virtualId, GeckoView.this));
+                    break;
+                case AUTO_FILL_NOTIFY_VIEW_EXITED:
+                    manager.notifyViewExited(GeckoView.this, virtualId);
+                    break;
+            }
+        }
+    }
 }