Bug 1615858 - Propagate historical touch data from Java to C++. r=kats,geckoview-reviewers,agi
authorMarkus Stange <mstange.moz@gmail.com>
Tue, 03 Nov 2020 19:48:43 +0000
changeset 555675 b4f668a31f18226c7183d538d55da73a8c56e55b
parent 555674 4c5fdaab5f27e72bfddeba2d17d62589c6cd92b3
child 555676 bf8313268c2024692d8263f04b7de3dd4450700f
push id37919
push userapavel@mozilla.com
push dateWed, 04 Nov 2020 04:15:21 +0000
treeherdermozilla-central@ad4d5535cb4f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskats, geckoview-reviewers, agi
bugs1615858
milestone84.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1615858 - Propagate historical touch data from Java to C++. r=kats,geckoview-reviewers,agi Differential Revision: https://phabricator.services.mozilla.com/D95650
mobile/android/geckoview/src/main/java/org/mozilla/geckoview/PanZoomController.java
widget/android/nsWindow.cpp
--- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/PanZoomController.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/PanZoomController.java
@@ -85,28 +85,110 @@ public class PanZoomController {
     public static final int INPUT_RESULT_HANDLED_CONTENT = 2;
 
     private SynthesizedEventState mPointerState;
 
     private ArrayList<Pair<Integer, MotionEvent>> mQueuedEvents;
 
     private boolean mSynthesizedEvent = false;
 
+    @WrapForJNI
+    private static class MotionEventData {
+        public final int action;
+        public final int actionIndex;
+        public final long time;
+        public final int metaState;
+        public final int pointerId[];
+        public final int historySize;
+        public final long historicalTime[];
+        public final float historicalX[];
+        public final float historicalY[];
+        public final float historicalOrientation[];
+        public final float historicalPressure[];
+        public final float historicalToolMajor[];
+        public final float historicalToolMinor[];
+        public final float x[];
+        public final float y[];
+        public final float orientation[];
+        public final float pressure[];
+        public final float toolMajor[];
+        public final float toolMinor[];
+
+        public MotionEventData(final MotionEvent event) {
+            final int count = event.getPointerCount();
+            action = event.getActionMasked();
+            actionIndex = event.getActionIndex();
+            time = event.getEventTime();
+            metaState = event.getMetaState();
+            historySize = event.getHistorySize();
+            historicalTime = new long[historySize];
+            historicalX = new float[historySize * count];
+            historicalY = new float[historySize * count];
+            historicalOrientation = new float[historySize * count];
+            historicalPressure = new float[historySize * count];
+            historicalToolMajor = new float[historySize * count];
+            historicalToolMinor = new float[historySize * count];
+            pointerId = new int[count];
+            x = new float[count];
+            y = new float[count];
+            orientation = new float[count];
+            pressure = new float[count];
+            toolMajor = new float[count];
+            toolMinor = new float[count];
+
+
+            for (int historyIndex = 0; historyIndex < historySize; historyIndex++) {
+                historicalTime[historyIndex] = event.getHistoricalEventTime(historyIndex);
+            }
+
+            final MotionEvent.PointerCoords coords = new MotionEvent.PointerCoords();
+            for (int i = 0; i < count; i++) {
+                pointerId[i] = event.getPointerId(i);
+
+                for (int historyIndex = 0; historyIndex < historySize; historyIndex++) {
+                    event.getHistoricalPointerCoords(i, historyIndex, coords);
+
+                    int historicalI = historyIndex * count + i;
+                    historicalX[historicalI] = coords.x;
+                    historicalY[historicalI] = coords.y;
+
+                    historicalOrientation[historicalI] = coords.orientation;
+                    historicalPressure[historicalI] = coords.pressure;
+
+                    // If we are converting to CSS pixels, we should adjust the radii as well.
+                    historicalToolMajor[historicalI] = coords.toolMajor;
+                    historicalToolMinor[historicalI] = coords.toolMinor;
+                }
+
+                event.getPointerCoords(i, coords);
+
+                x[i] = coords.x;
+                y[i] = coords.y;
+
+                orientation[i] = coords.orientation;
+                pressure[i] = coords.pressure;
+
+                // If we are converting to CSS pixels, we should adjust the radii as well.
+                toolMajor[i] = coords.toolMajor;
+                toolMinor[i] = coords.toolMinor;
+            }
+        }
+    }
+
     /* package */ final class NativeProvider extends JNIObject {
         @Override // JNIObject
         protected void disposeNative() {
             // Disposal happens in native code.
             throw new UnsupportedOperationException();
         }
 
         @WrapForJNI(calledFrom = "ui")
         private native void handleMotionEvent(
-               int action, int actionIndex, long time, int metaState,  float screenX, float screenY,
-               int pointerId[], float x[], float y[], float orientation[], float pressure[],
-               float toolMajor[], float toolMinor[], GeckoResult<Integer> result);
+               MotionEventData eventData, float screenX, float screenY,
+               GeckoResult<Integer> result);
 
         @WrapForJNI(calledFrom = "ui")
         private native @InputResult int handleScrollEvent(
                 long time, int metaState,
                 float x, float y,
                 float hScroll, float vScroll);
 
         @WrapForJNI(calledFrom = "ui")
@@ -159,65 +241,38 @@ public class PanZoomController {
             mQueuedEvents.add(new Pair<>(EVENT_SOURCE_MOTION, event));
             if (result != null) {
                 result.complete(INPUT_RESULT_HANDLED);
             }
             return;
         }
 
         final int action = event.getActionMasked();
-        final int count = event.getPointerCount();
 
         if (action == MotionEvent.ACTION_DOWN) {
             mLastDownTime = event.getDownTime();
         } else if (mLastDownTime != event.getDownTime()) {
             if (result != null) {
                 result.complete(INPUT_RESULT_UNHANDLED);
             }
             return;
         }
 
-        final int[] pointerId = new int[count];
-        final float[] x = new float[count];
-        final float[] y = new float[count];
-        final float[] orientation = new float[count];
-        final float[] pressure = new float[count];
-        final float[] toolMajor = new float[count];
-        final float[] toolMinor = new float[count];
-
-        final MotionEvent.PointerCoords coords = new MotionEvent.PointerCoords();
-
-        for (int i = 0; i < count; i++) {
-            pointerId[i] = event.getPointerId(i);
-            event.getPointerCoords(i, coords);
-
-            x[i] = coords.x;
-            y[i] = coords.y;
-
-            orientation[i] = coords.orientation;
-            pressure[i] = coords.pressure;
-
-            // If we are converting to CSS pixels, we should adjust the radii as well.
-            toolMajor[i] = coords.toolMajor;
-            toolMinor[i] = coords.toolMinor;
-        }
-
         final float screenX = event.getRawX() - event.getX();
         final float screenY = event.getRawY() - event.getY();
 
         // Take this opportunity to update screen origin of session. This gets
         // dispatched to the gecko thread, so we also pass the new screen x/y directly to apz.
         // If this is a synthesized touch, the screen offset is bogus so ignore it.
         if (!mSynthesizedEvent) {
             mSession.onScreenOriginChanged((int)screenX, (int)screenY);
         }
 
-        mNative.handleMotionEvent(action, event.getActionIndex(), event.getEventTime(),
-                                  event.getMetaState(), screenX, screenY, pointerId, x, y,
-                                  orientation, pressure, toolMajor, toolMinor, result);
+        final MotionEventData data = new MotionEventData(event);
+        mNative.handleMotionEvent(data, screenX, screenY, result);
     }
 
     private @InputResult int handleScrollEvent(final MotionEvent event) {
         if (!mAttached) {
             mQueuedEvents.add(new Pair<>(EVENT_SOURCE_SCROLL, event));
             return INPUT_RESULT_HANDLED;
         }
 
--- a/widget/android/nsWindow.cpp
+++ b/widget/android/nsWindow.cpp
@@ -505,27 +505,58 @@ class NPZCSupport final
       case nsEventStatus_eConsumeDoDefault:
         return ret;
       default:
         MOZ_ASSERT_UNREACHABLE("Unexpected nsEventStatus");
         return INPUT_RESULT_UNHANDLED;
     }
   }
 
+  // Convert MotionEvent touch radius and orientation into the format required
+  // by w3c touchevents.
+  // toolMajor and toolMinor span a rectangle that's oriented as per
+  // aOrientation, centered around the touch point.
+  static std::pair<float, ScreenSize> ConvertOrientationAndRadius(
+      float aOrientation, float aToolMajor, float aToolMinor) {
+    float angle = aOrientation * 180.0f / M_PI;
+    // w3c touchevents spec does not allow orientations == 90
+    // this shifts it to -90, which will be shifted to zero below
+    if (angle >= 90.0) {
+      angle -= 180.0f;
+    }
+
+    // w3c touchevent radii are given with an orientation between 0 and
+    // 90. The radii are found by removing the orientation and
+    // measuring the x and y radii of the resulting ellipse. For
+    // Android orientations >= 0 and < 90, use the y radius as the
+    // major radius, and x as the minor radius. However, for an
+    // orientation < 0, we have to shift the orientation by adding 90,
+    // and reverse which radius is major and minor.
+    ScreenSize radius;
+    if (angle < 0.0f) {
+      angle += 90.0f;
+      radius =
+          ScreenSize(int32_t(aToolMajor / 2.0f), int32_t(aToolMinor / 2.0f));
+    } else {
+      radius =
+          ScreenSize(int32_t(aToolMinor / 2.0f), int32_t(aToolMajor / 2.0f));
+    }
+
+    return std::make_pair(angle, radius);
+  }
+
   void HandleMotionEvent(
       const java::PanZoomController::NativeProvider::LocalRef& aInstance,
-      int32_t aAction, int32_t aActionIndex, int64_t aTime, int32_t aMetaState,
-      float aScreenX, float aScreenY, jni::IntArray::Param aPointerId,
-      jni::FloatArray::Param aX, jni::FloatArray::Param aY,
-      jni::FloatArray::Param aOrientation, jni::FloatArray::Param aPressure,
-      jni::FloatArray::Param aToolMajor, jni::FloatArray::Param aToolMinor,
+      jni::Object::Param aEventData, float aScreenX, float aScreenY,
       jni::Object::Param aResult) {
     MOZ_ASSERT(AndroidBridge::IsJavaUiThread());
 
     auto returnResult = java::GeckoResult::Ref::From(aResult);
+    auto eventData =
+        java::PanZoomController::MotionEventData::Ref::From(aEventData);
     RefPtr<IAPZCTreeManager> controller;
 
     if (auto window = mWindow.Access()) {
       nsWindow* gkWindow = window->GetNsWindow();
       if (gkWindow) {
         controller = gkWindow->mAPZC;
       }
     }
@@ -533,100 +564,130 @@ class NPZCSupport final
     if (!controller) {
       if (returnResult) {
         returnResult->Complete(
             java::sdk::Integer::ValueOf(INPUT_RESULT_UNHANDLED));
       }
       return;
     }
 
-    nsTArray<int32_t> pointerId(aPointerId->GetElements());
+    nsTArray<int32_t> pointerId(eventData->PointerId()->GetElements());
+    size_t pointerCount = pointerId.Length();
     MultiTouchInput::MultiTouchType type;
     size_t startIndex = 0;
-    size_t endIndex = pointerId.Length();
-
-    switch (aAction) {
+    size_t endIndex = pointerCount;
+
+    switch (eventData->Action()) {
       case java::sdk::MotionEvent::ACTION_DOWN:
       case java::sdk::MotionEvent::ACTION_POINTER_DOWN:
         type = MultiTouchInput::MULTITOUCH_START;
         break;
       case java::sdk::MotionEvent::ACTION_MOVE:
         type = MultiTouchInput::MULTITOUCH_MOVE;
         break;
       case java::sdk::MotionEvent::ACTION_UP:
       case java::sdk::MotionEvent::ACTION_POINTER_UP:
         // for pointer-up events we only want the data from
         // the one pointer that went up
         type = MultiTouchInput::MULTITOUCH_END;
-        startIndex = aActionIndex;
-        endIndex = aActionIndex + 1;
+        startIndex = eventData->ActionIndex();
+        endIndex = startIndex + 1;
         break;
       case java::sdk::MotionEvent::ACTION_OUTSIDE:
       case java::sdk::MotionEvent::ACTION_CANCEL:
         type = MultiTouchInput::MULTITOUCH_CANCEL;
         break;
       default:
         if (returnResult) {
           returnResult->Complete(
               java::sdk::Integer::ValueOf(INPUT_RESULT_UNHANDLED));
         }
         return;
     }
 
-    MultiTouchInput input(type, aTime, nsWindow::GetEventTimeStamp(aTime), 0);
-    input.modifiers = nsWindow::GetModifiers(aMetaState);
+    MultiTouchInput input(type, eventData->Time(),
+                          nsWindow::GetEventTimeStamp(eventData->Time()), 0);
+    input.modifiers = nsWindow::GetModifiers(eventData->MetaState());
     input.mTouches.SetCapacity(endIndex - startIndex);
     input.mScreenOffset =
         ExternalIntPoint(int32_t(floorf(aScreenX)), int32_t(floorf(aScreenY)));
 
-    nsTArray<float> x(aX->GetElements());
-    nsTArray<float> y(aY->GetElements());
-    nsTArray<float> orientation(aOrientation->GetElements());
-    nsTArray<float> pressure(aPressure->GetElements());
-    nsTArray<float> toolMajor(aToolMajor->GetElements());
-    nsTArray<float> toolMinor(aToolMinor->GetElements());
-
-    MOZ_ASSERT(pointerId.Length() == x.Length());
-    MOZ_ASSERT(pointerId.Length() == y.Length());
-    MOZ_ASSERT(pointerId.Length() == orientation.Length());
-    MOZ_ASSERT(pointerId.Length() == pressure.Length());
-    MOZ_ASSERT(pointerId.Length() == toolMajor.Length());
-    MOZ_ASSERT(pointerId.Length() == toolMinor.Length());
+    size_t historySize = eventData->HistorySize();
+    nsTArray<int64_t> historicalTime(
+        eventData->HistoricalTime()->GetElements());
+    MOZ_RELEASE_ASSERT(historicalTime.Length() == historySize);
+
+    // Each of these is |historySize| sets of |pointerCount| values.
+    size_t historicalDataCount = historySize * pointerCount;
+    nsTArray<float> historicalX(eventData->HistoricalX()->GetElements());
+    nsTArray<float> historicalY(eventData->HistoricalY()->GetElements());
+    nsTArray<float> historicalOrientation(
+        eventData->HistoricalOrientation()->GetElements());
+    nsTArray<float> historicalPressure(
+        eventData->HistoricalPressure()->GetElements());
+    nsTArray<float> historicalToolMajor(
+        eventData->HistoricalToolMajor()->GetElements());
+    nsTArray<float> historicalToolMinor(
+        eventData->HistoricalToolMinor()->GetElements());
+
+    MOZ_RELEASE_ASSERT(historicalX.Length() == historicalDataCount);
+    MOZ_RELEASE_ASSERT(historicalY.Length() == historicalDataCount);
+    MOZ_RELEASE_ASSERT(historicalOrientation.Length() == historicalDataCount);
+    MOZ_RELEASE_ASSERT(historicalPressure.Length() == historicalDataCount);
+    MOZ_RELEASE_ASSERT(historicalToolMajor.Length() == historicalDataCount);
+    MOZ_RELEASE_ASSERT(historicalToolMinor.Length() == historicalDataCount);
+
+    // Each of these is |pointerCount| values.
+    nsTArray<float> x(eventData->X()->GetElements());
+    nsTArray<float> y(eventData->Y()->GetElements());
+    nsTArray<float> orientation(eventData->Orientation()->GetElements());
+    nsTArray<float> pressure(eventData->Pressure()->GetElements());
+    nsTArray<float> toolMajor(eventData->ToolMajor()->GetElements());
+    nsTArray<float> toolMinor(eventData->ToolMinor()->GetElements());
+
+    MOZ_ASSERT(x.Length() == pointerCount);
+    MOZ_ASSERT(y.Length() == pointerCount);
+    MOZ_ASSERT(orientation.Length() == pointerCount);
+    MOZ_ASSERT(pressure.Length() == pointerCount);
+    MOZ_ASSERT(toolMajor.Length() == pointerCount);
+    MOZ_ASSERT(toolMinor.Length() == pointerCount);
 
     for (size_t i = startIndex; i < endIndex; i++) {
-      float orien = orientation[i] * 180.0f / M_PI;
-      // w3c touchevents spec does not allow orientations == 90
-      // this shifts it to -90, which will be shifted to zero below
-      if (orien >= 90.0) {
-        orien -= 180.0f;
+      float orien;
+      ScreenSize radius;
+      std::tie(orien, radius) = ConvertOrientationAndRadius(
+          orientation[i], toolMajor[i], toolMinor[i]);
+
+      ScreenIntPoint point(int32_t(floorf(x[i])), int32_t(floorf(y[i])));
+      SingleTouchData singleTouchData(pointerId[i], point, radius, orien,
+                                      pressure[i]);
+
+      for (size_t historyIndex = 0; historyIndex < historySize;
+           historyIndex++) {
+        size_t historicalI = historyIndex * pointerCount + i;
+        float historicalAngle;
+        ScreenSize historicalRadius;
+        std::tie(historicalAngle, historicalRadius) =
+            ConvertOrientationAndRadius(historicalOrientation[historicalI],
+                                        historicalToolMajor[historicalI],
+                                        historicalToolMinor[historicalI]);
+        ScreenIntPoint historicalPoint(
+            int32_t(floorf(historicalX[historicalI])),
+            int32_t(floorf(historicalY[historicalI])));
+        singleTouchData.mHistoricalData.AppendElement(
+            SingleTouchData::HistoricalTouchData{
+                nsWindow::GetEventTimeStamp(historicalTime[historyIndex]),
+                historicalPoint,
+                {},  // mLocalScreenPoint will be computed later by APZ
+                historicalRadius,
+                historicalAngle,
+                historicalPressure[historicalI]});
       }
 
-      nsIntPoint point =
-          nsIntPoint(int32_t(floorf(x[i])), int32_t(floorf(y[i])));
-
-      // w3c touchevent radii are given with an orientation between 0 and
-      // 90. The radii are found by removing the orientation and
-      // measuring the x and y radii of the resulting ellipse. For
-      // Android orientations >= 0 and < 90, use the y radius as the
-      // major radius, and x as the minor radius. However, for an
-      // orientation < 0, we have to shift the orientation by adding 90,
-      // and reverse which radius is major and minor.
-      gfx::Size radius;
-      if (orien < 0.0f) {
-        orien += 90.0f;
-        radius = gfx::Size(int32_t(toolMajor[i] / 2.0f),
-                           int32_t(toolMinor[i] / 2.0f));
-      } else {
-        radius = gfx::Size(int32_t(toolMinor[i] / 2.0f),
-                           int32_t(toolMajor[i] / 2.0f));
-      }
-
-      input.mTouches.AppendElement(SingleTouchData(
-          pointerId[i], ScreenIntPoint::FromUnknownPoint(point),
-          ScreenSize::FromUnknownSize(radius), orien, pressure[i]));
+      input.mTouches.AppendElement(singleTouchData);
     }
 
     APZEventResult result = controller->InputBridge()->ReceiveInputEvent(input);
     int32_t handled = (result.mHandledByRootApzc == Some(true))
                           ? INPUT_RESULT_HANDLED
                           : INPUT_RESULT_HANDLED_CONTENT;
 
     if (result.mStatus == nsEventStatus_eConsumeNoDefault) {