Bug 1486659 - p3: copy texture contents for remote allocated Surface. r=snorp
authorJohn Lin <jolin@mozilla.com>
Mon, 26 Nov 2018 19:41:31 +0000
changeset 507310 9c2834ca8823472c042a7c9bf10e06fadc5ee692
parent 507309 5d897c0c7a25a155fae7347954d8dc9b58bb5703
child 507311 c6399844f0318c7043ec4dca9dbd51c15f91784f
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssnorp
bugs1486659
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1486659 - p3: copy texture contents for remote allocated Surface. r=snorp Child processes cannot access textures allocated in the parent process, which is needed by the compositor to render video elements efficiently. Unfortunately, Android doesn't expose Sufrace buffers (sharable across processes) in the SDK/NDK as other platforms, so we need to generate extra texture/surface in the child process and update texture images through the surface, which is passed to the parent process for the remote texture to copy its contents into. Differential Revision: https://phabricator.services.mozilla.com/D11939
dom/media/platforms/android/RemoteDataDecoder.cpp
gfx/gl/SharedSurfaceEGL.cpp
gfx/layers/opengl/TextureClientOGL.cpp
mobile/android/geckoview/src/main/aidl/org/mozilla/gecko/gfx/ISurfaceAllocator.aidl
mobile/android/geckoview/src/main/aidl/org/mozilla/gecko/gfx/SyncConfig.aidl
mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/GeckoSurface.java
mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/GeckoSurfaceTexture.java
mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/SurfaceAllocator.java
mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/SurfaceAllocatorService.java
mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/SyncConfig.java
--- a/dom/media/platforms/android/RemoteDataDecoder.cpp
+++ b/dom/media/platforms/android/RemoteDataDecoder.cpp
@@ -114,17 +114,17 @@ class RemoteVideoDecoder : public Remote
       bool isEOS = !!(flags & MediaCodec::BUFFER_FLAG_END_OF_STREAM);
       if (!ok && !isEOS) {
         // Ignore output with no corresponding input.
         return;
       }
 
       if (ok && (size > 0 || presentationTimeUs >= 0)) {
         RefPtr<layers::Image> img = new SurfaceTextureImage(
-            mDecoder->mSurfaceHandle, inputInfo.mImageSize,
+            mDecoder->mImageHandle, inputInfo.mImageSize,
             false /* NOT continuous */, gl::OriginPos::BottomLeft);
 
         RefPtr<VideoData> v = VideoData::CreateFromImage(
             inputInfo.mDisplaySize, offset,
             TimeUnit::FromMicroseconds(presentationTimeUs),
             TimeUnit::FromMicroseconds(inputInfo.mDurationUs), img,
             !!(flags & MediaCodec::BUFFER_FLAG_SYNC_FRAME),
             TimeUnit::FromMicroseconds(presentationTimeUs));
@@ -163,17 +163,17 @@ class RemoteVideoDecoder : public Remote
   RefPtr<InitPromise> Init() override {
     mSurface = GeckoSurface::LocalRef(SurfaceAllocator::AcquireSurface(
         mConfig.mImage.width, mConfig.mImage.height, false));
     if (!mSurface) {
       return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
                                           __func__);
     }
 
-    mSurfaceHandle = mSurface->GetHandle();
+    mImageHandle = mSurface->GetImageHandle();
 
     // Register native methods.
     JavaCallbacksSupport::Init();
 
     mJavaCallbacks = CodecProxy::NativeCallbacks::New();
     JavaCallbacksSupport::AttachNative(
         mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(this));
 
@@ -251,17 +251,17 @@ class RemoteVideoDecoder : public Remote
 
   ConversionRequired NeedsConversion() const override {
     return ConversionRequired::kNeedAnnexB;
   }
 
  private:
   const VideoInfo mConfig;
   GeckoSurface::GlobalRef mSurface;
-  AndroidSurfaceTextureHandle mSurfaceHandle;
+  AndroidSurfaceTextureHandle mImageHandle;
   // Only accessed on reader's task queue.
   bool mIsCodecSupportAdaptivePlayback = false;
   // Can be accessed on any thread, but only written on during init.
   bool mIsHardwareAccelerated = false;
   // Accessed on mTaskQueue, reader's TaskQueue and Java callback tread.
   // SimpleMap however is thread-safe, so it's okay to do so.
   SimpleMap<InputInfo> mInputInfos;
   // Only accessed on the TaskQueue.
--- a/gfx/gl/SharedSurfaceEGL.cpp
+++ b/gfx/gl/SharedSurfaceEGL.cpp
@@ -275,17 +275,17 @@ bool
 SharedSurface_SurfaceTexture::IsBufferAvailable() const {
     return mSurface->GetAvailable();
 }
 
 bool
 SharedSurface_SurfaceTexture::ToSurfaceDescriptor(layers::SurfaceDescriptor* const out_descriptor)
 {
     *out_descriptor =
-        layers::SurfaceTextureDescriptor(mSurface->GetHandle(),
+        layers::SurfaceTextureDescriptor(mSurface->GetImageHandle(),
                                          mSize,
                                          gfx::SurfaceFormat::R8G8B8A8,
                                          false /* NOT continuous */,
                                          false /* Do not ignore transform */);
     return true;
 }
 
 ////////////////////////////////////////////////////////////////////////
--- a/gfx/layers/opengl/TextureClientOGL.cpp
+++ b/gfx/layers/opengl/TextureClientOGL.cpp
@@ -165,17 +165,17 @@ AndroidNativeWindowTextureData::FillInfo
   aInfo.supportsMoz2D = true;
   aInfo.canExposeMappedData = false;
   aInfo.canConcurrentlyReadLock = false;
 }
 
 bool
 AndroidNativeWindowTextureData::Serialize(SurfaceDescriptor& aOutDescriptor)
 {
-  aOutDescriptor = SurfaceTextureDescriptor(mSurface->GetHandle(),
+  aOutDescriptor = SurfaceTextureDescriptor(mSurface->GetImageHandle(),
                                             mSize,
                                             mFormat,
                                             false /* not continuous */,
                                             true /* ignore transform */);
   return true;
 }
 
 bool
--- a/mobile/android/geckoview/src/main/aidl/org/mozilla/gecko/gfx/ISurfaceAllocator.aidl
+++ b/mobile/android/geckoview/src/main/aidl/org/mozilla/gecko/gfx/ISurfaceAllocator.aidl
@@ -1,12 +1,15 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.gfx;
 
 import org.mozilla.gecko.gfx.GeckoSurface;
+import org.mozilla.gecko.gfx.SyncConfig;
 
 interface ISurfaceAllocator {
     GeckoSurface acquireSurface(in int width, in int height, in boolean singleBufferMode);
     void releaseSurface(in int handle);
+    void configureSync(in SyncConfig config);
+    void sync(in int handle);
 }
copy from mobile/android/geckoview/src/main/aidl/org/mozilla/gecko/gfx/ISurfaceAllocator.aidl
copy to mobile/android/geckoview/src/main/aidl/org/mozilla/gecko/gfx/SyncConfig.aidl
--- a/mobile/android/geckoview/src/main/aidl/org/mozilla/gecko/gfx/ISurfaceAllocator.aidl
+++ b/mobile/android/geckoview/src/main/aidl/org/mozilla/gecko/gfx/SyncConfig.aidl
@@ -1,12 +1,7 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.gfx;
 
-import org.mozilla.gecko.gfx.GeckoSurface;
-
-interface ISurfaceAllocator {
-    GeckoSurface acquireSurface(in int width, in int height, in boolean singleBufferMode);
-    void releaseSurface(in int handle);
-}
+parcelable SyncConfig;
--- a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/GeckoSurface.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/GeckoSurface.java
@@ -5,51 +5,52 @@
 
 package org.mozilla.gecko.gfx;
 
 import android.graphics.SurfaceTexture;
 
 import android.os.Parcel;
 import android.os.Parcelable;
 import android.view.Surface;
-import android.util.Log;
-
-import java.util.HashMap;
 
 import org.mozilla.gecko.annotation.WrapForJNI;
 
+import static org.mozilla.geckoview.BuildConfig.DEBUG_BUILD;
+
 public final class GeckoSurface extends Surface {
     private static final String LOGTAG = "GeckoSurface";
 
-    private static final HashMap<Integer, GeckoSurfaceTexture> sSurfaceTextures = new HashMap<Integer, GeckoSurfaceTexture>();
-
     private int mHandle;
     private boolean mIsSingleBuffer;
     private volatile boolean mIsAvailable;
     private boolean mOwned = true;
 
-    @WrapForJNI(exceptionMode = "nsresult")
+    private int mMyPid;
+    // Locally allocated surface/texture. Do not pass it over IPC.
+    private GeckoSurface mSyncSurface;
+
     public GeckoSurface(GeckoSurfaceTexture gst) {
         super(gst);
         mHandle = gst.getHandle();
         mIsSingleBuffer = gst.isSingleBuffer();
         mIsAvailable = true;
+        mMyPid = android.os.Process.myPid();
     }
 
     public GeckoSurface(Parcel p, SurfaceTexture dummy) {
         // A no-arg constructor exists, but is hidden in the SDK. We need to create a dummy
         // SurfaceTexture here in order to create the instance. This is used to transfer the
         // GeckoSurface across binder.
         super(dummy);
 
         readFromParcel(p);
         mHandle = p.readInt();
         mIsSingleBuffer = p.readByte() == 1 ? true : false;
         mIsAvailable = (p.readByte() == 1 ? true : false);
-
+        mMyPid = p.readInt();
         dummy.release();
     }
 
     public static final Parcelable.Creator<GeckoSurface> CREATOR = new Parcelable.Creator<GeckoSurface>() {
         public GeckoSurface createFromParcel(Parcel p) {
             return new GeckoSurface(p, new SurfaceTexture(0));
         }
 
@@ -59,34 +60,65 @@ public final class GeckoSurface extends 
     };
 
     @Override
     public void writeToParcel(Parcel out, int flags) {
         super.writeToParcel(out, flags);
         out.writeInt(mHandle);
         out.writeByte((byte) (mIsSingleBuffer ? 1 : 0));
         out.writeByte((byte) (mIsAvailable ? 1 : 0));
-
+        out.writeInt(mMyPid);
         mOwned = false;
     }
 
     @Override
     public void release() {
+        if (mSyncSurface != null) {
+            mSyncSurface.release();
+            GeckoSurfaceTexture gst = GeckoSurfaceTexture.lookup(mSyncSurface.getHandle());
+            if (gst != null) {
+                gst.decrementUse();
+            }
+            mSyncSurface = null;
+        }
+
         if (mOwned) {
             super.release();
         }
     }
 
     @WrapForJNI
-    public int getHandle() {
-        return mHandle;
+    public int getImageHandle() {
+        return mSyncSurface == null ? mHandle : mSyncSurface.getHandle();
     }
 
     @WrapForJNI
     public boolean getAvailable() {
         return mIsAvailable;
     }
 
     @WrapForJNI
     public void setAvailable(boolean available) {
         mIsAvailable = available;
     }
+
+    /* package */ boolean inProcess() {
+        return android.os.Process.myPid() == mMyPid;
+    }
+
+    /* package */ int getHandle() {
+        return mHandle;
+    }
+
+    /* package */ SyncConfig initSyncSurface(int width, int height) {
+        if (DEBUG_BUILD) {
+            if (inProcess()) {
+                throw new AssertionError("no need for sync when allocated in process");
+            }
+        }
+        GeckoSurfaceTexture texture = GeckoSurfaceTexture.acquire(false);
+        texture.setDefaultBufferSize(width, height);
+        texture.track(mHandle);
+        mSyncSurface = new GeckoSurface(texture);
+
+        return new SyncConfig(mHandle, mSyncSurface, width, height);
+    }
 }
--- a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/GeckoSurfaceTexture.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/GeckoSurfaceTexture.java
@@ -31,16 +31,17 @@ import org.mozilla.gecko.mozglue.JNIObje
 
     private long mAttachedContext;
     private int mTexName;
 
     private GeckoSurfaceTexture.Callbacks mListener;
     private AtomicInteger mUseCount;
     private boolean mFinalized;
 
+    private int mUpstream;
     private NativeGLBlitHelper mBlitter;
 
     private GeckoSurfaceTexture(int handle) {
         super(0);
         init(handle, false);
     }
 
     private GeckoSurfaceTexture(int handle, boolean singleBufferMode) {
@@ -107,27 +108,31 @@ import org.mozilla.gecko.mozglue.JNIObje
     public boolean isSingleBuffer() {
         return mIsSingleBuffer;
     }
 
     @Override
     @WrapForJNI
     public synchronized void updateTexImage() {
         try {
+            if (mUpstream != 0) {
+                SurfaceAllocator.sync(mUpstream);
+            }
             super.updateTexImage();
             if (mListener != null) {
                 mListener.onUpdateTexImage();
             }
         } catch (Exception e) {
             Log.w(LOGTAG, "updateTexImage() failed", e);
         }
     }
 
     @Override
     public synchronized void release() {
+        mUpstream = 0;
         if (mBlitter != null) {
             mBlitter.disposeNative();
         }
         try {
             super.release();
             synchronized (sSurfaceTextures) {
                 sSurfaceTextures.remove(mHandle);
             }
@@ -275,31 +280,35 @@ import org.mozilla.gecko.mozglue.JNIObje
 
     @WrapForJNI
     public static GeckoSurfaceTexture lookup(int handle) {
         synchronized (sSurfaceTextures) {
             return sSurfaceTextures.get(handle);
         }
     }
 
+    /* package */ void track(int upstream) {
+        mUpstream = upstream;
+    }
+
     /* package */ synchronized void configureSnapshot(GeckoSurface target, int width, int height) {
         mBlitter = NativeGLBlitHelper.create(mHandle, target, width, height);
     }
 
     /* package */ synchronized void takeSnapshot() {
         mBlitter.blit();
     }
 
     public interface Callbacks {
         void onUpdateTexImage();
         void onReleaseTexImage();
     }
 
     @WrapForJNI
-    public static class NativeGLBlitHelper extends JNIObject {
+    public static final class NativeGLBlitHelper extends JNIObject {
         public native static NativeGLBlitHelper create(int textureHandle,
                                                        GeckoSurface targetSurface,
                                                        int width,
                                                        int height);
         public native void blit();
 
         @Override
         protected native void disposeNative();
--- a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/SurfaceAllocator.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/SurfaceAllocator.java
@@ -5,22 +5,19 @@
 
 package org.mozilla.gecko.gfx;
 
 import android.content.ComponentName;
 import android.content.Context;
 import android.content.Intent;
 import android.content.ServiceConnection;
 
-import android.graphics.SurfaceTexture;
 import android.os.IBinder;
-import android.os.Process;
 import android.os.RemoteException;
 import android.util.Log;
-import android.view.Surface;
 
 import org.mozilla.gecko.annotation.WrapForJNI;
 import org.mozilla.gecko.GeckoAppShell;
 
 /* package */ final class SurfaceAllocator {
     private static final String LOGTAG = "SurfaceAllocator";
 
     private static SurfaceAllocatorConnection sConnection;
@@ -44,18 +41,22 @@ import org.mozilla.gecko.GeckoAppShell;
     @WrapForJNI
     public static GeckoSurface acquireSurface(int width, int height, boolean singleBufferMode) {
         try {
             ensureConnection();
 
             if (singleBufferMode && !GeckoSurfaceTexture.isSingleBufferSupported()) {
                 return null;
             }
-
-            return sConnection.getAllocator().acquireSurface(width, height, singleBufferMode);
+            ISurfaceAllocator allocator = sConnection.getAllocator();
+            GeckoSurface surface = allocator.acquireSurface(width, height, singleBufferMode);
+            if (surface != null && !surface.inProcess()) {
+                allocator.configureSync(surface.initSyncSurface(width, height));
+            }
+            return surface;
         } catch (Exception e) {
             Log.w(LOGTAG, "Failed to acquire GeckoSurface", e);
             return null;
         }
     }
 
     @WrapForJNI
     public static void disposeSurface(GeckoSurface surface) {
@@ -76,16 +77,32 @@ import org.mozilla.gecko.GeckoAppShell;
         // And now our Surface
         try {
             surface.release();
         } catch (Exception e) {
             Log.w(LOGTAG, "Failed to release surface", e);
         }
     }
 
+    public static void sync(int upstream) {
+        try {
+            ensureConnection();
+        } catch (Exception e) {
+            Log.w(LOGTAG, "Failed to sync texture, no connection");
+            return;
+        }
+
+        // Release the SurfaceTexture on the other side
+        try {
+            sConnection.getAllocator().sync(upstream);
+        } catch (RemoteException e) {
+            Log.w(LOGTAG, "Failed to sync texture", e);
+        }
+    }
+
     private static final class SurfaceAllocatorConnection implements ServiceConnection {
         private ISurfaceAllocator mAllocator;
 
         public synchronized ISurfaceAllocator getAllocator() {
             while (mAllocator == null) {
                 try {
                     this.wait();
                 } catch (InterruptedException e) { }
--- a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/SurfaceAllocatorService.java
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/SurfaceAllocatorService.java
@@ -4,18 +4,16 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.gfx;
 
 import android.app.Service;
 import android.content.Intent;
 import android.os.Binder;
 import android.os.IBinder;
-import android.os.RemoteException;
-import android.util.Log;
 
 public final class SurfaceAllocatorService extends Service {
 
     private static final String LOGTAG = "SurfaceAllocatorService";
 
     public int onStartCommand(final Intent intent, final int flags, final int startId) {
         return Service.START_STICKY;
     }
@@ -36,16 +34,30 @@ public final class SurfaceAllocatorServi
         }
 
         public void releaseSurface(int handle) {
             final GeckoSurfaceTexture gst = GeckoSurfaceTexture.lookup(handle);
             if (gst != null) {
                 gst.decrementUse();
             }
         }
+
+        public void configureSync(SyncConfig config) {
+            final GeckoSurfaceTexture gst = GeckoSurfaceTexture.lookup(config.sourceTextureHandle);
+            if (gst != null) {
+                gst.configureSnapshot(config.targetSurface, config.width, config.height);
+            }
+        }
+
+        public void sync(int handle) {
+            final GeckoSurfaceTexture gst = GeckoSurfaceTexture.lookup(handle);
+            if (gst != null) {
+                gst.takeSnapshot();
+            }
+        }
     };
 
     public IBinder onBind(final Intent intent) {
         return mBinder;
     }
 
     public boolean onUnbind(Intent intent) {
         return false;
new file mode 100644
--- /dev/null
+++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/gfx/SyncConfig.java
@@ -0,0 +1,54 @@
+package org.mozilla.gecko.gfx;
+
+import android.os.Parcel;
+import android.os.Parcelable;
+
+/* package */ final class SyncConfig implements Parcelable {
+    final int sourceTextureHandle;
+    final GeckoSurface targetSurface;
+    final int width;
+    final int height;
+
+    /* package */ SyncConfig(int sourceTextureHandle,
+                             GeckoSurface targetSurface,
+                             int width,
+                             int height) {
+        this.sourceTextureHandle = sourceTextureHandle;
+        this.targetSurface = targetSurface;
+        this.width = width;
+        this.height = height;
+    }
+
+    public static final Creator<SyncConfig> CREATOR =
+        new Creator<SyncConfig>() {
+            @Override
+            public SyncConfig createFromParcel(Parcel parcel) {
+                return new SyncConfig(parcel);
+            }
+
+            @Override
+            public SyncConfig[] newArray(int i) {
+                return new SyncConfig[i];
+            }
+        };
+
+    private SyncConfig(Parcel parcel) {
+        sourceTextureHandle = parcel.readInt();
+        targetSurface = GeckoSurface.CREATOR.createFromParcel(parcel);
+        width = parcel.readInt();
+        height = parcel.readInt();
+    }
+
+    @Override
+    public int describeContents() {
+        return 0;
+    }
+
+    @Override
+    public void writeToParcel(Parcel parcel, int flags) {
+        parcel.writeInt(sourceTextureHandle);
+        targetSurface.writeToParcel(parcel, flags);
+        parcel.writeInt(width);
+        parcel.writeInt(height);
+    }
+}