Bug 1359206 - Fix CompositionTime and CompositeUntil handling of WebRenderImageHost r=nical
authorsotaro <sotaro.ikeda.g@gmail.com>
Thu, 27 Apr 2017 09:34:54 -0700
changeset 569626 6b0f3a894ad20e7f1580867750481b19441f9754
parent 569625 258e92719dfe752cce90b050ff4c479a68ee0f97
child 569627 29f8f98fb7baf586f1b953e48c6eec235a7515aa
push id56240
push usermozilla@buttercookie.de
push dateThu, 27 Apr 2017 18:44:39 +0000
reviewersnical
bugs1359206
milestone55.0a1
Bug 1359206 - Fix CompositionTime and CompositeUntil handling of WebRenderImageHost r=nical
gfx/layers/composite/ImageComposite.cpp
gfx/layers/composite/ImageComposite.h
gfx/layers/composite/ImageHost.cpp
gfx/layers/wr/WebRenderBridgeParent.cpp
gfx/layers/wr/WebRenderBridgeParent.h
gfx/layers/wr/WebRenderCompositableHolder.h
gfx/layers/wr/WebRenderImageHost.cpp
gfx/layers/wr/WebRenderImageHost.h
--- a/gfx/layers/composite/ImageComposite.cpp
+++ b/gfx/layers/composite/ImageComposite.cpp
@@ -1,36 +1,35 @@
 /* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*-
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "ImageComposite.h"
 
-// this is also defined in ImageHost.cpp
-#define BIAS_TIME_MS 1.0
-
 namespace mozilla {
 
 using namespace gfx;
 
 namespace layers {
 
+/* static */ const float ImageComposite::BIAS_TIME_MS = 1.0f;
+
 ImageComposite::ImageComposite()
   : mLastFrameID(-1)
   , mLastProducerID(-1)
   , mBias(BIAS_NONE)
 {}
 
 ImageComposite::~ImageComposite()
 {
 }
 
-static TimeStamp
-GetBiasedTime(const TimeStamp& aInput, ImageComposite::Bias aBias)
+/* static */ TimeStamp
+ImageComposite::GetBiasedTime(const TimeStamp& aInput, ImageComposite::Bias aBias)
 {
   switch (aBias) {
   case ImageComposite::BIAS_NEGATIVE:
     return aInput - TimeDuration::FromMilliseconds(BIAS_TIME_MS);
   case ImageComposite::BIAS_POSITIVE:
     return aInput + TimeDuration::FromMilliseconds(BIAS_TIME_MS);
   default:
     return aInput;
@@ -115,10 +114,8 @@ const ImageComposite::TimedImage* ImageC
 ImageComposite::TimedImage* ImageComposite::ChooseImage()
 {
   int index = ChooseImageIndex();
   return index >= 0 ? &mImages[index] : nullptr;
 }
 
 } // namespace layers
 } // namespace mozilla
-
-#undef BIAS_TIME_MS
--- a/gfx/layers/composite/ImageComposite.h
+++ b/gfx/layers/composite/ImageComposite.h
@@ -15,16 +15,18 @@ namespace mozilla {
 namespace layers {
 
 /**
  * Implements Image selection logic.
  */
 class ImageComposite
 {
 public:
+  static const float BIAS_TIME_MS;
+
   explicit ImageComposite();
   ~ImageComposite();
 
   int32_t GetFrameID()
   {
     const TimedImage* img = ChooseImage();
     return img ? img->mFrameID : -1;
   }
@@ -42,16 +44,18 @@ public:
     // Don't apply bias to frame times
     BIAS_NONE,
     // Apply a negative bias to frame times to keep them before the vsync time
     BIAS_NEGATIVE,
     // Apply a positive bias to frame times to keep them after the vsync time
     BIAS_POSITIVE,
   };
 
+  static TimeStamp GetBiasedTime(const TimeStamp& aInput, ImageComposite::Bias aBias);
+
 protected:
   static Bias UpdateBias(const TimeStamp& aCompositionTime,
                          const TimeStamp& aCompositedImageTime,
                          const TimeStamp& aNextImageTime, // may be null
                          ImageComposite::Bias aBias);
 
   virtual TimeStamp GetCompositionTime() const = 0;
 
--- a/gfx/layers/composite/ImageHost.cpp
+++ b/gfx/layers/composite/ImageHost.cpp
@@ -11,19 +11,16 @@
 #include "mozilla/layers/Compositor.h"  // for Compositor
 #include "mozilla/layers/Effects.h"     // for TexturedEffect, Effect, etc
 #include "mozilla/layers/LayerManagerComposite.h"     // for TexturedEffect, Effect, etc
 #include "nsAString.h"
 #include "nsDebug.h"                    // for NS_WARNING, NS_ASSERTION
 #include "nsPrintfCString.h"            // for nsPrintfCString
 #include "nsString.h"                   // for nsAutoCString
 
-// this is also defined in ImageComposite.cpp
-#define BIAS_TIME_MS 1.0
-
 namespace mozilla {
 
 using namespace gfx;
 
 namespace layers {
 
 class ISurfaceAllocator;
 
@@ -471,10 +468,8 @@ ImageHost::GenEffect(const gfx::Sampling
   return CreateTexturedEffect(mCurrentTextureHost,
                               mCurrentTextureSource,
                               aSamplingFilter,
                               isAlphaPremultiplied);
 }
 
 } // namespace layers
 } // namespace mozilla
-
-#undef BIAS_TIME_MS
--- a/gfx/layers/wr/WebRenderBridgeParent.cpp
+++ b/gfx/layers/wr/WebRenderBridgeParent.cpp
@@ -16,17 +16,19 @@
 #include "mozilla/layers/Compositor.h"
 #include "mozilla/layers/CompositorBridgeParent.h"
 #include "mozilla/layers/CompositorThread.h"
 #include "mozilla/layers/CompositorVsyncScheduler.h"
 #include "mozilla/layers/ImageBridgeParent.h"
 #include "mozilla/layers/ImageDataSerializer.h"
 #include "mozilla/layers/TextureHost.h"
 #include "mozilla/layers/WebRenderCompositableHolder.h"
+#include "mozilla/layers/WebRenderImageHost.h"
 #include "mozilla/layers/WebRenderTextureHost.h"
+#include "mozilla/TimeStamp.h"
 #include "mozilla/webrender/RenderThread.h"
 #include "mozilla/widget/CompositorWidget.h"
 
 bool is_in_main_thread()
 {
   return NS_IsMainThread();
 }
 
@@ -396,34 +398,35 @@ WebRenderBridgeParent::RecvDPSyncEnd(con
 void
 WebRenderBridgeParent::ProcessWebRenderCommands(const gfx::IntSize &aSize,
                                                 InfallibleTArray<WebRenderParentCommand>& aCommands, const wr::Epoch& aEpoch,
                                                 const ByteBuffer& dl,
                                                 const WrBuiltDisplayListDescriptor& dlDesc,
                                                 const ByteBuffer& aux,
                                                 const WrAuxiliaryListsDescriptor& auxDesc)
 {
+  mCompositableHolder->SetCompositionTime(TimeStamp::Now());
 
   for (InfallibleTArray<WebRenderParentCommand>::index_type i = 0; i < aCommands.Length(); ++i) {
     const WebRenderParentCommand& cmd = aCommands[i];
     switch (cmd.type()) {
       case WebRenderParentCommand::TOpAddExternalImage: {
         const OpAddExternalImage& op = cmd.get_OpAddExternalImage();
         wr::ImageKey key = op.key();
         MOZ_ASSERT(mExternalImageIds.Get(wr::AsUint64(op.externalImageId())).get());
         MOZ_ASSERT(!mActiveKeys.Get(wr::AsUint64(key), nullptr));
         mActiveKeys.Put(wr::AsUint64(key), key);
 
-        RefPtr<CompositableHost> host = mExternalImageIds.Get(wr::AsUint64(op.externalImageId()));
+        RefPtr<WebRenderImageHost> host = mExternalImageIds.Get(wr::AsUint64(op.externalImageId()));
         if (!host) {
           NS_ERROR("CompositableHost does not exist");
           break;
         }
         // XXX select Texture for video in CompositeToTarget().
-        TextureHost* texture = host->GetAsTextureHost();
+        TextureHost* texture = host->GetAsTextureHostForComposite();
         if (!texture) {
           NS_ERROR("TextureHost does not exist");
           break;
         }
         WebRenderTextureHost* wrTexture = texture->AsWebRenderTextureHost();
         if (wrTexture) {
           if (wrTexture->IsWrappingNativeHandle()) {
             // XXX only for MacIOSurface right now.
@@ -582,54 +585,62 @@ WebRenderBridgeParent::RecvAddExternalIm
      return IPC_FAIL_NO_REASON(this);
   }
   RefPtr<CompositableHost> host = imageBridge->FindCompositable(aHandle);
   if (!host) {
     NS_ERROR("CompositableHost not found in the map!");
     return IPC_FAIL_NO_REASON(this);
   }
   MOZ_ASSERT(host->AsWebRenderImageHost());
-  if (!host->AsWebRenderImageHost()) {
+  WebRenderImageHost* wrHost = host->AsWebRenderImageHost();
+  if (!wrHost) {
     NS_ERROR("Incompatible CompositableHost");
     return IPC_OK();
   }
 
-  mExternalImageIds.Put(wr::AsUint64(aImageId), host);
+  wrHost->SetWrCompositableHolder(mCompositableHolder);
+  mExternalImageIds.Put(wr::AsUint64(aImageId), wrHost);
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
 WebRenderBridgeParent::RecvAddExternalImageIdForCompositable(const ExternalImageId& aImageId,
                                                              const CompositableHandle& aHandle)
 {
   if (mDestroyed) {
     return IPC_OK();
   }
   MOZ_ASSERT(!mExternalImageIds.Get(wr::AsUint64(aImageId)).get());
 
   RefPtr<CompositableHost> host = FindCompositable(aHandle);
   MOZ_ASSERT(host->AsWebRenderImageHost());
-  if (!host->AsWebRenderImageHost()) {
+  WebRenderImageHost* wrHost = host->AsWebRenderImageHost();
+  if (!wrHost) {
     NS_ERROR("Incompatible CompositableHost");
     return IPC_OK();
   }
 
-  mExternalImageIds.Put(wr::AsUint64(aImageId), host);
+  wrHost->SetWrCompositableHolder(mCompositableHolder);
+  mExternalImageIds.Put(wr::AsUint64(aImageId), wrHost);
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
 WebRenderBridgeParent::RecvRemoveExternalImageId(const ExternalImageId& aImageId)
 {
   if (mDestroyed) {
     return IPC_OK();
   }
   MOZ_ASSERT(mExternalImageIds.Get(wr::AsUint64(aImageId)).get());
+  WebRenderImageHost* wrHost = mExternalImageIds.Get(wr::AsUint64(aImageId)).get();
+  if (wrHost) {
+    wrHost->SetWrCompositableHolder(nullptr);
+  }
   mExternalImageIds.Remove(wr::AsUint64(aImageId));
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
 WebRenderBridgeParent::RecvSetLayerObserverEpoch(const uint64_t& aLayerObserverEpoch)
 {
@@ -703,16 +714,21 @@ WebRenderBridgeParent::CompositeToTarget
     if (!transformArray.IsEmpty() || !opacityArray.IsEmpty()) {
       mApi->GenerateFrame(opacityArray, transformArray);
       ScheduleComposition();
       return;
     }
   }
 
   mApi->GenerateFrame();
+
+  // XXX Enable it when async video is supported.
+  // if (!mCompositableHolder->GetCompositeUntilTime().IsNull()) {
+  //   ScheduleComposition();
+  // }
 }
 
 void
 WebRenderBridgeParent::HoldPendingTransactionId(uint32_t aWrEpoch, uint64_t aTransactionId)
 {
   // The transaction ID might get reset to 1 if the page gets reloaded, see
   // https://bugzilla.mozilla.org/show_bug.cgi?id=1145295#c41
   // Otherwise, it should be continually increasing.
@@ -823,16 +839,19 @@ WebRenderBridgeParent::ClearResources()
     }
     if (!mKeysToDelete.empty()) {
       // XXX Sync wait.
       mApi->WaitFlushed();
       DeleteOldImages();
     }
   }
   mCompositableHolder->RemovePipeline(mPipelineId);
+  for (auto iter = mExternalImageIds.Iter(); !iter.Done(); iter.Next()) {
+    iter.Data()->SetWrCompositableHolder(nullptr);
+  }
   mExternalImageIds.Clear();
 
   if (mWidget && mCompositorScheduler) {
     mCompositorScheduler->Destroy();
   }
   mCompositorScheduler = nullptr;
   mApi = nullptr;
   mCompositorBridge = nullptr;
--- a/gfx/layers/wr/WebRenderBridgeParent.h
+++ b/gfx/layers/wr/WebRenderBridgeParent.h
@@ -32,16 +32,17 @@ class WebRenderAPI;
 }
 
 namespace layers {
 
 class Compositor;
 class CompositorBridgeParentBase;
 class CompositorVsyncScheduler;
 class WebRenderCompositableHolder;
+class WebRenderImageHost;
 
 class WebRenderBridgeParent final : public PWebRenderBridgeParent
                                   , public CompositorVsyncSchedulerOwner
                                   , public CompositableParentManager
 {
 public:
   WebRenderBridgeParent(CompositorBridgeParentBase* aCompositorBridge,
                         const wr::PipelineId& aPipelineId,
@@ -204,17 +205,17 @@ private:
   wr::PipelineId mPipelineId;
   RefPtr<widget::CompositorWidget> mWidget;
   RefPtr<wr::WebRenderAPI> mApi;
   RefPtr<WebRenderCompositableHolder> mCompositableHolder;
   RefPtr<CompositorVsyncScheduler> mCompositorScheduler;
   std::vector<wr::ImageKey> mKeysToDelete;
   // XXX How to handle active keys of non-ExternalImages?
   nsDataHashtable<nsUint64HashKey, wr::ImageKey> mActiveKeys;
-  nsDataHashtable<nsUint64HashKey, RefPtr<CompositableHost>> mExternalImageIds;
+  nsDataHashtable<nsUint64HashKey, RefPtr<WebRenderImageHost>> mExternalImageIds;
   nsTArray<ImageCompositeNotificationInfo> mImageCompositeNotifications;
 
   // These fields keep track of the latest layer observer epoch values in the child and the
   // parent. mChildLayerObserverEpoch is the latest epoch value received from the child.
   // mParentLayerObserverEpoch is the latest epoch value that we have told TabParent about
   // (via ObserveLayerUpdate).
   uint64_t mChildLayerObserverEpoch;
   uint64_t mParentLayerObserverEpoch;
--- a/gfx/layers/wr/WebRenderCompositableHolder.h
+++ b/gfx/layers/wr/WebRenderCompositableHolder.h
@@ -34,16 +34,36 @@ protected:
   ~WebRenderCompositableHolder();
 
 public:
   void AddPipeline(const wr::PipelineId& aPipelineId);
   void RemovePipeline(const wr::PipelineId& aPipelineId);
   void HoldExternalImage(const wr::PipelineId& aPipelineId, const wr::Epoch& aEpoch, WebRenderTextureHost* aTexture);
   void Update(const wr::PipelineId& aPipelineId, const wr::Epoch& aEpoch);
 
+  TimeStamp GetCompositionTime() const {
+    return mCompositionTime;
+  }
+  void SetCompositionTime(TimeStamp aTimeStamp) {
+    mCompositionTime = aTimeStamp;
+    if (!mCompositionTime.IsNull() && !mCompositeUntilTime.IsNull() &&
+        mCompositionTime >= mCompositeUntilTime) {
+      mCompositeUntilTime = TimeStamp();
+    }
+  }
+  void CompositeUntil(TimeStamp aTimeStamp) {
+    if (mCompositeUntilTime.IsNull() ||
+        mCompositeUntilTime < aTimeStamp) {
+      mCompositeUntilTime = aTimeStamp;
+    }
+  }
+  TimeStamp GetCompositeUntilTime() const {
+    return mCompositeUntilTime;
+  }
+
 private:
 
   struct ForwardingTextureHost {
     ForwardingTextureHost(const wr::Epoch& aEpoch, TextureHost* aTexture)
       : mEpoch(aEpoch)
       , mTexture(aTexture)
     {}
     wr::Epoch mEpoch;
@@ -51,14 +71,22 @@ private:
   };
 
   struct PipelineTexturesHolder {
     // Holds forwarding WebRenderTextureHosts.
     std::queue<ForwardingTextureHost> mTextureHosts;
   };
 
   nsClassHashtable<nsUint64HashKey, PipelineTexturesHolder> mPipelineTexturesHolders;
+
+  // Render time for the current composition.
+  TimeStamp mCompositionTime;
+
+  // When nonnull, during rendering, some compositable indicated that it will
+  // change its rendering at this time. In order not to miss it, we composite
+  // on every vsync until this time occurs (this is the latest such time).
+  TimeStamp mCompositeUntilTime;
 };
 
 } // namespace layers
 } // namespace mozilla
 
 #endif /* MOZILLA_GFX_WEBRENDERCOMPOSITABLE_HOLDER_H */
--- a/gfx/layers/wr/WebRenderImageHost.cpp
+++ b/gfx/layers/wr/WebRenderImageHost.cpp
@@ -4,36 +4,39 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "WebRenderImageHost.h"
 
 #include "LayersLogging.h"
 #include "mozilla/layers/Compositor.h"  // for Compositor
 #include "mozilla/layers/Effects.h"     // for TexturedEffect, Effect, etc
 #include "mozilla/layers/LayerManagerComposite.h"     // for TexturedEffect, Effect, etc
+#include "mozilla/layers/WebRenderCompositableHolder.h"
 #include "nsAString.h"
 #include "nsDebug.h"                    // for NS_WARNING, NS_ASSERTION
 #include "nsPrintfCString.h"            // for nsPrintfCString
 #include "nsString.h"                   // for nsAutoCString
 
 namespace mozilla {
 
 using namespace gfx;
 
 namespace layers {
 
 class ISurfaceAllocator;
 
 WebRenderImageHost::WebRenderImageHost(const TextureInfo& aTextureInfo)
   : CompositableHost(aTextureInfo)
   , ImageComposite()
+  , mWrCompositableHolder(nullptr)
 {}
 
 WebRenderImageHost::~WebRenderImageHost()
 {
+  MOZ_ASSERT(!mWrCompositableHolder);
 }
 
 void
 WebRenderImageHost::UseTextureHost(const nsTArray<TimedTexture>& aTextures)
 {
   CompositableHost::UseTextureHost(aTextures);
   MOZ_ASSERT(aTextures.Length() >= 1);
 
@@ -57,16 +60,32 @@ WebRenderImageHost::UseTextureHost(const
     img.mFrameID = t.mFrameID;
     img.mProducerID = t.mProducerID;
     img.mTextureHost->SetCropRect(img.mPictureRect);
     img.mTextureHost->Updated();
   }
 
   mImages.SwapElements(newImages);
   newImages.Clear();
+
+  // Video producers generally send replacement images with the same frameID but
+  // slightly different timestamps in order to sync with the audio clock. This
+  // means that any CompositeUntil() call we made in Composite() may no longer
+  // guarantee that we'll composite until the next frame is ready. Fix that here.
+  if (mWrCompositableHolder && mLastFrameID >= 0) {
+    for (size_t i = 0; i < mImages.Length(); ++i) {
+      bool frameComesAfter = mImages[i].mFrameID > mLastFrameID ||
+                             mImages[i].mProducerID != mLastProducerID;
+      if (frameComesAfter && !mImages[i].mTimeStamp.IsNull()) {
+        mWrCompositableHolder->CompositeUntil(mImages[i].mTimeStamp +
+                           TimeDuration::FromMilliseconds(BIAS_TIME_MS));
+        break;
+      }
+    }
+  }
 }
 
 void
 WebRenderImageHost::UseComponentAlphaTextures(TextureHost* aTextureOnBlack,
                                               TextureHost* aTextureOnWhite)
 {
   MOZ_ASSERT_UNREACHABLE("unexpected to be called");
 }
@@ -90,30 +109,54 @@ WebRenderImageHost::RemoveTextureHost(Te
       mImages.RemoveElementAt(i);
     }
   }
 }
 
 TimeStamp
 WebRenderImageHost::GetCompositionTime() const
 {
-  // XXX temporary workaround
-  return TimeStamp::Now();
+  TimeStamp time;
+  if (mWrCompositableHolder) {
+    time = mWrCompositableHolder->GetCompositionTime();
+  }
+  return time;
 }
 
 TextureHost*
 WebRenderImageHost::GetAsTextureHost(IntRect* aPictureRect)
 {
   TimedImage* img = ChooseImage();
   if (img) {
     return img->mTextureHost;
   }
   return nullptr;
 }
 
+TextureHost*
+WebRenderImageHost::GetAsTextureHostForComposite()
+{
+  int imageIndex = ChooseImageIndex();
+  if (imageIndex < 0) {
+    return nullptr;
+  }
+
+  if (mWrCompositableHolder && uint32_t(imageIndex) + 1 < mImages.Length()) {
+    mWrCompositableHolder->CompositeUntil(mImages[imageIndex + 1].mTimeStamp + TimeDuration::FromMilliseconds(BIAS_TIME_MS));
+  }
+
+  TimedImage* img = &mImages[imageIndex];
+
+  if (mLastFrameID != img->mFrameID || mLastProducerID != img->mProducerID) {
+    mLastFrameID = img->mFrameID;
+    mLastProducerID = img->mProducerID;
+  }
+  return img->mTextureHost;
+}
+
 void WebRenderImageHost::Attach(Layer* aLayer,
                        TextureSourceProvider* aProvider,
                        AttachFlags aFlags)
 {
   MOZ_ASSERT_UNREACHABLE("unexpected to be called");
 }
 
 void
--- a/gfx/layers/wr/WebRenderImageHost.h
+++ b/gfx/layers/wr/WebRenderImageHost.h
@@ -7,16 +7,18 @@
 #define MOZILLA_GFX_WEBRENDERIMAGEHOST_H
 
 #include "CompositableHost.h"           // for CompositableHost
 #include "mozilla/layers/ImageComposite.h"  // for ImageComposite
 
 namespace mozilla {
 namespace layers {
 
+class WebRenderCompositableHolder;
+
 /**
  * ImageHost. Works with ImageClientSingle and ImageClientBuffered
  */
 class WebRenderImageHost : public CompositableHost,
                            public ImageComposite
 {
 public:
   explicit WebRenderImageHost(const TextureInfo& aTextureInfo);
@@ -60,17 +62,26 @@ public:
   virtual bool Lock() override;
 
   virtual void Unlock() override;
 
   virtual void CleanupResources() override;
 
   virtual WebRenderImageHost* AsWebRenderImageHost() override { return this; }
 
+  TextureHost* GetAsTextureHostForComposite();
+
+  void SetWrCompositableHolder(WebRenderCompositableHolder* aWrCompositableHolder)
+  {
+    mWrCompositableHolder = aWrCompositableHolder;
+  }
+
 protected:
   // ImageComposite
   virtual TimeStamp GetCompositionTime() const override;
+
+  WebRenderCompositableHolder* MOZ_NON_OWNING_REF mWrCompositableHolder;
 };
 
 } // namespace layers
 } // namespace mozilla
 
 #endif // MOZILLA_GFX_WEBRENDERIMAGEHOST_H