Bug 1208371 - Ensure a media element's ImageContainer is protected when playing a stream. r=mt,jesup
authorAndreas Pehrson <pehrsons@gmail.com>
Thu, 04 Feb 2016 09:27:09 +0800
changeset 348493 b8f87a9c54d893808c484fa80b0e09ef08d0ec7b
parent 348492 72c2bc247b28132ea50edc10067f61f6a2936c3e
child 348494 d599fc36575d5a02aca3b161728bce0bc8e2324b
push id14828
push userpehrsons@gmail.com
push dateThu, 07 Apr 2016 12:57:27 +0000
reviewersmt, jesup
bugs1208371
milestone48.0a1
Bug 1208371 - Ensure a media element's ImageContainer is protected when playing a stream. r=mt,jesup HTMLMediaElement needs special protection when playing a stream since its ImageContainer can outlive the video track of a stream. Consider for instance when a (cross-origin) video track is removed from a DOMMediaStream by a user and the remaining video track (non-CORS) does not yet contain any actual video frames. The HTMLMediaElement will display a frame from the removed track but the DOMMediaStream's principal has been updated to not include the principal from the removed track. With this patch we handle this by letting VideoFrameContainer notify HTMLMediaElement when it has flushed out all video frames belonging to a certain PrincipalHandle. I.e., when a new PrincipalHandle has been applied to the underlying ImageContainer. MozReview-Commit-ID: LvIZPl6Rdgj
dom/html/HTMLMediaElement.cpp
dom/html/HTMLMediaElement.h
dom/media/MediaStreamGraph.cpp
dom/media/VideoFrameContainer.cpp
dom/media/VideoFrameContainer.h
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -4188,24 +4188,63 @@ HTMLMediaElement::PrincipalChanged(DOMMe
   LOG(LogLevel::Info, ("HTMLMediaElement %p Stream principal changed.", this));
   nsContentUtils::CombineResourcePrincipals(&mSrcStreamVideoPrincipal,
                                             aStream->GetVideoPrincipal());
 
   LOG(LogLevel::Debug, ("HTMLMediaElement %p Stream video principal changed to "
                         "%p. Waiting for it to reach VideoFrameContainer before "
                         "setting.", this, aStream->GetVideoPrincipal()));
   if (mVideoFrameContainer) {
-    UpdateSrcStreamVideoPrincipal(aStream->GetVideoPrincipal());
+    UpdateSrcStreamVideoPrincipal(mVideoFrameContainer->GetLastPrincipalHandle());
   }
 }
 
 void
-HTMLMediaElement::UpdateSrcStreamVideoPrincipal(nsIPrincipal* aPrincipal)
-{
-  mSrcStreamVideoPrincipal = aPrincipal;
+HTMLMediaElement::UpdateSrcStreamVideoPrincipal(const PrincipalHandle& aPrincipalHandle)
+{
+  nsTArray<RefPtr<VideoStreamTrack>> videoTracks;
+  mSrcStream->GetVideoTracks(videoTracks);
+
+  PrincipalHandle handle(aPrincipalHandle);
+  bool matchesTrackPrincipal = false;
+  for (const RefPtr<VideoStreamTrack>& track : videoTracks) {
+    if (PrincipalHandleMatches(handle,
+                               track->GetPrincipal()) &&
+        !track->Ended()) {
+      // When the PrincipalHandle for the VideoFrameContainer changes to that of
+      // a track in mSrcStream we know that a removed track was displayed but
+      // is no longer so.
+      matchesTrackPrincipal = true;
+      LOG(LogLevel::Debug, ("HTMLMediaElement %p VideoFrameContainer's "
+                            "PrincipalHandle matches track %p. That's all we "
+                            "need.", this, track.get()));
+      break;
+    }
+  }
+
+  if (matchesTrackPrincipal) {
+    mSrcStreamVideoPrincipal = mSrcStream->GetVideoPrincipal();
+  }
+}
+
+void
+HTMLMediaElement::PrincipalHandleChangedForVideoFrameContainer(VideoFrameContainer* aContainer,
+                                                               const PrincipalHandle& aNewPrincipalHandle)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (!mSrcStream) {
+    return;
+  }
+
+  LOG(LogLevel::Debug, ("HTMLMediaElement %p PrincipalHandle changed in "
+                        "VideoFrameContainer.",
+                        this));
+
+  UpdateSrcStreamVideoPrincipal(aNewPrincipalHandle);
 }
 
 nsresult HTMLMediaElement::DispatchEvent(const nsAString& aName)
 {
   LOG_EVENT(LogLevel::Debug, ("%p Dispatching event %s", this,
                           NS_ConvertUTF16toUTF8(aName).get()));
 
   // Save events that occur while in the bfcache. These will be dispatched
--- a/dom/html/HTMLMediaElement.h
+++ b/dom/html/HTMLMediaElement.h
@@ -226,17 +226,22 @@ public:
   // Called by the media decoder and the video frame to get the
   // ImageContainer containing the video data.
   B2G_ACL_EXPORT virtual VideoFrameContainer* GetVideoFrameContainer() final override;
   layers::ImageContainer* GetImageContainer();
 
   // From PrincipalChangeObserver<DOMMediaStream>.
   void PrincipalChanged(DOMMediaStream* aStream) override;
 
-  void UpdateSrcStreamVideoPrincipal(nsIPrincipal* aPrincipal);
+  void UpdateSrcStreamVideoPrincipal(const PrincipalHandle& aPrincipalHandle);
+
+  // Called after the MediaStream we're playing rendered a frame to aContainer
+  // with a different principalHandle than the previous frame.
+  void PrincipalHandleChangedForVideoFrameContainer(VideoFrameContainer* aContainer,
+                                                    const PrincipalHandle& aNewPrincipalHandle);
 
   // Dispatch events
   virtual nsresult DispatchAsyncEvent(const nsAString& aName) final override;
 
   // Triggers a recomputation of readyState.
   void UpdateReadyState() override { UpdateReadyStateInternal(); }
 
   // Dispatch events that were raised while in the bfcache
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -913,16 +913,17 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
 
   if (aStream->mVideoOutputs.IsEmpty())
     return;
 
   TimeStamp currentTimeStamp = CurrentDriver()->GetCurrentTimeStamp();
 
   // Collect any new frames produced in this iteration.
   AutoTArray<ImageContainer::NonOwningImage,4> newImages;
+  PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
   RefPtr<Image> blackImage;
 
   MOZ_ASSERT(mProcessedTime >= aStream->mBufferStartTime, "frame position before buffer?");
   // We only look at the non-blocking interval
   StreamTime frameBufferTime = aStream->GraphTimeToStreamTime(mProcessedTime);
   StreamTime bufferEndTime = aStream->GraphTimeToStreamTime(aStream->mStartBlocking);
   StreamTime start;
   const VideoChunk* chunk;
@@ -976,28 +977,34 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
         }
       }
       if (blackImage) {
         image = blackImage;
       }
     }
     newImages.AppendElement(ImageContainer::NonOwningImage(image, targetTime));
 
+    lastPrincipalHandle = chunk->GetPrincipalHandle();
+
     aStream->mLastPlayedVideoFrame = *frame;
   }
 
   if (!aStream->mLastPlayedVideoFrame.GetImage())
     return;
 
   AutoTArray<ImageContainer::NonOwningImage,4> images;
   bool haveMultipleImages = false;
 
   for (uint32_t i = 0; i < aStream->mVideoOutputs.Length(); ++i) {
     VideoFrameContainer* output = aStream->mVideoOutputs[i];
 
+    bool principalHandleChanged =
+      lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+      lastPrincipalHandle != output->GetLastPrincipalHandle();
+
     // Find previous frames that may still be valid.
     AutoTArray<ImageContainer::OwningImage,4> previousImages;
     output->GetImageContainer()->GetCurrentImages(&previousImages);
     uint32_t j = previousImages.Length();
     if (j) {
       // Re-use the most recent frame before currentTimeStamp and subsequent,
       // always keeping at least one frame.
       do {
@@ -1027,16 +1034,22 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
                                           image.mTimeStamp, image.mFrameID));
     }
 
     // Add the frames from this iteration.
     for (auto& image : newImages) {
       image.mFrameID = output->NewFrameID();
       images.AppendElement(image);
     }
+
+    if (principalHandleChanged) {
+      output->UpdatePrincipalHandleForFrameID(lastPrincipalHandle,
+                                              newImages.LastElement().mFrameID);
+    }
+
     output->SetCurrentFrames(aStream->mLastPlayedVideoFrame.GetIntrinsicSize(),
                              images);
 
     nsCOMPtr<nsIRunnable> event =
       new VideoFrameContainerInvalidateRunnable(output);
     DispatchToMainThreadAfterStreamStateUpdate(event.forget());
 
     images.ClearAndRetainStorage();
--- a/dom/media/VideoFrameContainer.cpp
+++ b/dom/media/VideoFrameContainer.cpp
@@ -15,25 +15,43 @@ using namespace mozilla::layers;
 
 namespace mozilla {
 
 VideoFrameContainer::VideoFrameContainer(dom::HTMLMediaElement* aElement,
                                          already_AddRefed<ImageContainer> aContainer)
   : mElement(aElement),
     mImageContainer(aContainer), mMutex("nsVideoFrameContainer"),
     mFrameID(0),
-    mIntrinsicSizeChanged(false), mImageSizeChanged(false)
+    mIntrinsicSizeChanged(false), mImageSizeChanged(false),
+    mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE), mFrameIDForPendingPrincipalHandle(0)
 {
   NS_ASSERTION(aElement, "aElement must not be null");
   NS_ASSERTION(mImageContainer, "aContainer must not be null");
 }
 
 VideoFrameContainer::~VideoFrameContainer()
 {}
 
+PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle()
+{
+  MutexAutoLock lock(mMutex);
+  return mLastPrincipalHandle;
+}
+
+void VideoFrameContainer::UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
+                                                          const ImageContainer::FrameID& aFrameID)
+{
+  MutexAutoLock lock(mMutex);
+  if (mPendingPrincipalHandle == aPrincipalHandle) {
+    return;
+  }
+  mPendingPrincipalHandle = aPrincipalHandle;
+  mFrameIDForPendingPrincipalHandle = aFrameID;
+}
+
 void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize,
                                           Image* aImage,
                                           const TimeStamp& aTargetTime)
 {
   if (aImage) {
     MutexAutoLock lock(mMutex);
     AutoTArray<ImageContainer::NonOwningImage,1> imageList;
     imageList.AppendElement(
@@ -64,18 +82,43 @@ void VideoFrameContainer::SetCurrentFram
   gfx::IntSize oldFrameSize = mImageContainer->GetCurrentSize();
 
   // When using the OMX decoder, destruction of the current image can indirectly
   //  block on main thread I/O. If we let this happen while holding onto
   //  |mImageContainer|'s lock, then when the main thread then tries to
   //  composite it can then block on |mImageContainer|'s lock, causing a
   //  deadlock. We use this hack to defer the destruction of the current image
   //  until it is safe.
-  nsTArray<ImageContainer::OwningImage> kungFuDeathGrip;
-  mImageContainer->GetCurrentImages(&kungFuDeathGrip);
+  nsTArray<ImageContainer::OwningImage> oldImages;
+  mImageContainer->GetCurrentImages(&oldImages);
+
+  ImageContainer::FrameID lastFrameIDForOldPrincipalHandle =
+    mFrameIDForPendingPrincipalHandle - 1;
+  if (mPendingPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+       ((!oldImages.IsEmpty() &&
+          oldImages.LastElement().mFrameID >= lastFrameIDForOldPrincipalHandle) ||
+        (!aImages.IsEmpty() &&
+          aImages[0].mFrameID > lastFrameIDForOldPrincipalHandle))) {
+    // We are releasing the last FrameID prior to `lastFrameIDForOldPrincipalHandle`
+    // OR
+    // there are no FrameIDs prior to `lastFrameIDForOldPrincipalHandle` in the new
+    // set of images.
+    // This means that the old principal handle has been flushed out and we can
+    // notify our video element about this change.
+    RefPtr<VideoFrameContainer> self = this;
+    PrincipalHandle principalHandle = mPendingPrincipalHandle;
+    mLastPrincipalHandle = mPendingPrincipalHandle;
+    mPendingPrincipalHandle = PRINCIPAL_HANDLE_NONE;
+    mFrameIDForPendingPrincipalHandle = 0;
+    NS_DispatchToMainThread(NS_NewRunnableFunction([self, principalHandle]() {
+      if (self->mElement) {
+        self->mElement->PrincipalHandleChangedForVideoFrameContainer(self, principalHandle);
+      }
+    }));
+  }
 
   if (aImages.IsEmpty()) {
     mImageContainer->ClearAllImages();
   } else {
     mImageContainer->SetCurrentImages(aImages);
   }
   gfx::IntSize newFrameSize = mImageContainer->GetCurrentSize();
   if (oldFrameSize != newFrameSize) {
--- a/dom/media/VideoFrameContainer.h
+++ b/dom/media/VideoFrameContainer.h
@@ -8,16 +8,17 @@
 #define VIDEOFRAMECONTAINER_H_
 
 #include "mozilla/Mutex.h"
 #include "mozilla/TimeStamp.h"
 #include "gfxPoint.h"
 #include "nsCOMPtr.h"
 #include "nsAutoPtr.h"
 #include "ImageContainer.h"
+#include "MediaSegment.h"
 
 namespace mozilla {
 
 namespace dom {
 class HTMLMediaElement;
 } // namespace dom
 
 /**
@@ -37,16 +38,23 @@ public:
   typedef layers::Image Image;
 
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameContainer)
 
   VideoFrameContainer(dom::HTMLMediaElement* aElement,
                       already_AddRefed<ImageContainer> aContainer);
 
   // Call on any thread
+  // Returns the last principalHandle we notified mElement about.
+  PrincipalHandle GetLastPrincipalHandle();
+  // We will notify mElement that aPrincipalHandle has been applied when all
+  // FrameIDs prior to aFrameID have been flushed out.
+  // aFrameID is ignored if aPrincipalHandle already is our pending principalHandle.
+  void UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
+                                       const ImageContainer::FrameID& aFrameID);
   B2G_ACL_EXPORT void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
                        const TimeStamp& aTargetTime);
   void SetCurrentFrames(const gfx::IntSize& aIntrinsicSize,
                         const nsTArray<ImageContainer::NonOwningImage>& aImages);
   void ClearCurrentFrame(const gfx::IntSize& aIntrinsicSize)
   {
     SetCurrentFrames(aIntrinsicSize, nsTArray<ImageContainer::NonOwningImage>());
   }
@@ -106,13 +114,20 @@ protected:
   // and update the intrinsic size on the element, request a frame reflow and
   // then reset this flag.
   bool mIntrinsicSizeChanged;
   // True when the Image size has changed since the last time Invalidate() was
   // called. When set, the next call to Invalidate() will ensure that the
   // frame is fully invalidated instead of just invalidating for the image change
   // in the ImageLayer.
   bool mImageSizeChanged;
+  // The last PrincipalHandle we notified mElement about.
+  PrincipalHandle mLastPrincipalHandle;
+  // The PrincipalHandle the client has notified us is changing with FrameID
+  // mFrameIDForPendingPrincipalHandle.
+  PrincipalHandle mPendingPrincipalHandle;
+  // The FrameID for which mPendingPrincipal is first valid.
+  ImageContainer::FrameID mFrameIDForPendingPrincipalHandle;
 };
 
 } // namespace mozilla
 
 #endif /* VIDEOFRAMECONTAINER_H_ */