Bug 1201363 - Adding base class MediaStreamVideoSink and changing VideoFrameContainer to be inherited from MediaStreamVideoSink. r=jesup
authorctai <ctai@mozilla.com>
Thu, 24 Dec 2015 10:43:28 +0800
changeset 308251 d3295f400cd3ef369b5fe3e16c4f0918e2e7e9bf
parent 308250 79415ce5af65c6b207ae185825770ac039af13a8
child 308252 02fc34b73508ff8433cce0c96f98ae689a35bb7a
push id31092
push usercbook@mozilla.com
push dateFri, 05 Aug 2016 10:16:59 +0000
treeherderautoland@b97dd7dd3cb9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjesup
bugs1201363
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1201363 - Adding base class MediaStreamVideoSink and changing VideoFrameContainer to be inherited from MediaStreamVideoSink. r=jesup MediaStreamVideoSink is the base class of VideoFrameContainer, CaptureTask(ImageCapture), MediaStreamVideoRecorderSink(MediaRecoreder) and PipelineVideoSink(WebRTC-MediaPipelineTransmit). In this patch, I change VideoFrameContainer only. The rest of cases will be changed in latter patches of this bug. MozReview-Commit-ID: JNUke3fyCoN
dom/media/MediaStreamListener.h
dom/media/MediaStreamVideoSink.cpp
dom/media/MediaStreamVideoSink.h
dom/media/VideoFrameContainer.cpp
dom/media/VideoFrameContainer.h
dom/media/VideoSegment.h
dom/media/moz.build
--- a/dom/media/MediaStreamListener.h
+++ b/dom/media/MediaStreamListener.h
@@ -2,20 +2,24 @@
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef MOZILLA_MEDIASTREAMLISTENER_h_
 #define MOZILLA_MEDIASTREAMLISTENER_h_
 
+#include "StreamTracks.h"
+
 namespace mozilla {
 
+class AudioSegment;
 class MediaStream;
 class MediaStreamGraph;
+class VideoSegment;
 
 enum MediaStreamGraphEvent : uint32_t {
   EVENT_FINISHED,
   EVENT_REMOVED,
   EVENT_HAS_DIRECT_LISTENERS, // transition from no direct listeners
   EVENT_HAS_NO_DIRECT_LISTENERS,  // transition to no direct listeners
 };
 
new file mode 100644
--- /dev/null
+++ b/dom/media/MediaStreamVideoSink.cpp
@@ -0,0 +1,21 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaStreamVideoSink.h"
+
+#include "VideoSegment.h"
+
+namespace mozilla {
+void
+MediaStreamVideoSink::NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+                                              StreamTime aTrackOffset,
+                                              const MediaSegment& aMedia)
+{
+  if (aMedia.GetType() == MediaSegment::VIDEO) {
+    SetCurrentFrames(static_cast<const VideoSegment&>(aMedia));
+  }
+}
+
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/dom/media/MediaStreamVideoSink.h
@@ -0,0 +1,42 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MEDIASTREAMVIDEOSINK_H_
+#define MEDIASTREAMVIDEOSINK_H_
+
+#include "mozilla/Pair.h"
+
+#include "gfxPoint.h"
+#include "MediaStreamListener.h"
+
+namespace mozilla {
+
+class VideoFrameContainer;
+
+/**
+ * Base class of MediaStreamVideoSink family. This is the output of MediaStream.
+ */
+class MediaStreamVideoSink : public DirectMediaStreamTrackListener {
+public:
+  // Method of DirectMediaStreamTrackListener.
+  void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+                               StreamTime aTrackOffset,
+                               const MediaSegment& aMedia) override;
+
+  // Call on any thread
+  virtual void SetCurrentFrames(const VideoSegment& aSegment) = 0;
+  virtual void ClearFrames() = 0;
+
+  virtual VideoFrameContainer* AsVideoFrameContainer() { return nullptr; }
+  virtual void Invalidate() {}
+
+protected:
+  virtual ~MediaStreamVideoSink() {};
+};
+
+} // namespace mozilla
+
+#endif /* MEDIASTREAMVIDEOSINK_H_ */
--- a/dom/media/VideoFrameContainer.cpp
+++ b/dom/media/VideoFrameContainer.cpp
@@ -9,27 +9,33 @@
 #include "mozilla/dom/HTMLMediaElement.h"
 #include "nsIFrame.h"
 #include "nsDisplayList.h"
 #include "nsSVGEffects.h"
 
 using namespace mozilla::layers;
 
 namespace mozilla {
+PRLogModuleInfo* gVideoFrameContainerLog;
+#define CONTAINER_LOG(type, msg) MOZ_LOG(gVideoFrameContainerLog, type, msg)
 
 VideoFrameContainer::VideoFrameContainer(dom::HTMLMediaElement* aElement,
                                          already_AddRefed<ImageContainer> aContainer)
   : mElement(aElement),
     mImageContainer(aContainer), mMutex("nsVideoFrameContainer"),
+    mBlackImage(nullptr),
     mFrameID(0),
     mIntrinsicSizeChanged(false), mImageSizeChanged(false),
     mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE), mFrameIDForPendingPrincipalHandle(0)
 {
   NS_ASSERTION(aElement, "aElement must not be null");
   NS_ASSERTION(mImageContainer, "aContainer must not be null");
+  if (!gVideoFrameContainerLog) {
+    gVideoFrameContainerLog = PR_NewLogModule("VideoFrameContainer");
+  }
 }
 
 VideoFrameContainer::~VideoFrameContainer()
 {}
 
 PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle()
 {
   MutexAutoLock lock(mMutex);
@@ -42,16 +48,136 @@ void VideoFrameContainer::UpdatePrincipa
   MutexAutoLock lock(mMutex);
   if (mPendingPrincipalHandle == aPrincipalHandle) {
     return;
   }
   mPendingPrincipalHandle = aPrincipalHandle;
   mFrameIDForPendingPrincipalHandle = aFrameID;
 }
 
+static void
+SetImageToBlackPixel(PlanarYCbCrImage* aImage)
+{
+  uint8_t blackPixel[] = { 0x10, 0x80, 0x80 };
+
+  PlanarYCbCrData data;
+  data.mYChannel = blackPixel;
+  data.mCbChannel = blackPixel + 1;
+  data.mCrChannel = blackPixel + 2;
+  data.mYStride = data.mCbCrStride = 1;
+  data.mPicSize = data.mYSize = data.mCbCrSize = gfx::IntSize(1, 1);
+  aImage->CopyData(data);
+}
+
+class VideoFrameContainerInvalidateRunnable : public Runnable {
+public:
+  explicit VideoFrameContainerInvalidateRunnable(VideoFrameContainer* aVideoFrameContainer)
+    : mVideoFrameContainer(aVideoFrameContainer)
+  {}
+  NS_IMETHOD Run()
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+
+    mVideoFrameContainer->Invalidate();
+
+    return NS_OK;
+  }
+private:
+  RefPtr<VideoFrameContainer> mVideoFrameContainer;
+};
+
+void VideoFrameContainer::SetCurrentFrames(const VideoSegment& aSegment)
+{
+  if (aSegment.IsEmpty()) {
+    return;
+  }
+
+  MutexAutoLock lock(mMutex);
+
+  // Collect any new frames produced in this iteration.
+  AutoTArray<ImageContainer::NonOwningImage,4> newImages;
+  PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
+
+  VideoSegment::ConstChunkIterator iter(aSegment);
+  while (!iter.IsEnded()) {
+    VideoChunk chunk = *iter;
+
+    const VideoFrame* frame = &chunk.mFrame;
+    if (*frame == mLastPlayedVideoFrame) {
+      iter.Next();
+      continue;
+    }
+
+    Image* image = frame->GetImage();
+    CONTAINER_LOG(LogLevel::Verbose,
+                  ("VideoFrameContainer %p writing video frame %p (%d x %d)",
+                  this, image, frame->GetIntrinsicSize().width,
+                  frame->GetIntrinsicSize().height));
+
+    if (frame->GetForceBlack()) {
+      if (!mBlackImage) {
+        mBlackImage = GetImageContainer()->CreatePlanarYCbCrImage();
+        if (mBlackImage) {
+          // Sets the image to a single black pixel, which will be scaled to
+          // fill the rendered size.
+          SetImageToBlackPixel(mBlackImage->AsPlanarYCbCrImage());
+        }
+      }
+      if (mBlackImage) {
+        image = mBlackImage;
+      }
+    }
+    // Don't append null image to the newImages.
+    if (!image) {
+      iter.Next();
+      continue;
+    }
+    newImages.AppendElement(ImageContainer::NonOwningImage(image, chunk.mTimeStamp));
+
+    lastPrincipalHandle = chunk.GetPrincipalHandle();
+
+    mLastPlayedVideoFrame = *frame;
+    iter.Next();
+  }
+
+  // Don't update if there are no changes.
+  if (newImages.IsEmpty()) {
+    return;
+  }
+
+  AutoTArray<ImageContainer::NonOwningImage,4> images;
+
+  bool principalHandleChanged =
+     lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+     lastPrincipalHandle != GetLastPrincipalHandle();
+
+  // Add the frames from this iteration.
+  for (auto& image : newImages) {
+    image.mFrameID = NewFrameID();
+    images.AppendElement(image);
+  }
+
+  if (principalHandleChanged) {
+    UpdatePrincipalHandleForFrameID(lastPrincipalHandle,
+                                    newImages.LastElement().mFrameID);
+  }
+
+  SetCurrentFramesLocked(mLastPlayedVideoFrame.GetIntrinsicSize(), images);
+  nsCOMPtr<nsIRunnable> event =
+    new VideoFrameContainerInvalidateRunnable(this);
+  NS_DispatchToMainThread(event.forget());
+
+  images.ClearAndRetainStorage();
+}
+
+void VideoFrameContainer::ClearFrames()
+{
+  ClearFutureFrames();
+}
+
 void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize,
                                           Image* aImage,
                                           const TimeStamp& aTargetTime)
 {
   if (aImage) {
     MutexAutoLock lock(mMutex);
     AutoTArray<ImageContainer::NonOwningImage,1> imageList;
     imageList.AppendElement(
--- a/dom/media/VideoFrameContainer.h
+++ b/dom/media/VideoFrameContainer.h
@@ -8,60 +8,63 @@
 #define VIDEOFRAMECONTAINER_H_
 
 #include "mozilla/Mutex.h"
 #include "mozilla/TimeStamp.h"
 #include "gfxPoint.h"
 #include "nsCOMPtr.h"
 #include "ImageContainer.h"
 #include "MediaSegment.h"
+#include "MediaStreamVideoSink.h"
+#include "VideoSegment.h"
 
 namespace mozilla {
 
 namespace dom {
 class HTMLMediaElement;
 } // namespace dom
 
 /**
  * This object is used in the decoder backend threads and the main thread
  * to manage the "current video frame" state. This state includes timing data
  * and an intrinsic size (see below).
  * This has to be a thread-safe object since it's accessed by resource decoders
  * and other off-main-thread components. So we can't put this state in the media
  * element itself ... well, maybe we could, but it could be risky and/or
  * confusing.
  */
-class VideoFrameContainer {
-  ~VideoFrameContainer();
+class VideoFrameContainer : public MediaStreamVideoSink {
+  virtual ~VideoFrameContainer();
 
 public:
   typedef layers::ImageContainer ImageContainer;
   typedef layers::Image Image;
 
-  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameContainer)
-
   VideoFrameContainer(dom::HTMLMediaElement* aElement,
                       already_AddRefed<ImageContainer> aContainer);
 
   // Call on any thread
+  virtual void SetCurrentFrames(const VideoSegment& aSegment) override;
+  virtual void ClearFrames() override;
+  void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
+                       const TimeStamp& aTargetTime);
   // Returns the last principalHandle we notified mElement about.
   PrincipalHandle GetLastPrincipalHandle();
   // We will notify mElement that aPrincipalHandle has been applied when all
   // FrameIDs prior to aFrameID have been flushed out.
   // aFrameID is ignored if aPrincipalHandle already is our pending principalHandle.
   void UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle,
                                        const ImageContainer::FrameID& aFrameID);
-  void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage,
-                       const TimeStamp& aTargetTime);
   void SetCurrentFrames(const gfx::IntSize& aIntrinsicSize,
                         const nsTArray<ImageContainer::NonOwningImage>& aImages);
   void ClearCurrentFrame(const gfx::IntSize& aIntrinsicSize)
   {
     SetCurrentFrames(aIntrinsicSize, nsTArray<ImageContainer::NonOwningImage>());
   }
+  VideoFrameContainer* AsVideoFrameContainer() override { return this; }
 
   void ClearCurrentFrame();
   // Make the current frame the only frame in the container, i.e. discard
   // all future frames.
   void ClearFutureFrames();
   // Time in seconds by which the last painted video frame was late by.
   // E.g. if the last painted frame should have been painted at time t,
   // but was actually painted at t+n, this returns n in seconds. Threadsafe.
@@ -75,17 +78,17 @@ public:
     return ++mFrameID;
   }
 
   // Call on main thread
   enum {
     INVALIDATE_DEFAULT,
     INVALIDATE_FORCE
   };
-  void Invalidate() { InvalidateWithFlags(INVALIDATE_DEFAULT); }
+  void Invalidate() override { InvalidateWithFlags(INVALIDATE_DEFAULT); }
   void InvalidateWithFlags(uint32_t aFlags);
   ImageContainer* GetImageContainer();
   void ForgetElement() { mElement = nullptr; }
 
   uint32_t GetDroppedImageCount() { return mImageContainer->GetDroppedImageCount(); }
 
 protected:
   void SetCurrentFramesLocked(const gfx::IntSize& aIntrinsicSize,
@@ -93,25 +96,31 @@ protected:
 
   // Non-addreffed pointer to the element. The element calls ForgetElement
   // to clear this reference when the element is destroyed.
   dom::HTMLMediaElement* mElement;
   RefPtr<ImageContainer> mImageContainer;
 
   // mMutex protects all the fields below.
   Mutex mMutex;
+  // Once the frame is forced to black, we initialize mBlackImage for following
+  // frames.
+  RefPtr<Image> mBlackImage;
   // The intrinsic size is the ideal size which we should render the
   // ImageContainer's current Image at.
   // This can differ from the Image's actual size when the media resource
   // specifies that the Image should be stretched to have the correct aspect
   // ratio.
   gfx::IntSize mIntrinsicSize;
   // We maintain our own mFrameID which is auto-incremented at every
   // SetCurrentFrame() or NewFrameID() call.
   ImageContainer::FrameID mFrameID;
+  // We record the last played video frame to avoid playing the frame again
+  // with a different frame id.
+  VideoFrame mLastPlayedVideoFrame;
   // True when the intrinsic size has been changed by SetCurrentFrame() since
   // the last call to Invalidate().
   // The next call to Invalidate() will recalculate
   // and update the intrinsic size on the element, request a frame reflow and
   // then reset this flag.
   bool mIntrinsicSizeChanged;
   // True when the Image size has changed since the last time Invalidate() was
   // called. When set, the next call to Invalidate() will ensure that the
--- a/dom/media/VideoSegment.h
+++ b/dom/media/VideoSegment.h
@@ -132,13 +132,19 @@ public:
 
   // Segment-generic methods not in MediaSegmentBase
   static Type StaticType() { return VIDEO; }
 
   size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override
   {
     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
   }
+
+  bool IsEmpty() const
+  {
+    return mChunks.IsEmpty();
+  }
+
 };
 
 } // namespace mozilla
 
 #endif /* MOZILLA_VIDEOSEGMENT_H_ */
--- a/dom/media/moz.build
+++ b/dom/media/moz.build
@@ -122,16 +122,17 @@ EXPORTS += [
     'MediaQueue.h',
     'MediaRecorder.h',
     'MediaResource.h',
     'MediaResourceCallback.h',
     'MediaSegment.h',
     'MediaStatistics.h',
     'MediaStreamGraph.h',
     'MediaStreamListener.h',
+    'MediaStreamVideoSink.h',
     'MediaTimer.h',
     'MediaTrack.h',
     'MediaTrackList.h',
     'MP3Decoder.h',
     'MP3Demuxer.h',
     'MP3FrameParser.h',
     'NextFrameSeekTask.h',
     'nsIDocumentActivity.h',
@@ -234,16 +235,17 @@ UNIFIED_SOURCES += [
     'MediaPrefs.cpp',
     'MediaRecorder.cpp',
     'MediaResource.cpp',
     'MediaShutdownManager.cpp',
     'MediaStreamError.cpp',
     'MediaStreamGraph.cpp',
     'MediaStreamListener.cpp',
     'MediaStreamTrack.cpp',
+    'MediaStreamVideoSink.cpp',
     'MediaTimer.cpp',
     'MediaTrack.cpp',
     'MediaTrackList.cpp',
     'MP3Decoder.cpp',
     'MP3Demuxer.cpp',
     'MP3FrameParser.cpp',
     'NextFrameSeekTask.cpp',
     'QueueObject.cpp',