Bug 1201363 - Let MediaStreamVideoSink bind with particular video track. r=jesup
authorctai <ctai@mozilla.com>
Fri, 27 May 2016 14:33:50 +0800
changeset 308253 94a4304e69df741186ec14a9994bfa8adee62261
parent 308252 02fc34b73508ff8433cce0c96f98ae689a35bb7a
child 308254 8248c1bc1b17f0ab1b3090bd61493410f81cc546
push id31092
push usercbook@mozilla.com
push dateFri, 05 Aug 2016 10:16:59 +0000
treeherderautoland@b97dd7dd3cb9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjesup
bugs1201363
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1201363 - Let MediaStreamVideoSink bind with particular video track. r=jesup MozReview-Commit-ID: FcjnmDKuRQI
dom/camera/CameraPreviewMediaStream.cpp
dom/camera/CameraPreviewMediaStream.h
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
dom/media/TrackUnionStream.h
--- a/dom/camera/CameraPreviewMediaStream.cpp
+++ b/dom/camera/CameraPreviewMediaStream.cpp
@@ -55,28 +55,28 @@ CameraPreviewMediaStream::SetAudioOutput
 }
 
 void
 CameraPreviewMediaStream::RemoveAudioOutput(void* aKey)
 {
 }
 
 void
-CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   MutexAutoLock lock(mMutex);
   RefPtr<MediaStreamVideoSink> sink = aSink;
-  AddVideoOutputImpl(sink.forget());
+  AddVideoOutputImpl(sink.forget(), aID);
 }
 
 void
-CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   MutexAutoLock lock(mMutex);
-  RemoveVideoOutputImpl(aSink);
+  RemoveVideoOutputImpl(aSink, aID);
 }
 
 void
 CameraPreviewMediaStream::AddListener(MediaStreamListener* aListener)
 {
   MutexAutoLock lock(mMutex);
 
   MediaStreamListener* listener = *mListeners.AppendElement() = aListener;
@@ -121,18 +121,18 @@ CameraPreviewMediaStream::Destroy()
   DestroyImpl();
 }
 
 void
 CameraPreviewMediaStream::Invalidate()
 {
   MutexAutoLock lock(mMutex);
   --mInvalidatePending;
-  for (MediaStreamVideoSink* sink : mVideoOutputs) {
-    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+  for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
     if (!output) {
       continue;
     }
     output->Invalidate();
   }
 }
 
 void
@@ -163,18 +163,18 @@ CameraPreviewMediaStream::SetCurrentFram
       }
 
       DOM_CAMERA_LOGI("Update preview frame, %d invalidation(s) pending",
         mInvalidatePending);
     }
     mDiscardedFrames = 0;
 
     TimeStamp now = TimeStamp::Now();
-    for (MediaStreamVideoSink* sink : mVideoOutputs) {
-      VideoFrameContainer* output = sink->AsVideoFrameContainer();
+    for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+      VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
       if (!output) {
         continue;
       }
       output->SetCurrentFrame(aIntrinsicSize, aImage, now);
     }
 
     ++mInvalidatePending;
   }
@@ -182,18 +182,18 @@ CameraPreviewMediaStream::SetCurrentFram
   NS_DispatchToMainThread(NewRunnableMethod(this, &CameraPreviewMediaStream::Invalidate));
 }
 
 void
 CameraPreviewMediaStream::ClearCurrentFrame()
 {
   MutexAutoLock lock(mMutex);
 
-  for (MediaStreamVideoSink* sink : mVideoOutputs) {
-    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+  for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
     if (!output) {
       continue;
     }
     output->ClearCurrentFrame();
     NS_DispatchToMainThread(NewRunnableMethod(output, &VideoFrameContainer::Invalidate));
   }
 }
 
--- a/dom/camera/CameraPreviewMediaStream.h
+++ b/dom/camera/CameraPreviewMediaStream.h
@@ -38,26 +38,26 @@ protected:
  */
 class CameraPreviewMediaStream : public ProcessedMediaStream
 {
   typedef mozilla::layers::Image Image;
 
 public:
   CameraPreviewMediaStream();
 
-  virtual void AddAudioOutput(void* aKey) override;
-  virtual void SetAudioOutputVolume(void* aKey, float aVolume) override;
-  virtual void RemoveAudioOutput(void* aKey) override;
-  virtual void AddVideoOutput(MediaStreamVideoSink* aSink) override;
-  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink) override;
-  virtual void Suspend() override {}
-  virtual void Resume() override {}
-  virtual void AddListener(MediaStreamListener* aListener) override;
-  virtual void RemoveListener(MediaStreamListener* aListener) override;
-  virtual void Destroy() override;
+  void AddAudioOutput(void* aKey) override;
+  void SetAudioOutputVolume(void* aKey, float aVolume) override;
+  void RemoveAudioOutput(void* aKey) override;
+  void AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
+  void RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
+  void Suspend() override {}
+  void Resume() override {}
+  void AddListener(MediaStreamListener* aListener) override;
+  void RemoveListener(MediaStreamListener* aListener) override;
+  void Destroy() override;
   void OnPreviewStateChange(bool aActive);
 
   void Invalidate();
 
   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 
   // Call these on any thread.
   void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage);
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -1020,20 +1020,20 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
       TimeDuration::FromSeconds(MediaTimeToSeconds(frameTime - IterationEnd()));
 
     if (frame->GetForceBlack()) {
       if (!blackImage) {
         // Fixme: PlayVideo will be replaced in latter changeset
         // "Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack."
         // of this bug.
         // This is a temp workaround to pass the build and test.
-        if (!aStream->mVideoOutputs[0]->AsVideoFrameContainer()) {
+        if (!aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()) {
           return;
         }
-        blackImage = aStream->mVideoOutputs[0]->AsVideoFrameContainer()->
+        blackImage = aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()->
           GetImageContainer()->CreatePlanarYCbCrImage();
         if (blackImage) {
           // Sets the image to a single black pixel, which will be scaled to
           // fill the rendered size.
           SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage());
         }
       }
       if (blackImage) {
@@ -1048,18 +1048,18 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
   }
 
   if (!aStream->mLastPlayedVideoFrame.GetImage())
     return;
 
   AutoTArray<ImageContainer::NonOwningImage,4> images;
   bool haveMultipleImages = false;
 
-  for (MediaStreamVideoSink* sink : aStream->mVideoOutputs) {
-    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+  for (const TrackBound<MediaStreamVideoSink>& sink : aStream->mVideoOutputs) {
+    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
     if (!output) {
       continue;
     }
 
     bool principalHandleChanged =
       lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
       lastPrincipalHandle != output->GetLastPrincipalHandle();
 
@@ -2266,65 +2266,86 @@ MediaStream::RemoveAudioOutput(void* aKe
       mStream->RemoveAudioOutputImpl(mKey);
     }
     void* mKey;
   };
   GraphImpl()->AppendMessage(MakeUnique<Message>(this, aKey));
 }
 
 void
-MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink)
+MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
+                                TrackID aID)
 {
   RefPtr<MediaStreamVideoSink> sink = aSink;
   STREAM_LOG(LogLevel::Info, ("MediaStream %p Adding MediaStreamVideoSink %p as output",
                               this, sink.get()));
-  *mVideoOutputs.AppendElement() = sink.forget();
+  MOZ_ASSERT(aID != TRACK_NONE);
+   for (auto entry : mVideoOutputs) {
+     if (entry.mListener == sink &&
+         (entry.mTrackID == TRACK_ANY || entry.mTrackID == aID)) {
+       return;
+     }
+   }
+   TrackBound<MediaStreamVideoSink>* l = mVideoOutputs.AppendElement();
+   l->mListener = sink;
+   l->mTrackID = aID;
 }
 
 void
-MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink,
+                                   TrackID aID)
 {
   STREAM_LOG(LogLevel::Info, ("MediaStream %p Removing MediaStreamVideoSink %p as output",
                               this, aSink));
+  MOZ_ASSERT(aID != TRACK_NONE);
+
   // Ensure that any frames currently queued for playback by the compositor
   // are removed.
   aSink->ClearFrames();
-  mVideoOutputs.RemoveElement(aSink);
+  for (size_t i = 0; i < mVideoOutputs.Length(); ++i) {
+    if (mVideoOutputs[i].mListener == aSink &&
+        (mVideoOutputs[i].mTrackID == TRACK_ANY ||
+         mVideoOutputs[i].mTrackID == aID)) {
+      mVideoOutputs.RemoveElementAt(i);
+    }
+  }
 }
 
 void
-MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   class Message : public ControlMessage {
   public:
-    Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
-      ControlMessage(aStream), mSink(aSink) {}
+    Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+      ControlMessage(aStream), mSink(aSink), mID(aID) {}
     void Run() override
     {
-      mStream->AddVideoOutputImpl(mSink.forget());
+      mStream->AddVideoOutputImpl(mSink.forget(), mID);
     }
     RefPtr<MediaStreamVideoSink> mSink;
+    TrackID mID;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
 }
 
 void
-MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   class Message : public ControlMessage {
   public:
-    Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
-      ControlMessage(aStream), mSink(aSink) {}
+    Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+      ControlMessage(aStream), mSink(aSink), mID(aID) {}
     void Run() override
     {
-      mStream->RemoveVideoOutputImpl(mSink);
+      mStream->RemoveVideoOutputImpl(mSink, mID);
     }
     RefPtr<MediaStreamVideoSink> mSink;
+    TrackID mID;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
 }
 
 void
 MediaStream::Suspend()
 {
   class Message : public ControlMessage {
   public:
     explicit Message(MediaStream* aStream) :
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -12,17 +12,16 @@
 
 #include "mozilla/dom/AudioChannelBinding.h"
 
 #include "AudioStream.h"
 #include "nsTArray.h"
 #include "nsIRunnable.h"
 #include "VideoSegment.h"
 #include "StreamTracks.h"
-#include "MediaStreamVideoSink.h"
 #include "MainThreadUtils.h"
 #include "StreamTracks.h"
 #include "nsAutoPtr.h"
 #include "nsAutoRef.h"
 #include <speex/speex_resampler.h>
 
 class nsIRunnable;
 
@@ -162,18 +161,20 @@ class AudioNodeStream;
 class AudioSegment;
 class CameraPreviewMediaStream;
 class DirectMediaStreamListener;
 class DirectMediaStreamTrackListener;
 class MediaInputPort;
 class MediaStreamGraphImpl;
 class MediaStreamListener;
 class MediaStreamTrackListener;
+class MediaStreamVideoSink;
 class ProcessedMediaStream;
 class SourceMediaStream;
+class TrackUnionStream;
 
 enum MediaStreamGraphEvent : uint32_t;
 enum TrackEventCommand : uint32_t;
 
 /**
  * Helper struct for binding a track listener to a specific TrackID.
  */
 template<typename Listener>
@@ -289,18 +290,20 @@ public:
   // Currently only the first enabled audio track is played.
   // XXX change this so all enabled audio tracks are mixed and played.
   virtual void AddAudioOutput(void* aKey);
   virtual void SetAudioOutputVolume(void* aKey, float aVolume);
   virtual void RemoveAudioOutput(void* aKey);
   // Since a stream can be played multiple ways, we need to be able to
   // play to multiple MediaStreamVideoSinks.
   // Only the first enabled video track is played.
-  virtual void AddVideoOutput(MediaStreamVideoSink* aSink);
-  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink);
+  virtual void AddVideoOutput(MediaStreamVideoSink* aSink,
+                              TrackID aID = TRACK_ANY);
+  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink,
+                                 TrackID aID = TRACK_ANY);
   // Explicitly suspend. Useful for example if a media element is pausing
   // and we need to stop its stream emitting its buffered data. As soon as the
   // Suspend message reaches the graph, the stream stops processing. It
   // ignores its inputs and produces silence/no video until Resumed. Its
   // current time does not advance.
   virtual void Suspend();
   virtual void Resume();
   // Events will be dispatched by calling methods of aListener.
@@ -399,16 +402,17 @@ public:
 
   friend class MediaStreamGraphImpl;
   friend class MediaInputPort;
   friend class AudioNodeExternalInputStream;
 
   virtual SourceMediaStream* AsSourceStream() { return nullptr; }
   virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
   virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
+  virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; }
 
   // These Impl methods perform the core functionality of the control methods
   // above, on the media graph thread.
   /**
    * Stop all stream activity and disconnect it from all inputs and outputs.
    * This must be idempotent.
    */
   virtual void DestroyImpl();
@@ -419,18 +423,19 @@ public:
   void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
   void AddAudioOutputImpl(void* aKey);
   // Returns true if this stream has an audio output.
   bool HasAudioOutput()
   {
     return !mAudioOutputs.IsEmpty();
   }
   void RemoveAudioOutputImpl(void* aKey);
-  void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink);
-  void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink);
+  void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
+                          TrackID aID);
+  void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID);
   void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
   void RemoveListenerImpl(MediaStreamListener* aListener);
   void RemoveAllListenersImpl();
   virtual void AddTrackListenerImpl(already_AddRefed<MediaStreamTrackListener> aListener,
                                     TrackID aTrackID);
   virtual void RemoveTrackListenerImpl(MediaStreamTrackListener* aListener,
                                        TrackID aTrackID);
   virtual void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
@@ -579,17 +584,17 @@ protected:
 
   // Client-set volume of this stream
   struct AudioOutput {
     explicit AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
     void* mKey;
     float mVolume;
   };
   nsTArray<AudioOutput> mAudioOutputs;
-  nsTArray<RefPtr<MediaStreamVideoSink>> mVideoOutputs;
+  nsTArray<TrackBound<MediaStreamVideoSink>> mVideoOutputs;
   // We record the last played video frame to avoid playing the frame again
   // with a different frame id.
   VideoFrame mLastPlayedVideoFrame;
   nsTArray<RefPtr<MediaStreamListener> > mListeners;
   nsTArray<TrackBound<MediaStreamTrackListener>> mTrackListeners;
   nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
   nsTArray<TrackID> mDisabledTrackIDs;
 
--- a/dom/media/TrackUnionStream.h
+++ b/dom/media/TrackUnionStream.h
@@ -14,16 +14,19 @@ namespace mozilla {
 
 /**
  * See MediaStreamGraph::CreateTrackUnionStream.
  */
 class TrackUnionStream : public ProcessedMediaStream {
 public:
   explicit TrackUnionStream();
 
+  virtual TrackUnionStream* AsTrackUnionStream() override { return this; }
+  friend class DOMMediaStream;
+
   void RemoveInput(MediaInputPort* aPort) override;
   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;
 
   void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled) override;
 
   MediaStream* GetInputStreamFor(TrackID aTrackID) override;
   TrackID GetInputTrackIDFor(TrackID aTrackID) override;