Bug 1201363 - MediaStreamVideoSink for ImageCapture case. r=jesup
authorctai <ctai@mozilla.com>
Tue, 31 May 2016 13:53:49 +0800
changeset 308257 36328de44ea73990e3f4e103451b0d0c497d8faa
parent 308256 647046fe760c30364990fec0c3c4c40a78d7ef25
child 308258 d151c9b4885ace0e9bf9dea15e8312b6047100b0
push id31092
push usercbook@mozilla.com
push dateFri, 05 Aug 2016 10:16:59 +0000
treeherderautoland@b97dd7dd3cb9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjesup
bugs1201363
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1201363 - MediaStreamVideoSink for ImageCapture case. r=jesup Make CaptureTask to inherite from MediaStreamVideoSink. The main change is to move the logic of |NotifyQueuedTrackChanges| to |SetCurrentFrames|. The original image capture is not modified for support multiple video MediaStreamTracks. The design still used the track id in owned media stream. The should be fixed in the following bug if we still want to support ImageCapture in multiple video tracks case. MozReview-Commit-ID: Od4tHoR8Ef
dom/media/imagecapture/CaptureTask.cpp
dom/media/imagecapture/CaptureTask.h
dom/media/imagecapture/ImageCapture.cpp
--- a/dom/media/imagecapture/CaptureTask.cpp
+++ b/dom/media/imagecapture/CaptureTask.cpp
@@ -10,16 +10,42 @@
 #include "mozilla/dom/ImageEncoder.h"
 #include "mozilla/dom/MediaStreamTrack.h"
 #include "mozilla/dom/VideoStreamTrack.h"
 #include "gfxUtils.h"
 #include "nsThreadUtils.h"
 
 namespace mozilla {
 
+class CaptureTask::MediaStreamEventListener : public MediaStreamTrackListener
+{
+public:
+  explicit MediaStreamEventListener(CaptureTask* aCaptureTask)
+    : mCaptureTask(aCaptureTask) {};
+
+  // MediaStreamListener methods.
+  void NotifyEnded() override
+  {
+    if(!mCaptureTask->mImageGrabbedOrTrackEnd) {
+      mCaptureTask->PostTrackEndEvent();
+    }
+  }
+
+private:
+  CaptureTask* mCaptureTask;
+};
+
+CaptureTask::CaptureTask(dom::ImageCapture* aImageCapture)
+  : mImageCapture(aImageCapture)
+  , mEventListener(new MediaStreamEventListener(this))
+  , mImageGrabbedOrTrackEnd(false)
+  , mPrincipalChanged(false)
+{
+}
+
 nsresult
 CaptureTask::TaskComplete(already_AddRefed<dom::Blob> aBlob, nsresult aRv)
 {
   MOZ_ASSERT(NS_IsMainThread());
 
   DetachTrack();
 
   nsresult rv;
@@ -50,47 +76,45 @@ CaptureTask::TaskComplete(already_AddRef
 
 void
 CaptureTask::AttachTrack()
 {
   MOZ_ASSERT(NS_IsMainThread());
 
   dom::VideoStreamTrack* track = mImageCapture->GetVideoStreamTrack();
   track->AddPrincipalChangeObserver(this);
-  track->AddListener(this);
+  track->AddListener(mEventListener.get());
+  track->AddDirectListener(this);
 }
 
 void
 CaptureTask::DetachTrack()
 {
   MOZ_ASSERT(NS_IsMainThread());
 
   dom::VideoStreamTrack* track = mImageCapture->GetVideoStreamTrack();
   track->RemovePrincipalChangeObserver(this);
-  track->RemoveListener(this);
+  track->RemoveListener(mEventListener.get());
+  track->RemoveDirectListener(this);
 }
 
 void
 CaptureTask::PrincipalChanged(dom::MediaStreamTrack* aMediaStreamTrack)
 {
   MOZ_ASSERT(NS_IsMainThread());
   mPrincipalChanged = true;
 }
 
 void
-CaptureTask::NotifyQueuedChanges(MediaStreamGraph* aGraph,
-                                 StreamTime aTrackOffset,
-                                 const MediaSegment& aQueuedMedia)
+CaptureTask::SetCurrentFrames(const VideoSegment& aSegment)
 {
   if (mImageGrabbedOrTrackEnd) {
     return;
   }
 
-  MOZ_ASSERT(aQueuedMedia.GetType() == MediaSegment::VIDEO);
-
   // Callback for encoding complete, it calls on main thread.
   class EncodeComplete : public dom::EncodeCompleteCallback
   {
   public:
     explicit EncodeComplete(CaptureTask* aTask) : mTask(aTask) {}
 
     nsresult ReceiveBlob(already_AddRefed<dom::Blob> aBlob) override
     {
@@ -99,21 +123,23 @@ CaptureTask::NotifyQueuedChanges(MediaSt
       mTask = nullptr;
       return NS_OK;
     }
 
   protected:
     RefPtr<CaptureTask> mTask;
   };
 
-  VideoSegment* video =
-    const_cast<VideoSegment*> (static_cast<const VideoSegment*>(&aQueuedMedia));
-  VideoSegment::ChunkIterator iter(*video);
+  VideoSegment::ConstChunkIterator iter(aSegment);
+
+
+
   while (!iter.IsEnded()) {
     VideoChunk chunk = *iter;
+
     // Extract the first valid video frame.
     VideoFrame frame;
     if (!chunk.IsNull()) {
       RefPtr<layers::Image> image;
       if (chunk.mFrame.GetForceBlack()) {
         // Create a black image.
         image = VideoFrame::CreateBlackImage(chunk.mFrame.GetIntrinsicSize());
       } else {
@@ -137,24 +163,16 @@ CaptureTask::NotifyQueuedChanges(MediaSt
       }
       return;
     }
     iter.Next();
   }
 }
 
 void
-CaptureTask::NotifyEnded()
-{
-  if(!mImageGrabbedOrTrackEnd) {
-    PostTrackEndEvent();
-  }
-}
-
-void
 CaptureTask::PostTrackEndEvent()
 {
   mImageGrabbedOrTrackEnd = true;
 
   // Got track end or finish event, stop the task.
   class TrackEndRunnable : public Runnable
   {
   public:
--- a/dom/media/imagecapture/CaptureTask.h
+++ b/dom/media/imagecapture/CaptureTask.h
@@ -5,16 +5,17 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef CAPTURETASK_H
 #define CAPTURETASK_H
 
 #include "MediaStreamGraph.h"
 #include "MediaStreamListener.h"
 #include "PrincipalChangeObserver.h"
+#include "MediaStreamVideoSink.h"
 
 namespace mozilla {
 
 namespace dom {
 class Blob;
 class ImageCapture;
 class MediaStreamTrack;
 } // namespace dom
@@ -24,26 +25,25 @@ class MediaStreamTrack;
  * ImageEncoder. The whole procedures start at AttachTrack(), it will add this
  * class into MediaStream and retrieves an image in MediaStreamGraph thread.
  * Once the image is retrieved, it will be sent to ImageEncoder and the encoded
  * blob will be sent out via encoder callback in main thread.
  *
  * CaptureTask holds a reference of ImageCapture to ensure ImageCapture won't be
  * released during the period of the capturing process described above.
  */
-class CaptureTask : public MediaStreamTrackListener,
+class CaptureTask : public MediaStreamVideoSink,
                     public dom::PrincipalChangeObserver<dom::MediaStreamTrack>
 {
 public:
-  // MediaStreamTrackListener methods.
-  void NotifyQueuedChanges(MediaStreamGraph* aGraph,
-                           StreamTime aTrackOffset,
-                           const MediaSegment& aQueuedMedia) override;
+  class MediaStreamEventListener;
 
-  void NotifyEnded() override;
+  // MediaStreamVideoSink methods.
+  void SetCurrentFrames(const VideoSegment& aSegment) override;
+  void ClearFrames() override {}
 
   // PrincipalChangeObserver<MediaStreamTrack> method.
   void PrincipalChanged(dom::MediaStreamTrack* aMediaStreamTrack) override;
 
   // CaptureTask methods.
 
   // It is called when aBlob is ready to post back to script in company with
   // aRv == NS_OK. If aRv is not NS_OK, it will post an error event to script.
@@ -56,33 +56,32 @@ public:
   // It should be on main thread only.
   void AttachTrack();
 
   // Remove listeners from MediaStreamTrack and PrincipalChangeObserver.
   // It should be on main thread only.
   void DetachTrack();
 
   // CaptureTask should be created on main thread.
-  explicit CaptureTask(dom::ImageCapture* aImageCapture)
-    : mImageCapture(aImageCapture)
-    , mImageGrabbedOrTrackEnd(false)
-    , mPrincipalChanged(false) {}
+  explicit CaptureTask(dom::ImageCapture* aImageCapture);
 
 protected:
   virtual ~CaptureTask() {}
 
   // Post a runnable on main thread to end this task and call TaskComplete to post
   // error event to script. It is called off-main-thread.
   void PostTrackEndEvent();
 
   // The ImageCapture associates with this task. This reference count should not
   // change in this class unless it clears this reference after a blob or error
   // event to script.
   RefPtr<dom::ImageCapture> mImageCapture;
 
+  RefPtr<MediaStreamEventListener> mEventListener;
+
   // True when an image is retrieved from MediaStreamGraph or MediaStreamGraph
   // sends a track finish, end, or removed event.
   bool mImageGrabbedOrTrackEnd;
 
   // True after MediaStreamTrack principal changes while waiting for a photo
   // to finish and we should raise a security error.
   bool mPrincipalChanged;
 };
--- a/dom/media/imagecapture/ImageCapture.cpp
+++ b/dom/media/imagecapture/ImageCapture.cpp
@@ -142,18 +142,17 @@ ImageCapture::TakePhoto(ErrorResult& aRe
 
   // Try if MediaEngine supports taking photo.
   nsresult rv = TakePhotoByMediaEngine();
 
   // It falls back to MediaStreamGraph image capture if MediaEngine doesn't
   // support TakePhoto().
   if (rv == NS_ERROR_NOT_IMPLEMENTED) {
     IC_LOG("MediaEngine doesn't support TakePhoto(), it falls back to MediaStreamGraph.");
-    RefPtr<CaptureTask> task =
-      new CaptureTask(this);
+    RefPtr<CaptureTask> task = new CaptureTask(this);
 
     // It adds itself into MediaStreamGraph, so ImageCapture doesn't need to hold
     // the reference.
     task->AttachTrack();
   }
 }
 
 nsresult