Bug 801843: Change how video frames are inserted into getUserMedia streams to remove blocking r=roc,anant
authorRandell Jesup <rjesup@jesup.org>
Wed, 17 Oct 2012 05:46:40 -0400
changeset 110653 b5964726ad30b6d45de56d96d26478501009d681
parent 110652 af26f70eb73eb23ec83f421b7b3f832c507061bc
child 110654 b23d650f31fea9b7ee6aed12a8a0166e8e3fac85
push id93
push usernmatsakis@mozilla.com
push dateWed, 31 Oct 2012 21:26:57 +0000
reviewersroc, anant
bugs801843
milestone19.0a1
Bug 801843: Change how video frames are inserted into getUserMedia streams to remove blocking r=roc,anant
content/media/webrtc/MediaEngine.h
content/media/webrtc/MediaEngineDefault.cpp
content/media/webrtc/MediaEngineDefault.h
content/media/webrtc/MediaEngineWebRTC.h
content/media/webrtc/MediaEngineWebRTCAudio.cpp
content/media/webrtc/MediaEngineWebRTCVideo.cpp
dom/media/MediaManager.h
--- a/content/media/webrtc/MediaEngine.h
+++ b/content/media/webrtc/MediaEngine.h
@@ -68,16 +68,19 @@ public:
   virtual nsresult Start(SourceMediaStream*, TrackID) = 0;
 
   /* Take a snapshot from this source. In the case of video this is a single
    * image, and for audio, it is a snippet lasting aDuration milliseconds. The
    * duration argument is ignored for a MediaEngineVideoSource.
    */
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile) = 0;
 
+  /* Called when the stream wants more data */
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) = 0;
+
   /* Stop the device and release the corresponding MediaStream */
   virtual nsresult Stop() = 0;
 
   /* Return false if device is currently allocated or started */
   bool IsAvailable() {
     if (mState == kAllocated || mState == kStarted) {
       return false;
     } else {
--- a/content/media/webrtc/MediaEngineDefault.cpp
+++ b/content/media/webrtc/MediaEngineDefault.cpp
@@ -198,30 +198,46 @@ MediaEngineDefaultVideoSource::Notify(ns
 
   nsRefPtr<layers::PlanarYCbCrImage> image = mImage;
   segment.AppendFrame(image.forget(), USECS_PER_S / DEFAULT_FPS, gfxIntSize(DEFAULT_WIDTH, DEFAULT_HEIGHT));
   mSource->AppendToTrack(mTrackID, &segment);
 
   return NS_OK;
 }
 
-NS_IMPL_THREADSAFE_ISUPPORTS1(MediaEngineDefaultAudioSource, nsITimerCallback)
+void
+MediaEngineDefaultVideoSource::NotifyPull(MediaStreamGraph* aGraph,
+                                          StreamTime aDesiredTime)
+{
+  // Ignore - we push video data
+}
+
+
 /**
  * Default audio source.
  */
+NS_IMPL_THREADSAFE_ISUPPORTS1(MediaEngineDefaultAudioSource, nsITimerCallback)
+
 MediaEngineDefaultAudioSource::MediaEngineDefaultAudioSource()
   : mTimer(nullptr)
 {
   mState = kReleased;
 }
 
 MediaEngineDefaultAudioSource::~MediaEngineDefaultAudioSource()
 {}
 
 void
+MediaEngineDefaultAudioSource::NotifyPull(MediaStreamGraph* aGraph,
+                                          StreamTime aDesiredTime)
+{
+  // Ignore - we push audio data
+}
+
+void
 MediaEngineDefaultAudioSource::GetName(nsAString& aName)
 {
   aName.Assign(NS_LITERAL_STRING("Default Audio Device"));
   return;
 }
 
 void
 MediaEngineDefaultAudioSource::GetUUID(nsAString& aUUID)
--- a/content/media/webrtc/MediaEngineDefault.h
+++ b/content/media/webrtc/MediaEngineDefault.h
@@ -40,16 +40,17 @@ public:
   virtual void GetUUID(nsAString&);
 
   virtual const MediaEngineVideoOptions *GetOptions();
   virtual nsresult Allocate();
   virtual nsresult Deallocate();
   virtual nsresult Start(SourceMediaStream*, TrackID);
   virtual nsresult Stop();
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime);
 
   NS_DECL_ISUPPORTS
   NS_DECL_NSITIMERCALLBACK
 
   // Need something better...
   static const int DEFAULT_WIDTH=640;
   static const int DEFAULT_HEIGHT=480;
   static const int DEFAULT_FPS=30;
@@ -74,16 +75,17 @@ public:
   virtual void GetName(nsAString&);
   virtual void GetUUID(nsAString&);
 
   virtual nsresult Allocate();
   virtual nsresult Deallocate();
   virtual nsresult Start(SourceMediaStream*, TrackID);
   virtual nsresult Stop();
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime);
 
   NS_DECL_ISUPPORTS
   NS_DECL_NSITIMERCALLBACK
 
 protected:
   TrackID mTrackID;
   nsCOMPtr<nsITimer> mTimer;
 
--- a/content/media/webrtc/MediaEngineWebRTC.h
+++ b/content/media/webrtc/MediaEngineWebRTC.h
@@ -48,30 +48,31 @@ namespace mozilla {
 /**
  * The WebRTC implementation of the MediaEngine interface.
  */
 class MediaEngineWebRTCVideoSource : public MediaEngineVideoSource,
                                      public webrtc::ExternalRenderer,
                                      public nsRunnable
 {
 public:
-  static const int DEFAULT_VIDEO_FPS = 30;
+  static const int DEFAULT_VIDEO_FPS = 60;
   static const int DEFAULT_MIN_VIDEO_FPS = 10;
 
   // ViEExternalRenderer.
   virtual int FrameSizeChange(unsigned int, unsigned int, unsigned int);
   virtual int DeliverFrame(unsigned char*, int, uint32_t, int64_t);
 
   MediaEngineWebRTCVideoSource(webrtc::VideoEngine* aVideoEnginePtr,
     int aIndex, int aMinFps = DEFAULT_MIN_VIDEO_FPS)
     : mVideoEngine(aVideoEnginePtr)
     , mCaptureIndex(aIndex)
     , mCapabilityChosen(false)
     , mWidth(640)
     , mHeight(480)
+    , mLastEndTime(0)
     , mMonitor("WebRTCCamera.Monitor")
     , mFps(DEFAULT_VIDEO_FPS)
     , mMinFps(aMinFps)
     , mInitDone(false)
     , mInSnapshotMode(false)
     , mSnapshotPath(NULL) {
     mState = kReleased;
     Init();
@@ -81,16 +82,17 @@ public:
   virtual void GetName(nsAString&);
   virtual void GetUUID(nsAString&);
   virtual const MediaEngineVideoOptions *GetOptions();
   virtual nsresult Allocate();
   virtual nsresult Deallocate();
   virtual nsresult Start(SourceMediaStream*, TrackID);
   virtual nsresult Stop();
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime);
 
   NS_DECL_ISUPPORTS
 
   // This runnable is for creating a temporary file on the main thread.
   NS_IMETHODIMP
   Run()
   {
     nsCOMPtr<nsIFile> tmp;
@@ -123,26 +125,28 @@ private:
   webrtc::ViECapture* mViECapture;
   webrtc::ViERender* mViERender;
   webrtc::CaptureCapability mCapability; // Doesn't work on OS X.
 
   int mCaptureIndex;
   bool mCapabilityChosen;
   int mWidth, mHeight;
   TrackID mTrackID;
+  TrackTicks mLastEndTime;
 
   mozilla::ReentrantMonitor mMonitor; // Monitor for processing WebRTC frames.
   SourceMediaStream* mSource;
 
   int mFps; // Track rate (30 fps by default)
   int mMinFps; // Min rate we want to accept
   bool mInitDone;
   bool mInSnapshotMode;
   nsString* mSnapshotPath;
 
+  nsRefPtr<layers::Image> mImage;
   nsRefPtr<layers::ImageContainer> mImageContainer;
 
   PRLock* mSnapshotLock;
   PRCondVar* mSnapshotCondVar;
 
   // These are in UTF-8 but webrtc api uses char arrays
   char mDeviceName[KMaxDeviceNameLength];
   char mUniqueId[KMaxUniqueIdLength];
@@ -172,16 +176,17 @@ public:
   virtual void GetName(nsAString&);
   virtual void GetUUID(nsAString&);
 
   virtual nsresult Allocate();
   virtual nsresult Deallocate();
   virtual nsresult Start(SourceMediaStream*, TrackID);
   virtual nsresult Stop();
   virtual nsresult Snapshot(uint32_t aDuration, nsIDOMFile** aFile);
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime);
 
   // VoEMediaProcess.
   void Process(const int channel, const webrtc::ProcessingTypes type,
                WebRtc_Word16 audio10ms[], const int length,
                const int samplingFreq, const bool isStereo);
 
   NS_DECL_ISUPPORTS
 
--- a/content/media/webrtc/MediaEngineWebRTCAudio.cpp
+++ b/content/media/webrtc/MediaEngineWebRTCAudio.cpp
@@ -84,16 +84,17 @@ MediaEngineWebRTCAudioSource::Start(Sour
   }
 
   mSource = aStream;
 
   AudioSegment* segment = new AudioSegment();
   segment->Init(CHANNELS);
   mSource->AddTrack(aID, SAMPLE_FREQUENCY, 0, segment);
   mSource->AdvanceKnownTracksTime(STREAM_TIME_MAX);
+  LOG(("Initial audio"));
   mTrackID = aID;
 
   if (mVoEBase->StartReceive(mChannel)) {
     return NS_ERROR_FAILURE;
   }
   if (mVoEBase->StartSend(mChannel)) {
     return NS_ERROR_FAILURE;
   }
@@ -123,16 +124,23 @@ MediaEngineWebRTCAudioSource::Stop()
   if (mVoEBase->StopReceive(mChannel)) {
     return NS_ERROR_FAILURE;
   }
 
   mState = kStopped;
   return NS_OK;
 }
 
+void
+MediaEngineWebRTCAudioSource::NotifyPull(MediaStreamGraph* aGraph,
+                                         StreamTime aDesiredTime)
+{
+  // Ignore - we push audio data
+}
+
 nsresult
 MediaEngineWebRTCAudioSource::Snapshot(uint32_t aDuration, nsIDOMFile** aFile)
 {
    return NS_ERROR_NOT_IMPLEMENTED;
 }
 
 void
 MediaEngineWebRTCAudioSource::Init()
--- a/content/media/webrtc/MediaEngineWebRTCVideo.cpp
+++ b/content/media/webrtc/MediaEngineWebRTCVideo.cpp
@@ -31,18 +31,16 @@ MediaEngineWebRTCVideoSource::FrameSizeC
   return 0;
 }
 
 // ViEExternalRenderer Callback. Process every incoming frame here.
 int
 MediaEngineWebRTCVideoSource::DeliverFrame(
    unsigned char* buffer, int size, uint32_t time_stamp, int64_t render_time)
 {
-  ReentrantMonitorAutoEnter enter(mMonitor);
-
   if (mInSnapshotMode) {
     // Set the condition variable to false and notify Snapshot().
     PR_Lock(mSnapshotLock);
     mInSnapshotMode = false;
     PR_NotifyCondVar(mSnapshotCondVar);
     PR_Unlock(mSnapshotLock);
     return 0;
   }
@@ -50,16 +48,17 @@ MediaEngineWebRTCVideoSource::DeliverFra
   // Check for proper state.
   if (mState != kStarted) {
     LOG(("DeliverFrame: video not started"));
     return 0;
   }
 
   // Create a video frame and append it to the track.
   ImageFormat format = PLANAR_YCBCR;
+
   nsRefPtr<layers::Image> image = mImageContainer->CreateImage(&format, 1);
 
   layers::PlanarYCbCrImage* videoImage = static_cast<layers::PlanarYCbCrImage*>(image.get());
 
   uint8_t* frame = static_cast<uint8_t*> (buffer);
   const uint8_t lumaBpp = 8;
   const uint8_t chromaBpp = 4;
 
@@ -73,20 +72,56 @@ MediaEngineWebRTCVideoSource::DeliverFra
   data.mCbCrSize = gfxIntSize(mWidth/ 2, mHeight/ 2);
   data.mPicX = 0;
   data.mPicY = 0;
   data.mPicSize = gfxIntSize(mWidth, mHeight);
   data.mStereoMode = STEREO_MODE_MONO;
 
   videoImage->SetData(data);
 
+#ifdef LOG_ALL_FRAMES
+  static uint32_t frame_num = 0;
+  LOG(("frame %d; timestamp %u, render_time %lu", frame_num++, time_stamp, render_time));
+#endif
+
+  // we don't touch anything in 'this' until here (except for snapshot,
+  // which has it's own lock)
+  ReentrantMonitorAutoEnter enter(mMonitor);
+
+  // implicitly releases last image
+  mImage = image.forget();
+
+  return 0;
+}
+
+// Called if the graph thinks it's running out of buffered video; repeat
+// the last frame for whatever minimum period it think it needs.  Note that
+// this means that no *real* frame can be inserted during this period.
+void
+MediaEngineWebRTCVideoSource::NotifyPull(MediaStreamGraph* aGraph,
+                                         StreamTime aDesiredTime)
+{
   VideoSegment segment;
-  segment.AppendFrame(image.forget(), 1, gfxIntSize(mWidth, mHeight));
+
+  ReentrantMonitorAutoEnter enter(mMonitor);
+
+  if (mState != kStarted)
+    return;
+
+  // Note: we're not giving up mImage here
+  nsRefPtr<layers::Image> image = mImage;
+  TrackTicks target = TimeToTicksRoundUp(USECS_PER_S, aDesiredTime);
+  TrackTicks delta = target - mLastEndTime;
+#ifdef LOG_ALL_FRAMES
+  LOG(("NotifyPull, target = %lu, delta = %lu", (uint64_t) target, (uint64_t) delta));
+#endif
+  // NULL images are allowed
+  segment.AppendFrame(image ? image.forget() : nullptr, delta, gfxIntSize(mWidth, mHeight));
   mSource->AppendToTrack(mTrackID, &(segment));
-  return 0;
+  mLastEndTime = target;
 }
 
 void
 MediaEngineWebRTCVideoSource::ChooseCapability(uint32_t aWidth, uint32_t aHeight, uint32_t aMinFPS)
 {
   int num = mViECapture->NumberOfCapabilities(mUniqueId, KMaxUniqueIdLength);
 
   NS_WARN_IF_FALSE(!mCapabilityChosen,"Shouldn't select capability of a device twice");
@@ -208,18 +243,20 @@ MediaEngineWebRTCVideoSource::Start(Sour
   if (mState == kStarted) {
     return NS_OK;
   }
 
   mSource = aStream;
   mTrackID = aID;
 
   mImageContainer = layers::LayerManager::CreateImageContainer();
-  mSource->AddTrack(aID, mFps, 0, new VideoSegment());
+
+  mSource->AddTrack(aID, USECS_PER_S, 0, new VideoSegment());
   mSource->AdvanceKnownTracksTime(STREAM_TIME_MAX);
+  mLastEndTime = 0;
 
   error = mViERender->AddRenderer(mCaptureIndex, webrtc::kVideoI420, (webrtc::ExternalRenderer*)this);
   if (error == -1) {
     return NS_ERROR_FAILURE;
   }
 
   error = mViERender->StartRender(mCaptureIndex);
   if (error == -1) {
--- a/dom/media/MediaManager.h
+++ b/dom/media/MediaManager.h
@@ -105,41 +105,58 @@ public:
 
   void
   NotifyConsumptionChanged(MediaStreamGraph* aGraph, Consumption aConsuming)
   {
     if (aConsuming == CONSUMED) {
       nsresult rv;
 
       SourceMediaStream* stream = mStream->GetStream()->AsSourceStream();
+      stream->SetPullEnabled(true);
+
       if (mAudioSource) {
         rv = mAudioSource->Start(stream, kAudioTrack);
         if (NS_FAILED(rv)) {
           MM_LOG(("Starting audio failed, rv=%d",rv));
         }
       }
       if (mVideoSource) {
         rv = mVideoSource->Start(stream, kVideoTrack);
         if (NS_FAILED(rv)) {
           MM_LOG(("Starting video failed, rv=%d",rv));
         }
       }
+
       MM_LOG(("started all sources"));
       nsCOMPtr<GetUserMediaNotificationEvent> event =
         new GetUserMediaNotificationEvent(GetUserMediaNotificationEvent::STARTING);
 
       NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
       return;
     }
 
     // NOT_CONSUMED
     Invalidate();
     return;
   }
 
+  // Proxy NotifyPull() to sources
+  void
+  NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime)
+  {
+    // Currently audio sources ignore NotifyPull, but they could
+    // watch it especially for fake audio.
+    if (mAudioSource) {
+      mAudioSource->NotifyPull(aGraph, aDesiredTime);
+    }
+    if (mVideoSource) {
+      mVideoSource->NotifyPull(aGraph, aDesiredTime);
+    }
+  }
+
 private:
   nsRefPtr<MediaEngineSource> mAudioSource;
   nsRefPtr<MediaEngineSource> mVideoSource;
   nsCOMPtr<nsDOMMediaStream> mStream;
   bool mValid;
 };
 
 typedef nsTArray<nsRefPtr<GetUserMediaCallbackMediaStreamListener> > StreamListeners;