Bug 1080755 - Push video frames into MediaStreamGraph instead of waiting for pulls. r=padenot, a=lmandel
authorRandell Jesup <rjesup@jesup.org>
Thu, 23 Oct 2014 12:11:41 -0400
changeset 225829 22cfde2bf1ce
parent 225828 5a4dfee44717
child 225830 87081b44ad8a
child 225832 63587994bad9
child 225834 6cb4888905c9
push id4029
push userryanvm@gmail.com
push date2014-10-27 14:38 +0000
treeherdermozilla-beta@22cfde2bf1ce [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot, lmandel
bugs1080755
milestone34.0
Bug 1080755 - Push video frames into MediaStreamGraph instead of waiting for pulls. r=padenot, a=lmandel
content/media/webrtc/MediaEngineWebRTC.h
content/media/webrtc/MediaEngineWebRTCVideo.cpp
--- a/content/media/webrtc/MediaEngineWebRTC.h
+++ b/content/media/webrtc/MediaEngineWebRTC.h
@@ -97,16 +97,17 @@ public:
 #ifdef MOZ_B2G_CAMERA
   MediaEngineWebRTCVideoSource(int aIndex,
                                MediaSourceType aMediaSource = MediaSourceType::Camera)
     : mCameraControl(nullptr)
     , mCallbackMonitor("WebRTCCamera.CallbackMonitor")
     , mRotation(0)
     , mBackCamera(false)
     , mCaptureIndex(aIndex)
+    , mTrackID(0)
     , mMediaSource(aMediaSource)
     , mMonitor("WebRTCCamera.Monitor")
     , mWidth(0)
     , mHeight(0)
     , mHasDirectListeners(false)
     , mInitDone(false)
     , mInSnapshotMode(false)
     , mSnapshotPath(nullptr)
@@ -125,16 +126,17 @@ public:
    * XXX Investigate!  Especially for Android/B2G
    */
   virtual bool IsTextureSupported() { return false; }
 
   MediaEngineWebRTCVideoSource(webrtc::VideoEngine* aVideoEnginePtr, int aIndex,
                                MediaSourceType aMediaSource = MediaSourceType::Camera)
     : mVideoEngine(aVideoEnginePtr)
     , mCaptureIndex(aIndex)
+    , mTrackID(0)
     , mFps(-1)
     , mMinFps(-1)
     , mMediaSource(aMediaSource)
     , mMonitor("WebRTCCamera.Monitor")
     , mWidth(0)
     , mHeight(0)
     , mHasDirectListeners(false)
     , mInitDone(false)
@@ -215,16 +217,22 @@ public:
     return NS_OK;
   }
 
   void Refresh(int aIndex);
 
 protected:
   ~MediaEngineWebRTCVideoSource() { Shutdown(); }
 
+  // guts for appending data to the MSG track
+  virtual bool AppendToTrack(SourceMediaStream* aSource,
+                             layers::Image* aImage,
+                             TrackID aID,
+                             TrackTicks delta);
+
 private:
   // Initialize the needed Video engine interfaces.
   void Init();
   void Shutdown();
 
   // Engine variables.
 #ifdef MOZ_B2G_CAMERA
   mozilla::ReentrantMonitor mCallbackMonitor; // Monitor for camera callback handling
@@ -240,16 +248,17 @@ private:
   webrtc::VideoEngine* mVideoEngine; // Weak reference, don't free.
   webrtc::ViEBase* mViEBase;
   webrtc::ViECapture* mViECapture;
   webrtc::ViERender* mViERender;
 #endif
   webrtc::CaptureCapability mCapability; // Doesn't work on OS X.
 
   int mCaptureIndex;
+  TrackID mTrackID;
   int mFps; // Track rate (30 fps by default)
   int mMinFps; // Min rate we want to accept
   MediaSourceType mMediaSource; // source of media (camera | application | screen)
 
   // mMonitor protects mImage access/changes, and transitions of mState
   // from kStarted to kStopped (which are combined with EndTrack() and
   // image changes).  Note that mSources is not accessed from other threads
   // for video and is not protected.
--- a/content/media/webrtc/MediaEngineWebRTCVideo.cpp
+++ b/content/media/webrtc/MediaEngineWebRTCVideo.cpp
@@ -117,61 +117,90 @@ MediaEngineWebRTCVideoSource::DeliverFra
 
   // we don't touch anything in 'this' until here (except for snapshot,
   // which has it's own lock)
   MonitorAutoLock lock(mMonitor);
 
   // implicitly releases last image
   mImage = image.forget();
 
+  // Push the frame into the MSG with a minimal duration.  This will likely
+  // mean we'll still get NotifyPull calls which will then return the same
+  // frame again with a longer duration.  However, this means we won't
+  // fail to get the frame in and drop frames.
+
+  // XXX The timestamp for the frame should be base on the Capture time,
+  // not the MSG time, and MSG should never, ever block on a (realtime)
+  // video frame (or even really for streaming - audio yes, video probably no).
+  uint32_t len = mSources.Length();
+  for (uint32_t i = 0; i < len; i++) {
+    if (mSources[i]) {
+      AppendToTrack(mSources[i], mImage, mTrackID, 1); // shortest possible duration
+    }
+  }
+
   return 0;
 }
 #endif
 
+bool
+MediaEngineWebRTCVideoSource::AppendToTrack(SourceMediaStream* aSource,
+                                            layers::Image* aImage,
+                                            TrackID aID,
+                                            TrackTicks delta)
+{
+  MOZ_ASSERT(aSource);
+
+  VideoSegment segment;
+  nsRefPtr<layers::Image> image = aImage;
+  IntSize size(image ? mWidth : 0, image ? mHeight : 0);
+  segment.AppendFrame(image.forget(), delta, size);
+
+  // This is safe from any thread, and is safe if the track is Finished
+  // or Destroyed.
+  // This can fail if either a) we haven't added the track yet, or b)
+  // we've removed or finished the track.
+  return aSource->AppendToTrack(aID, &(segment));
+}
+
 // Called if the graph thinks it's running out of buffered video; repeat
 // the last frame for whatever minimum period it think it needs.  Note that
 // this means that no *real* frame can be inserted during this period.
 void
 MediaEngineWebRTCVideoSource::NotifyPull(MediaStreamGraph* aGraph,
-                                         SourceMediaStream *aSource,
+                                         SourceMediaStream* aSource,
                                          TrackID aID,
                                          StreamTime aDesiredTime,
                                          TrackTicks &aLastEndTime)
 {
   VideoSegment segment;
 
   MonitorAutoLock lock(mMonitor);
   // B2G does AddTrack, but holds kStarted until the hardware changes state.
   // So mState could be kReleased here.  We really don't care about the state,
   // though.
 
-  // Note: we're not giving up mImage here
-  nsRefPtr<layers::Image> image = mImage;
   TrackTicks target = aSource->TimeToTicksRoundUp(USECS_PER_S, aDesiredTime);
   TrackTicks delta = target - aLastEndTime;
   LOGFRAME(("NotifyPull, desired = %ld, target = %ld, delta = %ld %s", (int64_t) aDesiredTime,
-            (int64_t) target, (int64_t) delta, image ? "" : "<null>"));
+            (int64_t) target, (int64_t) delta, mImage ? "" : "<null>"));
 
   // Bug 846188 We may want to limit incoming frames to the requested frame rate
   // mFps - if you want 30FPS, and the camera gives you 60FPS, this could
   // cause issues.
   // We may want to signal if the actual frame rate is below mMinFPS -
   // cameras often don't return the requested frame rate especially in low
   // light; we should consider surfacing this so that we can switch to a
   // lower resolution (which may up the frame rate)
 
   // Don't append if we've already provided a frame that supposedly goes past the current aDesiredTime
   // Doing so means a negative delta and thus messes up handling of the graph
   if (delta > 0) {
     // nullptr images are allowed
-    IntSize size(image ? mWidth : 0, image ? mHeight : 0);
-    segment.AppendFrame(image.forget(), delta, size);
-    // This can fail if either a) we haven't added the track yet, or b)
-    // we've removed or finished the track.
-    if (aSource->AppendToTrack(aID, &(segment))) {
+    if (AppendToTrack(aSource, mImage, aID, delta)) {
       aLastEndTime = target;
     }
   }
 }
 
 static bool IsWithin(int32_t n, const ConstrainLongRange& aRange) {
   return aRange.mMin <= n && n <= aRange.mMax;
 }
@@ -542,18 +571,21 @@ MediaEngineWebRTCVideoSource::Start(Sour
 #ifdef MOZ_B2G_CAMERA
   NS_DispatchToMainThread(WrapRunnable(nsRefPtr<MediaEngineWebRTCVideoSource>(this),
                                        &MediaEngineWebRTCVideoSource::StartImpl,
                                        mCapability));
   mCallbackMonitor.Wait();
   if (mState != kStarted) {
     return NS_ERROR_FAILURE;
   }
+  mTrackID = aID;
 #else
   mState = kStarted;
+  mTrackID = aID;
+
   error = mViERender->AddRenderer(mCaptureIndex, webrtc::kVideoI420, (webrtc::ExternalRenderer*)this);
   if (error == -1) {
     return NS_ERROR_FAILURE;
   }
 
   error = mViERender->StartRender(mCaptureIndex);
   if (error == -1) {
     return NS_ERROR_FAILURE;
@@ -960,16 +992,31 @@ MediaEngineWebRTCVideoSource::RotateImag
   data.mPicY = 0;
   data.mPicSize = IntSize(dstWidth, dstHeight);
   data.mStereoMode = StereoMode::MONO;
 
   videoImage->SetDataNoCopy(data);
 
   // implicitly releases last image
   mImage = image.forget();
+
+  // Push the frame into the MSG with a minimal duration.  This will likely
+  // mean we'll still get NotifyPull calls which will then return the same
+  // frame again with a longer duration.  However, this means we won't
+  // fail to get the frame in and drop frames.
+
+  // XXX The timestamp for the frame should be base on the Capture time,
+  // not the MSG time, and MSG should never, ever block on a (realtime)
+  // video frame (or even really for streaming - audio yes, video probably no).
+  uint32_t len = mSources.Length();
+  for (uint32_t i = 0; i < len; i++) {
+    if (mSources[i]) {
+      AppendToTrack(mSources[i], mImage, mTrackID, 1); // shortest possible duration
+    }
+  }
 }
 
 bool
 MediaEngineWebRTCVideoSource::OnNewPreviewFrame(layers::Image* aImage, uint32_t aWidth, uint32_t aHeight) {
   {
     ReentrantMonitorAutoEnter sync(mCallbackMonitor);
     if (mState == kStopped) {
       return false;