Bug 1320101 - Support renegotiations with changes to TIAS and simulcast params r=pehrsons
authorAndreas Pehrson <pehrsons@gmail.com>
Thu, 08 Dec 2016 10:11:18 -1000
changeset 375063 45500e7a10c367be371038514a1e5c95ea0adb3f
parent 375062 84568b4acdccb31cc8c5c4d91a653b4d54f5b07d
child 375064 4ea7585240cfbecfa047fb578ba04478f2f19e1f
push id6996
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 20:48:21 +0000
treeherdermozilla-beta@d89512dab048 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspehrsons
bugs1320101, 1250326
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1320101 - Support renegotiations with changes to TIAS and simulcast params r=pehrsons Reland of bug 1320101 patch 5 due to partial backout via mismerge in bug 1250326 MozReview-Commit-ID: GNWRNnwX9pk
media/webrtc/signaling/src/common/EncodingConstraints.h
media/webrtc/signaling/src/media-conduit/CodecConfig.h
media/webrtc/signaling/src/media-conduit/VideoConduit.cpp
media/webrtc/signaling/src/media-conduit/VideoConduit.h
--- a/media/webrtc/signaling/src/common/EncodingConstraints.h
+++ b/media/webrtc/signaling/src/common/EncodingConstraints.h
@@ -37,16 +37,28 @@ public:
       maxBr == constraints.maxBr &&
       maxPps == constraints.maxPps &&
       maxMbps == constraints.maxMbps &&
       maxCpb == constraints.maxCpb &&
       maxDpb == constraints.maxDpb &&
       scaleDownBy == constraints.scaleDownBy;
   }
 
+  /**
+   * This returns true if the constraints affecting resolution are equal.
+   */
+  bool ResolutionEquals(const EncodingConstraints& constraints) const
+  {
+    return
+      maxWidth == constraints.maxWidth &&
+      maxHeight == constraints.maxHeight &&
+      maxFs == constraints.maxFs &&
+      scaleDownBy == constraints.scaleDownBy;
+  }
+
   uint32_t maxWidth;
   uint32_t maxHeight;
   uint32_t maxFps;
   uint32_t maxFs;
   uint32_t maxBr;
   uint32_t maxPps;
   uint32_t maxMbps; // macroblocks per second
   uint32_t maxCpb; // coded picture buffer size
--- a/media/webrtc/signaling/src/media-conduit/CodecConfig.h
+++ b/media/webrtc/signaling/src/media-conduit/CodecConfig.h
@@ -148,16 +148,30 @@ public:
       mProfile = (h264->profile_level_id & 0x00FF0000) >> 16;
       mConstraints = (h264->profile_level_id & 0x0000FF00) >> 8;
       mLevel = (h264->profile_level_id & 0x000000FF);
       mPacketizationMode = h264->packetization_mode;
       mSpropParameterSets = h264->sprop_parameter_sets;
     }
   }
 
+  bool ResolutionEquals(const VideoCodecConfig& aConfig) const
+  {
+    if (mSimulcastEncodings.size() != aConfig.mSimulcastEncodings.size()) {
+      return false;
+    }
+    for (size_t i = 0; i < mSimulcastEncodings.size(); ++i) {
+      if (!mSimulcastEncodings[i].constraints.ResolutionEquals(
+            aConfig.mSimulcastEncodings[i].constraints)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   // Nothing seems to use this right now. Do we intend to support this
   // someday?
   bool RtcpFbAckIsSet(const std::string& type) const
   {
     for (auto i = mAckFbTypes.begin(); i != mAckFbTypes.end(); ++i) {
       if (*i == type) {
         return true;
       }
--- a/media/webrtc/signaling/src/media-conduit/VideoConduit.cpp
+++ b/media/webrtc/signaling/src/media-conduit/VideoConduit.cpp
@@ -512,73 +512,98 @@ WebrtcVideoConduit::ConfigureSendMediaCo
   unsigned short width = 320;
   unsigned short height = 240;
   int max_framerate;
   if (codecConfig->mEncodingConstraints.maxFps > 0) {
     max_framerate = codecConfig->mEncodingConstraints.maxFps;
   } else {
     max_framerate = DEFAULT_VIDEO_MAX_FRAMERATE;
   }
+  // apply restrictions from maxMbps/etc
+  mSendingFramerate = SelectSendFrameRate(codecConfig,
+                                          max_framerate,
+                                          mSendingWidth,
+                                          mSendingHeight);
+
+  // So we can comply with b=TIAS/b=AS/maxbr=X when input resolution changes
+  mNegotiatedMaxBitrate = codecConfig->mTias / 1000;
 
   // width/height will be overridden on the first frame; they must be 'sane' for
   // SetSendCodec()
 
   if (mSendingWidth != 0) {
     // We're already in a call and are reconfiguring (perhaps due to
-    // ReplaceTrack).  Set to match the last frame we sent.
+    // ReplaceTrack).
+    bool resolutionChanged;
+    {
+      MutexAutoLock lock(mCodecMutex);
+      resolutionChanged = !mCurSendCodecConfig->ResolutionEquals(*codecConfig);
+    }
 
-    // We could also set mLastWidth to 0, to force immediate reconfig -
-    // more expensive, but perhaps less risk of missing something.  Really
-    // on ReplaceTrack we should just call ConfigureCodecMode(), and if the
-    // mode changed, we re-configure.
-    width = mSendingWidth;
-    height = mSendingHeight;
-    // max framerate remains the same
+    if (resolutionChanged) {
+      // We're already in a call and due to renegotiation an encoder parameter
+      // that requires reconfiguration has changed. Resetting these members
+      // triggers reconfig on the next frame.
+      mLastWidth = 0;
+      mLastHeight = 0;
+      mSendingWidth = 0;
+      mSendingHeight = 0;
+    } else {
+      // We're already in a call but changes don't require a reconfiguration.
+      // We update the resolutions in the send codec to match the current
+      // settings.  Framerate is already set.
+      width = mSendingWidth;
+      height = mSendingHeight;
+      // Bitrates are set in the loop below
+    }
   }
-  mSendingFramerate = std::max(mSendingFramerate,
-                               static_cast<unsigned int>(max_framerate));
-
-  // So we can comply with b=TIAS/b=AS/maxbr=X when input resolution changes
-  mNegotiatedMaxBitrate = codecConfig->mTias / 1000;
 
   for (size_t idx = streamCount - 1; streamCount > 0; idx--, streamCount--) {
     webrtc::VideoStream video_stream;
     VideoEncoderConfigBuilder::SimulcastStreamConfig simulcast_config;
     // Stream dimensions must be divisable by 2^(n-1), where n is the number of layers.
     // Each lower resolution layer is 1/2^(n-1) of the size of largest layer,
     // where n is the number of the layer
 
     // width/height will be overridden on the first frame; they must be 'sane' for
     // SetSendCodec()
     video_stream.width = width >> idx;
     video_stream.height = height >> idx;
-    video_stream.max_framerate = max_framerate;
+    video_stream.max_framerate = mSendingFramerate;
     auto& simulcastEncoding = codecConfig->mSimulcastEncodings[idx];
     // leave vector temporal_layer_thresholds_bps empty
     video_stream.temporal_layer_thresholds_bps.clear();
+    // Calculate these first
     video_stream.max_bitrate_bps = MinIgnoreZero(simulcastEncoding.constraints.maxBr,
                                                  kDefaultMaxBitrate_bps);
     video_stream.max_bitrate_bps = MinIgnoreZero((int) mPrefMaxBitrate*1000,
                                                  video_stream.max_bitrate_bps);
     video_stream.min_bitrate_bps = (mMinBitrate ? mMinBitrate : kDefaultMinBitrate_bps);
     if (video_stream.min_bitrate_bps > video_stream.max_bitrate_bps) {
       video_stream.min_bitrate_bps = video_stream.max_bitrate_bps;
     }
     video_stream.target_bitrate_bps = (mStartBitrate ? mStartBitrate : kDefaultStartBitrate_bps);
     if (video_stream.target_bitrate_bps > video_stream.max_bitrate_bps) {
       video_stream.target_bitrate_bps = video_stream.max_bitrate_bps;
     }
     if (video_stream.target_bitrate_bps < video_stream.min_bitrate_bps) {
       video_stream.target_bitrate_bps = video_stream.min_bitrate_bps;
     }
+    // We should use SelectBitrates here for the case of already-sending and no reconfig needed;
+    // overrides the calculations above
+    if (mSendingWidth) { // cleared if we need a reconfig
+      SelectBitrates(video_stream.width, video_stream.height,
+                     simulcastEncoding.constraints.maxBr,
+                     mLastFramerateTenths, video_stream);
+    }
 
     video_stream.max_qp = kQpMax;
     video_stream.SetRid(simulcastEncoding.rid);
     simulcast_config.jsScaleDownBy = simulcastEncoding.constraints.scaleDownBy;
-    simulcast_config.jsMaxBitrate = simulcastEncoding.constraints.maxBr;
+    simulcast_config.jsMaxBitrate = simulcastEncoding.constraints.maxBr; // bps
 
     if (codecConfig->mName == "H264") {
       if (codecConfig->mEncodingConstraints.maxMbps > 0) {
         // Not supported yet!
         CSFLogError(logTag, "%s H.264 max_mbps not supported yet", __FUNCTION__);
       }
     }
     mEncoderConfig.AddStream(video_stream, simulcast_config);
@@ -839,17 +864,16 @@ WebrtcVideoConduit::InitMain()
          mStartBitrate = temp;
          }
       }
       if (!NS_WARN_IF(NS_FAILED(branch->GetIntPref(
             "media.peerconnection.video.max_bitrate", &temp))))
       {
         if (temp >= 0) {
           mPrefMaxBitrate = temp;
-          mNegotiatedMaxBitrate = temp; // simplifies logic in SelectBitrate (don't have to do two limit tests)
         }
       }
       if (mMinBitrate != 0 && mMinBitrate < kViEMinCodecBitrate) {
         mMinBitrate = kViEMinCodecBitrate;
       }
       if (mStartBitrate < mMinBitrate) {
         mStartBitrate = mMinBitrate;
       }
@@ -1447,18 +1471,20 @@ WebrtcVideoConduit::SelectSendResolution
     // This will avoid us continually retrying this operation if it fails.
     // If the resolution changes, we'll try again.  In the meantime, we'll
     // keep using the old size in the encoder.
     mSendingWidth = width;
     mSendingHeight = height;
     changed = true;
   }
 
-  // uses mSendingWidth/Height
-  unsigned int framerate = SelectSendFrameRate(mSendingFramerate);
+  unsigned int framerate = SelectSendFrameRate(mCurSendCodecConfig,
+                                               mSendingFramerate,
+                                               mSendingWidth,
+                                               mSendingHeight);
   if (mSendingFramerate != framerate) {
     CSFLogDebug(logTag, "%s: framerate changing to %u (from %u)",
                 __FUNCTION__, framerate, mSendingFramerate);
     mSendingFramerate = framerate;
     changed = true;
   }
 
   if (changed) {
@@ -1530,19 +1556,22 @@ WebrtcVideoConduit::ReconfigureSendCodec
                 __FUNCTION__, width, height, static_cast<unsigned int>(video_stream.width),
                 static_cast<unsigned int>(video_stream.height), simStream.jsScaleDownBy);
 
     MOZ_ASSERT(simStream.jsScaleDownBy >= 1.0);
     uint32_t new_width = (width / simStream.jsScaleDownBy);
     uint32_t new_height = (height / simStream.jsScaleDownBy);
     video_stream.width = width;
     video_stream.height = height;
+    // XXX this should depend on the final values (below) of video_stream.width/height, not
+    // the current value calculated on the incoming framesize (largest simulcast layer)
     video_stream.max_framerate = mSendingFramerate;
     SelectBitrates(video_stream.width, video_stream.height,
-                   MinIgnoreZero(mNegotiatedMaxBitrate, simStream.jsMaxBitrate),
+                   // XXX formerly was MinIgnoreZero(mNegotiatedMaxBitrate, simStream.jsMaxBitrate),
+                   simStream.jsMaxBitrate,
                    mLastFramerateTenths, video_stream);
     CSFLogVerbose(logTag, "%s: new_width=%" PRIu32 " new_height=%" PRIu32,
                   __FUNCTION__, new_width, new_height);
     if (new_width != video_stream.width || new_height != video_stream.height) {
       if (mEncoderConfig.StreamCount() == 1) {
         CSFLogVerbose(logTag, "%s: ConstrainPreservingAspectRatio", __FUNCTION__);
         // Use less strict scaling in unicast. That way 320x240 / 3 = 106x79.
         ConstrainPreservingAspectRatio(new_width, new_height,
@@ -1569,42 +1598,37 @@ WebrtcVideoConduit::ReconfigureSendCodec
   if (frame) {
     // XXX I really don't like doing this from MainThread...
     mSendStream->Input()->IncomingCapturedFrame(*frame);
     CSFLogDebug(logTag, "%s Inserted a frame from reconfig lambda", __FUNCTION__);
   }
   return NS_OK;
 }
 
-// Invoked under lock of mCodecMutex!
 unsigned int
-WebrtcVideoConduit::SelectSendFrameRate(unsigned int framerate) const
+WebrtcVideoConduit::SelectSendFrameRate(const VideoCodecConfig* codecConfig,
+                                        unsigned int old_framerate,
+                                        unsigned short sending_width,
+                                        unsigned short sending_height) const
 {
-  mCodecMutex.AssertCurrentThreadOwns();
-  unsigned int new_framerate = framerate;
+  unsigned int new_framerate = old_framerate;
 
   // Limit frame rate based on max-mbps
-  if (mCurSendCodecConfig && mCurSendCodecConfig->mEncodingConstraints.maxMbps)
+  if (codecConfig && codecConfig->mEncodingConstraints.maxMbps)
   {
-    unsigned int cur_fs, mb_width, mb_height, max_fps;
+    unsigned int cur_fs, mb_width, mb_height;
 
-    mb_width = (mSendingWidth + 15) >> 4;
-    mb_height = (mSendingHeight + 15) >> 4;
+    mb_width = (sending_width + 15) >> 4;
+    mb_height = (sending_height + 15) >> 4;
 
     cur_fs = mb_width * mb_height;
     if (cur_fs > 0) { // in case no frames have been sent
-      max_fps = mCurSendCodecConfig->mEncodingConstraints.maxMbps / cur_fs;
-      if (max_fps < mSendingFramerate) {
-        new_framerate = max_fps;
-      }
+      new_framerate = codecConfig->mEncodingConstraints.maxMbps / cur_fs;
 
-      if (mCurSendCodecConfig->mEncodingConstraints.maxFps != 0 &&
-          mCurSendCodecConfig->mEncodingConstraints.maxFps < mSendingFramerate) {
-        new_framerate = mCurSendCodecConfig->mEncodingConstraints.maxFps;
-      }
+      new_framerate = MinIgnoreZero(new_framerate, codecConfig->mEncodingConstraints.maxFps);
     }
   }
   return new_framerate;
 }
 
 MediaConduitErrorCode
 WebrtcVideoConduit::SendVideoFrame(unsigned char* video_buffer,
                                    unsigned int video_length,
--- a/media/webrtc/signaling/src/media-conduit/VideoConduit.h
+++ b/media/webrtc/signaling/src/media-conduit/VideoConduit.h
@@ -184,17 +184,20 @@ public:
                                 webrtc::VideoFrame* frame);
 
   /**
    * Function to select and change the encoding frame rate based on incoming frame rate
    * and max-mbps setting.
    * @param current framerate
    * @result new framerate
    */
-  unsigned int SelectSendFrameRate(unsigned int framerate) const;
+  unsigned int SelectSendFrameRate(const VideoCodecConfig* codecConfig,
+                                   unsigned int old_framerate,
+                                   unsigned short sending_width,
+                                   unsigned short sending_height) const;
 
   /**
    * Function to deliver a capture video frame for encoding and transport
    * @param video_frame: pointer to captured video-frame.
    * @param video_frame_length: size of the frame
    * @param width, height: dimensions of the frame
    * @param video_type: Type of the video frame - I420, RAW
    * @param captured_time: timestamp when the frame was captured.