Bug 877954 - Push load state to media optimization. Add simple CPU adaption rules. r=jesup
authorGian-Carlo Pascutto <gpascutto@mozilla.com>
Thu, 13 Mar 2014 11:05:42 +0100
changeset 173404 7d879d9643edadb953af9ab1bb6c6c7273e0c480
parent 173403 519ae59fa49e5f2e796076c9a34f4624e67da6f0
child 173405 cb28fb0f9febd718fcb4afdc0fdce589db9177cd
push id26403
push userryanvm@gmail.com
push dateThu, 13 Mar 2014 19:32:04 +0000
treeherdermozilla-central@fe40387eba1a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjesup
bugs877954
milestone30.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 877954 - Push load state to media optimization. Add simple CPU adaption rules. r=jesup
media/webrtc/trunk/webrtc/modules/video_coding/main/interface/video_coding.h
media/webrtc/trunk/webrtc/modules/video_coding/main/source/media_optimization.cc
media/webrtc/trunk/webrtc/modules/video_coding/main/source/media_optimization.h
media/webrtc/trunk/webrtc/modules/video_coding/main/source/qm_select.cc
media/webrtc/trunk/webrtc/modules/video_coding/main/source/qm_select.h
media/webrtc/trunk/webrtc/modules/video_coding/main/source/video_coding_impl.cc
media/webrtc/trunk/webrtc/modules/video_coding/main/source/video_coding_impl.h
media/webrtc/trunk/webrtc/modules/video_coding/main/source/video_sender.cc
media/webrtc/trunk/webrtc/video_engine/vie_encoder.cc
--- a/media/webrtc/trunk/webrtc/modules/video_coding/main/interface/video_coding.h
+++ b/media/webrtc/trunk/webrtc/modules/video_coding/main/interface/video_coding.h
@@ -569,16 +569,19 @@ public:
     virtual void SetNackSettings(size_t max_nack_list_size,
                                  int max_packet_age_to_nack,
                                  int max_incomplete_time_ms) = 0;
 
     // Setting a desired delay to the VCM receiver. Video rendering will be
     // delayed by at least desired_delay_ms.
     virtual int SetMinReceiverDelay(int desired_delay_ms) = 0;
 
+    // Set current load state of the CPU
+    virtual void SetCPULoadState(CPULoadState state) = 0;
+
     // Enables recording of debugging information.
     virtual int StartDebugRecording(const char* file_name_utf8) = 0;
 
     // Disables recording of debugging information.
     virtual int StopDebugRecording() = 0;
 };
 
 }  // namespace webrtc
--- a/media/webrtc/trunk/webrtc/modules/video_coding/main/source/media_optimization.cc
+++ b/media/webrtc/trunk/webrtc/modules/video_coding/main/source/media_optimization.cc
@@ -383,16 +383,19 @@ void MediaOptimization::UpdateContentDat
 
 int32_t MediaOptimization::SelectQuality() {
   // Reset quantities for QM select.
   qm_resolution_->ResetQM();
 
   // Update QM will long-term averaged content metrics.
   qm_resolution_->UpdateContent(content_->LongTermAvgData());
 
+  // Update QM with current CPU load
+  qm_resolution_->SetCPULoadState(loadstate_);
+
   // Select quality mode.
   VCMResolutionScale* qm = NULL;
   int32_t ret = qm_resolution_->SelectResolution(&qm);
   if (ret < 0) {
     return ret;
   }
 
   // Check for updates to spatial/temporal modes.
@@ -579,10 +582,14 @@ void MediaOptimization::ProcessIncomingF
     const int64_t diff = now - incoming_frame_times_[num - 1];
     incoming_frame_rate_ = 1.0;
     if (diff > 0) {
       incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
     }
   }
 }
 
+void MediaOptimization::SetCPULoadState(CPULoadState state) {
+    loadstate_ = state;
+}
+
 }  // namespace media_optimization
 }  // namespace webrtc
--- a/media/webrtc/trunk/webrtc/modules/video_coding/main/source/media_optimization.h
+++ b/media/webrtc/trunk/webrtc/modules/video_coding/main/source/media_optimization.h
@@ -91,16 +91,19 @@ class MediaOptimization {
   // Returns the actual sent bit rate.
   uint32_t SentBitRate();
 
   // Informs Media Optimization of encoding output: Length and frame type.
   int32_t UpdateWithEncodedData(int encoded_length,
                                 uint32_t timestamp,
                                 FrameType encoded_frame_type);
 
+  // Informs Media Optimization of CPU Load state
+  void SetCPULoadState(CPULoadState state);
+
   // Registers a protection callback to be used to inform the user about the
   // protection methods used.
   int32_t RegisterProtectionCallback(
       VCMProtectionCallback* protection_callback);
 
   // Registers a quality settings callback to be used to inform VPM/user.
   int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* video_qmsettings);
 
@@ -176,14 +179,15 @@ class MediaOptimization {
   uint32_t avg_sent_framerate_;
   uint32_t key_frame_cnt_;
   uint32_t delta_frame_cnt_;
   scoped_ptr<VCMContentMetricsProcessing> content_;
   scoped_ptr<VCMQmResolution> qm_resolution_;
   int64_t last_qm_update_time_;
   int64_t last_change_time_;  // Content/user triggered.
   int num_layers_;
+  CPULoadState loadstate_;
 };  // End of MediaOptimization class declaration.
 
 }  // namespace media_optimization
 }  // namespace webrtc
 
 #endif  // WEBRTC_MODULES_VIDEO_CODING_MAIN_SOURCE_MEDIA_OPTIMIZATION_H_
--- a/media/webrtc/trunk/webrtc/modules/video_coding/main/source/qm_select.cc
+++ b/media/webrtc/trunk/webrtc/modules/video_coding/main/source/qm_select.cc
@@ -421,16 +421,21 @@ void VCMQmResolution::ComputeEncoderStat
   // 2) consistent under-shooting by encoder.
   if ((avg_rate_mismatch_ > kMaxRateMisMatch) &&
       (avg_rate_mismatch_sgn_ > kRateUnderShoot)) {
     encoder_state_ = kEasyEncoding;
   }
 }
 
 bool VCMQmResolution::GoingUpResolution() {
+  // We do not go up if we're already near max CPU load
+  if (loadstate_ == kLoadStressed) {
+    return false;
+  }
+
   // For going up, we check for undoing the previous down-sampling action.
 
   float fac_width = kFactorWidthSpatial[down_action_history_[0].spatial];
   float fac_height = kFactorHeightSpatial[down_action_history_[0].spatial];
   float fac_temp = kFactorTemporal[down_action_history_[0].temporal];
   // For going up spatially, we allow for going up by 3/4x3/4 at each stage.
   // So if the last spatial action was 1/2x1/2 it would be undone in 2 stages.
   // Modify the fac_width/height for this case.
@@ -501,18 +506,19 @@ bool VCMQmResolution::ConditionForGoingU
 
 bool VCMQmResolution::GoingDownResolution() {
   float estimated_transition_rate_down =
       GetTransitionRate(1.0f, 1.0f, 1.0f, 1.0f);
   float max_rate = kFrameRateFac[framerate_level_] * kMaxRateQm[image_type_];
   // Resolution reduction if:
   // (1) target rate is below transition rate, or
   // (2) encoder is in stressed state and target rate below a max threshold.
-  if ((avg_target_rate_ < estimated_transition_rate_down ) ||
-      (encoder_state_ == kStressedEncoding && avg_target_rate_ < max_rate)) {
+  if (loadstate_ == kLoadStressed
+      || (avg_target_rate_ < estimated_transition_rate_down)
+      || (encoder_state_ == kStressedEncoding && avg_target_rate_ < max_rate)) {
     // Get the down-sampling action: based on content class, and how low
     // average target rate is relative to transition rate.
     uint8_t spatial_fact =
         kSpatialAction[content_class_ +
                        9 * RateClass(estimated_transition_rate_down)];
     uint8_t temp_fact =
         kTemporalAction[content_class_ +
                         9 * RateClass(estimated_transition_rate_down)];
@@ -550,16 +556,32 @@ bool VCMQmResolution::GoingDownResolutio
       default: {
         assert(false);
       }
     }
     // Only allow for one action (spatial or temporal) at a given time.
     assert(action_.temporal == kNoChangeTemporal ||
            action_.spatial == kNoChangeSpatial);
 
+    // CPU stressed but we did not get an action yet, likely because our
+    // bitrate is too high relative to the target.
+    if (loadstate_ == kLoadStressed
+        && action_.temporal == kNoChangeTemporal
+        && action_.spatial == kNoChangeSpatial) {
+      // If FPS is high, allow dropping it to 20 fps.
+      if (avg_incoming_framerate_ >= 40) {
+        action_.temporal = kOneHalfTemporal;
+      } else if (avg_incoming_framerate_ >= 24) {
+        action_.temporal = kTwoThirdsTemporal;
+      } else {
+        // Drop resolution in all other cases.
+        action_.spatial = kOneHalfSpatialUniform;
+      }
+    }
+
     // Adjust cases not captured in tables, mainly based on frame rate, and
     // also check for odd frame sizes.
     AdjustAction();
 
     // Update down-sampling state.
     if (action_.spatial != kNoChangeSpatial ||
         action_.temporal != kNoChangeTemporal) {
       UpdateDownsamplingState(kDownResolution);
@@ -898,16 +920,20 @@ void VCMQmResolution::SelectSpatialDirec
   // Check for 2x1 selection.
   if (spatial_err_v < spatial_err_h * (1.0f - kSpatialErrVertVsHoriz) &&
       spatial_err_v < spatial_err * (1.0f - kSpatialErr2X2VsVert)) {
     qm_->spatial_width_fact = 1.0f;
     qm_->spatial_height_fact = 2.0f;
   }
 }
 
+void VCMQmResolution::SetCPULoadState(CPULoadState state) {
+  loadstate_ = state;
+}
+
 // ROBUSTNESS CLASS
 
 VCMQmRobustness::VCMQmRobustness() {
   Reset();
 }
 
 VCMQmRobustness::~VCMQmRobustness() {
 }
@@ -951,9 +977,10 @@ float VCMQmRobustness::AdjustFecFactor(u
 // Set the UEP (unequal-protection across packets) on/off for the FEC.
 bool VCMQmRobustness::SetUepProtection(uint8_t code_rate_delta,
                                        float total_rate,
                                        uint8_t packet_loss,
                                        bool frame_type) {
   // Default.
   return false;
 }
+
 }  // namespace
--- a/media/webrtc/trunk/webrtc/modules/video_coding/main/source/qm_select.h
+++ b/media/webrtc/trunk/webrtc/modules/video_coding/main/source/qm_select.h
@@ -226,16 +226,19 @@ class VCMQmResolution : public VCMQmMeth
                    float incoming_framerate,
                    uint8_t packet_loss);
 
   // Extract ST (spatio-temporal) resolution action.
   // Inputs: qm: Reference to the quality modes pointer.
   // Output: the spatial and/or temporal scale change.
   int SelectResolution(VCMResolutionScale** qm);
 
+  // Update with current system load
+  void SetCPULoadState(CPULoadState state);
+
  private:
   // Set the default resolution action.
   void SetDefaultAction();
 
   // Compute rates for the selection of down-sampling action.
   void ComputeRatesForSelection();
 
   // Compute the encoder state.
@@ -333,16 +336,17 @@ class VCMQmResolution : public VCMQmMeth
   EncoderState encoder_state_;
   ResolutionAction action_;
   // Short history of the down-sampling actions from the Initialize() state.
   // This is needed for going up in resolution. Since the total amount of
   // down-sampling actions are constrained, the length of the list need not be
   // large: i.e., (4/3) ^{kDownActionHistorySize} <= kMaxDownSample.
   ResolutionAction down_action_history_[kDownActionHistorySize];
   int num_layers_;
+  CPULoadState loadstate_;
 };
 
 // Robustness settings class.
 
 class VCMQmRobustness : public VCMQmMethod {
  public:
   VCMQmRobustness();
   ~VCMQmRobustness();
--- a/media/webrtc/trunk/webrtc/modules/video_coding/main/source/video_coding_impl.cc
+++ b/media/webrtc/trunk/webrtc/modules/video_coding/main/source/video_coding_impl.cc
@@ -297,16 +297,20 @@ class VideoCodingModuleImpl : public Vid
   void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) OVERRIDE {
     return receiver_->SetDecodeErrorMode(decode_error_mode);
   }
 
   virtual int SetMinReceiverDelay(int desired_delay_ms) OVERRIDE {
     return receiver_->SetMinReceiverDelay(desired_delay_ms);
   }
 
+  virtual void SetCPULoadState(CPULoadState state) OVERRIDE {
+    return sender_->SetCPULoadState(state);
+  }
+
   virtual int32_t SetReceiveChannelParameters(uint32_t rtt) OVERRIDE {
     return receiver_->SetReceiveChannelParameters(rtt);
   }
 
  private:
   scoped_ptr<vcm::VideoSender> sender_;
   scoped_ptr<vcm::VideoReceiver> receiver_;
   scoped_ptr<EventFactory> own_event_factory_;
--- a/media/webrtc/trunk/webrtc/modules/video_coding/main/source/video_coding_impl.h
+++ b/media/webrtc/trunk/webrtc/modules/video_coding/main/source/video_coding_impl.h
@@ -87,16 +87,18 @@ class VideoSender {
   int32_t EnableFrameDropper(bool enable);
   int32_t SentFrameCount(VCMFrameCount* frameCount) const;
 
   int SetSenderNackMode(SenderNackMode mode);
   int SetSenderReferenceSelection(bool enable);
   int SetSenderFEC(bool enable);
   int SetSenderKeyFramePeriod(int periodMs);
 
+  void SetCPULoadState(CPULoadState state);
+
   int StartDebugRecording(const char* file_name_utf8);
   int StopDebugRecording();
 
   int32_t TimeUntilNextProcess();
   int32_t Process();
 
  private:
   int32_t _id;
--- a/media/webrtc/trunk/webrtc/modules/video_coding/main/source/video_sender.cc
+++ b/media/webrtc/trunk/webrtc/modules/video_coding/main/source/video_sender.cc
@@ -415,10 +415,15 @@ int VideoSender::StopDebugRecording() {
   CriticalSectionScoped cs(_sendCritSect);
   if (_encoderInputFile != NULL) {
     fclose(_encoderInputFile);
     _encoderInputFile = NULL;
   }
   return VCM_OK;
 }
 
+void VideoSender::SetCPULoadState(CPULoadState state) {
+  CriticalSectionScoped cs(_sendCritSect);
+  _mediaOpt.SetCPULoadState(state);
+}
+
 }  // namespace vcm
 }  // namespace webrtc
--- a/media/webrtc/trunk/webrtc/video_engine/vie_encoder.cc
+++ b/media/webrtc/trunk/webrtc/video_engine/vie_encoder.cc
@@ -1095,16 +1095,17 @@ void ViEEncoder::OnNetworkChanged(const 
   default_rtp_rtcp_->SetTargetSendBitrate(stream_bitrates);
 }
 
 void ViEEncoder::onLoadStateChanged(CPULoadState load_state) {
     WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo,
                  ViEId(engine_id_, channel_id_),
                  "%s: load state changed to %d",
                  __FUNCTION__, (int)load_state);
+    vcm_.SetCPULoadState(load_state);
 }
 
 PacedSender* ViEEncoder::GetPacedSender() {
   return paced_sender_.get();
 }
 
 int32_t ViEEncoder::RegisterEffectFilter(ViEEffectFilter* effect_filter) {
   CriticalSectionScoped cs(callback_cs_.get());