Bug 907352 - Part 7: Width/height/frameRate gUM constraints. r=mt
authorJan-Ivar Bruaroey <jib@mozilla.com>
Fri, 18 Apr 2014 15:15:37 -0400
changeset 180797 6c46001c0f93ec2cb0c5254ea772bba334d280c6
parent 180796 093b21bd43f259da2d8934880e2a6e6e935f8e35
child 180798 ea77cc4fc2d8d6843a83f22a64994f4734f8217b
push id272
push userpvanderbeken@mozilla.com
push dateMon, 05 May 2014 16:31:18 +0000
reviewersmt
bugs907352
milestone31.0a1
Bug 907352 - Part 7: Width/height/frameRate gUM constraints. r=mt
content/media/webrtc/MediaEngine.h
content/media/webrtc/MediaEngineWebRTC.h
content/media/webrtc/MediaEngineWebRTCVideo.cpp
--- a/content/media/webrtc/MediaEngine.h
+++ b/content/media/webrtc/MediaEngine.h
@@ -127,21 +127,46 @@ public:
 
 protected:
   MediaEngineState mState;
 };
 
 /**
  * Video source and friends.
  */
-struct MediaEnginePrefs {
+class MediaEnginePrefs {
+public:
   int32_t mWidth;
   int32_t mHeight;
   int32_t mFPS;
   int32_t mMinFPS;
+
+  // mWidth and/or mHeight may be zero (=adaptive default), so use functions.
+
+  int32_t GetWidth(bool aHD = false) const {
+    return mWidth? mWidth : (mHeight?
+                             (mHeight * GetDefWidth(aHD)) / GetDefHeight(aHD) :
+                             GetDefWidth(aHD));
+  }
+
+  int32_t GetHeight(bool aHD = false) const {
+    return mHeight? mHeight : (mWidth?
+                               (mWidth * GetDefHeight(aHD)) / GetDefWidth(aHD) :
+                               GetDefHeight(aHD));
+  }
+private:
+  static int32_t GetDefWidth(bool aHD = false) {
+    return aHD ? MediaEngine::DEFAULT_169_VIDEO_WIDTH :
+                 MediaEngine::DEFAULT_43_VIDEO_WIDTH;
+  }
+
+  static int32_t GetDefHeight(bool aHD = false) {
+    return aHD ? MediaEngine::DEFAULT_169_VIDEO_HEIGHT :
+                 MediaEngine::DEFAULT_43_VIDEO_HEIGHT;
+  }
 };
 
 class MediaEngineVideoSource : public MediaEngineSource
 {
 public:
   virtual ~MediaEngineVideoSource() {}
 
   /* This call reserves but does not start the device. */
--- a/content/media/webrtc/MediaEngineWebRTC.h
+++ b/content/media/webrtc/MediaEngineWebRTC.h
@@ -245,16 +245,19 @@ private:
   bool mInSnapshotMode;
   nsString* mSnapshotPath;
 
   nsString mDeviceName;
   nsString mUniqueId;
 
   void ChooseCapability(const VideoTrackConstraintsN &aConstraints,
                         const MediaEnginePrefs &aPrefs);
+
+  void GuessCapability(const VideoTrackConstraintsN &aConstraints,
+                       const MediaEnginePrefs &aPrefs);
 };
 
 class MediaEngineWebRTCAudioSource : public MediaEngineAudioSource,
                                      public webrtc::VoEMediaProcess
 {
 public:
   MediaEngineWebRTCAudioSource(webrtc::VoiceEngine* aVoiceEnginePtr, int aIndex,
     const char* name, const char* uuid)
--- a/content/media/webrtc/MediaEngineWebRTCVideo.cpp
+++ b/content/media/webrtc/MediaEngineWebRTCVideo.cpp
@@ -16,16 +16,19 @@
 #include "libyuv.h"
 #include "mozilla/Hal.h"
 #include "ScreenOrientation.h"
 using namespace mozilla::dom;
 #endif
 namespace mozilla {
 
 using namespace mozilla::gfx;
+using dom::ConstrainLongRange;
+using dom::ConstrainDoubleRange;
+using dom::MediaTrackConstraintSet;
 
 #ifdef PR_LOGGING
 extern PRLogModuleInfo* GetMediaManagerLog();
 #define LOG(msg) PR_LOG(GetMediaManagerLog(), PR_LOG_DEBUG, msg)
 #define LOGFRAME(msg) PR_LOG(GetMediaManagerLog(), 6, msg)
 #else
 #define LOG(msg)
 #define LOGFRAME(msg)
@@ -162,51 +165,133 @@ MediaEngineWebRTCVideoSource::NotifyPull
     // This can fail if either a) we haven't added the track yet, or b)
     // we've removed or finished the track.
     if (aSource->AppendToTrack(aID, &(segment))) {
       aLastEndTime = target;
     }
   }
 }
 
+static bool IsWithin(int32_t n, const ConstrainLongRange& aRange) {
+  return aRange.mMin <= n && n <= aRange.mMax;
+}
+
+static bool IsWithin(double n, const ConstrainDoubleRange& aRange) {
+  return aRange.mMin <= n && n <= aRange.mMax;
+}
+
+static int32_t Clamp(int32_t n, const ConstrainLongRange& aRange) {
+  return std::max(aRange.mMin, std::min(n, aRange.mMax));
+}
+
+static bool
+AreIntersecting(const ConstrainLongRange& aA, const ConstrainLongRange& aB) {
+  return aA.mMax >= aB.mMin && aA.mMin <= aB.mMax;
+}
+
+static bool
+Intersect(ConstrainLongRange& aA, const ConstrainLongRange& aB) {
+  MOZ_ASSERT(AreIntersecting(aA, aB));
+  aA.mMin = std::max(aA.mMin, aB.mMin);
+  aA.mMax = std::min(aA.mMax, aB.mMax);
+  return true;
+}
+
+static bool SatisfyConstraintSet(const MediaTrackConstraintSet &aConstraints,
+                                 const webrtc::CaptureCapability& aCandidate) {
+  if (!IsWithin(aCandidate.width, aConstraints.mWidth) ||
+      !IsWithin(aCandidate.height, aConstraints.mHeight)) {
+    return false;
+  }
+  if (!IsWithin(aCandidate.maxFPS, aConstraints.mFrameRate)) {
+    return false;
+  }
+  return true;
+}
+
 void
 MediaEngineWebRTCVideoSource::ChooseCapability(
     const VideoTrackConstraintsN &aConstraints,
     const MediaEnginePrefs &aPrefs)
 {
 #ifdef MOZ_B2G_CAMERA
-  mCapability.width  = aPrefs.mWidth;
-  mCapability.height = aPrefs.mHeight;
+  return GuessCapability(aConstraints, aPrefs);
 #else
-  int num = mViECapture->NumberOfCapabilities(NS_ConvertUTF16toUTF8(mUniqueId).get(),
-                                              KMaxUniqueIdLength);
+  NS_ConvertUTF16toUTF8 uniqueId(mUniqueId);
+  int num = mViECapture->NumberOfCapabilities(uniqueId.get(), KMaxUniqueIdLength);
+  if (num <= 0) {
+    // Mac doesn't support capabilities.
+    return GuessCapability(aConstraints, aPrefs);
+  }
+
+  // The rest is the full algorithm for cameras that can list their capabilities.
 
-  LOG(("ChooseCapability: prefs: %dx%d @%d-%dfps", aPrefs.mWidth, aPrefs.mHeight, aPrefs.mFPS, aPrefs.mMinFPS));
+  LOG(("ChooseCapability: prefs: %dx%d @%d-%dfps",
+       aPrefs.mWidth, aPrefs.mHeight, aPrefs.mFPS, aPrefs.mMinFPS));
+
+  typedef nsTArray<uint8_t> SourceSet;
+
+  SourceSet candidateSet;
+  for (int i = 0; i < num; i++) {
+    candidateSet.AppendElement(i);
+  }
+
+  // Pick among capabilities: First apply required constraints.
 
-  int prefWidth = aPrefs.mWidth? aPrefs.mWidth : MediaEngine::DEFAULT_43_VIDEO_WIDTH;
-  int prefHeight = aPrefs.mHeight? aPrefs.mHeight : MediaEngine::DEFAULT_43_VIDEO_HEIGHT;
+  for (uint32_t i = 0; i < candidateSet.Length();) {
+    webrtc::CaptureCapability cap;
+    mViECapture->GetCaptureCapability(uniqueId.get(), KMaxUniqueIdLength,
+                                      candidateSet[i], cap);
+    if (!SatisfyConstraintSet(aConstraints.mRequired, cap)) {
+      candidateSet.RemoveElementAt(i);
+    } else {
+      ++i;
+    }
+  }
+
+  SourceSet tailSet;
+
+  // Then apply advanced (formerly known as optional) constraints.
+
+  if (aConstraints.mAdvanced.WasPassed()) {
+    auto &array = aConstraints.mAdvanced.Value();
 
-  if (num <= 0) {
-    // Set to default values
-    mCapability.width  = prefWidth;
-    mCapability.height = prefHeight;
-    mCapability.maxFPS = MediaEngine::DEFAULT_VIDEO_FPS;
+    for (uint32_t i = 0; i < array.Length(); i++) {
+      SourceSet rejects;
+      for (uint32_t j = 0; j < candidateSet.Length();) {
+        webrtc::CaptureCapability cap;
+        mViECapture->GetCaptureCapability(uniqueId.get(), KMaxUniqueIdLength,
+                                          candidateSet[j], cap);
+        if (!SatisfyConstraintSet(array[i], cap)) {
+          rejects.AppendElement(candidateSet[j]);
+          candidateSet.RemoveElementAt(j);
+        } else {
+          ++j;
+        }
+      }
+      (candidateSet.Length()? tailSet : candidateSet).MoveElementsFrom(rejects);
+    }
+  }
 
-    // Mac doesn't support capabilities.
-    return;
+  if (!candidateSet.Length()) {
+    candidateSet.AppendElement(0);
   }
 
+  int prefWidth = aPrefs.GetWidth();
+  int prefHeight = aPrefs.GetHeight();
+
   // Default is closest to available capability but equal to or below;
   // otherwise closest above.  Since we handle the num=0 case above and
   // take the first entry always, we can never exit uninitialized.
+
   webrtc::CaptureCapability cap;
   bool higher = true;
-  for (int i = 0; i < num; i++) {
+  for (uint32_t i = 0; i < candidateSet.Length(); i++) {
     mViECapture->GetCaptureCapability(NS_ConvertUTF16toUTF8(mUniqueId).get(),
-                                      KMaxUniqueIdLength, i, cap);
+                                      KMaxUniqueIdLength, candidateSet[i], cap);
     if (higher) {
       if (i == 0 ||
           (mCapability.width > cap.width && mCapability.height > cap.height)) {
         // closer than the current choice
         mCapability = cap;
         // FIXME: expose expected capture delay?
       }
       if (cap.width <= (uint32_t) prefWidth && cap.height <= (uint32_t) prefHeight) {
@@ -218,20 +303,102 @@ MediaEngineWebRTCVideoSource::ChooseCapa
         continue;
       }
       if (mCapability.width < cap.width && mCapability.height < cap.height) {
         mCapability = cap;
         // FIXME: expose expected capture delay?
       }
     }
   }
-  LOG(("chose cap %dx%d @%dfps", mCapability.width, mCapability.height, mCapability.maxFPS));
+  LOG(("chose cap %dx%d @%dfps",
+       mCapability.width, mCapability.height, mCapability.maxFPS));
 #endif
 }
 
+// A special version of the algorithm for cameras that don't list capabilities.
+
+void
+MediaEngineWebRTCVideoSource::GuessCapability(
+    const VideoTrackConstraintsN &aConstraints,
+    const MediaEnginePrefs &aPrefs)
+{
+  LOG(("GuessCapability: prefs: %dx%d @%d-%dfps",
+       aPrefs.mWidth, aPrefs.mHeight, aPrefs.mFPS, aPrefs.mMinFPS));
+
+  // In short: compound constraint-ranges and use pref as ideal.
+
+  ConstrainLongRange cWidth(aConstraints.mRequired.mWidth);
+  ConstrainLongRange cHeight(aConstraints.mRequired.mHeight);
+
+  if (aConstraints.mAdvanced.WasPassed()) {
+    const auto& advanced = aConstraints.mAdvanced.Value();
+    for (uint32_t i = 0; i < advanced.Length(); i++) {
+      if (AreIntersecting(cWidth, advanced[i].mWidth) &&
+          AreIntersecting(cHeight, advanced[i].mHeight)) {
+        Intersect(cWidth, advanced[i].mWidth);
+        Intersect(cHeight, advanced[i].mHeight);
+      }
+    }
+  }
+  // Detect Mac HD cams and give them some love in the form of a dynamic default
+  // since that hardware switches between 4:3 at low res and 16:9 at higher res.
+  //
+  // Logic is: if we're relying on defaults in aPrefs, then
+  // only use HD pref when non-HD pref is too small and HD pref isn't too big.
+
+  bool macHD = ((!aPrefs.mWidth || !aPrefs.mHeight) &&
+                mDeviceName.EqualsASCII("FaceTime HD Camera (Built-in)") &&
+                (aPrefs.GetWidth() < cWidth.mMin ||
+                 aPrefs.GetHeight() < cHeight.mMin) &&
+                !(aPrefs.GetWidth(true) > cWidth.mMax ||
+                  aPrefs.GetHeight(true) > cHeight.mMax));
+  int prefWidth = aPrefs.GetWidth(macHD);
+  int prefHeight = aPrefs.GetHeight(macHD);
+
+  // Clamp width and height without distorting inherent aspect too much.
+
+  if (IsWithin(prefWidth, cWidth) == IsWithin(prefHeight, cHeight)) {
+    // If both are within, we get the default (pref) aspect.
+    // If neither are within, we get the aspect of the enclosing constraint.
+    // Either are presumably reasonable (presuming constraints are sane).
+    mCapability.width = Clamp(prefWidth, cWidth);
+    mCapability.height = Clamp(prefHeight, cHeight);
+  } else {
+    // But if only one clips (e.g. width), the resulting skew is undesirable:
+    //       .------------.
+    //       | constraint |
+    //  .----+------------+----.
+    //  |    |            |    |
+    //  |pref|  result    |    |   prefAspect != resultAspect
+    //  |    |            |    |
+    //  '----+------------+----'
+    //       '------------'
+    //  So in this case, preserve prefAspect instead:
+    //  .------------.
+    //  | constraint |
+    //  .------------.
+    //  |pref        |             prefAspect is unchanged
+    //  '------------'
+    //  |            |
+    //  '------------'
+    if (IsWithin(prefWidth, cWidth)) {
+      mCapability.height = Clamp(prefHeight, cHeight);
+      mCapability.width = Clamp((mCapability.height * prefWidth) /
+                                prefHeight, cWidth);
+    } else {
+      mCapability.width = Clamp(prefWidth, cWidth);
+      mCapability.height = Clamp((mCapability.width * prefHeight) /
+                                 prefWidth, cHeight);
+    }
+  }
+  mCapability.maxFPS = MediaEngine::DEFAULT_VIDEO_FPS;
+  LOG(("chose cap %dx%d @%dfps",
+       mCapability.width, mCapability.height, mCapability.maxFPS));
+}
+
 void
 MediaEngineWebRTCVideoSource::GetName(nsAString& aName)
 {
   aName = mDeviceName;
 }
 
 void
 MediaEngineWebRTCVideoSource::GetUUID(nsAString& aUUID)