Bug 907352 - Part 7: Width/height/frameRate gUM constraints. r=mt
authorJan-Ivar Bruaroey <jib@mozilla.com>
Fri, 18 Apr 2014 15:15:37 -0400
changeset 200011 6c46001c0f93ec2cb0c5254ea772bba334d280c6
parent 200010 093b21bd43f259da2d8934880e2a6e6e935f8e35
child 200012 ea77cc4fc2d8d6843a83f22a64994f4734f8217b
push id486
push userasasaki@mozilla.com
push dateMon, 14 Jul 2014 18:39:42 +0000
treeherdermozilla-release@d33428174ff1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmt
bugs907352
milestone31.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 907352 - Part 7: Width/height/frameRate gUM constraints. r=mt
content/media/webrtc/MediaEngine.h
content/media/webrtc/MediaEngineWebRTC.h
content/media/webrtc/MediaEngineWebRTCVideo.cpp
--- a/content/media/webrtc/MediaEngine.h
+++ b/content/media/webrtc/MediaEngine.h
@@ -127,21 +127,46 @@ public:
 
 protected:
   MediaEngineState mState;
 };
 
 /**
  * Video source and friends.
  */
-struct MediaEnginePrefs {
+class MediaEnginePrefs {
+public:
   int32_t mWidth;
   int32_t mHeight;
   int32_t mFPS;
   int32_t mMinFPS;
+
+  // mWidth and/or mHeight may be zero (=adaptive default), so use functions.
+
+  int32_t GetWidth(bool aHD = false) const {
+    return mWidth? mWidth : (mHeight?
+                             (mHeight * GetDefWidth(aHD)) / GetDefHeight(aHD) :
+                             GetDefWidth(aHD));
+  }
+
+  int32_t GetHeight(bool aHD = false) const {
+    return mHeight? mHeight : (mWidth?
+                               (mWidth * GetDefHeight(aHD)) / GetDefWidth(aHD) :
+                               GetDefHeight(aHD));
+  }
+private:
+  static int32_t GetDefWidth(bool aHD = false) {
+    return aHD ? MediaEngine::DEFAULT_169_VIDEO_WIDTH :
+                 MediaEngine::DEFAULT_43_VIDEO_WIDTH;
+  }
+
+  static int32_t GetDefHeight(bool aHD = false) {
+    return aHD ? MediaEngine::DEFAULT_169_VIDEO_HEIGHT :
+                 MediaEngine::DEFAULT_43_VIDEO_HEIGHT;
+  }
 };
 
 class MediaEngineVideoSource : public MediaEngineSource
 {
 public:
   virtual ~MediaEngineVideoSource() {}
 
   /* This call reserves but does not start the device. */
--- a/content/media/webrtc/MediaEngineWebRTC.h
+++ b/content/media/webrtc/MediaEngineWebRTC.h
@@ -245,16 +245,19 @@ private:
   bool mInSnapshotMode;
   nsString* mSnapshotPath;
 
   nsString mDeviceName;
   nsString mUniqueId;
 
   void ChooseCapability(const VideoTrackConstraintsN &aConstraints,
                         const MediaEnginePrefs &aPrefs);
+
+  void GuessCapability(const VideoTrackConstraintsN &aConstraints,
+                       const MediaEnginePrefs &aPrefs);
 };
 
 class MediaEngineWebRTCAudioSource : public MediaEngineAudioSource,
                                      public webrtc::VoEMediaProcess
 {
 public:
   MediaEngineWebRTCAudioSource(webrtc::VoiceEngine* aVoiceEnginePtr, int aIndex,
     const char* name, const char* uuid)
--- a/content/media/webrtc/MediaEngineWebRTCVideo.cpp
+++ b/content/media/webrtc/MediaEngineWebRTCVideo.cpp
@@ -16,16 +16,19 @@
 #include "libyuv.h"
 #include "mozilla/Hal.h"
 #include "ScreenOrientation.h"
 using namespace mozilla::dom;
 #endif
 namespace mozilla {
 
 using namespace mozilla::gfx;
+using dom::ConstrainLongRange;
+using dom::ConstrainDoubleRange;
+using dom::MediaTrackConstraintSet;
 
 #ifdef PR_LOGGING
 extern PRLogModuleInfo* GetMediaManagerLog();
 #define LOG(msg) PR_LOG(GetMediaManagerLog(), PR_LOG_DEBUG, msg)
 #define LOGFRAME(msg) PR_LOG(GetMediaManagerLog(), 6, msg)
 #else
 #define LOG(msg)
 #define LOGFRAME(msg)
@@ -162,51 +165,133 @@ MediaEngineWebRTCVideoSource::NotifyPull
     // This can fail if either a) we haven't added the track yet, or b)
     // we've removed or finished the track.
     if (aSource->AppendToTrack(aID, &(segment))) {
       aLastEndTime = target;
     }
   }
 }
 
+static bool IsWithin(int32_t n, const ConstrainLongRange& aRange) {
+  return aRange.mMin <= n && n <= aRange.mMax;
+}
+
+static bool IsWithin(double n, const ConstrainDoubleRange& aRange) {
+  return aRange.mMin <= n && n <= aRange.mMax;
+}
+
+static int32_t Clamp(int32_t n, const ConstrainLongRange& aRange) {
+  return std::max(aRange.mMin, std::min(n, aRange.mMax));
+}
+
+static bool
+AreIntersecting(const ConstrainLongRange& aA, const ConstrainLongRange& aB) {
+  return aA.mMax >= aB.mMin && aA.mMin <= aB.mMax;
+}
+
+static bool
+Intersect(ConstrainLongRange& aA, const ConstrainLongRange& aB) {
+  MOZ_ASSERT(AreIntersecting(aA, aB));
+  aA.mMin = std::max(aA.mMin, aB.mMin);
+  aA.mMax = std::min(aA.mMax, aB.mMax);
+  return true;
+}
+
+static bool SatisfyConstraintSet(const MediaTrackConstraintSet &aConstraints,
+                                 const webrtc::CaptureCapability& aCandidate) {
+  if (!IsWithin(aCandidate.width, aConstraints.mWidth) ||
+      !IsWithin(aCandidate.height, aConstraints.mHeight)) {
+    return false;
+  }
+  if (!IsWithin(aCandidate.maxFPS, aConstraints.mFrameRate)) {
+    return false;
+  }
+  return true;
+}
+
 void
 MediaEngineWebRTCVideoSource::ChooseCapability(
     const VideoTrackConstraintsN &aConstraints,
     const MediaEnginePrefs &aPrefs)
 {
 #ifdef MOZ_B2G_CAMERA
-  mCapability.width  = aPrefs.mWidth;
-  mCapability.height = aPrefs.mHeight;
+  return GuessCapability(aConstraints, aPrefs);
 #else
-  int num = mViECapture->NumberOfCapabilities(NS_ConvertUTF16toUTF8(mUniqueId).get(),
-                                              KMaxUniqueIdLength);
+  NS_ConvertUTF16toUTF8 uniqueId(mUniqueId);
+  int num = mViECapture->NumberOfCapabilities(uniqueId.get(), KMaxUniqueIdLength);
+  if (num <= 0) {
+    // Mac doesn't support capabilities.
+    return GuessCapability(aConstraints, aPrefs);
+  }
+
+  // The rest is the full algorithm for cameras that can list their capabilities.
 
-  LOG(("ChooseCapability: prefs: %dx%d @%d-%dfps", aPrefs.mWidth, aPrefs.mHeight, aPrefs.mFPS, aPrefs.mMinFPS));
+  LOG(("ChooseCapability: prefs: %dx%d @%d-%dfps",
+       aPrefs.mWidth, aPrefs.mHeight, aPrefs.mFPS, aPrefs.mMinFPS));
+
+  typedef nsTArray<uint8_t> SourceSet;
+
+  SourceSet candidateSet;
+  for (int i = 0; i < num; i++) {
+    candidateSet.AppendElement(i);
+  }
+
+  // Pick among capabilities: First apply required constraints.
 
-  int prefWidth = aPrefs.mWidth? aPrefs.mWidth : MediaEngine::DEFAULT_43_VIDEO_WIDTH;
-  int prefHeight = aPrefs.mHeight? aPrefs.mHeight : MediaEngine::DEFAULT_43_VIDEO_HEIGHT;
+  for (uint32_t i = 0; i < candidateSet.Length();) {
+    webrtc::CaptureCapability cap;
+    mViECapture->GetCaptureCapability(uniqueId.get(), KMaxUniqueIdLength,
+                                      candidateSet[i], cap);
+    if (!SatisfyConstraintSet(aConstraints.mRequired, cap)) {
+      candidateSet.RemoveElementAt(i);
+    } else {
+      ++i;
+    }
+  }
+
+  SourceSet tailSet;
+
+  // Then apply advanced (formerly known as optional) constraints.
+
+  if (aConstraints.mAdvanced.WasPassed()) {
+    auto &array = aConstraints.mAdvanced.Value();
 
-  if (num <= 0) {
-    // Set to default values
-    mCapability.width  = prefWidth;
-    mCapability.height = prefHeight;
-    mCapability.maxFPS = MediaEngine::DEFAULT_VIDEO_FPS;
+    for (uint32_t i = 0; i < array.Length(); i++) {
+      SourceSet rejects;
+      for (uint32_t j = 0; j < candidateSet.Length();) {
+        webrtc::CaptureCapability cap;
+        mViECapture->GetCaptureCapability(uniqueId.get(), KMaxUniqueIdLength,
+                                          candidateSet[j], cap);
+        if (!SatisfyConstraintSet(array[i], cap)) {
+          rejects.AppendElement(candidateSet[j]);
+          candidateSet.RemoveElementAt(j);
+        } else {
+          ++j;
+        }
+      }
+      (candidateSet.Length()? tailSet : candidateSet).MoveElementsFrom(rejects);
+    }
+  }
 
-    // Mac doesn't support capabilities.
-    return;
+  if (!candidateSet.Length()) {
+    candidateSet.AppendElement(0);
   }
 
+  int prefWidth = aPrefs.GetWidth();
+  int prefHeight = aPrefs.GetHeight();
+
   // Default is closest to available capability but equal to or below;
   // otherwise closest above.  Since we handle the num=0 case above and
   // take the first entry always, we can never exit uninitialized.
+
   webrtc::CaptureCapability cap;
   bool higher = true;
-  for (int i = 0; i < num; i++) {
+  for (uint32_t i = 0; i < candidateSet.Length(); i++) {
     mViECapture->GetCaptureCapability(NS_ConvertUTF16toUTF8(mUniqueId).get(),
-                                      KMaxUniqueIdLength, i, cap);
+                                      KMaxUniqueIdLength, candidateSet[i], cap);
     if (higher) {
       if (i == 0 ||
           (mCapability.width > cap.width && mCapability.height > cap.height)) {
         // closer than the current choice
         mCapability = cap;
         // FIXME: expose expected capture delay?
       }
       if (cap.width <= (uint32_t) prefWidth && cap.height <= (uint32_t) prefHeight) {
@@ -218,20 +303,102 @@ MediaEngineWebRTCVideoSource::ChooseCapa
         continue;
       }
       if (mCapability.width < cap.width && mCapability.height < cap.height) {
         mCapability = cap;
         // FIXME: expose expected capture delay?
       }
     }
   }
-  LOG(("chose cap %dx%d @%dfps", mCapability.width, mCapability.height, mCapability.maxFPS));
+  LOG(("chose cap %dx%d @%dfps",
+       mCapability.width, mCapability.height, mCapability.maxFPS));
 #endif
 }
 
+// A special version of the algorithm for cameras that don't list capabilities.
+
+void
+MediaEngineWebRTCVideoSource::GuessCapability(
+    const VideoTrackConstraintsN &aConstraints,
+    const MediaEnginePrefs &aPrefs)
+{
+  LOG(("GuessCapability: prefs: %dx%d @%d-%dfps",
+       aPrefs.mWidth, aPrefs.mHeight, aPrefs.mFPS, aPrefs.mMinFPS));
+
+  // In short: compound constraint-ranges and use pref as ideal.
+
+  ConstrainLongRange cWidth(aConstraints.mRequired.mWidth);
+  ConstrainLongRange cHeight(aConstraints.mRequired.mHeight);
+
+  if (aConstraints.mAdvanced.WasPassed()) {
+    const auto& advanced = aConstraints.mAdvanced.Value();
+    for (uint32_t i = 0; i < advanced.Length(); i++) {
+      if (AreIntersecting(cWidth, advanced[i].mWidth) &&
+          AreIntersecting(cHeight, advanced[i].mHeight)) {
+        Intersect(cWidth, advanced[i].mWidth);
+        Intersect(cHeight, advanced[i].mHeight);
+      }
+    }
+  }
+  // Detect Mac HD cams and give them some love in the form of a dynamic default
+  // since that hardware switches between 4:3 at low res and 16:9 at higher res.
+  //
+  // Logic is: if we're relying on defaults in aPrefs, then
+  // only use HD pref when non-HD pref is too small and HD pref isn't too big.
+
+  bool macHD = ((!aPrefs.mWidth || !aPrefs.mHeight) &&
+                mDeviceName.EqualsASCII("FaceTime HD Camera (Built-in)") &&
+                (aPrefs.GetWidth() < cWidth.mMin ||
+                 aPrefs.GetHeight() < cHeight.mMin) &&
+                !(aPrefs.GetWidth(true) > cWidth.mMax ||
+                  aPrefs.GetHeight(true) > cHeight.mMax));
+  int prefWidth = aPrefs.GetWidth(macHD);
+  int prefHeight = aPrefs.GetHeight(macHD);
+
+  // Clamp width and height without distorting inherent aspect too much.
+
+  if (IsWithin(prefWidth, cWidth) == IsWithin(prefHeight, cHeight)) {
+    // If both are within, we get the default (pref) aspect.
+    // If neither are within, we get the aspect of the enclosing constraint.
+    // Either are presumably reasonable (presuming constraints are sane).
+    mCapability.width = Clamp(prefWidth, cWidth);
+    mCapability.height = Clamp(prefHeight, cHeight);
+  } else {
+    // But if only one clips (e.g. width), the resulting skew is undesirable:
+    //       .------------.
+    //       | constraint |
+    //  .----+------------+----.
+    //  |    |            |    |
+    //  |pref|  result    |    |   prefAspect != resultAspect
+    //  |    |            |    |
+    //  '----+------------+----'
+    //       '------------'
+    //  So in this case, preserve prefAspect instead:
+    //  .------------.
+    //  | constraint |
+    //  .------------.
+    //  |pref        |             prefAspect is unchanged
+    //  '------------'
+    //  |            |
+    //  '------------'
+    if (IsWithin(prefWidth, cWidth)) {
+      mCapability.height = Clamp(prefHeight, cHeight);
+      mCapability.width = Clamp((mCapability.height * prefWidth) /
+                                prefHeight, cWidth);
+    } else {
+      mCapability.width = Clamp(prefWidth, cWidth);
+      mCapability.height = Clamp((mCapability.width * prefHeight) /
+                                 prefWidth, cHeight);
+    }
+  }
+  mCapability.maxFPS = MediaEngine::DEFAULT_VIDEO_FPS;
+  LOG(("chose cap %dx%d @%dfps",
+       mCapability.width, mCapability.height, mCapability.maxFPS));
+}
+
 void
 MediaEngineWebRTCVideoSource::GetName(nsAString& aName)
 {
   aName = mDeviceName;
 }
 
 void
 MediaEngineWebRTCVideoSource::GetUUID(nsAString& aUUID)