Bug 907352 - Part 4: Normalized constraints to relieve downstream width/height/frameRate implementation. r=mt
authorJan-Ivar Bruaroey <jib@mozilla.com>
Fri, 18 Apr 2014 15:16:08 -0400
changeset 198942 f0edcb56a33aa4eff4f7aac6fd93ab93a33655f1
parent 198941 c0b31eca151e8bf2a9b2371b826f3ce10c445436
child 198943 66205061c13f7266750be6e0389796a386f24b44
push id3624
push userasasaki@mozilla.com
push dateMon, 09 Jun 2014 21:49:01 +0000
treeherdermozilla-beta@b1a5da15899a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmt
bugs907352
milestone31.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 907352 - Part 4: Normalized constraints to relieve downstream width/height/frameRate implementation. r=mt
content/media/webrtc/MediaTrackConstraints.h
content/media/webrtc/moz.build
dom/media/MediaManager.cpp
dom/media/tests/mochitest/constraints.js
dom/webidl/Constraints.webidl
dom/webidl/MediaTrackConstraintSet.webidl
new file mode 100644
--- /dev/null
+++ b/content/media/webrtc/MediaTrackConstraints.h
@@ -0,0 +1,93 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file should not be included by other includes, as it contains code
+
+#ifndef MEDIATRACKCONSTRAINTS_H_
+#define MEDIATRACKCONSTRAINTS_H_
+
+#include "mozilla/dom/MediaStreamTrackBinding.h"
+
+namespace mozilla {
+
+// Normalized internal version of MediaTrackConstraints to simplify downstream
+// processing. This implementation-only helper is included as needed by both
+// MediaManager (for gUM camera selection) and MediaEngine (for applyConstraints).
+
+template<typename T>
+class MediaTrackConstraintsN : public dom::MediaTrackConstraints
+{
+public:
+  typedef T Kind;
+  dom::Sequence<Kind> mRequireN;
+  bool mUnsupportedRequirement;
+  MediaTrackConstraintSet mRequired;
+  dom::Sequence<MediaTrackConstraintSet> mNonrequired;
+
+  MediaTrackConstraintsN(const dom::MediaTrackConstraints &aOther,
+                         const dom::EnumEntry* aStrings)
+  : dom::MediaTrackConstraints(aOther)
+  , mUnsupportedRequirement(false)
+  , mStrings(aStrings)
+  {
+    if (mRequire.WasPassed()) {
+      auto& array = mRequire.Value();
+      for (uint32_t i = 0; i < array.Length(); i++) {
+        auto value = ToEnum(array[i]);
+        if (value != Kind::Other) {
+          mRequireN.AppendElement(value);
+        } else {
+          mUnsupportedRequirement = true;
+        }
+      }
+    }
+  }
+protected:
+  MediaTrackConstraintSet& Triage(const Kind kind) {
+    if (mRequireN.IndexOf(kind) != mRequireN.NoIndex) {
+      return mRequired;
+    } else {
+      mNonrequired.AppendElement(MediaTrackConstraintSet());
+      return mNonrequired[mNonrequired.Length()-1];
+    }
+  }
+private:
+  Kind ToEnum(const nsAString& aSrc) {
+    for (size_t i = 0; mStrings[i].value; i++) {
+      if (aSrc.EqualsASCII(mStrings[i].value)) {
+        return Kind(i);
+      }
+    }
+    return Kind::Other;
+  }
+  const dom::EnumEntry* mStrings;
+};
+
+struct AudioTrackConstraintsN :
+  public MediaTrackConstraintsN<dom::SupportedAudioConstraints>
+{
+  AudioTrackConstraintsN(const dom::MediaTrackConstraints &aOther)
+  : MediaTrackConstraintsN<dom::SupportedAudioConstraints>(aOther, // B2G ICS compiler bug
+                           dom::SupportedAudioConstraintsValues::strings) {}
+};
+
+struct VideoTrackConstraintsN :
+    public MediaTrackConstraintsN<dom::SupportedVideoConstraints>
+{
+  VideoTrackConstraintsN(const dom::MediaTrackConstraints &aOther)
+  : MediaTrackConstraintsN<dom::SupportedVideoConstraints>(aOther,
+                           dom::SupportedVideoConstraintsValues::strings) {
+    if (mFacingMode.WasPassed()) {
+      Triage(Kind::FacingMode).mFacingMode.Construct(mFacingMode.Value());
+    }
+    // Reminder: add handling for new constraints both here & SatisfyConstraintSet
+    Triage(Kind::Width).mWidth = mWidth;
+    Triage(Kind::Height).mHeight = mHeight;
+    Triage(Kind::FrameRate).mFrameRate = mFrameRate;
+  }
+};
+
+}
+
+#endif /* MEDIATRACKCONSTRAINTS_H_ */
--- a/content/media/webrtc/moz.build
+++ b/content/media/webrtc/moz.build
@@ -4,16 +4,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 XPIDL_MODULE = 'content_webrtc'
 
 EXPORTS += [
     'MediaEngine.h',
     'MediaEngineDefault.h',
+    'MediaTrackConstraints.h',
 ]
 
 if CONFIG['MOZ_WEBRTC']:
     EXPORTS += ['AudioOutputObserver.h',
                 'LoadManager.h',
                 'LoadManagerFactory.h',
                 'LoadMonitor.h',
                 'MediaEngineWebRTC.h']
--- a/dom/media/MediaManager.cpp
+++ b/dom/media/MediaManager.cpp
@@ -23,16 +23,17 @@
 #include "nsIDocument.h"
 #include "nsISupportsPrimitives.h"
 #include "nsIInterfaceRequestorUtils.h"
 #include "mozilla/Types.h"
 #include "mozilla/dom/ContentChild.h"
 #include "mozilla/dom/MediaStreamBinding.h"
 #include "mozilla/dom/MediaStreamTrackBinding.h"
 #include "mozilla/dom/GetUserMediaRequestBinding.h"
+#include "MediaTrackConstraints.h"
 
 #include "Latency.h"
 
 // For PR_snprintf
 #include "prprf.h"
 
 #include "nsJSUtils.h"
 #include "nsDOMFile.h"
@@ -75,16 +76,18 @@ GetMediaManagerLog()
 #endif
 
 using dom::MediaStreamConstraints;         // Outside API (contains JSObject)
 using dom::MediaTrackConstraintSet;        // Mandatory or optional constraints
 using dom::MediaTrackConstraints;          // Raw mMandatory (as JSObject)
 using dom::GetUserMediaRequest;
 using dom::Sequence;
 using dom::OwningBooleanOrMediaTrackConstraints;
+using dom::SupportedAudioConstraints;
+using dom::SupportedVideoConstraints;
 
 ErrorCallbackRunnable::ErrorCallbackRunnable(
   nsCOMPtr<nsIDOMGetUserMediaSuccessCallback>& aSuccess,
   nsCOMPtr<nsIDOMGetUserMediaErrorCallback>& aError,
   const nsAString& aErrorMsg, uint64_t aWindowID)
   : mErrorMsg(aErrorMsg)
   , mWindowID(aWindowID)
   , mManager(MediaManager::GetInstance())
@@ -650,30 +653,16 @@ GetInvariant(const OwningBooleanOrMediaT
       aUnion.GetAsMediaTrackConstraints() : empty;
 }
 
 /**
  * Helper functions that implement the constraints algorithm from
  * http://dev.w3.org/2011/webrtc/editor/getusermedia.html#methods-5
  */
 
-#define lengthof(a) (sizeof(a) / sizeof(*a))
-
-static auto
-GetSupportedConstraintNames(const MediaEngineVideoSource *) ->
-    decltype((dom::SupportedVideoConstraintsValues::strings)) {
-  return dom::SupportedVideoConstraintsValues::strings;
-}
-
-static auto
-GetSupportedConstraintNames(const MediaEngineAudioSource *) ->
-    decltype((dom::SupportedAudioConstraintsValues::strings)) {
-  return dom::SupportedAudioConstraintsValues::strings;
-}
-
 // Reminder: add handling for new constraints both here and in GetSources below!
 
 static bool SatisfyConstraintSet(const MediaEngineVideoSource *,
                                  const MediaTrackConstraintSet &aConstraints,
                                  nsIMediaDevice &aCandidate)
 {
   if (aConstraints.mFacingMode.WasPassed()) {
     nsString s;
@@ -690,71 +679,29 @@ static bool SatisfyConstraintSet(const M
 static bool SatisfyConstraintSet(const MediaEngineAudioSource *,
                                  const MediaTrackConstraintSet &aConstraints,
                                  nsIMediaDevice &aCandidate)
 {
   // TODO: Add audio-specific constraints
   return true;
 }
 
-// Triage constraints into required and nonrequired + detect missing requireds
-
-class TriageHelper
-{
-public:
-  TriageHelper(const nsTArray<nsString>& aRequire)
-  : mRequire(aRequire)
-  , mNumRequirementsMet(0) {}
-
-  MediaTrackConstraintSet& Triage(dom::SupportedVideoConstraints kind) {
-    return Triage(NS_ConvertUTF8toUTF16(
-        dom::SupportedVideoConstraintsValues::strings[uint32_t(kind)].value));
-  }
-  MediaTrackConstraintSet& Triage(dom::SupportedAudioConstraints kind) {
-    return Triage(NS_ConvertUTF8toUTF16(
-        dom::SupportedAudioConstraintsValues::strings[uint32_t(kind)].value));
-  }
-private:
-  MediaTrackConstraintSet& Triage(const nsAString &name) {
-    if (mRequire.IndexOf(name) != mRequire.NoIndex) {
-      mNumRequirementsMet++;
-      return mRequired;
-    } else {
-      return mNonrequired;
-    }
-  }
-public:
-  bool RequirementsAreMet() {
-    MOZ_ASSERT(mNumRequirementsMet <= mRequire.Length());
-    return mNumRequirementsMet == mRequire.Length();
-  }
-  MediaTrackConstraintSet mRequired;
-  MediaTrackConstraintSet mNonrequired;
-private:
-  const nsTArray<nsString> mRequire;
-  uint32_t mNumRequirementsMet;
-};
-
 typedef nsTArray<nsCOMPtr<nsIMediaDevice> > SourceSet;
 
 // Source getter that constrains list returned
 
-template<class SourceType>
+template<class SourceType, class ConstraintsType>
 static SourceSet *
   GetSources(MediaEngine *engine,
-             const OwningBooleanOrMediaTrackConstraints &aConstraints,
+             ConstraintsType &aConstraints,
              void (MediaEngine::* aEnumerate)(nsTArray<nsRefPtr<SourceType> >*),
              char* media_device_name = nullptr)
 {
   ScopedDeletePtr<SourceSet> result(new SourceSet);
 
-  if (!IsOn(aConstraints)) {
-    return result.forget();
-  }
-
   const SourceType * const type = nullptr;
   nsString deviceName;
   // First collect sources
   SourceSet candidateSet;
   {
     nsTArray<nsRefPtr<SourceType> > sources;
     (engine->*aEnumerate)(&sources);
     /**
@@ -777,78 +724,50 @@ static SourceSet *
 #ifdef DEBUG
       }
 #endif
     }
   }
 
   // Apply constraints to the list of sources.
 
-  auto& c = GetInvariant(aConstraints);
-  const nsTArray<nsString> empty;
-  const auto &require = c.mRequire.WasPassed()? c.mRequire.Value() : empty;
-  {
+  auto& c = aConstraints;
+  if (c.mUnsupportedRequirement) {
     // Check upfront the names of required constraints that are unsupported for
     // this media-type. The spec requires these to fail, so getting them out of
     // the way early provides a necessary invariant for the remaining algorithm
     // which maximizes code-reuse by ignoring constraints of the other type
     // (specifically, SatisfyConstraintSet is reused for the advanced algorithm
     // where the spec requires it to ignore constraints of the other type)
-
-    const auto& supported = GetSupportedConstraintNames(type);
-    for (uint32_t i = 0; i < require.Length(); i++) {
-      bool found = false;
-      // EnumType arrays have a zero-terminator entry at the end. Skip.
-      for (size_t j = 0; j < sizeof(supported)/sizeof(*supported) - 1; j++) {
-        if (require[i].EqualsASCII(supported[j].value)) {
-          found = true;
-          break;
-        }
-      }
-      if (!found) {
-        return result.forget();
-      }
-    }
-  }
-
-  // Before we start, triage constraints into required and nonrequired.
-  // This part is type-agnostic because it can be.
-  // Reminder: add handling for new constraints both here & SatisfyConstraintSet
-
-  TriageHelper helper(require);
-
-  if (c.mFacingMode.WasPassed()) {
-    helper.Triage(dom::SupportedVideoConstraints::FacingMode).
-        mFacingMode.Construct(c.mFacingMode.Value());
-  }
-  if (c.mWidth.mMin.WasPassed() || c.mWidth.mMax.WasPassed()) {
-    helper.Triage(dom::SupportedVideoConstraints::Width).mWidth = c.mWidth;
-  }
-  if (c.mHeight.mMin.WasPassed() || c.mHeight.mMax.WasPassed()) {
-    helper.Triage(dom::SupportedVideoConstraints::Height).mHeight = c.mHeight;
-  }
-  if (c.mFrameRate.mMin.WasPassed() || c.mFrameRate.mMax.WasPassed()) {
-    helper.Triage(dom::SupportedVideoConstraints::FrameRate).mFrameRate =
-        c.mFrameRate;
-  }
-  if (!helper.RequirementsAreMet()) {
     return result.forget();
   }
 
   // Now on to the actual algorithm: First apply required constraints.
 
   for (uint32_t i = 0; i < candidateSet.Length();) {
     // Overloading instead of template specialization keeps things local
-    if (!SatisfyConstraintSet(type, helper.mRequired, *candidateSet[i])) {
+    if (!SatisfyConstraintSet(type, c.mRequired, *candidateSet[i])) {
       candidateSet.RemoveElementAt(i);
     } else {
       ++i;
     }
   }
 
+  // TODO(jib): Proper non-ordered handling of nonrequired constraints (907352)
+  //
+  // For now, put nonrequired constraints at tail of Advanced list.
+  // This isn't entirely accurate, as order will matter, but few will notice
+  // the difference until we get camera selection and a few more constraints.
+  if (c.mNonrequired.Length()) {
+    if (!c.mAdvanced.WasPassed()) {
+      c.mAdvanced.Construct();
+    }
+    c.mAdvanced.Value().MoveElementsFrom(c.mNonrequired);
+  }
+
   // Then apply advanced (formerly known as optional) constraints.
   //
   // These are only effective when there are multiple sources to pick from.
   // Spec as-of-this-writing says to run algorithm on "all possible tracks
   // of media type T that the browser COULD RETURN" (emphasis added).
   //
   // We think users ultimately control which devices we could return, so after
   // determining the webpage's preferred list, we add the remaining choices
@@ -856,34 +775,33 @@ static SourceSet *
   // i.e. if the user had any one of them as their sole device (enabled).
   //
   // This avoids users having to unplug/disable devices should a webpage pick
   // the wrong one (UX-fail). Webpage-preferred devices will be listed first.
 
   SourceSet tailSet;
 
   if (c.mAdvanced.WasPassed()) {
-    const auto &array = c.mAdvanced.Value();
+    auto &array = c.mAdvanced.Value();
+
     for (int i = 0; i < int(array.Length()); i++) {
       SourceSet rejects;
       for (uint32_t j = 0; j < candidateSet.Length();) {
         if (!SatisfyConstraintSet(type, array[i], *candidateSet[j])) {
           rejects.AppendElement(candidateSet[j]);
           candidateSet.RemoveElementAt(j);
         } else {
           ++j;
         }
       }
       (candidateSet.Length()? tailSet : candidateSet).MoveElementsFrom(rejects);
     }
   }
 
-  // Finally, order any remaining sources by how many nonrequired constraints
-  // they satisfy. TODO(jib): TBD once we implement >1 constraint (Bug 907352)
-
+  // TODO: Proper non-ordered handling of nonrequired constraints (Bug 907352)
 
   result->MoveElementsFrom(candidateSet);
   result->MoveElementsFrom(tailSet);
   return result.forget();
 }
 
 /**
  * Runs on a seperate thread and is responsible for enumerating devices.
@@ -1048,31 +966,33 @@ public:
   }
 
   nsresult
   SelectDevice(MediaEngine* backend)
   {
     MOZ_ASSERT(mSuccess);
     MOZ_ASSERT(mError);
     if (mConstraints.mPicture || IsOn(mConstraints.mVideo)) {
-      ScopedDeletePtr<SourceSet> sources (GetSources(backend,
-          mConstraints.mVideo, &MediaEngine::EnumerateVideoDevices));
+      VideoTrackConstraintsN constraints(GetInvariant(mConstraints.mVideo));
+      ScopedDeletePtr<SourceSet> sources (GetSources(backend, constraints,
+          &MediaEngine::EnumerateVideoDevices));
 
       if (!sources->Length()) {
         Fail(NS_LITERAL_STRING("NO_DEVICES_FOUND"));
         return NS_ERROR_FAILURE;
       }
       // Pick the first available device.
       mVideoDevice = do_QueryObject((*sources)[0]);
       LOG(("Selected video device"));
     }
 
     if (IsOn(mConstraints.mAudio)) {
-      ScopedDeletePtr<SourceSet> sources (GetSources(backend,
-          mConstraints.mAudio, &MediaEngine::EnumerateAudioDevices));
+      AudioTrackConstraintsN constraints(GetInvariant(mConstraints.mAudio));
+      ScopedDeletePtr<SourceSet> sources (GetSources(backend, constraints,
+          &MediaEngine::EnumerateAudioDevices));
 
       if (!sources->Length()) {
         Fail(NS_LITERAL_STRING("NO_DEVICES_FOUND"));
         return NS_ERROR_FAILURE;
       }
       // Pick the first available device.
       mAudioDevice = do_QueryObject((*sources)[0]);
       LOG(("Selected audio device"));
@@ -1198,23 +1118,29 @@ public:
     NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");
 
     nsRefPtr<MediaEngine> backend;
     if (mConstraints.mFake)
       backend = new MediaEngineDefault();
     else
       backend = mManager->GetBackend(mWindowId);
 
-    ScopedDeletePtr<SourceSet> final (GetSources(backend, mConstraints.mVideo,
-                                          &MediaEngine::EnumerateVideoDevices,
-                                          mLoopbackVideoDevice));
-    {
-      ScopedDeletePtr<SourceSet> s (GetSources(backend, mConstraints.mAudio,
-                                        &MediaEngine::EnumerateAudioDevices,
-                                        mLoopbackAudioDevice));
+    ScopedDeletePtr<SourceSet> final(new SourceSet);
+    if (IsOn(mConstraints.mVideo)) {
+      VideoTrackConstraintsN constraints(GetInvariant(mConstraints.mVideo));
+      ScopedDeletePtr<SourceSet> s(GetSources(backend, constraints,
+          &MediaEngine::EnumerateVideoDevices,
+          mLoopbackVideoDevice));
+      final->MoveElementsFrom(*s);
+    }
+    if (IsOn(mConstraints.mAudio)) {
+      AudioTrackConstraintsN constraints(GetInvariant(mConstraints.mAudio));
+      ScopedDeletePtr<SourceSet> s (GetSources(backend, constraints,
+          &MediaEngine::EnumerateAudioDevices,
+          mLoopbackAudioDevice));
       final->MoveElementsFrom(*s);
     }
     NS_DispatchToMainThread(new DeviceSuccessCallbackRunnable(mWindowId,
                                                               mSuccess, mError,
                                                               final.forget()));
     // DeviceSuccessCallbackRunnable should have taken these.
     MOZ_ASSERT(!mSuccess && !mError);
     return NS_OK;
--- a/dom/media/tests/mochitest/constraints.js
+++ b/dom/media/tests/mochitest/constraints.js
@@ -14,22 +14,16 @@ var common_tests = [
   { message: "unknown required constraint on video fails",
     constraints: { video: { somethingUnknown:0, require:["somethingUnknown"] },
                    fake: true },
     error: "NO_DEVICES_FOUND" },
   { message: "unknown required constraint on audio fails",
     constraints: { audio: { somethingUnknown:0, require:["somethingUnknown"] },
                    fake: true },
     error: "NO_DEVICES_FOUND" },
-  { message: "missing required constraint on video fails",
-    constraints: { video: { require:["facingMode"] }, fake: true },
-    error: "NO_DEVICES_FOUND" },
-  { message: "missing required constraint on audio fails",
-    constraints: { audio: { require:["facingMode"] }, fake: true },
-    error: "NO_DEVICES_FOUND" },
   { message: "video overconstrained by facingMode fails",
     constraints: { video: { facingMode:'left', require:["facingMode"] },
                    fake: true },
     error: "NO_DEVICES_FOUND" },
   { message: "audio overconstrained by facingMode fails",
     constraints: { audio: { facingMode:'left', require:["facingMode"] },
                    fake: true },
     error: "NO_DEVICES_FOUND" },
--- a/dom/webidl/Constraints.webidl
+++ b/dom/webidl/Constraints.webidl
@@ -10,16 +10,16 @@
 enum VideoFacingModeEnum {
     "user",
     "environment",
     "left",
     "right"
 };
 
 dictionary ConstrainLongRange {
-    long min;
-    long max;
+    long min = -2147483647; // +1 works around windows compiler bug
+    long max = 2147483647;
 };
 
 dictionary ConstrainDoubleRange {
-    double min;
-    double max;
+    unrestricted double min = -Infinity;
+    unrestricted double max = Infinity;
 };
--- a/dom/webidl/MediaTrackConstraintSet.webidl
+++ b/dom/webidl/MediaTrackConstraintSet.webidl
@@ -3,24 +3,25 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/.
  *
  * The origin of this IDL file is
  * http://dev.w3.org/2011/webrtc/editor/getusermedia.html
  */
 
 enum SupportedVideoConstraints {
+    "other",
     "facingMode",
     "width",
     "height",
-    "frameRate"
+    "frameRate",
 };
 
 enum SupportedAudioConstraints {
-    "dummy"
+    "other"
 };
 
 dictionary MediaTrackConstraintSet {
     ConstrainLongRange width;
     ConstrainLongRange height;
     ConstrainDoubleRange frameRate;
     ConstrainVideoFacingMode facingMode;
 };