Bug 882145: Implements gUM constraints. First constraint: facingMode. r=jesup
☠☠ backed out by c833bcf12ad8 ☠ ☠
authorJan-Ivar Bruaroey <jib@mozilla.com>
Sat, 14 Sep 2013 00:57:33 -0400
changeset 160136 bc4ee7cbd7bdc90069832d670ce55fc9e793edc5
parent 160135 cf373e408a6b2ce0d05fadde2611140aee2ec624
child 160137 f9c7040dddba2fa56ba7665fba55d5c000be2e9a
push id2961
push userlsblakk@mozilla.com
push dateMon, 28 Oct 2013 21:59:28 +0000
treeherdermozilla-beta@73ef4f13486f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjesup
bugs882145
milestone26.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 882145: Implements gUM constraints. First constraint: facingMode. r=jesup
dom/media/MediaManager.cpp
dom/media/MediaManager.h
dom/media/nsIDOMNavigatorUserMedia.idl
--- a/dom/media/MediaManager.cpp
+++ b/dom/media/MediaManager.cpp
@@ -65,16 +65,17 @@ GetMediaManagerLog()
 #endif
 
 using dom::MediaStreamConstraints;         // Outside API (contains JSObject)
 using dom::MediaStreamConstraintsInternal; // Storable supported constraints
 using dom::MediaTrackConstraintsInternal;  // Video or audio constraints
 using dom::MediaTrackConstraintSet;        // Mandatory or optional constraints
 using dom::MediaTrackConstraints;          // Raw mMandatory (as JSObject)
 using dom::GetUserMediaRequest;
+using dom::Sequence;
 
 // Used to compare raw MediaTrackConstraintSet against normalized dictionary
 // version to detect member differences, e.g. unsupported constraints.
 
 static nsresult CompareDictionaries(JSContext* aCx, JSObject *aA,
                                     const MediaTrackConstraintSet &aB,
                                     nsString *aDifference)
 {
@@ -304,16 +305,38 @@ protected:
   nsRefPtr<GetUserMediaCallbackMediaStreamListener> mListener;
 };
 
 /**
  * nsIMediaDevice implementation.
  */
 NS_IMPL_ISUPPORTS1(MediaDevice, nsIMediaDevice)
 
+MediaDevice::MediaDevice(MediaEngineVideoSource* aSource)
+  : mHasFacingMode(false)
+  , mSource(aSource) {
+  mType.Assign(NS_LITERAL_STRING("video"));
+  mSource->GetName(mName);
+  mSource->GetUUID(mID);
+
+  // Kludge to test user-facing cameras on OSX.
+  if (mName.Find(NS_LITERAL_STRING("Face")) != -1) {
+    mHasFacingMode = true;
+    mFacingMode = dom::VideoFacingModeEnum::User;
+  }
+}
+
+MediaDevice::MediaDevice(MediaEngineAudioSource* aSource)
+  : mHasFacingMode(false)
+  , mSource(aSource) {
+  mType.Assign(NS_LITERAL_STRING("audio"));
+  mSource->GetName(mName);
+  mSource->GetUUID(mID);
+}
+
 NS_IMETHODIMP
 MediaDevice::GetName(nsAString& aName)
 {
   aName.Assign(mName);
   return NS_OK;
 }
 
 NS_IMETHODIMP
@@ -325,16 +348,28 @@ MediaDevice::GetType(nsAString& aType)
 
 NS_IMETHODIMP
 MediaDevice::GetId(nsAString& aID)
 {
   aID.Assign(mID);
   return NS_OK;
 }
 
+NS_IMETHODIMP
+MediaDevice::GetFacingMode(nsAString& aFacingMode)
+{
+  if (mHasFacingMode) {
+    aFacingMode.Assign(NS_ConvertUTF8toUTF16(
+        dom::VideoFacingModeEnumValues::strings[uint32_t(mFacingMode)].value));
+  } else {
+    aFacingMode.Truncate(0);
+  }
+  return NS_OK;
+}
+
 MediaEngineSource*
 MediaDevice::GetSource()
 {
   return mSource;
 }
 
 /**
  * A subclass that we only use to stash internal pointers to MediaStreamGraph objects
@@ -576,58 +611,162 @@ private:
   nsRefPtr<MediaEngineSource> mAudioSource;
   nsRefPtr<MediaEngineSource> mVideoSource;
   uint64_t mWindowID;
   nsRefPtr<GetUserMediaCallbackMediaStreamListener> mListener;
   nsRefPtr<MediaManager> mManager; // get ref to this when creating the runnable
 };
 
 /**
+ * Helper functions that implement the constraints algorithm from
+ * http://dev.w3.org/2011/webrtc/editor/getusermedia.html#methods-5
+ */
+
+static bool SatisfyConstraint(const MediaEngineVideoSource *,
+                              const MediaTrackConstraintSet &aConstraints,
+                              nsIMediaDevice &aCandidate)
+{
+  if (aConstraints.mFacingMode.WasPassed()) {
+    nsString s;
+    aCandidate.GetFacingMode(s);
+    if (!s.EqualsASCII(dom::VideoFacingModeEnumValues::strings[
+        uint32_t(aConstraints.mFacingMode.Value())].value)) {
+      return false;
+    }
+  }
+  // TODO: Add more video-specific constraints
+  return true;
+}
+
+static bool SatisfyConstraint(const MediaEngineAudioSource *,
+                              const MediaTrackConstraintSet &aConstraints,
+                              nsIMediaDevice &aCandidate)
+{
+  // TODO: Add audio-specific constraints
+  return true;
+}
+
+typedef nsTArray<nsCOMPtr<nsIMediaDevice> > SourceSet;
+
+// Source getter that constrains list returned
+
+template<class SourceType>
+static SourceSet *
+  GetSources(MediaEngine *engine,
+             const MediaTrackConstraintsInternal &aConstraints,
+             void (MediaEngine::* aEnumerate)(nsTArray<nsRefPtr<SourceType> >*))
+{
+  const SourceType * const type = nullptr;
+
+  // First collect sources
+  SourceSet candidateSet;
+  {
+    nsTArray<nsRefPtr<SourceType> > sources;
+    (engine->*aEnumerate)(&sources);
+
+    /**
+      * We're allowing multiple tabs to access the same camera for parity
+      * with Chrome.  See bug 811757 for some of the issues surrounding
+      * this decision.  To disallow, we'd filter by IsAvailable() as we used
+      * to.
+      */
+
+    for (uint32_t len = sources.Length(), i = 0; i < len; i++) {
+      candidateSet.AppendElement(new MediaDevice(sources[i]));
+    }
+  }
+
+  // Then apply mandatory constraints
+
+  // Note: Iterator must be signed as it can dip below zero
+  for (int i = 0; i < int(candidateSet.Length()); i++) {
+    // Overloading instead of template specialization keeps things local
+    if (!SatisfyConstraint(type, aConstraints.mMandatory, *candidateSet[i])) {
+      candidateSet.RemoveElementAt(i--);
+    }
+  }
+
+  // Then apply optional constraints.
+  //
+  // These are only effective when there are multiple sources to pick from.
+  // Spec as-of-this-writing says to run algorithm on "all possible tracks
+  // of media type T that the browser COULD RETURN" (emphasis added).
+  //
+  // We think users ultimately control which devices we could return, so after
+  // determining the webpage's preferred list, we add the remaining choices
+  // to the tail, reasoning that they would all have passed individually,
+  // i.e. if the user had any one of them as their sole device (enabled).
+  //
+  // This avoids users having to unplug/disable devices should a webpage pick
+  // the wrong one (UX-fail). Webpage-preferred devices will be listed first.
+
+  SourceSet tailSet;
+
+  if (aConstraints.mOptional.WasPassed()) {
+    const Sequence<MediaTrackConstraintSet> &array = aConstraints.mOptional.Value();
+    for (int i = 0; i < int(array.Length()); i++) {
+      SourceSet rejects;
+      // Note: Iterator must be signed as it can dip below zero
+      for (int j = 0; j < int(candidateSet.Length()); j++) {
+        if (!SatisfyConstraint(type, array[i], *candidateSet[j])) {
+          rejects.AppendElement(candidateSet[j]);
+          candidateSet.RemoveElementAt(j--);
+        }
+      }
+      (candidateSet.Length()? tailSet : candidateSet).MoveElementsFrom(rejects);
+    }
+  }
+
+  SourceSet *result = new SourceSet;
+  result->MoveElementsFrom(candidateSet);
+  result->MoveElementsFrom(tailSet);
+  return result;
+}
+
+/**
  * Runs on a seperate thread and is responsible for enumerating devices.
  * Depending on whether a picture or stream was asked for, either
  * ProcessGetUserMedia or ProcessGetUserMediaSnapshot is called, and the results
  * are sent back to the DOM.
  *
  * Do not run this on the main thread. The success and error callbacks *MUST*
  * be dispatched on the main thread!
  */
 class GetUserMediaRunnable : public nsRunnable
 {
 public:
-  GetUserMediaRunnable(bool aAudio, bool aVideo, bool aPicture,
+  GetUserMediaRunnable(
+    const MediaStreamConstraintsInternal& aConstraints,
     already_AddRefed<nsIDOMGetUserMediaSuccessCallback> aSuccess,
     already_AddRefed<nsIDOMGetUserMediaErrorCallback> aError,
     uint64_t aWindowID, GetUserMediaCallbackMediaStreamListener *aListener,
     MediaEnginePrefs &aPrefs)
-    : mAudio(aAudio)
-    , mVideo(aVideo)
-    , mPicture(aPicture)
+    : mConstraints(aConstraints)
     , mSuccess(aSuccess)
     , mError(aError)
     , mWindowID(aWindowID)
     , mListener(aListener)
     , mPrefs(aPrefs)
     , mDeviceChosen(false)
     , mBackendChosen(false)
     , mManager(MediaManager::GetInstance())
   {}
 
   /**
    * The caller can also choose to provide their own backend instead of
    * using the one provided by MediaManager::GetBackend.
    */
-  GetUserMediaRunnable(bool aAudio, bool aVideo,
+  GetUserMediaRunnable(
+    const MediaStreamConstraintsInternal& aConstraints,
     already_AddRefed<nsIDOMGetUserMediaSuccessCallback> aSuccess,
     already_AddRefed<nsIDOMGetUserMediaErrorCallback> aError,
     uint64_t aWindowID, GetUserMediaCallbackMediaStreamListener *aListener,
     MediaEnginePrefs &aPrefs,
     MediaEngine* aBackend)
-    : mAudio(aAudio)
-    , mVideo(aVideo)
-    , mPicture(false)
+    : mConstraints(aConstraints)
     , mSuccess(aSuccess)
     , mError(aError)
     , mWindowID(aWindowID)
     , mListener(aListener)
     , mPrefs(aPrefs)
     , mDeviceChosen(false)
     , mBackendChosen(true)
     , mBackend(aBackend)
@@ -654,31 +793,33 @@ public:
     if (!mDeviceChosen) {
       nsresult rv = SelectDevice();
       if (rv != NS_OK) {
         return rv;
       }
     }
 
     // It is an error if audio or video are requested along with picture.
-    if (mPicture && (mAudio || mVideo)) {
+    if (mConstraints.mPicture && (mConstraints.mAudio || mConstraints.mVideo)) {
       NS_DispatchToMainThread(new ErrorCallbackRunnable(
         mSuccess, mError, NS_LITERAL_STRING("NOT_SUPPORTED_ERR"), mWindowID
       ));
       return NS_OK;
     }
 
-    if (mPicture) {
+    if (mConstraints.mPicture) {
       ProcessGetUserMediaSnapshot(mVideoDevice->GetSource(), 0);
       return NS_OK;
     }
 
     // There's a bug in the permission code that can leave us with mAudio but no audio device
-    ProcessGetUserMedia((mAudio && mAudioDevice) ? mAudioDevice->GetSource() : nullptr,
-                        (mVideo && mVideoDevice) ? mVideoDevice->GetSource() : nullptr);
+    ProcessGetUserMedia(((mConstraints.mAudio && mAudioDevice) ?
+                         mAudioDevice->GetSource() : nullptr),
+                        ((mConstraints.mVideo && mVideoDevice) ?
+                         mVideoDevice->GetSource() : nullptr));
     return NS_OK;
   }
 
   nsresult
   Denied(const nsAString& aErrorMsg)
   {
       // We add a disabled listener to the StreamListeners array until accepted
       // If this was the only active MediaStream, remove the window from the list.
@@ -719,79 +860,41 @@ public:
     mVideoDevice = aVideoDevice;
     mDeviceChosen = true;
     return NS_OK;
   }
 
   nsresult
   SelectDevice()
   {
-    bool found = false;
-    uint32_t count;
-    if (mPicture || mVideo) {
-      nsTArray<nsRefPtr<MediaEngineVideoSource> > videoSources;
-      mBackend->EnumerateVideoDevices(&videoSources);
+    if (mConstraints.mPicture || mConstraints.mVideo) {
+      ScopedDeletePtr<SourceSet> sources (GetSources(mBackend,
+          mConstraints.mVideom, &MediaEngine::EnumerateVideoDevices));
 
-      count = videoSources.Length();
-      if (count <= 0) {
+      if (!sources->Length()) {
         NS_DispatchToMainThread(new ErrorCallbackRunnable(
-          mSuccess, mError, NS_LITERAL_STRING("NO_DEVICES_FOUND"), mWindowID
-        ));
+          mSuccess, mError, NS_LITERAL_STRING("NO_DEVICES_FOUND"), mWindowID));
         return NS_ERROR_FAILURE;
       }
-
-      /**
-       * We're allowing multiple tabs to access the same camera for parity
-       * with Chrome.  See bug 811757 for some of the issues surrounding
-       * this decision.  To disallow, we'd filter by IsAvailable() as we used
-       * to.
-       */
       // Pick the first available device.
-      for (uint32_t i = 0; i < count; i++) {
-        nsRefPtr<MediaEngineVideoSource> vSource = videoSources[i];
-        found = true;
-        mVideoDevice = new MediaDevice(videoSources[i]);
-        break;
-      }
-
-      if (!found) {
-        NS_DispatchToMainThread(new ErrorCallbackRunnable(
-          mSuccess, mError, NS_LITERAL_STRING("HARDWARE_UNAVAILABLE"), mWindowID
-        ));
-        return NS_ERROR_FAILURE;
-      }
+      mVideoDevice = do_QueryObject((*sources)[0]);
       LOG(("Selected video device"));
     }
 
-    found = false;
-    if (mAudio) {
-      nsTArray<nsRefPtr<MediaEngineAudioSource> > audioSources;
-      mBackend->EnumerateAudioDevices(&audioSources);
+    if (mConstraints.mAudio) {
+      ScopedDeletePtr<SourceSet> sources (GetSources(mBackend,
+          mConstraints.mAudiom, &MediaEngine::EnumerateAudioDevices));
 
-      count = audioSources.Length();
-      if (count <= 0) {
+      if (!sources->Length()) {
         NS_DispatchToMainThread(new ErrorCallbackRunnable(
-          mSuccess, mError, NS_LITERAL_STRING("NO_DEVICES_FOUND"), mWindowID
-        ));
+          mSuccess, mError, NS_LITERAL_STRING("NO_DEVICES_FOUND"), mWindowID));
         return NS_ERROR_FAILURE;
       }
-
-      for (uint32_t i = 0; i < count; i++) {
-        nsRefPtr<MediaEngineAudioSource> aSource = audioSources[i];
-        found = true;
-        mAudioDevice = new MediaDevice(audioSources[i]);
-        break;
-      }
-
-      if (!found) {
-        NS_DispatchToMainThread(new ErrorCallbackRunnable(
-          mSuccess, mError, NS_LITERAL_STRING("HARDWARE_UNAVAILABLE"), mWindowID
-        ));
-        return NS_ERROR_FAILURE;
-      }
+      // Pick the first available device.
+      mAudioDevice = do_QueryObject((*sources)[0]);
       LOG(("Selected audio device"));
     }
 
     return NS_OK;
   }
 
   /**
    * Allocates a video or audio device and returns a MediaStream via
@@ -855,19 +958,17 @@ public:
 
     NS_DispatchToMainThread(new SuccessCallbackRunnable(
       mSuccess, mError, file, mWindowID
     ));
     return;
   }
 
 private:
-  bool mAudio;
-  bool mVideo;
-  bool mPicture;
+  MediaStreamConstraintsInternal mConstraints;
 
   already_AddRefed<nsIDOMGetUserMediaSuccessCallback> mSuccess;
   already_AddRefed<nsIDOMGetUserMediaErrorCallback> mError;
   uint64_t mWindowID;
   nsRefPtr<GetUserMediaCallbackMediaStreamListener> mListener;
   nsRefPtr<MediaDevice> mAudioDevice;
   nsRefPtr<MediaDevice> mVideoDevice;
   MediaEnginePrefs mPrefs;
@@ -898,48 +999,27 @@ public:
     , mError(aError)
     , mManager(MediaManager::GetInstance())
     , mWindowId(aWindowId) {}
 
   NS_IMETHOD
   Run()
   {
     NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");
-
-    uint32_t audioCount, videoCount, i;
-
-    nsTArray<nsRefPtr<MediaEngineVideoSource> > videoSources;
-    mManager->GetBackend(mWindowId)->EnumerateVideoDevices(&videoSources);
-    videoCount = videoSources.Length();
-
-    nsTArray<nsRefPtr<MediaEngineAudioSource> > audioSources;
-    mManager->GetBackend(mWindowId)->EnumerateAudioDevices(&audioSources);
-    audioCount = audioSources.Length();
-
-    nsTArray<nsCOMPtr<nsIMediaDevice> > *devices =
-      new nsTArray<nsCOMPtr<nsIMediaDevice> >;
+    MediaEngine *backend = mManager->GetBackend(mWindowId);
 
-    /**
-     * We're allowing multiple tabs to access the same camera for parity
-     * with Chrome.  See bug 811757 for some of the issues surrounding
-     * this decision.  To disallow, we'd filter by IsAvailable() as we used
-     * to.
-     */
-    for (i = 0; i < videoCount; i++) {
-      MediaEngineVideoSource *vSource = videoSources[i];
-      devices->AppendElement(new MediaDevice(vSource));
+    ScopedDeletePtr<SourceSet> final (GetSources(backend, mConstraints.mVideom,
+                                          &MediaEngine::EnumerateVideoDevices));
+    {
+      ScopedDeletePtr<SourceSet> s (GetSources(backend, mConstraints.mAudiom,
+                                        &MediaEngine::EnumerateAudioDevices));
+      final->MoveElementsFrom(*s);
     }
-    for (i = 0; i < audioCount; i++) {
-      MediaEngineAudioSource *aSource = audioSources[i];
-      devices->AppendElement(new MediaDevice(aSource));
-    }
-
-    NS_DispatchToMainThread(new DeviceSuccessCallbackRunnable(
-      mSuccess, mError, devices // give ownership of the nsTArray to the runnable
-    ));
+    NS_DispatchToMainThread(new DeviceSuccessCallbackRunnable(mSuccess, mError,
+                                                              final.forget()));
     return NS_OK;
   }
 
 private:
   MediaStreamConstraintsInternal mConstraints;
   already_AddRefed<nsIGetUserMediaDevicesSuccessCallback> mSuccess;
   already_AddRefed<nsIDOMGetUserMediaErrorCallback> mError;
   nsRefPtr<MediaManager> mManager;
@@ -1157,24 +1237,22 @@ MediaManager::GetUserMedia(JSContext* aC
   }
 
   /**
    * Pass runnables along to GetUserMediaRunnable so it can add the
    * MediaStreamListener to the runnable list.
    */
   if (c.mFake) {
     // Fake stream from default backend.
-    gUMRunnable = new GetUserMediaRunnable(
-      c.mAudio, c.mVideo, onSuccess.forget(),
+    gUMRunnable = new GetUserMediaRunnable(c, onSuccess.forget(),
       onError.forget(), windowID, listener, mPrefs, new MediaEngineDefault());
   } else {
     // Stream from default device from WebRTC backend.
-    gUMRunnable = new GetUserMediaRunnable(
-      c.mAudio, c.mVideo, c.mPicture,
-      onSuccess.forget(), onError.forget(), windowID, listener, mPrefs);
+    gUMRunnable = new GetUserMediaRunnable(c, onSuccess.forget(),
+      onError.forget(), windowID, listener, mPrefs);
   }
 
 #ifdef MOZ_B2G_CAMERA
   if (mCameraManager == nullptr) {
     aPrivileged = nsDOMCameraManager::CheckPermission(aWindow);
     if (aPrivileged) {
       mCameraManager = nsDOMCameraManager::CreateInstance(aWindow);
     }
--- a/dom/media/MediaManager.h
+++ b/dom/media/MediaManager.h
@@ -351,35 +351,27 @@ typedef nsTArray<nsRefPtr<GetUserMediaCa
 typedef nsClassHashtable<nsUint64HashKey, StreamListeners> WindowTable;
 
 class MediaDevice : public nsIMediaDevice
 {
 public:
   NS_DECL_THREADSAFE_ISUPPORTS
   NS_DECL_NSIMEDIADEVICE
 
-  MediaDevice(MediaEngineVideoSource* aSource) {
-    mSource = aSource;
-    mType.Assign(NS_LITERAL_STRING("video"));
-    mSource->GetName(mName);
-    mSource->GetUUID(mID);
-  }
-  MediaDevice(MediaEngineAudioSource* aSource) {
-    mSource = aSource;
-    mType.Assign(NS_LITERAL_STRING("audio"));
-    mSource->GetName(mName);
-    mSource->GetUUID(mID);
-  }
+  MediaDevice(MediaEngineVideoSource* aSource);
+  MediaDevice(MediaEngineAudioSource* aSource);
   virtual ~MediaDevice() {}
 
   MediaEngineSource* GetSource();
 private:
   nsString mName;
   nsString mType;
   nsString mID;
+  bool mHasFacingMode;
+  dom::VideoFacingModeEnum mFacingMode;
   nsRefPtr<MediaEngineSource> mSource;
 };
 
 class MediaManager MOZ_FINAL : public nsIMediaManagerService,
                                public nsIObserver
 {
 public:
   static already_AddRefed<MediaManager> GetInstance();
--- a/dom/media/nsIDOMNavigatorUserMedia.idl
+++ b/dom/media/nsIDOMNavigatorUserMedia.idl
@@ -1,22 +1,23 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "nsISupports.idl"
 #include "nsIVariant.idl"
 #include "nsIDOMMediaStream.idl"
 
-[scriptable, builtinclass, uuid(6de854f9-acf8-4383-b464-4803631ef309)]
+[scriptable, builtinclass, uuid(4af2bdb7-1547-4d10-8886-02a78c3c0b83)]
 interface nsIMediaDevice : nsISupports
 {
   readonly attribute DOMString type;
   readonly attribute DOMString name;
   readonly attribute DOMString id;
+  readonly attribute DOMString facingMode;
 };
 
 [scriptable, function, uuid(24544878-d35e-4962-8c5f-fb84e97bdfee)]
 interface nsIGetUserMediaDevicesSuccessCallback : nsISupports
 {
   void onSuccess(in nsIVariant devices);
 };