Bug 983504 - Enumeration & MediaEngine changes for screen sharing. r=jesup,mt
authorGian-Carlo Pascutto <gpascutto@mozilla.com>
Mon, 07 Jul 2014 09:46:00 +0200
changeset 216330 47007b47379f75a868fd98d4c5244538d9699b37
parent 216329 e8491346dfbd193078598872646883fd197a34b4
child 216331 77ec03dc289a751d711368b8c8bc6a10433674bc
push id515
push userraliiev@mozilla.com
push dateMon, 06 Oct 2014 12:51:51 +0000
treeherdermozilla-release@267c7a481bef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjesup, mt
bugs983504
milestone33.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 983504 - Enumeration & MediaEngine changes for screen sharing. r=jesup,mt
content/media/webrtc/MediaEngine.h
content/media/webrtc/MediaEngineDefault.cpp
content/media/webrtc/MediaEngineDefault.h
content/media/webrtc/MediaEngineWebRTC.cpp
content/media/webrtc/MediaEngineWebRTC.h
--- a/content/media/webrtc/MediaEngine.h
+++ b/content/media/webrtc/MediaEngine.h
@@ -4,16 +4,17 @@
 
 #ifndef MEDIAENGINE_H_
 #define MEDIAENGINE_H_
 
 #include "mozilla/RefPtr.h"
 #include "nsIDOMFile.h"
 #include "DOMMediaStream.h"
 #include "MediaStreamGraph.h"
+#include "mozilla/dom/MediaStreamTrackBinding.h"
 
 namespace mozilla {
 
 struct VideoTrackConstraintsN;
 struct AudioTrackConstraintsN;
 
 /**
  * Abstract interface for managing audio and video devices. Each platform
@@ -49,21 +50,23 @@ public:
   static const int DEFAULT_43_VIDEO_WIDTH = 640;
   static const int DEFAULT_43_VIDEO_HEIGHT = 480;
   static const int DEFAULT_169_VIDEO_WIDTH = 1280;
   static const int DEFAULT_169_VIDEO_HEIGHT = 720;
   static const int DEFAULT_AUDIO_TIMER_MS = 10;
 
   /* Populate an array of video sources in the nsTArray. Also include devices
    * that are currently unavailable. */
-  virtual void EnumerateVideoDevices(nsTArray<nsRefPtr<MediaEngineVideoSource> >*) = 0;
+  virtual void EnumerateVideoDevices(dom::MediaSourceEnum,
+                                     nsTArray<nsRefPtr<MediaEngineVideoSource> >*) = 0;
 
   /* Populate an array of audio sources in the nsTArray. Also include devices
    * that are currently unavailable. */
-  virtual void EnumerateAudioDevices(nsTArray<nsRefPtr<MediaEngineAudioSource> >*) = 0;
+  virtual void EnumerateAudioDevices(dom::MediaSourceEnum,
+                                     nsTArray<nsRefPtr<MediaEngineAudioSource> >*) = 0;
 
 protected:
   virtual ~MediaEngine() {}
 };
 
 /**
  * Common abstract base class for audio and video sources.
  */
@@ -177,16 +180,19 @@ private:
   }
 };
 
 class MediaEngineVideoSource : public MediaEngineSource
 {
 public:
   virtual ~MediaEngineVideoSource() {}
 
+  virtual const dom::MediaSourceEnum GetMediaSource() {
+      return dom::MediaSourceEnum::Camera;
+  }
   /* This call reserves but does not start the device. */
   virtual nsresult Allocate(const VideoTrackConstraintsN &aConstraints,
                             const MediaEnginePrefs &aPrefs) = 0;
 };
 
 /**
  * Audio source and friends.
  */
--- a/content/media/webrtc/MediaEngineDefault.cpp
+++ b/content/media/webrtc/MediaEngineDefault.cpp
@@ -473,34 +473,43 @@ MediaEngineDefaultAudioSource::Notify(ns
   channels.AppendElement(dest);
   segment.AppendFrames(buffer.forget(), channels, AUDIO_FRAME_LENGTH);
   mSource->AppendToTrack(mTrackID, &segment);
 
   return NS_OK;
 }
 
 void
-MediaEngineDefault::EnumerateVideoDevices(nsTArray<nsRefPtr<MediaEngineVideoSource> >* aVSources) {
+MediaEngineDefault::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource,
+                                          nsTArray<nsRefPtr<MediaEngineVideoSource> >* aVSources) {
   MutexAutoLock lock(mMutex);
 
+  // only supports camera sources (for now).  See Bug 1038241
+  if (aMediaSource != dom::MediaSourceEnum::Camera) {
+    return;
+  }
+
   // We once had code here to find a VideoSource with the same settings and re-use that.
   // This no longer is possible since the resolution is being set in Allocate().
 
   nsRefPtr<MediaEngineVideoSource> newSource = new MediaEngineDefaultVideoSource();
   mVSources.AppendElement(newSource);
   aVSources->AppendElement(newSource);
 
   return;
 }
 
 void
-MediaEngineDefault::EnumerateAudioDevices(nsTArray<nsRefPtr<MediaEngineAudioSource> >* aASources) {
+MediaEngineDefault::EnumerateAudioDevices(dom::MediaSourceEnum aMediaSource,
+                                          nsTArray<nsRefPtr<MediaEngineAudioSource> >* aASources) {
   MutexAutoLock lock(mMutex);
   int32_t len = mASources.Length();
 
+  // aMediaSource is ignored for audio devices (for now).
+
   for (int32_t i = 0; i < len; i++) {
     nsRefPtr<MediaEngineAudioSource> source = mASources.ElementAt(i);
     if (source->IsAvailable()) {
       aASources->AppendElement(source);
     }
   }
 
   // All streams are currently busy, just make a new one.
--- a/content/media/webrtc/MediaEngineDefault.h
+++ b/content/media/webrtc/MediaEngineDefault.h
@@ -133,18 +133,20 @@ protected:
 
 class MediaEngineDefault : public MediaEngine
 {
 public:
   MediaEngineDefault()
   : mMutex("mozilla::MediaEngineDefault")
   {}
 
-  virtual void EnumerateVideoDevices(nsTArray<nsRefPtr<MediaEngineVideoSource> >*);
-  virtual void EnumerateAudioDevices(nsTArray<nsRefPtr<MediaEngineAudioSource> >*);
+  virtual void EnumerateVideoDevices(dom::MediaSourceEnum,
+                                     nsTArray<nsRefPtr<MediaEngineVideoSource> >*);
+  virtual void EnumerateAudioDevices(dom::MediaSourceEnum,
+                                     nsTArray<nsRefPtr<MediaEngineAudioSource> >*);
 
 private:
   ~MediaEngineDefault() {}
 
   Mutex mMutex;
   // protected with mMutex:
 
   nsTArray<nsRefPtr<MediaEngineVideoSource> > mVSources;
--- a/content/media/webrtc/MediaEngineWebRTC.cpp
+++ b/content/media/webrtc/MediaEngineWebRTC.cpp
@@ -40,22 +40,25 @@ GetUserMediaLog()
 #endif
 
 #undef LOG
 #define LOG(args) PR_LOG(GetUserMediaLog(), PR_LOG_DEBUG, args)
 
 namespace mozilla {
 
 MediaEngineWebRTC::MediaEngineWebRTC(MediaEnginePrefs &aPrefs)
-  : mMutex("mozilla::MediaEngineWebRTC")
-  , mVideoEngine(nullptr)
-  , mVoiceEngine(nullptr)
-  , mVideoEngineInit(false)
-  , mAudioEngineInit(false)
-  , mHasTabVideoSource(false)
+    : mMutex("mozilla::MediaEngineWebRTC")
+    , mScreenEngine(nullptr)
+    , mAppEngine(nullptr)
+    , mVideoEngine(nullptr)
+    , mVoiceEngine(nullptr)
+    , mVideoEngineInit(false)
+    , mAudioEngineInit(false)
+    , mScreenEngineInit(false)
+    , mAppEngineInit(false)
 {
 #ifndef MOZ_B2G_CAMERA
   nsCOMPtr<nsIComponentRegistrar> compMgr;
   NS_GetComponentRegistrar(getter_AddRefs(compMgr));
   if (compMgr) {
     compMgr->IsContractIDRegistered(NS_TABSOURCESERVICE_CONTRACTID, &mHasTabVideoSource);
   }
 #else
@@ -64,21 +67,28 @@ MediaEngineWebRTC::MediaEngineWebRTC(Med
   // XXX
   gFarendObserver = new AudioOutputObserver();
 
   NS_NewNamedThread("AudioGUM", getter_AddRefs(mThread));
   MOZ_ASSERT(mThread);
 }
 
 void
-MediaEngineWebRTC::EnumerateVideoDevices(nsTArray<nsRefPtr<MediaEngineVideoSource> >* aVSources)
+MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource,
+                                         nsTArray<nsRefPtr<MediaEngineVideoSource> >* aVSources)
 {
-#ifdef MOZ_B2G_CAMERA
+  // We spawn threads to handle gUM runnables, so we must protect the member vars
   MutexAutoLock lock(mMutex);
 
+ #ifdef MOZ_B2G_CAMERA
+  if (aMediaSource != dom::MediaSourceEnum::Camera) {
+    // only supports camera sources
+    return;
+  }
+
   /**
    * We still enumerate every time, in case a new device was plugged in since
    * the last call. TODO: Verify that WebRTC actually does deal with hotplugging
    * new devices (with or without new engine creation) and accordingly adjust.
    * Enumeration is not neccessary if GIPS reports the same set of devices
    * for a given instance of the engine. Likewise, if a device was plugged out,
    * mVideoSources must be updated.
    */
@@ -97,58 +107,86 @@ MediaEngineWebRTC::EnumerateVideoDevices
     }
 
     nsRefPtr<MediaEngineWebRTCVideoSource> vSource;
     NS_ConvertUTF8toUTF16 uuid(cameraName);
     if (mVideoSources.Get(uuid, getter_AddRefs(vSource))) {
       // We've already seen this device, just append.
       aVSources->AppendElement(vSource.get());
     } else {
-      vSource = new MediaEngineWebRTCVideoSource(i);
+      vSource = new MediaEngineWebRTCVideoSource(i, mediaSourceType);
       mVideoSources.Put(uuid, vSource); // Hashtable takes ownership.
       aVSources->AppendElement(vSource);
     }
   }
 
   return;
 #else
   ScopedCustomReleasePtr<webrtc::ViEBase> ptrViEBase;
   ScopedCustomReleasePtr<webrtc::ViECapture> ptrViECapture;
-
-  // We spawn threads to handle gUM runnables, so we must protect the member vars
-  MutexAutoLock lock(mMutex);
+  webrtc::Config configSet;
+  webrtc::VideoEngine *videoEngine = nullptr;
+  bool *videoEngineInit = nullptr;
 
 #ifdef MOZ_WIDGET_ANDROID
   // get the JVM
   JavaVM *jvm = mozilla::AndroidBridge::Bridge()->GetVM();
 
   if (webrtc::VideoEngine::SetAndroidObjects(jvm) != 0) {
     LOG(("VieCapture:SetAndroidObjects Failed"));
     return;
   }
 #endif
-  if (!mVideoEngine) {
-    if (!(mVideoEngine = webrtc::VideoEngine::Create())) {
-      return;
-    }
+
+  switch (aMediaSource) {
+    case dom::MediaSourceEnum::Application:
+      mAppEngineConfig.Set<webrtc::CaptureDeviceInfo>(
+          new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Application));
+      if (!mAppEngine) {
+        if (!(mAppEngine = webrtc::VideoEngine::Create(mAppEngineConfig))) {
+          return;
+        }
+      }
+      videoEngine = mAppEngine;
+      videoEngineInit = &mAppEngineInit;
+      break;
+    case dom::MediaSourceEnum::Screen:
+      mScreenEngineConfig.Set<webrtc::CaptureDeviceInfo>(
+          new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Screen));
+      if (!mScreenEngine) {
+        if (!(mScreenEngine = webrtc::VideoEngine::Create(mScreenEngineConfig))) {
+          return;
+        }
+      }
+      videoEngine = mScreenEngine;
+      videoEngineInit = &mScreenEngineInit;
+      break;
+    case dom::MediaSourceEnum::Camera:
+      // fall through
+    default:
+      if (!mVideoEngine) {
+        if (!(mVideoEngine = webrtc::VideoEngine::Create())) {
+          return;
+        }
+      }
+      videoEngine = mVideoEngine;
+      videoEngineInit = &mVideoEngineInit;
+      break;
   }
 
-  ptrViEBase = webrtc::ViEBase::GetInterface(mVideoEngine);
+  ptrViEBase = webrtc::ViEBase::GetInterface(videoEngine);
   if (!ptrViEBase) {
     return;
   }
+  if (ptrViEBase->Init() < 0) {
+    return;
+  }
+  *videoEngineInit = true;
 
-  if (!mVideoEngineInit) {
-    if (ptrViEBase->Init() < 0) {
-      return;
-    }
-    mVideoEngineInit = true;
-  }
-
-  ptrViECapture = webrtc::ViECapture::GetInterface(mVideoEngine);
+  ptrViECapture = webrtc::ViECapture::GetInterface(videoEngine);
   if (!ptrViECapture) {
     return;
   }
 
   /**
    * We still enumerate every time, in case a new device was plugged in since
    * the last call. TODO: Verify that WebRTC actually does deal with hotplugging
    * new devices (with or without new engine creation) and accordingly adjust.
@@ -202,31 +240,32 @@ MediaEngineWebRTC::EnumerateVideoDevices
     }
 
     nsRefPtr<MediaEngineWebRTCVideoSource> vSource;
     NS_ConvertUTF8toUTF16 uuid(uniqueId);
     if (mVideoSources.Get(uuid, getter_AddRefs(vSource))) {
       // We've already seen this device, just append.
       aVSources->AppendElement(vSource.get());
     } else {
-      vSource = new MediaEngineWebRTCVideoSource(mVideoEngine, i);
+      vSource = new MediaEngineWebRTCVideoSource(videoEngine, i, aMediaSource);
       mVideoSources.Put(uuid, vSource); // Hashtable takes ownership.
       aVSources->AppendElement(vSource);
     }
   }
 
   if (mHasTabVideoSource)
     aVSources->AppendElement(new MediaEngineTabVideoSource());
 
   return;
 #endif
 }
 
 void
-MediaEngineWebRTC::EnumerateAudioDevices(nsTArray<nsRefPtr<MediaEngineAudioSource> >* aASources)
+MediaEngineWebRTC::EnumerateAudioDevices(dom::MediaSourceEnum aMediaSource,
+                                         nsTArray<nsRefPtr<MediaEngineAudioSource> >* aASources)
 {
   ScopedCustomReleasePtr<webrtc::VoEBase> ptrVoEBase;
   ScopedCustomReleasePtr<webrtc::VoEHardware> ptrVoEHw;
   // We spawn threads to handle gUM runnables, so we must protect the member vars
   MutexAutoLock lock(mMutex);
 
 #ifdef MOZ_WIDGET_ANDROID
   jobject context = mozilla::AndroidBridge::Bridge()->GetGlobalContextRef();
@@ -318,24 +357,33 @@ MediaEngineWebRTC::Shutdown()
 
   // Clear callbacks before we go away since the engines may outlive us
   if (mVideoEngine) {
     mVideoSources.Clear();
     mVideoEngine->SetTraceCallback(nullptr);
     webrtc::VideoEngine::Delete(mVideoEngine);
   }
 
+  if (mScreenEngine) {
+    webrtc::VideoEngine::Delete(mScreenEngine);
+  }
+  if (mAppEngine) {
+    webrtc::VideoEngine::Delete(mAppEngine);
+  }
+
   if (mVoiceEngine) {
     mAudioSources.Clear();
     mVoiceEngine->SetTraceCallback(nullptr);
     webrtc::VoiceEngine::Delete(mVoiceEngine);
   }
 
   mVideoEngine = nullptr;
   mVoiceEngine = nullptr;
+  mScreenEngine = nullptr;
+  mAppEngine = nullptr;
 
   if (mThread) {
     mThread->Shutdown();
     mThread = nullptr;
   }
 }
 
 }
--- a/content/media/webrtc/MediaEngineWebRTC.h
+++ b/content/media/webrtc/MediaEngineWebRTC.h
@@ -23,19 +23,19 @@
 #include "VideoUtils.h"
 #include "MediaEngine.h"
 #include "VideoSegment.h"
 #include "AudioSegment.h"
 #include "StreamBuffer.h"
 #include "MediaStreamGraph.h"
 
 #include "MediaEngineWrapper.h"
-
+#include "mozilla/dom/MediaStreamTrackBinding.h"
 // WebRTC library includes follow
-
+#include "webrtc/common.h"
 // Audio Engine
 #include "webrtc/voice_engine/include/voe_base.h"
 #include "webrtc/voice_engine/include/voe_codec.h"
 #include "webrtc/voice_engine/include/voe_hardware.h"
 #include "webrtc/voice_engine/include/voe_network.h"
 #include "webrtc/voice_engine/include/voe_audio_processing.h"
 #include "webrtc/voice_engine/include/voe_volume_control.h"
 #include "webrtc/voice_engine/include/voe_external_media.h"
@@ -90,22 +90,24 @@ class MediaEngineWebRTCVideoSource : pub
                                    , public CameraControlListener
                                    , public mozilla::hal::ScreenConfigurationObserver
 #else
                                    , public webrtc::ExternalRenderer
 #endif
 {
 public:
 #ifdef MOZ_B2G_CAMERA
-  MediaEngineWebRTCVideoSource(int aIndex)
+  MediaEngineWebRTCVideoSource(int aIndex,
+                               dom::MediaSourceEnum aMediaSource = dom::MediaSourceEnum::Camera)
     : mCameraControl(nullptr)
     , mCallbackMonitor("WebRTCCamera.CallbackMonitor")
     , mRotation(0)
     , mBackCamera(false)
     , mCaptureIndex(aIndex)
+    , mMediaSource(aMediaSource)
     , mMonitor("WebRTCCamera.Monitor")
     , mWidth(0)
     , mHeight(0)
     , mHasDirectListeners(false)
     , mInitDone(false)
     , mInSnapshotMode(false)
     , mSnapshotPath(nullptr)
   {
@@ -119,21 +121,23 @@ public:
                            void *handle);
   /**
    * Does DeliverFrame() support a null buffer and non-null handle
    * (video texture)?
    * XXX Investigate!  Especially for Android/B2G
    */
   virtual bool IsTextureSupported() { return false; }
 
-  MediaEngineWebRTCVideoSource(webrtc::VideoEngine* aVideoEnginePtr, int aIndex)
+  MediaEngineWebRTCVideoSource(webrtc::VideoEngine* aVideoEnginePtr, int aIndex,
+                               dom::MediaSourceEnum aMediaSource = dom::MediaSourceEnum::Camera)
     : mVideoEngine(aVideoEnginePtr)
     , mCaptureIndex(aIndex)
     , mFps(-1)
     , mMinFps(-1)
+    , mMediaSource(aMediaSource)
     , mMonitor("WebRTCCamera.Monitor")
     , mWidth(0)
     , mHeight(0)
     , mHasDirectListeners(false)
     , mInitDone(false)
     , mInSnapshotMode(false)
     , mSnapshotPath(nullptr) {
     MOZ_ASSERT(aVideoEnginePtr);
@@ -160,16 +164,20 @@ public:
                           TrackID aId,
                           StreamTime aDesiredTime,
                           TrackTicks &aLastEndTime);
 
   virtual bool IsFake() {
     return false;
   }
 
+  virtual const dom::MediaSourceEnum GetMediaSource() {
+    return mMediaSource;
+  }
+
 #ifndef MOZ_B2G_CAMERA
   NS_DECL_THREADSAFE_ISUPPORTS
 #else
   // We are subclassed from CameraControlListener, which implements a
   // threadsafe reference-count for us.
   NS_DECL_ISUPPORTS_INHERITED
 
   void OnHardwareStateChange(HardwareState aState);
@@ -234,16 +242,17 @@ private:
   webrtc::ViECapture* mViECapture;
   webrtc::ViERender* mViERender;
 #endif
   webrtc::CaptureCapability mCapability; // Doesn't work on OS X.
 
   int mCaptureIndex;
   int mFps; // Track rate (30 fps by default)
   int mMinFps; // Min rate we want to accept
+  dom::MediaSourceEnum mMediaSource; // source of media (camera | application | screen)
 
   // mMonitor protects mImage access/changes, and transitions of mState
   // from kStarted to kStopped (which are combined with EndTrack() and
   // image changes).  Note that mSources is not accessed from other threads
   // for video and is not protected.
   Monitor mMonitor; // Monitor for processing WebRTC frames.
   int mWidth, mHeight;
   nsRefPtr<layers::Image> mImage;
@@ -376,40 +385,49 @@ class MediaEngineWebRTC : public MediaEn
 {
 public:
   MediaEngineWebRTC(MediaEnginePrefs &aPrefs);
 
   // Clients should ensure to clean-up sources video/audio sources
   // before invoking Shutdown on this class.
   void Shutdown();
 
-  virtual void EnumerateVideoDevices(nsTArray<nsRefPtr<MediaEngineVideoSource> >*);
-  virtual void EnumerateAudioDevices(nsTArray<nsRefPtr<MediaEngineAudioSource> >*);
-
+  virtual void EnumerateVideoDevices(dom::MediaSourceEnum,
+                                    nsTArray<nsRefPtr<MediaEngineVideoSource> >*);
+  virtual void EnumerateAudioDevices(dom::MediaSourceEnum,
+                                    nsTArray<nsRefPtr<MediaEngineAudioSource> >*);
 private:
   ~MediaEngineWebRTC() {
     Shutdown();
 #ifdef MOZ_B2G_CAMERA
     AsyncLatencyLogger::Get()->Release();
 #endif
     // XXX
     gFarendObserver = nullptr;
   }
 
   nsCOMPtr<nsIThread> mThread;
 
   Mutex mMutex;
+
   // protected with mMutex:
-
+  webrtc::VideoEngine* mScreenEngine;
+  webrtc::VideoEngine* mAppEngine;
   webrtc::VideoEngine* mVideoEngine;
   webrtc::VoiceEngine* mVoiceEngine;
 
+  // specialized configurations
+  webrtc::Config mAppEngineConfig;
+  webrtc::Config mScreenEngineConfig;
+
   // Need this to avoid unneccesary WebRTC calls while enumerating.
   bool mVideoEngineInit;
   bool mAudioEngineInit;
+  bool mScreenEngineInit;
+  bool mAppEngineInit;
   bool mHasTabVideoSource;
 
   // Store devices we've already seen in a hashtable for quick return.
   // Maps UUID to MediaEngineSource (one set for audio, one for video).
   nsRefPtrHashtable<nsStringHashKey, MediaEngineWebRTCVideoSource > mVideoSources;
   nsRefPtrHashtable<nsStringHashKey, MediaEngineWebRTCAudioSource > mAudioSources;
 };