Bug 1164463 - Clean up MediaManager shutdown to be reliable and avoid holding locks while Joining a thread. r=jib
authorRandell Jesup <rjesup@jesup.org>
Fri, 29 May 2015 14:28:03 -0400
changeset 278722 7aec151a5d61ac96d517adc73f723db0a73ae0b6
parent 278721 f7c43c293359135ca00e2b75a32f1e230bed9a8b
child 278723 a62605b318383d83e944b664af30292d606c7a00
push id897
push userjlund@mozilla.com
push dateMon, 14 Sep 2015 18:56:12 +0000
treeherdermozilla-release@9411e2d2b214 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjib
bugs1164463
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1164463 - Clean up MediaManager shutdown to be reliable and avoid holding locks while Joining a thread. r=jib
dom/media/MediaManager.cpp
dom/media/MediaManager.h
dom/media/webrtc/MediaEngine.h
dom/media/webrtc/MediaEngineCameraVideoSource.h
dom/media/webrtc/MediaEngineDefault.h
dom/media/webrtc/MediaEngineTabVideoSource.h
dom/media/webrtc/MediaEngineWebRTC.cpp
dom/media/webrtc/MediaEngineWebRTC.h
dom/media/webrtc/MediaEngineWebRTCAudio.cpp
dom/media/webrtc/MediaEngineWebRTCVideo.cpp
--- a/dom/media/MediaManager.cpp
+++ b/dom/media/MediaManager.cpp
@@ -119,16 +119,18 @@ using dom::MediaTrackConstraintSet;
 using dom::MediaTrackConstraints;
 using dom::MediaStreamError;
 using dom::GetUserMediaRequest;
 using dom::Sequence;
 using dom::OwningBooleanOrMediaTrackConstraints;
 using dom::SupportedAudioConstraints;
 using dom::SupportedVideoConstraints;
 
+static Atomic<bool> sInShutdown;
+
 static bool
 HostInDomain(const nsCString &aHost, const nsCString &aPattern)
 {
   int32_t patternOffset = 0;
   int32_t hostOffset = 0;
 
   // Act on '*.' wildcard in the left-most position in a domain pattern.
   if (aPattern.Length() > 2 && aPattern[0] == '*' && aPattern[1] == '.') {
@@ -941,25 +943,26 @@ public:
         branch->GetIntPref("media.getusermedia.playout_delay", &playout_delay);
       }
     }
 #endif
     // Create a media stream.
     nsRefPtr<nsDOMUserMediaStream> trackunion =
       nsDOMUserMediaStream::CreateTrackUnionStream(window, mListener,
                                                    mAudioSource, mVideoSource);
-    if (!trackunion) {
+    if (!trackunion || sInShutdown) {
       nsCOMPtr<nsIDOMGetUserMediaErrorCallback> onFailure = mOnFailure.forget();
       LOG(("Returning error for getUserMedia() - no stream"));
 
       nsGlobalWindow* window = nsGlobalWindow::GetInnerWindowWithId(mWindowID);
       if (window) {
         nsRefPtr<MediaStreamError> error = new MediaStreamError(window,
             NS_LITERAL_STRING("InternalError"),
-            NS_LITERAL_STRING("No stream."));
+            sInShutdown ? NS_LITERAL_STRING("In shutdown") :
+                          NS_LITERAL_STRING("No stream."));
         onFailure->OnError(error);
       }
       return NS_OK;
     }
     trackunion->AudioConfig(aec_on, (uint32_t) aec,
                             agc_on, (uint32_t) agc,
                             noise_on, (uint32_t) noise,
                             playout_delay);
@@ -1004,17 +1007,17 @@ public:
                            agc_on, (uint32_t) agc,
                            noise_on, (uint32_t) noise,
                            playout_delay);
 
     // Dispatch to the media thread to ask it to start the sources,
     // because that can take a while.
     // Pass ownership of trackunion to the MediaOperationTask
     // to ensure it's kept alive until the MediaOperationTask runs (at least).
-    MediaManager::GetMessageLoop()->PostTask(FROM_HERE,
+    MediaManager::PostTask(FROM_HERE,
       new MediaOperationTask(MEDIA_START, mListener, trackunion,
                              tracksAvailableCallback,
                              mAudioSource, mVideoSource, false, mWindowID,
                              mOnFailure.forget()));
 
     // We won't need mOnFailure now.
     mOnFailure = nullptr;
     return NS_OK;
@@ -1607,17 +1610,17 @@ MediaManager::Get() {
     if (!sSingleton->mMediaThread->StartWithOptions(options)) {
       MOZ_CRASH();
     }
 
     LOG(("New Media thread for gum"));
 
     nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
     if (obs) {
-      obs->AddObserver(sSingleton, "xpcom-shutdown", false);
+      obs->AddObserver(sSingleton, "xpcom-will-shutdown", false);
       obs->AddObserver(sSingleton, "getUserMedia:response:allow", false);
       obs->AddObserver(sSingleton, "getUserMedia:response:deny", false);
       obs->AddObserver(sSingleton, "getUserMedia:revoke", false);
       obs->AddObserver(sSingleton, "phone-state-changed", false);
     }
     // else MediaManager won't work properly and will leak (see bug 837874)
     nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
     if (prefs) {
@@ -1639,22 +1642,27 @@ MediaManager::GetIfExists() {
 MediaManager::GetInstance()
 {
   // so we can have non-refcounted getters
   nsRefPtr<MediaManager> service = MediaManager::Get();
   return service.forget();
 }
 
 /* static */
-MessageLoop*
-MediaManager::GetMessageLoop()
+void
+MediaManager::PostTask(const tracked_objects::Location& from_here, Task* task)
 {
+  if (sInShutdown) {
+    // Can't safely delete task here since it may have items with specific
+    // thread-release requirements.
+    return;
+  }
   NS_ASSERTION(Get(), "MediaManager singleton?");
   NS_ASSERTION(Get()->mMediaThread, "No thread yet");
-  return Get()->mMediaThread->message_loop();
+  Get()->mMediaThread->message_loop()->PostTask(from_here, task);
 }
 
 /* static */ nsresult
 MediaManager::NotifyRecordingStatusChange(nsPIDOMWindow* aWindow,
                                           const nsString& aMsg,
                                           const bool& aIsAudio,
                                           const bool& aIsVideo)
 {
@@ -1778,16 +1786,19 @@ MediaManager::GetUserMedia(
     // Fake stream from default backend.
     task = new GetUserMediaTask(c, onSuccess.forget(),
       onFailure.forget(), windowID, listener, mPrefs, new MediaEngineDefault(c.mFakeTracks));
   } else {
     // Stream from default device from WebRTC backend.
     task = new GetUserMediaTask(c, onSuccess.forget(),
       onFailure.forget(), windowID, listener, mPrefs);
   }
+  if (sInShutdown) {
+    return task->Denied(NS_LITERAL_STRING("In shutdown"));
+  }
 
   nsIURI* docURI = aWindow->GetDocumentURI();
 
   bool isLoop = false;
   nsCOMPtr<nsIURI> loopURI;
   nsresult rv = NS_NewURI(getter_AddRefs(loopURI), "about:loopconversation");
   NS_ENSURE_SUCCESS(rv, rv);
   rv = docURI->EqualsExceptRef(loopURI, &isLoop);
@@ -1884,17 +1895,17 @@ MediaManager::GetUserMedia(
   if (mCameraManager == nullptr) {
     mCameraManager = nsDOMCameraManager::CreateInstance(aWindow);
   }
 #endif
 
   // XXX No full support for picture in Desktop yet (needs proper UI)
   if (privileged ||
       (fake && !Preferences::GetBool("media.navigator.permission.fake"))) {
-    MediaManager::GetMessageLoop()->PostTask(FROM_HERE, task.forget());
+    MediaManager::PostTask(FROM_HERE, task.forget());
   } else {
     bool isHTTPS = false;
     if (docURI) {
       docURI->SchemeIs("https", &isHTTPS);
     }
 
     // Check if this site has persistent permissions.
     nsresult rv;
@@ -1969,16 +1980,17 @@ MediaManager::GetUserMediaDevices(nsPIDO
   nsIDOMGetUserMediaErrorCallback* aOnFailure,
   uint64_t aInnerWindowID,
   bool aPrivileged)
 {
   NS_ASSERTION(NS_IsMainThread(), "Only call on main thread");
 
   NS_ENSURE_TRUE(aOnFailure, NS_ERROR_NULL_POINTER);
   NS_ENSURE_TRUE(aOnSuccess, NS_ERROR_NULL_POINTER);
+  NS_ENSURE_TRUE(!sInShutdown, NS_ERROR_FAILURE);
 
   nsCOMPtr<nsIGetUserMediaDevicesSuccessCallback> onSuccess(aOnSuccess);
   nsCOMPtr<nsIDOMGetUserMediaErrorCallback> onFailure(aOnFailure);
 
   // Check if the preference for using loopback devices is enabled.
   nsAdoptingCString loopbackAudioDevice =
     Preferences::GetCString("media.audio_loopback_dev");
   nsAdoptingCString loopbackVideoDevice =
@@ -1989,17 +2001,17 @@ MediaManager::GetUserMediaDevices(nsPIDO
   nsCString origin;
   nsPrincipal::GetOriginForURI(aWindow->GetDocumentURI(), origin);
   bool inPrivateBrowsing;
   {
     nsCOMPtr<nsIDocument> doc = aWindow->GetDoc();
     nsCOMPtr<nsILoadContext> loadContext = doc->GetLoadContext();
     inPrivateBrowsing = loadContext && loadContext->UsePrivateBrowsing();
   }
-  MediaManager::GetMessageLoop()->PostTask(FROM_HERE,
+  MediaManager::PostTask(FROM_HERE,
     new GetUserMediaDevicesTask(
       aConstraints, onSuccess.forget(), onFailure.forget(),
       (aInnerWindowID ? aInnerWindowID : aWindow->WindowID()),
       loopbackAudioDevice, loopbackVideoDevice, aPrivileged, origin,
       inPrivateBrowsing, useFakeStreams));
 
   return NS_OK;
 }
@@ -2022,16 +2034,17 @@ MediaManager::EnumerateDevices(nsPIDOMWi
 MediaEngine*
 MediaManager::GetBackend(uint64_t aWindowId)
 {
   // Plugin backends as appropriate. The default engine also currently
   // includes picture support for Android.
   // This IS called off main-thread.
   MutexAutoLock lock(mMutex);
   if (!mBackend) {
+    MOZ_RELEASE_ASSERT(!sInShutdown);  // we should never create a new backend in shutdown
 #if defined(MOZ_WEBRTC)
     mBackend = new MediaEngineWebRTC(mPrefs);
 #else
     mBackend = new MediaEngineDefault();
 #endif
   }
   return mBackend;
 }
@@ -2197,69 +2210,97 @@ MediaManager::Observe(nsISupports* aSubj
 
   if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) {
     nsCOMPtr<nsIPrefBranch> branch( do_QueryInterface(aSubject) );
     if (branch) {
       GetPrefs(branch,NS_ConvertUTF16toUTF8(aData).get());
       LOG(("%s: %dx%d @%dfps (min %d)", __FUNCTION__,
            mPrefs.mWidth, mPrefs.mHeight, mPrefs.mFPS, mPrefs.mMinFPS));
     }
-  } else if (!strcmp(aTopic, "xpcom-shutdown")) {
-    obs->RemoveObserver(this, "xpcom-shutdown");
+  } else if (!strcmp(aTopic, "xpcom-will-shutdown")) {
+    sInShutdown = true;
+
+    obs->RemoveObserver(this, "xpcom-will-shutdown");
     obs->RemoveObserver(this, "getUserMedia:response:allow");
     obs->RemoveObserver(this, "getUserMedia:response:deny");
     obs->RemoveObserver(this, "getUserMedia:revoke");
 
     nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
     if (prefs) {
       prefs->RemoveObserver("media.navigator.video.default_width", this);
       prefs->RemoveObserver("media.navigator.video.default_height", this);
       prefs->RemoveObserver("media.navigator.video.default_fps", this);
       prefs->RemoveObserver("media.navigator.video.default_minfps", this);
     }
 
+    // Close off any remaining active windows.
+    GetActiveWindows()->Clear();
+    mActiveCallbacks.Clear();
+    mCallIds.Clear();
+    {
+      MutexAutoLock lock(mMutex);
+      if (mBackend) {
+        mBackend->Shutdown(); // ok to invoke multiple times
+      }
+    }
+
     // Because mMediaThread is not an nsThread, we must dispatch to it so it can
     // clean up BackgroundChild. Continue stopping thread once this is done.
 
     class ShutdownTask : public Task
     {
     public:
-      explicit ShutdownTask(nsRunnable* aReply) : mReply(aReply) {}
+      ShutdownTask(TemporaryRef<MediaEngine> aBackend,
+                   nsRunnable* aReply)
+        : mReply(aReply)
+        , mBackend(aBackend) {}
     private:
       virtual void
       Run()
       {
+        LOG(("MediaManager Thread Shutdown"));
         MOZ_ASSERT(MediaManager::IsInMediaThread());
         mozilla::ipc::BackgroundChild::CloseForCurrentThread();
-        NS_DispatchToMainThread(mReply);
+        // must explicitly do this before dispatching the reply, since the reply may kill us with Stop()
+        mBackend = nullptr; // last reference, will invoke Shutdown() again
+
+        if (NS_FAILED(NS_DispatchToMainThread(mReply))) {
+          LOG(("Will leak thread: DispatchToMainthread of reply runnable failed in MediaManager shutdown"));
+        }
       }
       nsRefPtr<nsRunnable> mReply;
+      RefPtr<MediaEngine> mBackend;
     };
 
     // Post ShutdownTask to execute on mMediaThread and pass in a lambda
     // callback to be executed back on this thread once it is done.
     //
     // The lambda callback "captures" the 'this' pointer for member access.
     // This is safe since this is guaranteed to be here since sSingleton isn't
     // cleared until the lambda function clears it.
 
-    MediaManager::GetMessageLoop()->PostTask(FROM_HERE, new ShutdownTask(
-        media::NewRunnableFrom([this]() mutable {
-      // Close off any remaining active windows.
+    // note that this == sSingleton
+    nsRefPtr<MediaManager> that(sSingleton);
+    // Release the backend (and call Shutdown()) from within the MediaManager thread
+    RefPtr<MediaEngine> temp;
+    {
       MutexAutoLock lock(mMutex);
-      GetActiveWindows()->Clear();
-      mActiveCallbacks.Clear();
-      mCallIds.Clear();
-      LOG(("Releasing MediaManager singleton and thread"));
-      // Note: won't be released immediately as the Observer has a ref to us
-      sSingleton = nullptr;
+      temp = mBackend.forget();
+    }
+    // Don't use MediaManager::PostTask() because we're sInShutdown=true here!
+    mMediaThread->message_loop()->PostTask(FROM_HERE, new ShutdownTask(
+        temp.forget(),
+        media::NewRunnableFrom([this, that]() mutable {
+      LOG(("MediaManager shutdown lambda running, releasing MediaManager singleton and thread"));
       if (mMediaThread) {
         mMediaThread->Stop();
       }
-      mBackend = nullptr;
+      // we hold a ref to 'that' which is the same as sSingleton
+      sSingleton = nullptr;
+
       return NS_OK;
     })));
     return NS_OK;
 
   } else if (!strcmp(aTopic, "getUserMedia:response:allow")) {
     nsString key(aData);
     nsAutoPtr<GetUserMediaTask> task;
     mActiveCallbacks.RemoveAndForget(key, task);
@@ -2294,18 +2335,21 @@ MediaManager::Observe(nsISupports* aSubj
             task->SetAudioDevice(static_cast<AudioDevice*>(device.get()));
           } else {
             NS_WARNING("Unknown device type in getUserMedia");
           }
         }
       }
     }
 
+    if (sInShutdown) {
+      return task->Denied(NS_LITERAL_STRING("In shutdown"));
+    }
     // Reuse the same thread to save memory.
-    MediaManager::GetMessageLoop()->PostTask(FROM_HERE, task.forget());
+    MediaManager::PostTask(FROM_HERE, task.forget());
     return NS_OK;
 
   } else if (!strcmp(aTopic, "getUserMedia:response:deny")) {
     nsString errorMessage(NS_LITERAL_STRING("PermissionDeniedError"));
 
     if (aSubject) {
       nsCOMPtr<nsISupportsString> msg(do_QueryInterface(aSubject));
       MOZ_ASSERT(msg);
@@ -2493,18 +2537,17 @@ MediaManager::MediaCaptureWindowState(ns
 }
 
 NS_IMETHODIMP
 MediaManager::SanitizeDeviceIds(int64_t aSinceWhen)
 {
   NS_ASSERTION(NS_IsMainThread(), "Only call on main thread");
   LOG(("%s: sinceWhen = %llu", __FUNCTION__, aSinceWhen));
 
-  MediaManager::GetMessageLoop()->PostTask(FROM_HERE,
-    new SanitizeDeviceIdsTask(aSinceWhen));
+  MediaManager::PostTask(FROM_HERE, new SanitizeDeviceIdsTask(aSinceWhen));
   return NS_OK;
 }
 
 static void
 StopScreensharingCallback(MediaManager *aThis,
                           uint64_t aWindowID,
                           StreamListeners *aListeners,
                           void *aData)
@@ -2611,35 +2654,33 @@ void
 GetUserMediaCallbackMediaStreamListener::AudioConfig(bool aEchoOn,
               uint32_t aEcho,
               bool aAgcOn, uint32_t aAGC,
               bool aNoiseOn, uint32_t aNoise,
               int32_t aPlayoutDelay)
 {
   if (mAudioSource) {
 #ifdef MOZ_WEBRTC
-    mMediaThread->message_loop()->PostTask(FROM_HERE,
+    MediaManager::PostTask(FROM_HERE,
       NewRunnableMethod(mAudioSource.get(), &MediaEngineSource::Config,
                         aEchoOn, aEcho, aAgcOn, aAGC, aNoiseOn,
                         aNoise, aPlayoutDelay));
-#else
-    unused << mMediaThread;
 #endif
   }
 }
 
 // Can be invoked from EITHER MainThread or MSG thread
 void
 GetUserMediaCallbackMediaStreamListener::Invalidate()
 {
   // We can't take a chance on blocking here, so proxy this to another
   // thread.
   // Pass a ref to us (which is threadsafe) so it can query us for the
   // source stream info.
-  MediaManager::GetMessageLoop()->PostTask(FROM_HERE,
+  MediaManager::PostTask(FROM_HERE,
     new MediaOperationTask(MEDIA_STOP,
                            this, nullptr, nullptr,
                            mAudioSource, mVideoSource,
                            mFinished, mWindowID, nullptr));
 }
 
 // Doesn't kill audio
 // XXX refactor to combine with Invalidate()?
@@ -2647,17 +2688,17 @@ void
 GetUserMediaCallbackMediaStreamListener::StopScreenWindowSharing()
 {
   NS_ASSERTION(NS_IsMainThread(), "Only call on main thread");
   if (mVideoSource && !mStopped &&
       (mVideoSource->GetMediaSource() == dom::MediaSourceEnum::Screen ||
        mVideoSource->GetMediaSource() == dom::MediaSourceEnum::Application ||
        mVideoSource->GetMediaSource() == dom::MediaSourceEnum::Window)) {
     // Stop the whole stream if there's no audio; just the video track if we have both
-    MediaManager::GetMessageLoop()->PostTask(FROM_HERE,
+    MediaManager::PostTask(FROM_HERE,
       new MediaOperationTask(mAudioSource ? MEDIA_STOP_TRACK : MEDIA_STOP,
                              this, nullptr, nullptr,
                              nullptr, mVideoSource,
                              mFinished, mWindowID, nullptr));
   }
 }
 
 // Stop backend for track
@@ -2665,17 +2706,17 @@ GetUserMediaCallbackMediaStreamListener:
 void
 GetUserMediaCallbackMediaStreamListener::StopTrack(TrackID aID, bool aIsAudio)
 {
   if (((aIsAudio && mAudioSource) ||
        (!aIsAudio && mVideoSource)) && !mStopped)
   {
     // XXX to support multiple tracks of a type in a stream, this should key off
     // the TrackID and not just the type
-    MediaManager::GetMessageLoop()->PostTask(FROM_HERE,
+    MediaManager::PostTask(FROM_HERE,
       new MediaOperationTask(MEDIA_STOP_TRACK,
                              this, nullptr, nullptr,
                              aIsAudio  ? mAudioSource : nullptr,
                              !aIsAudio ? mVideoSource : nullptr,
                              mFinished, mWindowID, nullptr));
   } else {
     LOG(("gUM track %d ended, but we don't have type %s",
          aID, aIsAudio ? "audio" : "video"));
@@ -2691,17 +2732,17 @@ GetUserMediaCallbackMediaStreamListener:
   NS_DispatchToMainThread(new GetUserMediaListenerRemove(mWindowID, this));
 }
 
 // Called from the MediaStreamGraph thread
 void
 GetUserMediaCallbackMediaStreamListener::NotifyDirectListeners(MediaStreamGraph* aGraph,
                                                                bool aHasListeners)
 {
-  MediaManager::GetMessageLoop()->PostTask(FROM_HERE,
+  MediaManager::PostTask(FROM_HERE,
     new MediaOperationTask(MEDIA_DIRECT_LISTENERS,
                            this, nullptr, nullptr,
                            mAudioSource, mVideoSource,
                            aHasListeners, mWindowID, nullptr));
 }
 
 // Called from the MediaStreamGraph thread
 // this can be in response to our own RemoveListener() (via ::Remove()), or
--- a/dom/media/MediaManager.h
+++ b/dom/media/MediaManager.h
@@ -66,16 +66,17 @@ public:
     , mWindowID(aWindowID)
     , mStopped(false)
     , mFinished(false)
     , mLock("mozilla::GUMCMSL")
     , mRemoved(false) {}
 
   ~GetUserMediaCallbackMediaStreamListener()
   {
+    unused << mMediaThread;
     // It's OK to release mStream on any thread; they have thread-safe
     // refcounts.
   }
 
   void Activate(already_AddRefed<SourceMediaStream> aStream,
     MediaEngineSource* aAudioSource,
     MediaEngineSource* aVideoSource)
   {
@@ -509,17 +510,17 @@ class MediaManager final : public nsIMed
 public:
   static already_AddRefed<MediaManager> GetInstance();
 
   // NOTE: never Dispatch(....,NS_DISPATCH_SYNC) to the MediaManager
   // thread from the MainThread, as we NS_DISPATCH_SYNC to MainThread
   // from MediaManager thread.
   static MediaManager* Get();
   static MediaManager* GetIfExists();
-  static MessageLoop* GetMessageLoop();
+  static void PostTask(const tracked_objects::Location& from_here, Task* task);
 #ifdef DEBUG
   static bool IsInMediaThread();
 #endif
 
   static bool Exists()
   {
     return !!sSingleton;
   }
--- a/dom/media/webrtc/MediaEngine.h
+++ b/dom/media/webrtc/MediaEngine.h
@@ -59,16 +59,18 @@ public:
   virtual void EnumerateVideoDevices(dom::MediaSourceEnum,
                                      nsTArray<nsRefPtr<MediaEngineVideoSource> >*) = 0;
 
   /* Populate an array of audio sources in the nsTArray. Also include devices
    * that are currently unavailable. */
   virtual void EnumerateAudioDevices(dom::MediaSourceEnum,
                                      nsTArray<nsRefPtr<MediaEngineAudioSource> >*) = 0;
 
+  virtual void Shutdown() = 0;
+
 protected:
   virtual ~MediaEngine() {}
 };
 
 /**
  * Common abstract base class for audio and video sources.
  */
 class MediaEngineSource : public nsISupports
@@ -76,16 +78,18 @@ class MediaEngineSource : public nsISupp
 public:
   // code inside webrtc.org assumes these sizes; don't use anything smaller
   // without verifying it's ok
   static const unsigned int kMaxDeviceNameLength = 128;
   static const unsigned int kMaxUniqueIdLength = 256;
 
   virtual ~MediaEngineSource() {}
 
+  virtual void Shutdown() = 0;
+
   /* Populate the human readable name of this device in the nsAString */
   virtual void GetName(nsAString&) = 0;
 
   /* Populate the UUID of this device in the nsAString */
   virtual void GetUUID(nsAString&) = 0;
 
   /* Release the device back to the system. */
   virtual nsresult Deallocate() = 0;
@@ -240,16 +244,17 @@ protected:
 class MediaEngineAudioSource : public MediaEngineSource
 {
 public:
   virtual ~MediaEngineAudioSource() {}
 
   /* This call reserves but does not start the device. */
   virtual nsresult Allocate(const dom::MediaTrackConstraints &aConstraints,
                             const MediaEnginePrefs &aPrefs) = 0;
+
 protected:
   explicit MediaEngineAudioSource(MediaEngineState aState)
     : MediaEngineSource(aState) {}
   MediaEngineAudioSource()
     : MediaEngineSource(kReleased) {}
 
 };
 
--- a/dom/media/webrtc/MediaEngineCameraVideoSource.h
+++ b/dom/media/webrtc/MediaEngineCameraVideoSource.h
@@ -56,16 +56,18 @@ public:
   virtual nsresult TakePhoto(PhotoCallback* aCallback) override
   {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
 
   uint32_t GetBestFitnessDistance(
       const nsTArray<const dom::MediaTrackConstraintSet*>& aConstraintSets) override;
 
+  virtual void Shutdown() override {};
+
 protected:
   struct CapabilityCandidate {
     explicit CapabilityCandidate(uint8_t index, uint32_t distance = 0)
     : mIndex(index), mDistance(distance) {}
 
     size_t mIndex;
     uint32_t mDistance;
   };
--- a/dom/media/webrtc/MediaEngineDefault.h
+++ b/dom/media/webrtc/MediaEngineDefault.h
@@ -31,16 +31,18 @@ class MediaEngineDefault;
  * The default implementation of the MediaEngine interface.
  */
 class MediaEngineDefaultVideoSource : public nsITimerCallback,
                                       public MediaEngineVideoSource
 {
 public:
   MediaEngineDefaultVideoSource();
 
+  virtual void Shutdown() override {};
+
   virtual void GetName(nsAString&) override;
   virtual void GetUUID(nsAString&) override;
 
   virtual nsresult Allocate(const dom::MediaTrackConstraints &aConstraints,
                             const MediaEnginePrefs &aPrefs) override;
   virtual nsresult Deallocate() override;
   virtual nsresult Start(SourceMediaStream*, TrackID) override;
   virtual nsresult Stop(SourceMediaStream*, TrackID) override;
@@ -99,16 +101,18 @@ protected:
 class SineWaveGenerator;
 
 class MediaEngineDefaultAudioSource : public nsITimerCallback,
                                       public MediaEngineAudioSource
 {
 public:
   MediaEngineDefaultAudioSource();
 
+  virtual void Shutdown() override {};
+
   virtual void GetName(nsAString&) override;
   virtual void GetUUID(nsAString&) override;
 
   virtual nsresult Allocate(const dom::MediaTrackConstraints &aConstraints,
                             const MediaEnginePrefs &aPrefs) override;
   virtual nsresult Deallocate() override;
   virtual nsresult Start(SourceMediaStream*, TrackID) override;
   virtual nsresult Stop(SourceMediaStream*, TrackID) override;
@@ -153,25 +157,33 @@ class MediaEngineDefault : public MediaE
 {
 public:
   explicit MediaEngineDefault(bool aHasFakeTracks = false)
     : mHasFakeTracks(aHasFakeTracks)
     , mMutex("mozilla::MediaEngineDefault")
   {}
 
   virtual void EnumerateVideoDevices(dom::MediaSourceEnum,
-                                     nsTArray<nsRefPtr<MediaEngineVideoSource> >*);
+                                     nsTArray<nsRefPtr<MediaEngineVideoSource> >*) override;
   virtual void EnumerateAudioDevices(dom::MediaSourceEnum,
-                                     nsTArray<nsRefPtr<MediaEngineAudioSource> >*);
+                                     nsTArray<nsRefPtr<MediaEngineAudioSource> >*) override;
+  virtual void Shutdown() {
+    MutexAutoLock lock(mMutex);
+
+    mVSources.Clear();
+    mASources.Clear();
+  };
 
 protected:
   bool mHasFakeTracks;
 
 private:
-  ~MediaEngineDefault() {}
+  ~MediaEngineDefault() {
+    Shutdown();
+  }
 
   Mutex mMutex;
   // protected with mMutex:
 
   nsTArray<nsRefPtr<MediaEngineVideoSource> > mVSources;
   nsTArray<nsRefPtr<MediaEngineAudioSource> > mASources;
 };
 
--- a/dom/media/webrtc/MediaEngineTabVideoSource.h
+++ b/dom/media/webrtc/MediaEngineTabVideoSource.h
@@ -13,16 +13,17 @@ namespace mozilla {
 
 class MediaEngineTabVideoSource : public MediaEngineVideoSource, nsIDOMEventListener, nsITimerCallback {
   public:
     NS_DECL_THREADSAFE_ISUPPORTS
     NS_DECL_NSIDOMEVENTLISTENER
     NS_DECL_NSITIMERCALLBACK
     MediaEngineTabVideoSource();
 
+    virtual void Shutdown() override {};
     virtual void GetName(nsAString_internal&) override;
     virtual void GetUUID(nsAString_internal&) override;
     virtual nsresult Allocate(const dom::MediaTrackConstraints &,
                               const mozilla::MediaEnginePrefs&) override;
     virtual nsresult Deallocate() override;
     virtual nsresult Start(mozilla::SourceMediaStream*, mozilla::TrackID) override;
     virtual void SetDirectListeners(bool aHasDirectListeners) override {};
     virtual void NotifyPull(mozilla::MediaStreamGraph*, mozilla::SourceMediaStream*, mozilla::TrackID, mozilla::StreamTime) override;
--- a/dom/media/webrtc/MediaEngineWebRTC.cpp
+++ b/dom/media/webrtc/MediaEngineWebRTC.cpp
@@ -269,20 +269,19 @@ MediaEngineWebRTC::EnumerateVideoDevices
       aVSources->AppendElement(vSource.get());
     } else {
       vSource = new MediaEngineWebRTCVideoSource(videoEngine, i, aMediaSource);
       mVideoSources.Put(uuid, vSource); // Hashtable takes ownership.
       aVSources->AppendElement(vSource);
     }
   }
 
-  if (mHasTabVideoSource || dom::MediaSourceEnum::Browser == aMediaSource)
+  if (mHasTabVideoSource || dom::MediaSourceEnum::Browser == aMediaSource) {
     aVSources->AppendElement(new MediaEngineTabVideoSource());
-
-  return;
+  }
 #endif
 }
 
 void
 MediaEngineWebRTC::EnumerateAudioDevices(dom::MediaSourceEnum aMediaSource,
                                          nsTArray<nsRefPtr<MediaEngineAudioSource> >* aASources)
 {
   ScopedCustomReleasePtr<webrtc::VoEBase> ptrVoEBase;
@@ -367,25 +366,53 @@ MediaEngineWebRTC::EnumerateAudioDevices
         mThread, mVoiceEngine, i, deviceName, uniqueId
       );
       mAudioSources.Put(uuid, aSource); // Hashtable takes ownership.
       aASources->AppendElement(aSource);
     }
   }
 }
 
+static PLDHashOperator
+ClearVideoSource (const nsAString&, // unused
+                  MediaEngineVideoSource* aData,
+                  void *userArg)
+{
+  if (aData) {
+    aData->Shutdown();
+  }
+  return PL_DHASH_NEXT;
+}
+
+static PLDHashOperator
+ClearAudioSource (const nsAString&, // unused
+                  MediaEngineWebRTCAudioSource* aData,
+                  void *userArg)
+{
+  if (aData) {
+    aData->Shutdown();
+  }
+  return PL_DHASH_NEXT;
+}
+
 void
 MediaEngineWebRTC::Shutdown()
 {
   // This is likely paranoia
   MutexAutoLock lock(mMutex);
 
-  // Clear callbacks before we go away since the engines may outlive us
+  LOG(("%s", __FUNCTION__));
+  // Shutdown all the sources, since we may have dangling references to the
+  // sources in nsDOMUserMediaStreams waiting for GC/CC
+  mVideoSources.EnumerateRead(ClearVideoSource, nullptr);
+  mAudioSources.EnumerateRead(ClearAudioSource, nullptr);
   mVideoSources.Clear();
   mAudioSources.Clear();
+
+  // Clear callbacks before we go away since the engines may outlive us
   if (mVideoEngine) {
     mVideoEngine->SetTraceCallback(nullptr);
     webrtc::VideoEngine::Delete(mVideoEngine);
   }
 
   if (mScreenEngine) {
     mScreenEngine->SetTraceCallback(nullptr);
     webrtc::VideoEngine::Delete(mScreenEngine);
--- a/dom/media/webrtc/MediaEngineWebRTC.h
+++ b/dom/media/webrtc/MediaEngineWebRTC.h
@@ -104,29 +104,30 @@ public:
   }
   virtual nsresult TakePhoto(PhotoCallback* aCallback) override
   {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
 
   void Refresh(int aIndex);
 
+  virtual void Shutdown() override;
+
 protected:
   ~MediaEngineWebRTCVideoSource() { Shutdown(); }
 
 private:
   // Initialize the needed Video engine interfaces.
   void Init();
-  void Shutdown();
 
   // Engine variables.
   webrtc::VideoEngine* mVideoEngine; // Weak reference, don't free.
-  webrtc::ViEBase* mViEBase;
-  webrtc::ViECapture* mViECapture;
-  webrtc::ViERender* mViERender;
+  ScopedCustomReleasePtr<webrtc::ViEBase> mViEBase;
+  ScopedCustomReleasePtr<webrtc::ViECapture> mViECapture;
+  ScopedCustomReleasePtr<webrtc::ViERender> mViERender;
 
   int mMinFps; // Min rate we want to accept
   dom::MediaSourceEnum mMediaSource; // source of media (camera | application | screen)
 
   size_t NumCapabilities() override;
   void GetCapability(size_t aIndex, webrtc::CaptureCapability& aOut) override;
 };
 
@@ -190,22 +191,23 @@ public:
 
   // VoEMediaProcess.
   void Process(int channel, webrtc::ProcessingTypes type,
                int16_t audio10ms[], int length,
                int samplingFreq, bool isStereo) override;
 
   NS_DECL_THREADSAFE_ISUPPORTS
 
+  virtual void Shutdown() override;
+
 protected:
   ~MediaEngineWebRTCAudioSource() { Shutdown(); }
 
 private:
   void Init();
-  void Shutdown();
 
   webrtc::VoiceEngine* mVoiceEngine;
   ScopedCustomReleasePtr<webrtc::VoEBase> mVoEBase;
   ScopedCustomReleasePtr<webrtc::VoEExternalMedia> mVoERender;
   ScopedCustomReleasePtr<webrtc::VoENetwork> mVoENetwork;
   ScopedCustomReleasePtr<webrtc::VoEAudioProcessing> mVoEProcessing;
 
   // mMonitor protects mSources[] access/changes, and transitions of mState
@@ -234,22 +236,22 @@ private:
 
 class MediaEngineWebRTC : public MediaEngine
 {
 public:
   explicit MediaEngineWebRTC(MediaEnginePrefs& aPrefs);
 
   // Clients should ensure to clean-up sources video/audio sources
   // before invoking Shutdown on this class.
-  void Shutdown();
+  void Shutdown() override;
 
   virtual void EnumerateVideoDevices(dom::MediaSourceEnum,
-                                    nsTArray<nsRefPtr<MediaEngineVideoSource> >*);
+                                     nsTArray<nsRefPtr<MediaEngineVideoSource>>*) override;
   virtual void EnumerateAudioDevices(dom::MediaSourceEnum,
-                                    nsTArray<nsRefPtr<MediaEngineAudioSource> >*);
+                                     nsTArray<nsRefPtr<MediaEngineAudioSource>>*) override;
 private:
   ~MediaEngineWebRTC() {
     Shutdown();
 #if defined(MOZ_B2G_CAMERA) && defined(MOZ_WIDGET_GONK)
     AsyncLatencyLogger::Get()->Release();
 #endif
     gFarendObserver = nullptr;
   }
--- a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp
+++ b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp
@@ -473,21 +473,22 @@ MediaEngineWebRTCAudioSource::Init()
   }
 }
 
 void
 MediaEngineWebRTCAudioSource::Shutdown()
 {
   if (!mInitDone) {
     // duplicate these here in case we failed during Init()
-    if (mChannel != -1) {
+    if (mChannel != -1 && mVoENetwork) {
       mVoENetwork->DeRegisterExternalTransport(mChannel);
     }
 
     delete mNullTransport;
+    mNullTransport = nullptr;
     return;
   }
 
   if (mState == kStarted) {
     SourceMediaStream *source;
     bool empty;
 
     while (1) {
@@ -509,16 +510,17 @@ MediaEngineWebRTCAudioSource::Shutdown()
   }
 
   mVoEBase->Terminate();
   if (mChannel != -1) {
     mVoENetwork->DeRegisterExternalTransport(mChannel);
   }
 
   delete mNullTransport;
+  mNullTransport = nullptr;
 
   mVoEProcessing = nullptr;
   mVoENetwork = nullptr;
   mVoERender = nullptr;
   mVoEBase = nullptr;
 
   mState = kReleased;
   mInitDone = false;
--- a/dom/media/webrtc/MediaEngineWebRTCVideo.cpp
+++ b/dom/media/webrtc/MediaEngineWebRTCVideo.cpp
@@ -261,17 +261,17 @@ MediaEngineWebRTCVideoSource::Deallocate
     // event isn't needed).
     // XXX Note if MainThread Dispatch()es NS_DISPATCH_SYNC to us we can deadlock.
     // XXX It might be nice to only do this if we're in shutdown...  Hard to be
     // sure when that is though.
     // Thread safety: a) we call this synchronously, and don't use ViECapture from
     // another thread anywhere else, b) ViEInputManager::DestroyCaptureDevice() grabs
     // an exclusive object lock and deletes it in a critical section, so all in all
     // this should be safe threadwise.
-    NS_DispatchToMainThread(WrapRunnable(mViECapture,
+    NS_DispatchToMainThread(WrapRunnable(mViECapture.get(),
                                          &webrtc::ViECapture::ReleaseCaptureDevice,
                                          mCaptureIndex),
                             NS_DISPATCH_SYNC);
 #else
     mViECapture->ReleaseCaptureDevice(mCaptureIndex);
 #endif
     mState = kReleased;
     LOG(("Video device %d deallocated", mCaptureIndex));
@@ -417,19 +417,20 @@ MediaEngineWebRTCVideoSource::Shutdown()
       Stop(source, kVideoTrack); // XXX change to support multiple tracks
     }
     MOZ_ASSERT(mState == kStopped);
   }
 
   if (mState == kAllocated || mState == kStopped) {
     Deallocate();
   }
-  mViECapture->Release();
-  mViERender->Release();
-  mViEBase->Release();
+  mViECapture = nullptr;
+  mViERender = nullptr;
+  mViEBase = nullptr;
+
   mState = kReleased;
   mInitDone = false;
 }
 
 void MediaEngineWebRTCVideoSource::Refresh(int aIndex) {
   // NOTE: mCaptureIndex might have changed when allocated!
   // Use aIndex to update information, but don't change mCaptureIndex!!
   // Caller looked up this source by uniqueId, so it shouldn't change