Bug 1149494 - Part 1: Add a listener directly to the unblocked input stream that reports the size of the first non-empty frame seen. r=pehrsons, a=sledru
authorRobert O'Callahan <robert@ocallahan.org>
Wed, 08 Apr 2015 17:51:21 +1200
changeset 258409 d46cb3b3ebb3
parent 258408 d13016a31d6f
child 258410 c821f76bf302
push id4661
push userryanvm@gmail.com
push date2015-04-09 18:39 +0000
treeherdermozilla-beta@3f5e298cb641 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspehrsons, sledru
bugs1149494
milestone38.0
Bug 1149494 - Part 1: Add a listener directly to the unblocked input stream that reports the size of the first non-empty frame seen. r=pehrsons, a=sledru
dom/html/HTMLMediaElement.cpp
dom/html/HTMLMediaElement.h
dom/media/MediaSegment.h
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -2031,17 +2031,16 @@ HTMLMediaElement::LookupMediaElementURIT
       }
     }
   }
   return nullptr;
 }
 
 HTMLMediaElement::HTMLMediaElement(already_AddRefed<mozilla::dom::NodeInfo>& aNodeInfo)
   : nsGenericHTMLElement(aNodeInfo),
-    mSrcStreamListener(nullptr),
     mCurrentLoadID(0),
     mNetworkState(nsIDOMHTMLMediaElement::NETWORK_EMPTY),
     mReadyState(nsIDOMHTMLMediaElement::HAVE_NOTHING),
     mLastNextFrameStatus(NEXT_FRAME_UNINITIALIZED),
     mLoadWaitStatus(NOT_WAITING),
     mVolume(1.0),
     mPreloadAction(PRELOAD_UNDEFINED),
     mMediaSize(-1,-1),
@@ -2944,16 +2943,71 @@ private:
   bool mHaveCurrentData;
   bool mBlocked;
 
   // mMutex protects the fields below; they can be accessed on any thread
   Mutex mMutex;
   bool mPendingNotifyOutput;
 };
 
+/**
+ * This listener observes the first video frame to arrive with a non-empty size,
+ * and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size.
+ */
+class HTMLMediaElement::StreamSizeListener : public MediaStreamListener {
+public:
+  explicit StreamSizeListener(HTMLMediaElement* aElement) :
+    mElement(aElement),
+    mMutex("HTMLMediaElement::StreamSizeListener")
+  {}
+  void Forget() { mElement = nullptr; }
+
+  void ReceivedSize()
+  {
+    if (!mElement) {
+      return;
+    }
+    gfxIntSize size;
+    {
+      MutexAutoLock lock(mMutex);
+      size = mInitialSize;
+    }
+    nsRefPtr<HTMLMediaElement> deathGrip = mElement;
+    mElement->UpdateInitialMediaSize(size);
+  }
+  virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
+                                        StreamTime aTrackOffset,
+                                        uint32_t aTrackEvents,
+                                        const MediaSegment& aQueuedMedia) override
+  {
+    MutexAutoLock lock(mMutex);
+    if (mInitialSize != gfxIntSize(0,0) ||
+        aQueuedMedia.GetType() != MediaSegment::VIDEO) {
+      return;
+    }
+    const VideoSegment& video = static_cast<const VideoSegment&>(aQueuedMedia);
+    for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
+      if (c->mFrame.GetIntrinsicSize() != gfxIntSize(0,0)) {
+        mInitialSize = c->mFrame.GetIntrinsicSize();
+        nsCOMPtr<nsIRunnable> event =
+          NS_NewRunnableMethod(this, &StreamSizeListener::ReceivedSize);
+        aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
+      }
+    }
+  }
+
+private:
+  // These fields may only be accessed on the main thread
+  HTMLMediaElement* mElement;
+
+  // mMutex protects the fields below; they can be accessed on any thread
+  Mutex mMutex;
+  gfxIntSize mInitialSize;
+};
+
 class HTMLMediaElement::MediaStreamTracksAvailableCallback:
     public DOMMediaStream::OnTracksAvailableCallback
 {
 public:
   explicit MediaStreamTracksAvailableCallback(HTMLMediaElement* aElement):
       DOMMediaStream::OnTracksAvailableCallback(),
       mElement(aElement)
     {}
@@ -2964,17 +3018,18 @@ public:
     mElement->NotifyMediaStreamTracksAvailable(aStream);
   }
 private:
   HTMLMediaElement* mElement;
 };
 
 void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream)
 {
-  NS_ASSERTION(!mSrcStream && !mSrcStreamListener, "Should have been ended already");
+  NS_ASSERTION(!mSrcStream && !mMediaStreamListener && !mMediaStreamSizeListener,
+               "Should have been ended already");
 
   mSrcStream = aStream;
 
   nsIDOMWindow* window = OwnerDoc()->GetInnerWindow();
   if (!window) {
     return;
   }
 
@@ -2997,18 +3052,23 @@ void HTMLMediaElement::SetupSrcMediaStre
 
   nsRefPtr<MediaStream> stream = mSrcStream->GetStream();
   if (stream) {
     stream->SetAudioChannelType(mAudioChannel);
   }
 
   // XXX if we ever support capturing the output of a media element which is
   // playing a stream, we'll need to add a CombineWithPrincipal call here.
-  mSrcStreamListener = new StreamListener(this);
-  GetSrcMediaStream()->AddListener(mSrcStreamListener);
+  mMediaStreamListener = new StreamListener(this);
+  mMediaStreamSizeListener = new StreamSizeListener(this);
+
+  GetSrcMediaStream()->AddListener(mMediaStreamListener);
+  // Listen for an initial image size on mSrcStream so we can get results even
+  // if we block the mPlaybackStream.
+  stream->AddListener(mMediaStreamSizeListener);
   if (mPaused) {
     GetSrcMediaStream()->ChangeExplicitBlockerCount(1);
   }
   if (mPausedForInactiveDocumentOrChannel) {
     GetSrcMediaStream()->ChangeExplicitBlockerCount(1);
   }
 
   ChangeNetworkState(nsIDOMHTMLMediaElement::NETWORK_IDLE);
@@ -3031,27 +3091,32 @@ void HTMLMediaElement::SetupSrcMediaStre
 
   // FirstFrameLoaded() will be called when the stream has current data.
 }
 
 void HTMLMediaElement::EndSrcMediaStreamPlayback()
 {
   MediaStream* stream = GetSrcMediaStream();
   if (stream) {
-    stream->RemoveListener(mSrcStreamListener);
+    stream->RemoveListener(mMediaStreamListener);
+  }
+  if (mSrcStream->GetStream()) {
+    mSrcStream->GetStream()->RemoveListener(mMediaStreamSizeListener);
   }
   mSrcStream->DisconnectTrackListListeners(AudioTracks(), VideoTracks());
 
   if (mPlaybackStreamInputPort) {
     mPlaybackStreamInputPort->Destroy();
   }
 
   // Kill its reference to this element
-  mSrcStreamListener->Forget();
-  mSrcStreamListener = nullptr;
+  mMediaStreamListener->Forget();
+  mMediaStreamListener = nullptr;
+  mMediaStreamSizeListener->Forget();
+  mMediaStreamSizeListener = nullptr;
   if (stream) {
     stream->RemoveAudioOutput(this);
   }
   VideoFrameContainer* container = GetVideoFrameContainer();
   if (container) {
     if (stream) {
       stream->RemoveVideoOutput(container);
     }
@@ -3773,26 +3838,33 @@ void HTMLMediaElement::NotifyDecoderPrin
 
   for (uint32_t i = 0; i < mOutputStreams.Length(); ++i) {
     OutputMediaStream* ms = &mOutputStreams[i];
     ms->mStream->SetCORSMode(mCORSMode);
     ms->mStream->CombineWithPrincipal(principal);
   }
 }
 
-void HTMLMediaElement::UpdateMediaSize(nsIntSize size)
+void HTMLMediaElement::UpdateMediaSize(const nsIntSize& aSize)
 {
-  if (IsVideo() && mReadyState != HAVE_NOTHING && mMediaSize != size) {
+  if (IsVideo() && mReadyState != HAVE_NOTHING && mMediaSize != aSize) {
     DispatchAsyncEvent(NS_LITERAL_STRING("resize"));
   }
 
-  mMediaSize = size;
+  mMediaSize = aSize;
   UpdateReadyStateForData(mLastNextFrameStatus);
 }
 
+void HTMLMediaElement::UpdateInitialMediaSize(const nsIntSize& aSize)
+{
+  if (mMediaSize == nsIntSize(-1, -1)) {
+    UpdateMediaSize(aSize);
+  }
+}
+
 void HTMLMediaElement::SuspendOrResumeElement(bool aPauseElement, bool aSuspendEvents)
 {
   if (aPauseElement != mPausedForInactiveDocumentOrChannel) {
     mPausedForInactiveDocumentOrChannel = aPauseElement;
     if (aPauseElement) {
       if (mMediaSource) {
         ReportMSETelemetry();
 #ifdef MOZ_EME
--- a/dom/html/HTMLMediaElement.h
+++ b/dom/html/HTMLMediaElement.h
@@ -255,17 +255,20 @@ public:
   // Returns null if nothing is playing.
   already_AddRefed<nsIPrincipal> GetCurrentPrincipal();
 
   // called to notify that the principal of the decoder's media resource has changed.
   virtual void NotifyDecoderPrincipalChanged() final override;
 
   // Update the visual size of the media. Called from the decoder on the
   // main thread when/if the size changes.
-  void UpdateMediaSize(nsIntSize size);
+  void UpdateMediaSize(const nsIntSize& aSize);
+  // Like UpdateMediaSize, but only updates the size if no size has yet
+  // been set.
+  void UpdateInitialMediaSize(const nsIntSize& aSize);
 
   // Returns the CanPlayStatus indicating if we can handle the
   // full MIME type including the optional codecs parameter.
   static CanPlayStatus GetCanPlay(const nsAString& aType);
 
   /**
    * Called when a child source element is added to this media element. This
    * may queue a task to run the select resource algorithm if appropriate.
@@ -636,16 +639,17 @@ public:
   bool IsBeingDestroyed();
 
 protected:
   virtual ~HTMLMediaElement();
 
   class MediaLoadListener;
   class MediaStreamTracksAvailableCallback;
   class StreamListener;
+  class StreamSizeListener;
 
   virtual void GetItemValueText(DOMString& text) override;
   virtual void SetItemValueText(const nsAString& text) override;
 
   class WakeLockBoolWrapper {
   public:
     explicit WakeLockBoolWrapper(bool val = false)
       : mValue(val), mCanPlay(true), mOuter(nullptr) {}
@@ -1040,18 +1044,22 @@ protected:
   // Holds references to the DOM wrappers for the MediaStreams that we're
   // writing to.
   struct OutputMediaStream {
     nsRefPtr<DOMMediaStream> mStream;
     bool mFinishWhenEnded;
   };
   nsTArray<OutputMediaStream> mOutputStreams;
 
-  // Holds a reference to the MediaStreamListener attached to mSrcStream.
-  nsRefPtr<StreamListener> mSrcStreamListener;
+  // Holds a reference to the MediaStreamListener attached to mPlaybackStream
+  // (or mSrcStream if mPlaybackStream is null).
+  nsRefPtr<StreamListener> mMediaStreamListener;
+  // Holds a reference to the size-getting MediaStreamListener attached to
+  // mSrcStream.
+  nsRefPtr<StreamSizeListener> mMediaStreamSizeListener;
 
   // Holds a reference to the MediaSource supplying data for playback.
   nsRefPtr<MediaSource> mMediaSource;
 
   // Holds a reference to the first channel we open to the media resource.
   // Once the decoder is created, control over the channel passes to the
   // decoder, and we null out this reference. We must store this in case
   // we need to cancel the channel before control of it passes to the decoder.
--- a/dom/media/MediaSegment.h
+++ b/dom/media/MediaSegment.h
@@ -260,16 +260,28 @@ public:
     bool IsEnded() { return mIndex >= mSegment.mChunks.Length(); }
     void Next() { ++mIndex; }
     Chunk& operator*() { return mSegment.mChunks[mIndex]; }
     Chunk* operator->() { return &mSegment.mChunks[mIndex]; }
   private:
     MediaSegmentBase<C, Chunk>& mSegment;
     uint32_t mIndex;
   };
+  class ConstChunkIterator {
+  public:
+    explicit ConstChunkIterator(const MediaSegmentBase<C, Chunk>& aSegment)
+      : mSegment(aSegment), mIndex(0) {}
+    bool IsEnded() { return mIndex >= mSegment.mChunks.Length(); }
+    void Next() { ++mIndex; }
+    const Chunk& operator*() { return mSegment.mChunks[mIndex]; }
+    const Chunk* operator->() { return &mSegment.mChunks[mIndex]; }
+  private:
+    const MediaSegmentBase<C, Chunk>& mSegment;
+    uint32_t mIndex;
+  };
 
   void RemoveLeading(StreamTime aDuration)
   {
     RemoveLeading(aDuration, 0);
   }
 
 #ifdef MOZILLA_INTERNAL_API
   void GetStartTime(TimeStamp &aTime) {