author | ctai <ctai@mozilla.com> |
Thu, 24 Dec 2015 10:43:28 +0800 | |
changeset 308251 | d3295f400cd3ef369b5fe3e16c4f0918e2e7e9bf |
parent 308250 | 79415ce5af65c6b207ae185825770ac039af13a8 |
child 308252 | 02fc34b73508ff8433cce0c96f98ae689a35bb7a |
push id | 31092 |
push user | cbook@mozilla.com |
push date | Fri, 05 Aug 2016 10:16:59 +0000 |
treeherder | autoland@b97dd7dd3cb9 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | jesup |
bugs | 1201363 |
milestone | 51.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/dom/media/MediaStreamListener.h +++ b/dom/media/MediaStreamListener.h @@ -2,20 +2,24 @@ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef MOZILLA_MEDIASTREAMLISTENER_h_ #define MOZILLA_MEDIASTREAMLISTENER_h_ +#include "StreamTracks.h" + namespace mozilla { +class AudioSegment; class MediaStream; class MediaStreamGraph; +class VideoSegment; enum MediaStreamGraphEvent : uint32_t { EVENT_FINISHED, EVENT_REMOVED, EVENT_HAS_DIRECT_LISTENERS, // transition from no direct listeners EVENT_HAS_NO_DIRECT_LISTENERS, // transition to no direct listeners };
new file mode 100644 --- /dev/null +++ b/dom/media/MediaStreamVideoSink.cpp @@ -0,0 +1,21 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaStreamVideoSink.h" + +#include "VideoSegment.h" + +namespace mozilla { +void +MediaStreamVideoSink::NotifyRealtimeTrackData(MediaStreamGraph* aGraph, + StreamTime aTrackOffset, + const MediaSegment& aMedia) +{ + if (aMedia.GetType() == MediaSegment::VIDEO) { + SetCurrentFrames(static_cast<const VideoSegment&>(aMedia)); + } +} + +} // namespace mozilla
new file mode 100644 --- /dev/null +++ b/dom/media/MediaStreamVideoSink.h @@ -0,0 +1,42 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MEDIASTREAMVIDEOSINK_H_ +#define MEDIASTREAMVIDEOSINK_H_ + +#include "mozilla/Pair.h" + +#include "gfxPoint.h" +#include "MediaStreamListener.h" + +namespace mozilla { + +class VideoFrameContainer; + +/** + * Base class of MediaStreamVideoSink family. This is the output of MediaStream. + */ +class MediaStreamVideoSink : public DirectMediaStreamTrackListener { +public: + // Method of DirectMediaStreamTrackListener. + void NotifyRealtimeTrackData(MediaStreamGraph* aGraph, + StreamTime aTrackOffset, + const MediaSegment& aMedia) override; + + // Call on any thread + virtual void SetCurrentFrames(const VideoSegment& aSegment) = 0; + virtual void ClearFrames() = 0; + + virtual VideoFrameContainer* AsVideoFrameContainer() { return nullptr; } + virtual void Invalidate() {} + +protected: + virtual ~MediaStreamVideoSink() {}; +}; + +} // namespace mozilla + +#endif /* MEDIASTREAMVIDEOSINK_H_ */
--- a/dom/media/VideoFrameContainer.cpp +++ b/dom/media/VideoFrameContainer.cpp @@ -9,27 +9,33 @@ #include "mozilla/dom/HTMLMediaElement.h" #include "nsIFrame.h" #include "nsDisplayList.h" #include "nsSVGEffects.h" using namespace mozilla::layers; namespace mozilla { +PRLogModuleInfo* gVideoFrameContainerLog; +#define CONTAINER_LOG(type, msg) MOZ_LOG(gVideoFrameContainerLog, type, msg) VideoFrameContainer::VideoFrameContainer(dom::HTMLMediaElement* aElement, already_AddRefed<ImageContainer> aContainer) : mElement(aElement), mImageContainer(aContainer), mMutex("nsVideoFrameContainer"), + mBlackImage(nullptr), mFrameID(0), mIntrinsicSizeChanged(false), mImageSizeChanged(false), mPendingPrincipalHandle(PRINCIPAL_HANDLE_NONE), mFrameIDForPendingPrincipalHandle(0) { NS_ASSERTION(aElement, "aElement must not be null"); NS_ASSERTION(mImageContainer, "aContainer must not be null"); + if (!gVideoFrameContainerLog) { + gVideoFrameContainerLog = PR_NewLogModule("VideoFrameContainer"); + } } VideoFrameContainer::~VideoFrameContainer() {} PrincipalHandle VideoFrameContainer::GetLastPrincipalHandle() { MutexAutoLock lock(mMutex); @@ -42,16 +48,136 @@ void VideoFrameContainer::UpdatePrincipa MutexAutoLock lock(mMutex); if (mPendingPrincipalHandle == aPrincipalHandle) { return; } mPendingPrincipalHandle = aPrincipalHandle; mFrameIDForPendingPrincipalHandle = aFrameID; } +static void +SetImageToBlackPixel(PlanarYCbCrImage* aImage) +{ + uint8_t blackPixel[] = { 0x10, 0x80, 0x80 }; + + PlanarYCbCrData data; + data.mYChannel = blackPixel; + data.mCbChannel = blackPixel + 1; + data.mCrChannel = blackPixel + 2; + data.mYStride = data.mCbCrStride = 1; + data.mPicSize = data.mYSize = data.mCbCrSize = gfx::IntSize(1, 1); + aImage->CopyData(data); +} + +class VideoFrameContainerInvalidateRunnable : public Runnable { +public: + explicit VideoFrameContainerInvalidateRunnable(VideoFrameContainer* aVideoFrameContainer) + : mVideoFrameContainer(aVideoFrameContainer) + {} + NS_IMETHOD Run() + { + MOZ_ASSERT(NS_IsMainThread()); + + mVideoFrameContainer->Invalidate(); + + return NS_OK; + } +private: + RefPtr<VideoFrameContainer> mVideoFrameContainer; +}; + +void VideoFrameContainer::SetCurrentFrames(const VideoSegment& aSegment) +{ + if (aSegment.IsEmpty()) { + return; + } + + MutexAutoLock lock(mMutex); + + // Collect any new frames produced in this iteration. + AutoTArray<ImageContainer::NonOwningImage,4> newImages; + PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE; + + VideoSegment::ConstChunkIterator iter(aSegment); + while (!iter.IsEnded()) { + VideoChunk chunk = *iter; + + const VideoFrame* frame = &chunk.mFrame; + if (*frame == mLastPlayedVideoFrame) { + iter.Next(); + continue; + } + + Image* image = frame->GetImage(); + CONTAINER_LOG(LogLevel::Verbose, + ("VideoFrameContainer %p writing video frame %p (%d x %d)", + this, image, frame->GetIntrinsicSize().width, + frame->GetIntrinsicSize().height)); + + if (frame->GetForceBlack()) { + if (!mBlackImage) { + mBlackImage = GetImageContainer()->CreatePlanarYCbCrImage(); + if (mBlackImage) { + // Sets the image to a single black pixel, which will be scaled to + // fill the rendered size. + SetImageToBlackPixel(mBlackImage->AsPlanarYCbCrImage()); + } + } + if (mBlackImage) { + image = mBlackImage; + } + } + // Don't append null image to the newImages. + if (!image) { + iter.Next(); + continue; + } + newImages.AppendElement(ImageContainer::NonOwningImage(image, chunk.mTimeStamp)); + + lastPrincipalHandle = chunk.GetPrincipalHandle(); + + mLastPlayedVideoFrame = *frame; + iter.Next(); + } + + // Don't update if there are no changes. + if (newImages.IsEmpty()) { + return; + } + + AutoTArray<ImageContainer::NonOwningImage,4> images; + + bool principalHandleChanged = + lastPrincipalHandle != PRINCIPAL_HANDLE_NONE && + lastPrincipalHandle != GetLastPrincipalHandle(); + + // Add the frames from this iteration. + for (auto& image : newImages) { + image.mFrameID = NewFrameID(); + images.AppendElement(image); + } + + if (principalHandleChanged) { + UpdatePrincipalHandleForFrameID(lastPrincipalHandle, + newImages.LastElement().mFrameID); + } + + SetCurrentFramesLocked(mLastPlayedVideoFrame.GetIntrinsicSize(), images); + nsCOMPtr<nsIRunnable> event = + new VideoFrameContainerInvalidateRunnable(this); + NS_DispatchToMainThread(event.forget()); + + images.ClearAndRetainStorage(); +} + +void VideoFrameContainer::ClearFrames() +{ + ClearFutureFrames(); +} + void VideoFrameContainer::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage, const TimeStamp& aTargetTime) { if (aImage) { MutexAutoLock lock(mMutex); AutoTArray<ImageContainer::NonOwningImage,1> imageList; imageList.AppendElement(
--- a/dom/media/VideoFrameContainer.h +++ b/dom/media/VideoFrameContainer.h @@ -8,60 +8,63 @@ #define VIDEOFRAMECONTAINER_H_ #include "mozilla/Mutex.h" #include "mozilla/TimeStamp.h" #include "gfxPoint.h" #include "nsCOMPtr.h" #include "ImageContainer.h" #include "MediaSegment.h" +#include "MediaStreamVideoSink.h" +#include "VideoSegment.h" namespace mozilla { namespace dom { class HTMLMediaElement; } // namespace dom /** * This object is used in the decoder backend threads and the main thread * to manage the "current video frame" state. This state includes timing data * and an intrinsic size (see below). * This has to be a thread-safe object since it's accessed by resource decoders * and other off-main-thread components. So we can't put this state in the media * element itself ... well, maybe we could, but it could be risky and/or * confusing. */ -class VideoFrameContainer { - ~VideoFrameContainer(); +class VideoFrameContainer : public MediaStreamVideoSink { + virtual ~VideoFrameContainer(); public: typedef layers::ImageContainer ImageContainer; typedef layers::Image Image; - NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameContainer) - VideoFrameContainer(dom::HTMLMediaElement* aElement, already_AddRefed<ImageContainer> aContainer); // Call on any thread + virtual void SetCurrentFrames(const VideoSegment& aSegment) override; + virtual void ClearFrames() override; + void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage, + const TimeStamp& aTargetTime); // Returns the last principalHandle we notified mElement about. PrincipalHandle GetLastPrincipalHandle(); // We will notify mElement that aPrincipalHandle has been applied when all // FrameIDs prior to aFrameID have been flushed out. // aFrameID is ignored if aPrincipalHandle already is our pending principalHandle. void UpdatePrincipalHandleForFrameID(const PrincipalHandle& aPrincipalHandle, const ImageContainer::FrameID& aFrameID); - void SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Image* aImage, - const TimeStamp& aTargetTime); void SetCurrentFrames(const gfx::IntSize& aIntrinsicSize, const nsTArray<ImageContainer::NonOwningImage>& aImages); void ClearCurrentFrame(const gfx::IntSize& aIntrinsicSize) { SetCurrentFrames(aIntrinsicSize, nsTArray<ImageContainer::NonOwningImage>()); } + VideoFrameContainer* AsVideoFrameContainer() override { return this; } void ClearCurrentFrame(); // Make the current frame the only frame in the container, i.e. discard // all future frames. void ClearFutureFrames(); // Time in seconds by which the last painted video frame was late by. // E.g. if the last painted frame should have been painted at time t, // but was actually painted at t+n, this returns n in seconds. Threadsafe. @@ -75,17 +78,17 @@ public: return ++mFrameID; } // Call on main thread enum { INVALIDATE_DEFAULT, INVALIDATE_FORCE }; - void Invalidate() { InvalidateWithFlags(INVALIDATE_DEFAULT); } + void Invalidate() override { InvalidateWithFlags(INVALIDATE_DEFAULT); } void InvalidateWithFlags(uint32_t aFlags); ImageContainer* GetImageContainer(); void ForgetElement() { mElement = nullptr; } uint32_t GetDroppedImageCount() { return mImageContainer->GetDroppedImageCount(); } protected: void SetCurrentFramesLocked(const gfx::IntSize& aIntrinsicSize, @@ -93,25 +96,31 @@ protected: // Non-addreffed pointer to the element. The element calls ForgetElement // to clear this reference when the element is destroyed. dom::HTMLMediaElement* mElement; RefPtr<ImageContainer> mImageContainer; // mMutex protects all the fields below. Mutex mMutex; + // Once the frame is forced to black, we initialize mBlackImage for following + // frames. + RefPtr<Image> mBlackImage; // The intrinsic size is the ideal size which we should render the // ImageContainer's current Image at. // This can differ from the Image's actual size when the media resource // specifies that the Image should be stretched to have the correct aspect // ratio. gfx::IntSize mIntrinsicSize; // We maintain our own mFrameID which is auto-incremented at every // SetCurrentFrame() or NewFrameID() call. ImageContainer::FrameID mFrameID; + // We record the last played video frame to avoid playing the frame again + // with a different frame id. + VideoFrame mLastPlayedVideoFrame; // True when the intrinsic size has been changed by SetCurrentFrame() since // the last call to Invalidate(). // The next call to Invalidate() will recalculate // and update the intrinsic size on the element, request a frame reflow and // then reset this flag. bool mIntrinsicSizeChanged; // True when the Image size has changed since the last time Invalidate() was // called. When set, the next call to Invalidate() will ensure that the
--- a/dom/media/VideoSegment.h +++ b/dom/media/VideoSegment.h @@ -132,13 +132,19 @@ public: // Segment-generic methods not in MediaSegmentBase static Type StaticType() { return VIDEO; } size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override { return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } + + bool IsEmpty() const + { + return mChunks.IsEmpty(); + } + }; } // namespace mozilla #endif /* MOZILLA_VIDEOSEGMENT_H_ */
--- a/dom/media/moz.build +++ b/dom/media/moz.build @@ -122,16 +122,17 @@ EXPORTS += [ 'MediaQueue.h', 'MediaRecorder.h', 'MediaResource.h', 'MediaResourceCallback.h', 'MediaSegment.h', 'MediaStatistics.h', 'MediaStreamGraph.h', 'MediaStreamListener.h', + 'MediaStreamVideoSink.h', 'MediaTimer.h', 'MediaTrack.h', 'MediaTrackList.h', 'MP3Decoder.h', 'MP3Demuxer.h', 'MP3FrameParser.h', 'NextFrameSeekTask.h', 'nsIDocumentActivity.h', @@ -234,16 +235,17 @@ UNIFIED_SOURCES += [ 'MediaPrefs.cpp', 'MediaRecorder.cpp', 'MediaResource.cpp', 'MediaShutdownManager.cpp', 'MediaStreamError.cpp', 'MediaStreamGraph.cpp', 'MediaStreamListener.cpp', 'MediaStreamTrack.cpp', + 'MediaStreamVideoSink.cpp', 'MediaTimer.cpp', 'MediaTrack.cpp', 'MediaTrackList.cpp', 'MP3Decoder.cpp', 'MP3Demuxer.cpp', 'MP3FrameParser.cpp', 'NextFrameSeekTask.cpp', 'QueueObject.cpp',