Bug 1108950 part.4 - Integrated with MediaStreamGraph. r?roc draft
authorctai <ctai@mozilla.com>
Tue, 01 Sep 2015 10:19:19 +0800
changeset 289367 a498cb75e5f2742d0a5d80674d45560c3219a8f5
parent 289366 0c32eaa61e28c3ffa1fede73aaf21a74d8c63f76
child 289368 3056e58a5dbabd365b3dfa9b86c13c5f47b6e7c2
push id4976
push userctai@mozilla.com
push dateTue, 01 Sep 2015 02:32:48 +0000
reviewersroc
bugs1108950
milestone43.0a1
Bug 1108950 part.4 - Integrated with MediaStreamGraph. r?roc
dom/media/DOMMediaStream.cpp
dom/media/DOMMediaStream.h
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
dom/media/TrackUnionStream.cpp
dom/media/TrackUnionStream.h
--- a/dom/media/DOMMediaStream.cpp
+++ b/dom/media/DOMMediaStream.cpp
@@ -431,16 +431,21 @@ DOMMediaStream::CreateDOMTrack(TrackID a
   case MediaSegment::VIDEO:
     track = new VideoStreamTrack(this, aTrackID);
     break;
   default:
     MOZ_CRASH("Unhandled track type");
   }
   mTracks.AppendElement(track);
 
+  // Make MediaStream not only keep TrackID. The VideoMonitorEvent need id of
+  // MediaStreamTrack.
+  nsString domTrackID;
+  track->GetId(domTrackID);
+  mStream->AppendDOMTarckIDMapping(aTrackID, domTrackID);
   return track;
 }
 
 MediaStreamTrack*
 DOMMediaStream::BindDOMTrack(TrackID aTrackID, MediaSegment::Type aType)
 {
   MediaStreamTrack* track = nullptr;
   bool bindSuccess = false;
--- a/dom/media/DOMMediaStream.h
+++ b/dom/media/DOMMediaStream.h
@@ -212,16 +212,21 @@ public:
   static already_AddRefed<DOMMediaStream> CreateAudioCaptureStream(
     nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr);
 
   void SetLogicalStreamStartTime(StreamTime aTime)
   {
     mLogicalStreamStartTime = aTime;
   }
 
+  StreamTime GetLogicalStreamStartTime()
+  {
+    return mLogicalStreamStartTime;
+  }
+
   // Notifications from StreamListener.
   // BindDOMTrack should only be called when it's safe to run script.
   MediaStreamTrack* BindDOMTrack(TrackID aTrackID, MediaSegment::Type aType);
   MediaStreamTrack* CreateDOMTrack(TrackID aTrackID, MediaSegment::Type aType);
   MediaStreamTrack* GetDOMTrackFor(TrackID aTrackID);
 
   class OnTracksAvailableCallback {
   public:
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -2413,16 +2413,45 @@ MediaStream::AddMainThreadListener(MainT
   nsRefPtr<nsRunnable> runnable = new NotifyRunnable(this);
   if (NS_WARN_IF(NS_FAILED(NS_DispatchToMainThread(runnable)))) {
     return;
   }
 
   mNotificationMainThreadRunnable = runnable;
 }
 
+void MediaStream::AppendDOMTarckIDMapping(TrackID aTrackID, const nsString& aDOMTrackID)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  class Message : public ControlMessage {
+  public:
+    Message(MediaStream* aStream, TrackID aTrackID, const nsString& aDOMTrackID) :
+      ControlMessage(aStream), mTrackID(aTrackID), mDOMTrackID(aDOMTrackID) {}
+    virtual void Run()
+    {
+      mStream->mDOMTrackIDMapping.AppendElement(TrackIDPair(mTrackID, mDOMTrackID));
+    }
+    TrackID mTrackID;
+    nsString mDOMTrackID;
+  };
+  GraphImpl()->AppendMessage(new Message(this, aTrackID, aDOMTrackID));
+}
+
+void MediaStream::QueryDOMTrackID(TrackID aTrackID, nsString& aRetVal)
+{
+  SetDOMStringToNull(aRetVal);
+  for (TrackIDPair pair : mDOMTrackIDMapping) {
+    if (aTrackID == pair.first()) {
+      aRetVal = pair.second();
+      return;
+    }
+  }
+}
+
 void
 SourceMediaStream::DestroyImpl()
 {
   // Hold mMutex while mGraph is reset so that other threads holding mMutex
   // can null-check know that the graph will not destroyed.
   MutexAutoLock lock(mMutex);
   MediaStream::DestroyImpl();
 }
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -3,16 +3,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef MOZILLA_MEDIASTREAMGRAPH_H_
 #define MOZILLA_MEDIASTREAMGRAPH_H_
 
 #include "mozilla/LinkedList.h"
 #include "mozilla/Mutex.h"
+#include "mozilla/Pair.h"
 #include "mozilla/TaskQueue.h"
 
 #include "mozilla/dom/AudioChannelBinding.h"
 
 #include "AudioSegment.h"
 #include "AudioStream.h"
 #include "nsTArray.h"
 #include "nsIRunnable.h"
@@ -577,16 +578,19 @@ public:
   }
 
   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
 
   void SetAudioChannelType(dom::AudioChannel aType) { mAudioChannelType = aType; }
   dom::AudioChannel AudioChannelType() const { return mAudioChannelType; }
 
+  void AppendDOMTarckIDMapping(TrackID aTrackID, const nsString& aDOMTrackID);
+  void QueryDOMTrackID(TrackID aTrackID, nsString& aRetVal);
+
 protected:
   void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime)
   {
     mBufferStartTime += aBlockedTime;
     mExplicitBlockerCount.AdvanceCurrentTime(aCurrentTime);
 
     mBuffer.ForgetUpTo(aCurrentTime - mBufferStartTime);
   }
@@ -649,16 +653,20 @@ protected:
   // not been blocked before mCurrentTime (its mBufferStartTime is increased
   // as necessary to account for that time instead) --- this avoids us having to
   // record the entire history of the stream's blocking-ness in mBlocked.
   TimeVarying<GraphTime,bool,5> mBlocked;
 
   // MediaInputPorts to which this is connected
   nsTArray<MediaInputPort*> mConsumers;
 
+  // Store the mapping of TrackID to DOMTrackID.
+  typedef Pair<TrackID, nsString> TrackIDPair;
+  nsAutoTArray<TrackIDPair, 2> mDOMTrackIDMapping;
+
   // Where audio output is going. There is one AudioOutputStream per
   // audio track.
   struct AudioOutputStream
   {
     // When we started audio playback for this track.
     // Add mStream->GetPosition() to find the current audio playback position.
     GraphTime mAudioPlaybackStartTime;
     // Amount of time that we've wanted to play silence because of the stream
--- a/dom/media/TrackUnionStream.cpp
+++ b/dom/media/TrackUnionStream.cpp
@@ -28,18 +28,25 @@
 #include "webaudio/MediaStreamAudioDestinationNode.h"
 #include <algorithm>
 #include "DOMMediaStream.h"
 #include "GeckoProfiler.h"
 #ifdef MOZ_WEBRTC
 #include "AudioOutputObserver.h"
 #endif
 
+#include "jsapi.h"
+#include "MediaStreamTrack.h"
+#include "mozilla/dom/ImageBitmap.h"
+#include "nsQueryObject.h"
+#include "VideoProcessEvent.h"
 #include "WorkerPrivate.h"
 #include "WorkerRunnable.h"
+#include "WorkerPrivate.h"
+#include "WorkerScope.h"
 
 using namespace mozilla::layers;
 using namespace mozilla::dom;
 using namespace mozilla::gfx;
 
 namespace mozilla {
 
 #ifdef STREAM_LOG
@@ -145,16 +152,40 @@ TrackUnionStream::TrackUnionStream(DOMMe
       mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTime(aTo));
     }
     if (allHaveCurrentData) {
       // We can make progress if we're not blocked
       mHasCurrentData = true;
     }
   }
 
+  void TrackUnionStream::UpdateWorkerStatusImpl(WorkerPrivate* aWorker,
+                                                TrackID aTrackID,
+                                                bool aHasOnFlyEvent)
+  {
+    for (uint32_t i = 0; i < mTrackMap.Length(); ++i) {
+      TrackMapEntry* entry = &mTrackMap[i];
+      if (entry->mOutputTrackID == aTrackID) {
+        for (uint32_t j = 0; j < entry->mWorkerMonitorInfos.Length(); ++j) {
+          WorkerInformation& info = entry->mWorkerMonitorInfos[j];
+          if (aWorker == info.mWorker) {
+            info.mOnFlyStatus = aHasOnFlyEvent;
+            // Check whether the latest image dispatched or not. If the
+            // dispatched is not the latest source frame, dispatch the latest
+            // frame.
+            if (info.mDispatchedImage != entry->mLastImage) {
+              DispatchToVideoWorkerMonitor(entry);
+            }
+            return;
+          }
+        }
+      }
+    }
+  }
+
   // Forward SetTrackEnabled(output_track_id, enabled) to the Source MediaStream,
   // translating the output track ID into the correct ID in the source.
   void TrackUnionStream::ForwardTrackEnabled(TrackID aOutputID, bool aEnabled)
   {
     for (int32_t i = mTrackMap.Length() - 1; i >= 0; --i) {
       if (mTrackMap[i].mOutputTrackID == aOutputID) {
         mTrackMap[i].mInputPort->GetSource()->
           SetTrackEnabled(mTrackMap[i].mInputTrackID, aEnabled);
@@ -229,16 +260,20 @@ TrackUnionStream::TrackUnionStream(DOMMe
             return;
           }
         }
         WorkerInformation* info = entry->mWorkerMonitorInfos.AppendElement();
         info->mWorker = aWorker;
         info->mDispatchedImage = nullptr;
         info->mWorkerFeature = new VideoWorkerFeature(aWorker, this);
         MOZ_ASSERT(info->mWorkerFeature);
+        info->mOnFlyStatus = false;
+        MediaStream* destStream = entry->mInputPort->GetDestination();
+        destStream->QueryDOMTrackID(aTrackID, entry->mDOMTrackID);
+        MOZ_ASSERT(!entry->mDOMTrackID.IsVoid());
 
         // |WorkerPrivate::AddFeature| could be called in worker thread only.
         // We need an WorkerRunnable to add the WorkerFeature to the worker.
         // This runnable is dispatched from MSG thread, not from parent thread,
         // main thread or worker thread. So the BusyCount balance is controlled
         // by the runnable itself. Also we already added BusyCount while the
         // |VideoStreamTrack::AddVideoMonitor| is called. The worker shhould be
         // kept alive at this point.
@@ -491,18 +526,20 @@ TrackUnionStream::TrackUnionStream(DOMMe
                                   *segment);
     }
     segment->AppendNullData(outputStart);
     StreamBuffer::Track* track =
       &mBuffer.AddTrack(id, outputStart, segment.forget());
     STREAM_LOG(LogLevel::Debug, ("TrackUnionStream %p adding track %d for input stream %p track %d, start ticks %lld",
                               this, id, aPort->GetSource(), aTrack->GetID(),
                               (long long)outputStart));
-
     TrackMapEntry* map = mTrackMap.AppendElement();
+    map->mDOMTrackID.SetIsVoid(true);
+    map->mLastImage = nullptr;
+    map->mLastImageStreamTime = 0;
     map->mEndOfConsumedInputTicks = 0;
     map->mEndOfLastInputIntervalInInputStream = -1;
     map->mEndOfLastInputIntervalInOutputStream = -1;
     map->mInputPort = aPort;
     map->mInputTrackID = aTrack->GetID();
     map->mOutputTrackID = track->GetID();
     map->mSegment = aTrack->GetSegment()->CreateEmptyClone();
     return mTrackMap.Length() - 1;
@@ -520,16 +557,174 @@ TrackUnionStream::TrackUnionStream(DOMMe
       segment = outputTrack->GetSegment()->CreateEmptyClone();
       l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(), offset,
                                   MediaStreamListener::TRACK_EVENT_ENDED,
                                   *segment);
     }
     outputTrack->SetEnded();
   }
 
+  void TrackUnionStream::UpdateWorkerStatus(WorkerPrivate* aWorker,
+                                            TrackID aTrackID,
+                                            bool aHasOnFlyEvent)
+  {
+    class Message final : public ControlMessage
+    {
+    public:
+      Message(TrackUnionStream* aStream,
+              WorkerPrivate* aWorker,
+              TrackID aTrackID,
+              bool aHasOnFlyEvent)
+        : ControlMessage(aStream)
+        , mWorker(aWorker)
+        , mTrackID(aTrackID)
+        , mHasOnFlyEvent(aHasOnFlyEvent)
+      {}
+      virtual void Run() override
+      {
+        mStream->AsTrackUnionStream()->UpdateWorkerStatusImpl(mWorker, mTrackID, mHasOnFlyEvent);
+      }
+      WorkerPrivate* mWorker;
+      TrackID mTrackID;
+      bool mHasOnFlyEvent;
+    };
+    // The MediaStream might be shutting down. If the Stream is destroyed, we
+    // don't need to append message to the graph.
+    if (this->IsDestroyed()) {
+      return;
+    }
+    GraphImpl()->AppendMessage(new Message(this, aWorker, aTrackID, aHasOnFlyEvent));
+  }
+
+  // This runnable is dispatched from MSG thread, not from parent thread,
+  // main thread or worker thread. So the BusyCount balance is controlled
+  // by the runnable itself.
+  class VideoMonitorEventRunnable final : public WorkerRunnable {
+  public:
+    VideoMonitorEventRunnable(WorkerPrivate* aWorkerPrivate,
+                              TargetAndBusyBehavior aBehavior,
+                              TrackUnionStream* aStream,
+                              TrackID aTrackID,
+                              double aPlaybackTime,
+                              const nsString& aID,
+                              layers::Image* aImage)
+    :WorkerRunnable(aWorkerPrivate, aBehavior),
+     mStream(aStream),
+     mImage(aImage),
+     mTrackID(aTrackID),
+     mPlaybackTime(aPlaybackTime),
+     mID(aID)
+    {
+    }
+
+  private:
+    virtual bool
+    WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override
+    {
+      aWorkerPrivate->AssertIsOnWorkerThread();
+      aWorkerPrivate->ModifyBusyCountFromWorker(aCx, true);
+
+      // The callback is returned. Set the mOnFlyStatus flag to false which
+      // allowing following event flow.
+      class SetOnFlyEventStatusFalseRunnable : public nsRunnable
+      {
+      public:
+        SetOnFlyEventStatusFalseRunnable(TrackUnionStream* aStream,
+                                         WorkerPrivate* aWorker,
+                                         TrackID aTrackID)
+          : mStream(aStream)
+          , mWorker(aWorker)
+          , mTrackID(aTrackID)
+        {
+          MOZ_ASSERT(mStream);
+          MOZ_ASSERT(mWorker);
+        }
+
+        NS_IMETHOD
+        Run(void) override
+        {
+          // Set the mOnFlyStatus to false;
+          mStream->UpdateWorkerStatus(mWorker, mTrackID, false);
+          return NS_OK;
+        }
+        TrackUnionStream* mStream;
+        WorkerPrivate* mWorker;
+        TrackID mTrackID;
+      };
+      // We need to recover the onFlyEvent status whether the event is executed
+      // successfully or not.
+      nsRefPtr<SetOnFlyEventStatusFalseRunnable> runnable = new
+        SetOnFlyEventStatusFalseRunnable(mStream, aWorkerPrivate, mTrackID);
+      MOZ_ASSERT(runnable);
+
+      DOMEventTargetHelper* target = aWorkerPrivate->GlobalScope();
+      nsRefPtr<VideoMonitorEvent> event =
+        new VideoMonitorEvent(target, nullptr, nullptr);
+      nsresult rv = event->InitEvent(mPlaybackTime, mID, mImage);
+      if (NS_FAILED(rv)) {
+        xpc::Throw(aCx, rv);
+        // The ControlMessage is only allowed to send from main thread.
+        NS_DispatchToMainThread(runnable);
+        return false;
+      }
+      event->SetTrusted(true);
+
+      nsCOMPtr<nsIDOMEvent> domEvent = do_QueryObject(event);
+
+      nsEventStatus dummy = nsEventStatus_eIgnore;
+      // Update the memory usage information to JS context before dispathing
+      // event.
+      if (aCx) {
+        int32_t width = mImage->GetSize().width;
+        int32_t height = mImage->GetSize().height;
+        // We don't know exactly format of mImage. So just set the maximum
+        // possible size.
+        JS_updateMallocCounter(aCx, width * height * 4);
+      }
+      rv = target->DispatchDOMEvent(nullptr, domEvent, nullptr, &dummy);
+      NS_WARN_IF(NS_FAILED(rv));
+
+      // The ControlMessage is only allowed to send from main thread.
+      rv = NS_DispatchToMainThread(runnable);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return false;
+      }
+      return true;
+    }
+
+    void
+    PostRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate, bool aRunResult)
+    {
+      aWorkerPrivate->ModifyBusyCountFromWorker(aCx, false);
+    }
+
+    // Override |PreDispatch| and |PostDispatch| to avoid triggering
+    // |AssertIsOnParentThread| because of dispatching from MSG thread.
+    bool
+    PreDispatch(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override
+    {
+      return true;
+    }
+
+    void
+    PostDispatch(JSContext* aCx, WorkerPrivate* aWorkerPrivate,
+                 bool aDispatchResult) override
+    {
+    }
+
+    ~VideoMonitorEventRunnable(){
+    }
+
+    TrackUnionStream* mStream;
+    nsRefPtr<Image> mImage;
+    TrackID mTrackID;
+    double mPlaybackTime;
+    nsString mID;
+  };
+
   void TrackUnionStream::CopyTrackData(StreamBuffer::Track* aInputTrack,
                      uint32_t aMapIndex, GraphTime aFrom, GraphTime aTo,
                      bool* aOutputTrackFinished)
   {
     TrackMapEntry* map = &mTrackMap[aMapIndex];
     StreamBuffer::Track* outputTrack = mBuffer.FindTrack(map->mOutputTrackID);
     MOZ_ASSERT(outputTrack && !outputTrack->IsEnded(), "Can't copy to ended track");
 
@@ -567,23 +762,68 @@ TrackUnionStream::TrackUnionStream(DOMMe
         segment->AppendNullData(ticks);
       } else {
         if (GraphImpl()->StreamSuspended(source)) {
           segment->AppendNullData(aTo - aFrom);
         } else {
           MOZ_ASSERT(outputTrack->GetEnd() == GraphTimeToStreamTime(interval.mStart),
                      "Samples missing");
           StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart);
-          segment->AppendSlice(*aInputTrack->GetSegment(),
-                               std::min(inputTrackEndPoint, inputStart),
-                               std::min(inputTrackEndPoint, inputEnd));
+          if (segment->GetType() == MediaSegment::AUDIO){
+            segment->AppendSlice(*aInputTrack->GetSegment(),
+                                 std::min(inputTrackEndPoint, inputStart),
+                                 std::min(inputTrackEndPoint, inputEnd));
+          } else {
+            nsAutoPtr<MediaSegment> tmpSegment;
+            tmpSegment = aInputTrack->GetSegment()->CreateEmptyClone();
+            tmpSegment->AppendSlice(*aInputTrack->GetSegment(),
+                                    std::min(inputTrackEndPoint, inputStart),
+                                    std::min(inputTrackEndPoint, inputEnd));
+            VideoSegment* videoSegment = static_cast<VideoSegment*>(tmpSegment.get());
+            for (VideoSegment::ChunkIterator i(*videoSegment); !i.IsEnded(); i.Next()) {
+              VideoChunk& chunk = *i;
+              VideoFrame& frame = chunk.mFrame;
+              VideoFrame::Image* image = frame.GetImage();
+              if (image != map->mLastImage) {
+                map->mLastImage = image;
+                map->mLastImageStreamTime = std::min(inputTrackEndPoint, source->GraphTimeToStreamTime(interval.mStart));
+                DispatchToVideoWorkerMonitor(map);
+              }
+            }
+            segment->AppendFrom(videoSegment);
+          }
         }
       }
       ApplyTrackDisabling(outputTrack->GetID(), segment);
       for (uint32_t j = 0; j < mListeners.Length(); ++j) {
         MediaStreamListener* l = mListeners[j];
         l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(),
                                     outputStart, 0, *segment);
       }
       outputTrack->GetSegment()->AppendFrom(segment);
     }
   }
+
+  void TrackUnionStream::DispatchToVideoWorkerMonitor(TrackMapEntry* aMapEntry)
+  {
+    if (aMapEntry->mWorkerMonitorInfos.IsEmpty()) {
+      return;
+    }
+    for (uint32_t i = 0; i < aMapEntry->mWorkerMonitorInfos.Length(); ++i) {
+      WorkerInformation& info = aMapEntry->mWorkerMonitorInfos[i];
+      if (!info.mOnFlyStatus) {
+        nsRefPtr<VideoMonitorEventRunnable> runnable =
+          new VideoMonitorEventRunnable(info.mWorker,
+                                        workers::WorkerRunnable::WorkerThreadUnchangedBusyCount,
+                                        this,
+                                        aMapEntry->mOutputTrackID,
+                                        StreamTimeToSeconds(aMapEntry->mLastImageStreamTime),
+                                        aMapEntry->mDOMTrackID,
+                                        aMapEntry->mLastImage);
+        if (NS_WARN_IF(!runnable->Dispatch(nullptr))) {
+          return;
+        }
+        info.mDispatchedImage = aMapEntry->mLastImage;
+        info.mOnFlyStatus = true;
+      }
+    }
+  }
 } // namespace mozilla
--- a/dom/media/TrackUnionStream.h
+++ b/dom/media/TrackUnionStream.h
@@ -41,16 +41,18 @@ public:
   virtual TrackUnionStream* AsTrackUnionStream() override { return this; };
 
   void AddVideoMonitor(dom::Promise* aPromise, WorkerPrivate* aWorker, TrackID aTrackID);
 
   // |RemoveVideoMonitor| can be called in |VideoWorkerFeature::Notify|. There
   // is no Promise in such case. So use maybe to handle it.
   void RemoveVideoMonitor(Maybe<nsRefPtr<dom::Promise>> aPromise, WorkerPrivate* aWorker, Maybe<TrackID> aTrackID, bool aNeedRemoveFeature = true);
 
+  void UpdateWorkerStatus(WorkerPrivate* aWorker, TrackID aTrackID, bool aHasOnFlyEvent);
+
 protected:
   virtual ~TrackUnionStream();
 
   struct WorkerInformation; // Forward declaration for VideoWorkerFeature.
 
   class VideoWorkerFeature final : public WorkerFeature
   {
   public:
@@ -83,16 +85,29 @@ protected:
     }
 
     WorkerPrivate* mWorker;
     nsRefPtr<VideoWorkerFeature> mWorkerFeature;
     // Memorized dispatched image. Because the complexity in each worker might be
     // different, keep the latest dispatched image to determine whether dispatch
     // the new event once the worker finished the job.
     nsRefPtr<VideoFrame::Image> mDispatchedImage;
+    // True when there is still an on-fly event. We need this for recording the
+    // worker status. Because the processing capability of event handler
+    // "onvideoprocess" might slower than the new frame coming rate, that might
+    // cause a "back pressure" problem. So use this flag for flow control.
+    // Right now, the drop frame mechanism is below:
+    // If a video processing or monitoring callback is still processing frame N
+    // when frame N+1 becomes available, save the pointer of frame N in
+    // mDispatchedImage, and keep the frame N+1 in TrackMapEntry::mLastImage.
+    // Once the callback function compeltes, the frame N+1 will be sent to the
+    // event handler. If frame N+2 arrives while N is still being processed,
+    // frame N+1 is effectively dropped and save the pointer of frame N+2 to
+    // TrackMapEntry::mLastImage.
+    bool mOnFlyStatus;
   };
 
   // Only non-ended tracks are allowed to persist in this map.
   struct TrackMapEntry {
     // mEndOfConsumedInputTicks is the end of the input ticks that we've consumed.
     // 0 if we haven't consumed any yet.
     StreamTime mEndOfConsumedInputTicks;
     // mEndOfLastInputIntervalInInputStream is the timestamp for the end of the
@@ -113,16 +128,22 @@ protected:
     TrackID mOutputTrackID;
     nsAutoPtr<MediaSegment> mSegment;
     // TrackUnionStream will hold TrackMapEntrys. Each TrackMapEntry has
     // multiple WorkerInformation. The WorkerInformation contains
     // VideoWorkerFeature object. The WorkerInformation is added into
     // mWorkerMonitorInfos when the AddWorkerMonitor is called. And it is
     // removed by RemoveWorkerMonitor.
     nsTArray<WorkerInformation> mWorkerMonitorInfos;
+    // The latest frame from source stream.
+    nsRefPtr<VideoFrame::Image> mLastImage;
+    nsString mDOMTrackID;
+    // The stream of latest frame from source stream.
+    StreamTime mLastImageStreamTime;
+
   };
 
   // Add the track to this stream, retaining its TrackID if it has never
   // been previously used in this stream, allocating a new TrackID otherwise.
   uint32_t AddTrack(MediaInputPort* aPort, StreamBuffer::Track* aTrack,
                     GraphTime aFrom);
   void EndTrack(uint32_t aIndex);
   void CopyTrackData(StreamBuffer::Track* aInputTrack,
@@ -137,13 +158,17 @@ protected:
 
   // Sorted array of used TrackIDs that require manual tracking.
   nsTArray<TrackID> mUsedTracks;
 
 private:
   // MSG thread only methods
   void AddVideoMonitorImpl(WorkerPrivate* aWorker, TrackID aTrackID);
   void RemoveVideoMonitorImpl(WorkerPrivate* aWorker, Maybe<TrackID> aTrackID, bool aNeedRemoveFeature);
+
+  void UpdateWorkerStatusImpl(WorkerPrivate* aWorker, TrackID aTrackID, bool aHasOnFlyEvent);
+
+  void DispatchToVideoWorkerMonitor(TrackMapEntry* aMapEntry);
 };
 
 } // namespace mozilla
 
 #endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */