Bug 1537554 - part2 : let track track handle adding `current cue` and `other cue` if it's not disable. r=jya a=pascalc
authorAlastor Wu <alwu@mozilla.com>
Tue, 26 Mar 2019 07:50:50 +0000
changeset 525778 00a14c57b0ad27c22f5301f8a8a50ed8c274f91f
parent 525777 3f25e13283021c46039f566437967260e5cf4e32
child 525779 0b745b93a430983cdcd6c748e6e2098d012e1969
push id2032
push userffxbld-merge
push dateMon, 13 May 2019 09:36:57 +0000
treeherdermozilla-release@455c1065dcbe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjya, pascalc
bugs1537554
milestone67.0
Bug 1537554 - part2 : let track track handle adding `current cue` and `other cue` if it's not disable. r=jya a=pascalc According to the spec [1], `current cues` and `other cues` should only contain cues from `hidden` or `showing` text tracks. In this patch, text track would be responsible to add `current cues` and `other cues` to the cues list by calling `GetCurrentCuesAndOtherCues()`. If the text track is disabled, then it won't add any cues to the cues list. In addition, in order to reduce the size of `other cues` (as actually we don't need to process all cues in the `other cues`), we use the time interval to only get the cues which are overlapping with the time interval. [1] https://html.spec.whatwg.org/multipage/media.html#time-marches-on Differential Revision: https://phabricator.services.mozilla.com/D24475
dom/html/TextTrackManager.cpp
dom/html/TextTrackManager.h
dom/media/TextTrack.cpp
dom/media/TextTrack.h
dom/media/TextTrackCueList.cpp
dom/media/TextTrackCueList.h
--- a/dom/html/TextTrackManager.cpp
+++ b/dom/html/TextTrackManager.cpp
@@ -94,43 +94,42 @@ bool CompareTextTracks::LessThan(TextTra
     case MediaResourceSpecific:
       // No rules for Media Resource Specific tracks yet.
       break;
   }
   return true;
 }
 
 NS_IMPL_CYCLE_COLLECTION(TextTrackManager, mMediaElement, mTextTracks,
-                         mPendingTextTracks, mNewCues, mLastActiveCues)
+                         mPendingTextTracks, mNewCues)
 
 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager)
   NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener)
 NS_INTERFACE_MAP_END
 
 NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager)
 NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager)
 
 StaticRefPtr<nsIWebVTTParserWrapper> TextTrackManager::sParserWrapper;
 
 TextTrackManager::TextTrackManager(HTMLMediaElement* aMediaElement)
     : mMediaElement(aMediaElement),
       mHasSeeked(false),
-      mLastTimeMarchesOnCalled(0.0),
+      mLastTimeMarchesOnCalled(media::TimeUnit::Zero()),
       mTimeMarchesOnDispatched(false),
       mUpdateCueDisplayDispatched(false),
       performedTrackSelection(false),
       mCueTelemetryReported(false),
       mShutdown(false) {
   nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
 
   NS_ENSURE_TRUE_VOID(parentObject);
   WEBVTT_LOG("Create TextTrackManager");
   nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
   mNewCues = new TextTrackCueList(window);
-  mLastActiveCues = new TextTrackCueList(window);
   mTextTracks = new TextTrackList(window, this);
   mPendingTextTracks = new TextTrackList(window, this);
 
   if (!sParserWrapper) {
     nsCOMPtr<nsIWebVTTParserWrapper> parserWrapper =
         do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID);
     MOZ_ASSERT(parserWrapper, "Can't create nsIWebVTTParserWrapper");
     sParserWrapper = parserWrapper;
@@ -229,19 +228,20 @@ void TextTrackManager::RemoveTextTrack(T
     }
     TimeMarchesOn();
   }
 }
 
 void TextTrackManager::DidSeek() {
   WEBVTT_LOG("DidSeek");
   if (mMediaElement) {
-    mLastTimeMarchesOnCalled = mMediaElement->CurrentTime();
+    mLastTimeMarchesOnCalled =
+        media::TimeUnit::FromSeconds(mMediaElement->CurrentTime());
     WEBVTT_LOGV("DidSeek set mLastTimeMarchesOnCalled %lf",
-                mLastTimeMarchesOnCalled);
+                mLastTimeMarchesOnCalled.ToSeconds());
   }
   mHasSeeked = true;
 }
 
 void TextTrackManager::UpdateCueDisplay() {
   WEBVTT_LOG("UpdateCueDisplay");
   mUpdateCueDisplayDispatched = false;
 
@@ -638,67 +638,61 @@ void TextTrackManager::TimeMarchesOn() {
 
   if (mMediaElement &&
       (!(mMediaElement->GetPlayedOrSeeked()) || mMediaElement->Seeking())) {
     WEBVTT_LOG("TimeMarchesOn seeking or post return");
     return;
   }
 
   // Step 3.
-  double currentPlaybackTime = mMediaElement->CurrentTime();
+  auto currentPlaybackTime =
+      media::TimeUnit::FromSeconds(mMediaElement->CurrentTime());
   bool hasNormalPlayback = !mHasSeeked;
   mHasSeeked = false;
   WEBVTT_LOG(
       "TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf "
       "hasNormalPlayback %d",
-      mLastTimeMarchesOnCalled, currentPlaybackTime, hasNormalPlayback);
+      mLastTimeMarchesOnCalled.ToSeconds(), currentPlaybackTime.ToSeconds(),
+      hasNormalPlayback);
 
   // Step 1, 2.
   RefPtr<TextTrackCueList> currentCues = new TextTrackCueList(window);
   RefPtr<TextTrackCueList> otherCues = new TextTrackCueList(window);
-  bool dummy;
-  for (uint32_t index = 0; index < mTextTracks->Length(); ++index) {
-    TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
-    if (ttrack && ttrack->Mode() != TextTrackMode::Disabled) {
-      // TODO: call GetCueListByTimeInterval on mNewCues?
-      ttrack->GetCurrentCueList(currentCues);
+
+  // The reason we collect other cues is (1) to change active cues to inactive,
+  // (2) find missing cues, so we actually no need to process all cues. We just
+  // need to handle cues which are in the time interval [lastTime:currentTime]
+  // or [currentTime:lastTime] (seeking forward). That can help us to reduce the
+  // size of other cues, which can improve execution time.
+  auto start = std::min(mLastTimeMarchesOnCalled, currentPlaybackTime);
+  auto end = std::max(mLastTimeMarchesOnCalled, currentPlaybackTime);
+  media::TimeInterval interval(start, end);
+  WEBVTT_LOGV("TimeMarchesOn Time interval [%f:%f]", start.ToSeconds(),
+              end.ToSeconds());
+  for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
+    TextTrack* track = (*mTextTracks)[idx];
+    if (track) {
+      track->GetCurrentCuesAndOtherCues(currentCues, otherCues, interval);
     }
   }
-  WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues->Length());
-  // Populate otherCues with 'non-active" cues.
-  if (hasNormalPlayback) {
-    if (currentPlaybackTime < mLastTimeMarchesOnCalled) {
-      // TODO: Add log and find the root cause why the
-      // playback position goes backward.
-      mLastTimeMarchesOnCalled = currentPlaybackTime;
-    }
-    media::Interval<double> interval(mLastTimeMarchesOnCalled,
-                                     currentPlaybackTime);
-    otherCues = mNewCues->GetCueListByTimeInterval(interval);
-    ;
-  } else {
-    // Seek case. Put the mLastActiveCues into otherCues.
-    otherCues = mLastActiveCues;
-  }
-  for (uint32_t i = 0; i < currentCues->Length(); ++i) {
-    TextTrackCue* cue = (*currentCues)[i];
-    otherCues->RemoveCue(*cue);
-  }
-  WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues->Length());
+
   // Step 4.
   RefPtr<TextTrackCueList> missedCues = new TextTrackCueList(window);
   if (hasNormalPlayback) {
     for (uint32_t i = 0; i < otherCues->Length(); ++i) {
       TextTrackCue* cue = (*otherCues)[i];
-      if (cue->StartTime() >= mLastTimeMarchesOnCalled &&
-          cue->EndTime() <= currentPlaybackTime) {
+      if (cue->StartTime() >= mLastTimeMarchesOnCalled.ToSeconds() &&
+          cue->EndTime() <= currentPlaybackTime.ToSeconds()) {
         missedCues->AddCue(*cue);
       }
     }
   }
+
+  WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues->Length());
+  WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues->Length());
   WEBVTT_LOGV("TimeMarchesOn missedCues %d", missedCues->Length());
   // Step 5. Empty now.
   // TODO: Step 6: fire timeupdate?
 
   // Step 7. Abort steps if condition 1, 2, 3 are satisfied.
   // 1. All of the cues in current cues have their active flag set.
   // 2. None of the cues in other cues have their active flag set.
   // 3. Missed cues is empty.
@@ -715,17 +709,17 @@ void TextTrackManager::TimeMarchesOn() {
       c2 = false;
       break;
     }
   }
   bool c3 = (missedCues->Length() == 0);
   if (c1 && c2 && c3) {
     mLastTimeMarchesOnCalled = currentPlaybackTime;
     WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf",
-               mLastTimeMarchesOnCalled);
+               mLastTimeMarchesOnCalled.ToSeconds());
     return;
   }
 
   // Step 8. Respect PauseOnExit flag if not seek.
   if (hasNormalPlayback) {
     for (uint32_t i = 0; i < otherCues->Length(); ++i) {
       TextTrackCue* cue = (*otherCues)[i];
       if (cue && cue->PauseOnExit() && cue->GetActive()) {
@@ -815,17 +809,16 @@ void TextTrackManager::TimeMarchesOn() {
       HTMLTrackElement* trackElement = ttrack->GetTrackElement();
       if (trackElement) {
         trackElement->DispatchTrackRunnable(NS_LITERAL_STRING("cuechange"));
       }
     }
   }
 
   mLastTimeMarchesOnCalled = currentPlaybackTime;
-  mLastActiveCues = currentCues;
 
   // Step 18.
   UpdateCueDisplay();
 }
 
 void TextTrackManager::NotifyCueUpdated(TextTrackCue* aCue) {
   // TODO: Add/Reorder the cue to mNewCues if we have some optimization?
   WEBVTT_LOG("NotifyCueUpdated, cue=%p", aCue);
@@ -833,31 +826,31 @@ void TextTrackManager::NotifyCueUpdated(
   // For the case "Texttrack.mode = hidden/showing", if the mode
   // changing between showing and hidden, TimeMarchesOn
   // doesn't render the cue. Call DispatchUpdateCueDisplay() explicitly.
   DispatchUpdateCueDisplay();
 }
 
 void TextTrackManager::NotifyReset() {
   WEBVTT_LOG("NotifyReset");
-  mLastTimeMarchesOnCalled = 0.0;
+  mLastTimeMarchesOnCalled = media::TimeUnit::Zero();
 }
 
 void TextTrackManager::ReportTelemetryForTrack(TextTrack* aTextTrack) const {
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(aTextTrack);
   MOZ_ASSERT(mTextTracks->Length() > 0);
 
   TextTrackKind kind = aTextTrack->Kind();
   Telemetry::Accumulate(Telemetry::WEBVTT_TRACK_KINDS, uint32_t(kind));
 }
 
 void TextTrackManager::ReportTelemetryForCue() {
   MOZ_ASSERT(NS_IsMainThread());
-  MOZ_ASSERT(!mNewCues->IsEmpty() || !mLastActiveCues->IsEmpty());
+  MOZ_ASSERT(!mNewCues->IsEmpty());
 
   if (!mCueTelemetryReported) {
     Telemetry::Accumulate(Telemetry::WEBVTT_USED_VTT_CUES, 1);
     mCueTelemetryReported = true;
   }
 }
 
 bool TextTrackManager::IsLoaded() {
--- a/dom/html/TextTrackManager.h
+++ b/dom/html/TextTrackManager.h
@@ -7,16 +7,17 @@
 #ifndef mozilla_dom_TextTrackManager_h
 #define mozilla_dom_TextTrackManager_h
 
 #include "mozilla/dom/TextTrack.h"
 #include "mozilla/dom/TextTrackList.h"
 #include "mozilla/dom/TextTrackCueList.h"
 #include "mozilla/StaticPtr.h"
 #include "nsContentUtils.h"
+#include "TimeUnits.h"
 
 class nsIWebVTTParserWrapper;
 
 namespace mozilla {
 namespace dom {
 
 class HTMLMediaElement;
 
@@ -113,24 +114,22 @@ class TextTrackManager final : public ns
   // List of the TextTrackManager's owning HTMLMediaElement's TextTracks.
   RefPtr<TextTrackList> mTextTracks;
   // List of text track objects awaiting loading.
   RefPtr<TextTrackList> mPendingTextTracks;
   // List of newly introduced Text Track cues.
 
   // Contain all cues for a MediaElement. Not sorted.
   RefPtr<TextTrackCueList> mNewCues;
-  // The active cues for the last TimeMarchesOn iteration.
-  RefPtr<TextTrackCueList> mLastActiveCues;
 
   // True if the media player playback changed due to seeking prior to and
   // during running the "Time Marches On" algorithm.
   bool mHasSeeked;
   // Playback position at the time of last "Time Marches On" call
-  double mLastTimeMarchesOnCalled;
+  media::TimeUnit mLastTimeMarchesOnCalled;
 
   bool mTimeMarchesOnDispatched;
   bool mUpdateCueDisplayDispatched;
 
   static StaticRefPtr<nsIWebVTTParserWrapper> sParserWrapper;
 
   bool performedTrackSelection;
 
--- a/dom/media/TextTrack.cpp
+++ b/dom/media/TextTrack.cpp
@@ -12,18 +12,19 @@
 #include "mozilla/dom/TextTrackCueList.h"
 #include "mozilla/dom/TextTrackRegion.h"
 #include "mozilla/dom/HTMLMediaElement.h"
 #include "mozilla/dom/HTMLTrackElement.h"
 #include "nsGlobalWindow.h"
 
 extern mozilla::LazyLogModule gTextTrackLog;
 
-#define WEBVTT_LOG(msg, ...) \
-  MOZ_LOG(gTextTrackLog, LogLevel::Debug, ("TextTrack=%p, " msg, this, ##__VA_ARGS__))
+#define WEBVTT_LOG(msg, ...)              \
+  MOZ_LOG(gTextTrackLog, LogLevel::Debug, \
+          ("TextTrack=%p, " msg, this, ##__VA_ARGS__))
 
 namespace mozilla {
 namespace dom {
 
 static const char* ToStateStr(const TextTrackMode aMode) {
   switch (aMode) {
     case TextTrackMode::Disabled:
       return "DISABLED";
@@ -274,43 +275,65 @@ bool TextTrack::IsLoaded() {
   }
   return (mReadyState >= Loaded);
 }
 
 void TextTrack::NotifyCueActiveStateChanged(TextTrackCue* aCue) {
   MOZ_ASSERT(aCue);
   if (aCue->GetActive()) {
     MOZ_ASSERT(!mActiveCueList->IsCueExist(aCue));
-    WEBVTT_LOG("NotifyCueActiveStateChanged, add cue %p to the active list", aCue);
+    WEBVTT_LOG("NotifyCueActiveStateChanged, add cue %p to the active list",
+               aCue);
     mActiveCueList->AddCue(*aCue);
   } else {
     MOZ_ASSERT(mActiveCueList->IsCueExist(aCue));
-    WEBVTT_LOG("NotifyCueActiveStateChanged, remove cue %p from the active list", aCue);
+    WEBVTT_LOG(
+        "NotifyCueActiveStateChanged, remove cue %p from the active list",
+        aCue);
     mActiveCueList->RemoveCue(*aCue);
   }
 }
 
-void TextTrack::GetCurrentCueList(RefPtr<TextTrackCueList>& aCueList) const {
+void TextTrack::GetCurrentCuesAndOtherCues(
+    RefPtr<TextTrackCueList>& aCurrentCues,
+    RefPtr<TextTrackCueList>& aOtherCues,
+    const media::TimeInterval& aInterval) const {
   const HTMLMediaElement* mediaElement = GetMediaElement();
   if (!mediaElement) {
     return;
   }
 
+  if (Mode() == TextTrackMode::Disabled) {
+    return;
+  }
+
   // According to `time marches on` step1, current cue list contains the cues
   // whose start times are less than or equal to the current playback position
   // and whose end times are greater than the current playback position.
   // https://html.spec.whatwg.org/multipage/media.html#time-marches-on
-  MOZ_ASSERT(aCueList);
+  MOZ_ASSERT(aCurrentCues && aOtherCues);
   const double playbackTime = mediaElement->CurrentTime();
   for (uint32_t idx = 0; idx < mCueList->Length(); idx++) {
     TextTrackCue* cue = (*mCueList)[idx];
     if (cue->StartTime() <= playbackTime && cue->EndTime() > playbackTime) {
-      WEBVTT_LOG("Add cue %p [%f:%f] to current cue list",
-                 cue, cue->StartTime(), cue->EndTime());
-      aCueList->AddCue(*cue);
+      WEBVTT_LOG("Add cue %p [%f:%f] to current cue list", cue,
+                 cue->StartTime(), cue->EndTime());
+      aCurrentCues->AddCue(*cue);
+    } else {
+      media::TimeInterval cueInterval(
+          media::TimeUnit::FromSeconds(cue->StartTime()),
+          media::TimeUnit::FromSeconds(cue->EndTime()));
+      // cues are completely outside the time interval.
+      if (!aInterval.Touches(cueInterval)) {
+        continue;
+      }
+      // contains any cues which are overlapping within the time interval.
+      WEBVTT_LOG("Add cue %p [%f:%f] to other cue list", cue, cue->StartTime(),
+                 cue->EndTime());
+      aOtherCues->AddCue(*cue);
     }
   }
 }
 
 HTMLMediaElement* TextTrack::GetMediaElement() const {
   return mTextTrackList ? mTextTrackList->GetMediaElement() : nullptr;
 }
 
--- a/dom/media/TextTrack.h
+++ b/dom/media/TextTrack.h
@@ -7,16 +7,17 @@
 #ifndef mozilla_dom_TextTrack_h
 #define mozilla_dom_TextTrack_h
 
 #include "mozilla/DOMEventTargetHelper.h"
 #include "mozilla/dom/TextTrackBinding.h"
 #include "nsCOMPtr.h"
 #include "nsCycleCollectionParticipant.h"
 #include "nsString.h"
+#include "TimeUnits.h"
 
 namespace mozilla {
 namespace dom {
 
 class TextTrackList;
 class TextTrackCue;
 class TextTrackCueList;
 class HTMLTrackElement;
@@ -98,20 +99,26 @@ class TextTrack final : public DOMEventT
   void DispatchAsyncTrustedEvent(const nsString& aEventName);
 
   bool IsLoaded();
 
   // Called when associated cue's active flag has been changed, and then we
   // would add or remove the cue to the active cue list.
   void NotifyCueActiveStateChanged(TextTrackCue* aCue);
 
-  // Use this function to request current cues which start time are less than or
+  // Use this function to request current cues, which start time are less than or
   // equal to the current playback position and whose end times are greater than
-  // the current playback position.
-  void GetCurrentCueList(RefPtr<TextTrackCueList>& aCueList) const;
+  // the current playback position, and other cues, which are not in the current
+  // cues. Because there would be LOTS of cues in the other cues, and we don't
+  // actually need all of them. Therefore, we use a time interval to get the
+  // cues which are overlapping within the time interval.
+  void GetCurrentCuesAndOtherCues(
+    RefPtr<TextTrackCueList>& aCurrentCues,
+    RefPtr<TextTrackCueList>& aOtherCues,
+    const media::TimeInterval& aInterval) const;
 
  private:
   ~TextTrack();
 
   HTMLMediaElement* GetMediaElement() const;
 
   RefPtr<TextTrackList> mTextTrackList;
 
--- a/dom/media/TextTrackCueList.cpp
+++ b/dom/media/TextTrackCueList.cpp
@@ -100,29 +100,16 @@ void TextTrackCueList::GetArray(nsTArray
 }
 
 void TextTrackCueList::SetCuesInactive() {
   for (uint32_t i = 0; i < mList.Length(); ++i) {
     mList[i]->SetActive(false);
   }
 }
 
-already_AddRefed<TextTrackCueList> TextTrackCueList::GetCueListByTimeInterval(
-    media::Interval<double>& aInterval) {
-  RefPtr<TextTrackCueList> output = new TextTrackCueList(mParent);
-  for (uint32_t i = 0; i < mList.Length(); ++i) {
-    TextTrackCue* cue = mList[i];
-    if (cue->StartTime() <= aInterval.mEnd &&
-        aInterval.mStart <= cue->EndTime()) {
-      output->AddCue(*cue);
-    }
-  }
-  return output.forget();
-}
-
 void TextTrackCueList::NotifyCueUpdated(TextTrackCue* aCue) {
   if (aCue) {
     mList.RemoveElement(aCue);
     mList.InsertElementSorted(aCue, CompareCuesByTime());
   }
 }
 
 bool TextTrackCueList::IsCueExist(TextTrackCue* aCue) {
--- a/dom/media/TextTrackCueList.h
+++ b/dom/media/TextTrackCueList.h
@@ -7,17 +7,16 @@
 #ifndef mozilla_dom_TextTrackCueList_h
 #define mozilla_dom_TextTrackCueList_h
 
 #include "nsTArray.h"
 #include "nsCOMPtr.h"
 #include "nsCycleCollectionParticipant.h"
 #include "nsWrapperCache.h"
 #include "mozilla/ErrorResult.h"
-#include "Intervals.h"
 
 namespace mozilla {
 namespace dom {
 
 class TextTrackCue;
 
 class TextTrackCueList final : public nsISupports, public nsWrapperCache {
  public:
@@ -48,18 +47,16 @@ class TextTrackCueList final : public ns
   void RemoveCue(TextTrackCue& aCue);
   void RemoveCue(TextTrackCue& aCue, ErrorResult& aRv);
   void RemoveCueAt(uint32_t aIndex);
   void RemoveAll();
   void GetArray(nsTArray<RefPtr<TextTrackCue>>& aCues);
 
   void SetCuesInactive();
 
-  already_AddRefed<TextTrackCueList> GetCueListByTimeInterval(
-      media::Interval<double>& aInterval);
   void NotifyCueUpdated(TextTrackCue* aCue);
   bool IsCueExist(TextTrackCue* aCue);
   nsTArray<RefPtr<TextTrackCue>>& GetCuesArray();
 
  private:
   ~TextTrackCueList();
 
   nsCOMPtr<nsISupports> mParent;