--- a/content/media/mediasource/MediaSourceDecoder.cpp
+++ b/content/media/mediasource/MediaSourceDecoder.cpp
@@ -83,41 +83,36 @@ public:
 
   bool DecodeVideoFrame(bool& aKeyFrameSkip, int64_t aTimeThreshold) MOZ_OVERRIDE
   {
     if (!GetVideoReader()) {
       MSE_DEBUG("%p DecodeVideoFrame called with no video reader", this);
       MOZ_ASSERT(mPendingDecoders.IsEmpty());
       return false;
     }
+
+    if (MaybeSwitchVideoReaders(aTimeThreshold)) {
+      GetVideoReader()->DecodeToTarget(aTimeThreshold);
+    }
+
     bool rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
 
     nsAutoTArray<VideoData*, 10> video;
     GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video);
     for (uint32_t i = 0; i < video.Length(); ++i) {
       VideoQueue().Push(video[i]);
     }
     GetVideoReader()->VideoQueue().Empty();
 
     if (rv) {
       return true;
     }
 
     MSE_DEBUG("%p MSR::DecodeVF %d (%p) returned false (readers=%u)",
               this, mActiveVideoDecoder, mDecoders[mActiveVideoDecoder].get(), mDecoders.Length());
-    if (SwitchVideoReaders(aTimeThreshold)) {
-      rv = GetVideoReader()->DecodeVideoFrame(aKeyFrameSkip, aTimeThreshold);
-
-      nsAutoTArray<VideoData*, 10> video;
-      GetVideoReader()->VideoQueue().GetElementsAfter(-1, &video);
-      for (uint32_t i = 0; i < video.Length(); ++i) {
-        VideoQueue().Push(video[i]);
-      }
-      GetVideoReader()->VideoQueue().Empty();
-    }
     return rv;
   }
 
   bool HasVideo() MOZ_OVERRIDE
   {
     return mInfo.HasVideo();
   }
 
@@ -131,39 +126,37 @@ public:
                 int64_t aCurrentTime) MOZ_OVERRIDE;
   nsresult GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime) MOZ_OVERRIDE;
   already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
                                                       MediaSourceDecoder* aParentDecoder);
 
   void CallDecoderInitialization();
 
 private:
-  bool SwitchVideoReaders(int64_t aTimeThreshold) {
+  bool MaybeSwitchVideoReaders(int64_t aTimeThreshold) {
+    ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
     MOZ_ASSERT(mActiveVideoDecoder != -1);
-    // XXX: We switch when the first reader is depleted, but it might be
-    // better to switch as soon as the next reader is ready to decode and
-    // has data for the current media time.
-    ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
-
-    GetVideoReader()->SetIdle();
 
     WaitForPendingDecoders();
 
     for (uint32_t i = mActiveVideoDecoder + 1; i < mDecoders.Length(); ++i) {
       if (!mDecoders[i]->GetReader()->GetMediaInfo().HasVideo()) {
         continue;
       }
-      mActiveVideoDecoder = i;
-      MSE_DEBUG("%p MSR::DecodeVF switching to %d", this, mActiveVideoDecoder);
+      if (aTimeThreshold >= mDecoders[i]->GetMediaStartTime()) {
+        GetVideoReader()->SetIdle();
+
+        mActiveVideoDecoder = i;
+        MSE_DEBUG("%p MSR::DecodeVF switching to %d", this, mActiveVideoDecoder);
 
-      GetVideoReader()->SetActive();
-      GetVideoReader()->DecodeToTarget(aTimeThreshold);
+        GetVideoReader()->SetActive();
+        return true;
+      }
+    }
 
-      return true;
-    }
     return false;
   }
 
   MediaDecoderReader* GetAudioReader() {
     if (mActiveAudioDecoder == -1) {
       return nullptr;
     }
     return mDecoders[mActiveAudioDecoder]->GetReader();
@@ -332,30 +325,36 @@ MediaSourceReader::CallDecoderInitializa
     nsRefPtr<SubBufferDecoder> decoder = mPendingDecoders[i];
     MediaDecoderReader* reader = decoder->GetReader();
     MSE_DEBUG("%p: Initializating subdecoder %p reader %p", this, decoder.get(), reader);
 
     reader->SetActive();
     MediaInfo mi;
     nsAutoPtr<MetadataTags> tags; // TODO: Handle metadata.
     nsresult rv;
+    int64_t startTime = 0;
     {
       ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor());
       rv = reader->ReadMetadata(&mi, getter_Transfers(tags));
+      if (NS_SUCCEEDED(rv)) {
+        reader->FindStartTime(startTime);
+      }
     }
     reader->SetIdle();
     if (NS_FAILED(rv)) {
       // XXX: Need to signal error back to owning SourceBuffer.
       MSE_DEBUG("%p: Reader %p failed to initialize, rv=%x", this, reader, rv);
       continue;
     }
+    decoder->SetMediaStartTime(startTime);
 
     bool active = false;
     if (mi.HasVideo() || mi.HasAudio()) {
-      MSE_DEBUG("%p: Reader %p has video=%d audio=%d", this, reader, mi.HasVideo(), mi.HasAudio());
+      MSE_DEBUG("%p: Reader %p has video=%d audio=%d startTime=%lld",
+                this, reader, mi.HasVideo(), mi.HasAudio(), startTime);
       active = true;
     }
 
     if (active) {
       mDecoders.AppendElement(decoder);
     } else {
       MSE_DEBUG("%p: Reader %p not activated", this, reader);
     }

--- a/content/media/mediasource/SourceBuffer.cpp
+++ b/content/media/mediasource/SourceBuffer.cpp
@@ -191,26 +191,20 @@ SourceBuffer::GetBuffered(ErrorResult& a
     aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);
     return nullptr;
   }
   nsRefPtr<TimeRanges> ranges = new TimeRanges();
   for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
     nsRefPtr<TimeRanges> r = new TimeRanges();
     mDecoders[i]->GetBuffered(r);
     if (r->Length() > 0) {
-      MSE_DEBUG("%p GetBuffered decoder=%u Length=%u Start=%f End=%f", this, i, r->Length(),
-                r->GetStartTime(), r->GetEndTime());
       ranges->Add(r->GetStartTime(), r->GetEndTime());
-    } else {
-      MSE_DEBUG("%p GetBuffered decoder=%u Length=%u", this, i, r->Length());
     }
   }
   ranges->Normalize();
-  MSE_DEBUG("%p GetBuffered Length=%u Start=%f End=%f", this, ranges->Length(),
-            ranges->GetStartTime(), ranges->GetEndTime());
   return ranges.forget();
 }
 
 void
 SourceBuffer::SetAppendWindowStart(double aAppendWindowStart, ErrorResult& aRv)
 {
   if (!IsAttached() || mUpdating) {
     aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR);

--- a/content/media/mediasource/SubBufferDecoder.h
+++ b/content/media/mediasource/SubBufferDecoder.h
@@ -16,16 +16,17 @@ class MediaSourceDecoder;
 
 class SubBufferDecoder : public BufferDecoder
 {
 public:
   // This class holds a weak pointer to MediaResource.  It's the responsibility
   // of the caller to manage the memory of the MediaResource object.
   SubBufferDecoder(MediaResource* aResource, MediaSourceDecoder* aParentDecoder)
     : BufferDecoder(aResource), mParentDecoder(aParentDecoder), mReader(nullptr)
+    , mMediaDuration(-1), mMediaStartTime(0)
   {
   }
 
   void SetReader(MediaDecoderReader* aReader)
   {
     MOZ_ASSERT(!mReader);
     mReader = aReader;
   }
@@ -67,17 +68,28 @@ public:
   // cached data. Returns -1 if no such value is computable.
   int64_t ConvertToByteOffset(double aTime);
 
   int64_t GetMediaDuration() MOZ_OVERRIDE
   {
     return mMediaDuration;
   }
 
+  int64_t GetMediaStartTime()
+  {
+    return mMediaStartTime;
+  }
+
+  void SetMediaStartTime(int64_t aMediaStartTime)
+  {
+    mMediaStartTime = aMediaStartTime;
+  }
+
 private:
   MediaSourceDecoder* mParentDecoder;
   nsAutoPtr<MediaDecoderReader> mReader;
   int64_t mMediaDuration;
+  int64_t mMediaStartTime;
 };
 
 } // namespace mozilla
 
 #endif /* MOZILLA_SUBBUFFERDECODER_H_ */