Bug 1465619 - Part 11. Add support for recycling animated image frames. r=tnikkel
authorAndrew Osmond <aosmond@mozilla.com>
Mon, 04 Jun 2018 08:33:16 -0400
changeset 490713 9c8e6aaa017625dd61942d3100316798c8c2ef61
parent 490712 453d21db2a306f4800e8be4d77f6d61000cb9a0c
child 490714 0bacfc8c286f858c822d309fcb49104d83191793
push id247
push userfmarier@mozilla.com
push dateSat, 27 Oct 2018 01:06:44 +0000
reviewerstnikkel
bugs1465619
milestone65.0a1
Bug 1465619 - Part 11. Add support for recycling animated image frames. r=tnikkel This is what we have been working towards in all of the previous parts in the series. This subclasses AnimationFrameDiscardingQueue to save the discarded frames for recycling by the decoder, if the frame is marked as supporting recycling. Differential Revision: https://phabricator.services.mozilla.com/D7516
gfx/thebes/gfxPrefs.h
image/AnimationFrameBuffer.cpp
image/AnimationFrameBuffer.h
image/AnimationSurfaceProvider.cpp
image/AnimationSurfaceProvider.h
modules/libpref/init/all.js
--- a/gfx/thebes/gfxPrefs.h
+++ b/gfx/thebes/gfxPrefs.h
@@ -540,17 +540,18 @@ private:
 #if defined(XP_MACOSX)
   DECL_GFX_PREF(Live, "gl.multithreaded",                      GLMultithreaded, bool, false);
 #endif
   DECL_GFX_PREF(Live, "gl.require-hardware",                   RequireHardwareGL, bool, false);
   DECL_GFX_PREF(Live, "gl.use-tls-is-current",                 UseTLSIsCurrent, int32_t, 0);
 
   DECL_GFX_PREF(Live, "image.animated.decode-on-demand.threshold-kb", ImageAnimatedDecodeOnDemandThresholdKB, uint32_t, 20480);
   DECL_GFX_PREF(Live, "image.animated.decode-on-demand.batch-size", ImageAnimatedDecodeOnDemandBatchSize, uint32_t, 6);
-  DECL_GFX_PREF(Live, "image.animated.generate-full-frames",   ImageAnimatedGenerateFullFrames, bool, false);
+  DECL_GFX_PREF(Live, "image.animated.decode-on-demand.recycle", ImageAnimatedDecodeOnDemandRecycle, bool, false);
+  DECL_GFX_PREF(Once, "image.animated.generate-full-frames",   ImageAnimatedGenerateFullFrames, bool, false);
   DECL_GFX_PREF(Live, "image.animated.resume-from-last-displayed", ImageAnimatedResumeFromLastDisplayed, bool, false);
   DECL_GFX_PREF(Live, "image.cache.factor2.threshold-surfaces", ImageCacheFactor2ThresholdSurfaces, int32_t, -1);
   DECL_GFX_PREF(Live, "image.cache.max-rasterized-svg-threshold-kb", ImageCacheMaxRasterizedSVGThresholdKB, int32_t, 90*1024);
   DECL_GFX_PREF(Once, "image.cache.size",                      ImageCacheSize, int32_t, 5*1024*1024);
   DECL_GFX_PREF(Once, "image.cache.timeweight",                ImageCacheTimeWeight, int32_t, 500);
   DECL_GFX_PREF(Live, "image.decode-immediately.enabled",      ImageDecodeImmediatelyEnabled, bool, false);
   DECL_GFX_PREF(Live, "image.downscale-during-decode.enabled", ImageDownscaleDuringDecodeEnabled, bool, true);
   DECL_GFX_PREF(Live, "image.infer-src-animation.threshold-ms", ImageInferSrcAnimationThresholdMS, uint32_t, 2000);
--- a/image/AnimationFrameBuffer.cpp
+++ b/image/AnimationFrameBuffer.cpp
@@ -315,10 +315,118 @@ AnimationFrameDiscardingQueue::AddSizeOf
       [&](AddSizeOfCbData& aMetadata) {
         aMetadata.index = i;
         aCallback(aMetadata);
       }
     );
   }
 }
 
+AnimationFrameRecyclingQueue::AnimationFrameRecyclingQueue(AnimationFrameRetainedBuffer&& aQueue)
+  : AnimationFrameDiscardingQueue(std::move(aQueue))
+{
+  // In an ideal world, we would always save the already displayed frames for
+  // recycling but none of the frames were marked as recyclable. We will incur
+  // the extra allocation cost for a few more frames.
+  mRecycling = true;
+}
+
+void
+AnimationFrameRecyclingQueue::AddSizeOfExcludingThis(MallocSizeOf aMallocSizeOf,
+                                                     const AddSizeOfCb& aCallback)
+{
+  AnimationFrameDiscardingQueue::AddSizeOfExcludingThis(aMallocSizeOf,
+                                                        aCallback);
+
+  for (const RecycleEntry& entry : mRecycle) {
+    if (entry.mFrame) {
+      entry.mFrame->AddSizeOfExcludingThis(aMallocSizeOf,
+        [&](AddSizeOfCbData& aMetadata) {
+          aMetadata.index = 0; // Frame is not applicable
+          aCallback(aMetadata);
+        }
+      );
+    }
+  }
+}
+
+void
+AnimationFrameRecyclingQueue::AdvanceInternal()
+{
+  MOZ_ASSERT(!mDisplay.empty());
+  MOZ_ASSERT(mDisplay.front());
+
+  RefPtr<imgFrame>& front = mDisplay.front();
+
+  // The first frame should always have a dirty rect that matches the frame
+  // rect. As such, we should use mFirstFrameRefreshArea instead for recycle
+  // rect calculations.
+  MOZ_ASSERT_IF(mGetIndex == 1,
+                front->GetRect().IsEqualEdges(front->GetDirtyRect()));
+
+  RecycleEntry newEntry(mGetIndex == 1 ? mFirstFrameRefreshArea
+                                       : front->GetDirtyRect());
+
+  // If we are allowed to recycle the frame, then we should save it before the
+  // base class's AdvanceInternal discards it.
+  if (front->ShouldRecycle()) {
+    // Calculate the recycle rect for the recycled frame. This is the cumulative
+    // dirty rect of all of the frames ahead of us to be displayed, and to be
+    // used for recycling. Or in other words, the dirty rect between the
+    // recycled frame and the decoded frame which reuses the buffer.
+    for (const RefPtr<imgFrame>& frame : mDisplay) {
+      newEntry.mRecycleRect = newEntry.mRecycleRect.Union(frame->GetDirtyRect());
+    }
+    for (const RecycleEntry& entry : mRecycle) {
+      newEntry.mRecycleRect = newEntry.mRecycleRect.Union(entry.mDirtyRect);
+    }
+
+    newEntry.mFrame = std::move(front);
+  }
+
+  // Even if the frame itself isn't saved, we want the dirty rect to calculate
+  // the recycle rect for future recycled frames.
+  mRecycle.push_back(std::move(newEntry));
+  AnimationFrameDiscardingQueue::AdvanceInternal();
+}
+
+bool
+AnimationFrameRecyclingQueue::ResetInternal()
+{
+  mRecycle.clear();
+  return AnimationFrameDiscardingQueue::ResetInternal();
+}
+
+RawAccessFrameRef
+AnimationFrameRecyclingQueue::RecycleFrame(gfx::IntRect& aRecycleRect)
+{
+  if (mRecycle.empty()) {
+    return RawAccessFrameRef();
+  }
+
+  RawAccessFrameRef frame;
+  if (mRecycle.front().mFrame) {
+    frame = mRecycle.front().mFrame->RawAccessRef();
+    if (frame) {
+      aRecycleRect = mRecycle.front().mRecycleRect;
+    }
+  }
+
+  mRecycle.pop_front();
+  return frame;
+}
+
+bool
+AnimationFrameRecyclingQueue::MarkComplete(const gfx::IntRect& aFirstFrameRefreshArea)
+{
+  bool continueDecoding =
+    AnimationFrameDiscardingQueue::MarkComplete(aFirstFrameRefreshArea);
+
+  MOZ_ASSERT_IF(!mRedecodeError,
+                mFirstFrameRefreshArea.IsEmpty() ||
+                mFirstFrameRefreshArea.IsEqualEdges(aFirstFrameRefreshArea));
+
+  mFirstFrameRefreshArea = aFirstFrameRefreshArea;
+  return continueDecoding;
+}
+
 } // namespace image
 } // namespace mozilla
--- a/image/AnimationFrameBuffer.h
+++ b/image/AnimationFrameBuffer.h
@@ -417,12 +417,80 @@ protected:
   /// Queue storing frames to be displayed by the animator. The first frame in
   /// the queue is the currently displayed frame.
   std::deque<RefPtr<imgFrame>> mDisplay;
 
   /// The first frame which is never discarded, and preferentially reused.
   RefPtr<imgFrame> mFirstFrame;
 };
 
+/**
+ * An AnimationFrameRecyclingQueue will only retain up to mBatch * 2 frames.
+ * When the animation advances, it will place the old current frame into a
+ * recycling queue to be reused for a future allocation. This only works for
+ * animated images where we decoded full sized frames into their own buffers,
+ * so that the buffers are all identically sized and contain the complete frame
+ * data.
+ */
+class AnimationFrameRecyclingQueue final : public AnimationFrameDiscardingQueue
+{
+public:
+  explicit AnimationFrameRecyclingQueue(AnimationFrameRetainedBuffer&& aQueue);
+
+  bool MarkComplete(const gfx::IntRect& aFirstFrameRefreshArea) override;
+  void AddSizeOfExcludingThis(MallocSizeOf aMallocSizeOf,
+                              const AddSizeOfCb& aCallback) override;
+
+  RawAccessFrameRef RecycleFrame(gfx::IntRect& aRecycleRect) override;
+
+  struct RecycleEntry {
+    explicit RecycleEntry(const gfx::IntRect &aDirtyRect)
+      : mDirtyRect(aDirtyRect)
+    { }
+
+    RecycleEntry(RecycleEntry&& aOther)
+      : mFrame(std::move(aOther.mFrame))
+      , mDirtyRect(aOther.mDirtyRect)
+      , mRecycleRect(aOther.mRecycleRect)
+    {
+    }
+
+    RecycleEntry& operator=(RecycleEntry&& aOther)
+    {
+      mFrame = std::move(aOther.mFrame);
+      mDirtyRect = aOther.mDirtyRect;
+      mRecycleRect = aOther.mRecycleRect;
+      return *this;
+    }
+
+    RecycleEntry(const RecycleEntry& aOther) = delete;
+    RecycleEntry& operator=(const RecycleEntry& aOther) = delete;
+
+    RefPtr<imgFrame> mFrame;   // The frame containing the buffer to recycle.
+    gfx::IntRect mDirtyRect;   // The dirty rect of the frame itself.
+    gfx::IntRect mRecycleRect; // The dirty rect between the recycled frame and
+                               // the future frame that will be written to it.
+  };
+
+  const std::deque<RecycleEntry>& Recycle() const { return mRecycle; }
+  const gfx::IntRect& FirstFrameRefreshArea() const
+  {
+    return mFirstFrameRefreshArea;
+  }
+
+protected:
+  void AdvanceInternal() override;
+  bool ResetInternal() override;
+
+  /// Queue storing frames to be recycled by the decoder to produce its future
+  /// frames. May contain up to mBatch frames, where the last frame in the queue
+  /// is adjacent to the first frame in the mDisplay queue.
+  std::deque<RecycleEntry> mRecycle;
+
+  /// The first frame refresh area. This is used instead of the dirty rect for
+  /// the last frame when transitioning back to the first frame.
+  gfx::IntRect mFirstFrameRefreshArea;
+};
+
 } // namespace image
 } // namespace mozilla
 
 #endif // mozilla_image_AnimationFrameBuffer_h
--- a/image/AnimationSurfaceProvider.cpp
+++ b/image/AnimationSurfaceProvider.cpp
@@ -1,16 +1,17 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "AnimationSurfaceProvider.h"
 
 #include "gfxPrefs.h"
+#include "mozilla/gfx/gfxVars.h"
 #include "nsProxyRelease.h"
 
 #include "DecodePool.h"
 #include "Decoder.h"
 
 using namespace mozilla::gfx;
 
 namespace mozilla {
@@ -47,16 +48,20 @@ AnimationSurfaceProvider::AnimationSurfa
   size_t batch = gfxPrefs::ImageAnimatedDecodeOnDemandBatchSize();
 
   mFrames.reset(new AnimationFrameRetainedBuffer(threshold, batch, aCurrentFrame));
 }
 
 AnimationSurfaceProvider::~AnimationSurfaceProvider()
 {
   DropImageReference();
+
+  if (mDecoder) {
+    mDecoder->SetFrameRecycler(nullptr);
+  }
 }
 
 void
 AnimationSurfaceProvider::DropImageReference()
 {
   if (!mImage) {
     return;  // Nothing to do.
   }
@@ -392,23 +397,36 @@ AnimationSurfaceProvider::CheckForNewFra
 
 void
 AnimationSurfaceProvider::RequestFrameDiscarding()
 {
   mDecodingMutex.AssertCurrentThreadOwns();
   mFramesMutex.AssertCurrentThreadOwns();
   MOZ_ASSERT(mDecoder);
 
-  if (mFrames->MayDiscard()) {
+  if (mFrames->MayDiscard() || mFrames->IsRecycling()) {
     MOZ_ASSERT_UNREACHABLE("Already replaced frame queue!");
     return;
   }
 
   auto oldFrameQueue = static_cast<AnimationFrameRetainedBuffer*>(mFrames.get());
-  mFrames.reset(new AnimationFrameDiscardingQueue(std::move(*oldFrameQueue)));
+
+  // We only recycle if it is a full frame. Partial frames may be sized
+  // differently from each other. We do not support recycling with WebRender
+  // and shared surfaces at this time as there is additional synchronization
+  // required to know when it is safe to recycle.
+  MOZ_ASSERT(!mDecoder->GetFrameRecycler());
+  if (gfxPrefs::ImageAnimatedDecodeOnDemandRecycle() &&
+      mDecoder->ShouldBlendAnimation() &&
+      (!gfxVars::GetUseWebRenderOrDefault() || !gfxPrefs::ImageMemShared())) {
+    mFrames.reset(new AnimationFrameRecyclingQueue(std::move(*oldFrameQueue)));
+    mDecoder->SetFrameRecycler(this);
+  } else {
+    mFrames.reset(new AnimationFrameDiscardingQueue(std::move(*oldFrameQueue)));
+  }
 }
 
 void
 AnimationSurfaceProvider::AnnounceSurfaceAvailable()
 {
   mFramesMutex.AssertNotCurrentThreadOwns();
   MOZ_ASSERT(mImage);
 
@@ -459,10 +477,18 @@ bool
 AnimationSurfaceProvider::ShouldPreferSyncRun() const
 {
   MutexAutoLock lock(mDecodingMutex);
   MOZ_ASSERT(mDecoder);
 
   return mDecoder->ShouldSyncDecode(gfxPrefs::ImageMemDecodeBytesAtATime());
 }
 
+RawAccessFrameRef
+AnimationSurfaceProvider::RecycleFrame(gfx::IntRect& aRecycleRect)
+{
+  MutexAutoLock lock(mFramesMutex);
+  MOZ_ASSERT(mFrames->IsRecycling());
+  return mFrames->RecycleFrame(aRecycleRect);
+}
+
 } // namespace image
 } // namespace mozilla
--- a/image/AnimationSurfaceProvider.h
+++ b/image/AnimationSurfaceProvider.h
@@ -7,32 +7,34 @@
  * An ISurfaceProvider for animated images.
  */
 
 #ifndef mozilla_image_AnimationSurfaceProvider_h
 #define mozilla_image_AnimationSurfaceProvider_h
 
 #include "mozilla/UniquePtr.h"
 
+#include "Decoder.h"
 #include "FrameAnimator.h"
 #include "IDecodingTask.h"
 #include "ISurfaceProvider.h"
 #include "AnimationFrameBuffer.h"
 
 namespace mozilla {
 namespace image {
 
 /**
  * An ISurfaceProvider that manages the decoding of animated images and
  * dynamically generates surfaces for the current playback state of the
  * animation.
  */
 class AnimationSurfaceProvider final
   : public ISurfaceProvider
   , public IDecodingTask
+  , public IDecoderFrameRecycler
 {
 public:
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AnimationSurfaceProvider, override)
 
   AnimationSurfaceProvider(NotNull<RasterImage*> aImage,
                            const SurfaceKey& aSurfaceKey,
                            NotNull<Decoder*> aDecoder,
                            size_t aCurrentFrame);
@@ -75,16 +77,23 @@ protected:
 public:
   void Run() override;
   bool ShouldPreferSyncRun() const override;
 
   // Full decodes are low priority compared to metadata decodes because they
   // don't block layout or page load.
   TaskPriority Priority() const override { return TaskPriority::eLow; }
 
+  //////////////////////////////////////////////////////////////////////////////
+  // IDecoderFrameRecycler implementation.
+  //////////////////////////////////////////////////////////////////////////////
+
+public:
+  RawAccessFrameRef RecycleFrame(gfx::IntRect& aRecycleRect) override;
+
 private:
   virtual ~AnimationSurfaceProvider();
 
   void DropImageReference();
   void AnnounceSurfaceAvailable();
   void FinishDecoding();
   void RequestFrameDiscarding();
 
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -4637,16 +4637,21 @@ pref("toolkit.zoomManager.zoomValues", "
 // before it starts to discard already displayed frames and redecode them as
 // necessary.
 pref("image.animated.decode-on-demand.threshold-kb", 20480);
 
 // The minimum number of frames we want to have buffered ahead of an
 // animation's currently displayed frame.
 pref("image.animated.decode-on-demand.batch-size", 6);
 
+// Whether we should recycle already displayed frames instead of discarding
+// them. This saves on the allocation itself, and may be able to reuse the
+// contents as well. Only applies if generating full frames.
+pref("image.animated.decode-on-demand.recycle", true);
+
 // Whether we should generate full frames at decode time or partial frames which
 // are combined at display time (historical behavior and default).
 pref("image.animated.generate-full-frames", false);
 
 // Resume an animated image from the last displayed frame rather than
 // advancing when out of view.
 pref("image.animated.resume-from-last-displayed", true);