--- a/dom/media/ipc/RemoteAudioDecoder.cpp
+++ b/dom/media/ipc/RemoteAudioDecoder.cpp
@@ -75,18 +75,17 @@ MediaResult RemoteAudioDecoderChild::Ini
 }
 
 RemoteAudioDecoderParent::RemoteAudioDecoderParent(
     RemoteDecoderManagerParent* aParent, const AudioInfo& aAudioInfo,
     const CreateDecoderParams::OptionSet& aOptions,
     TaskQueue* aManagerTaskQueue, TaskQueue* aDecodeTaskQueue, bool* aSuccess,
     nsCString* aErrorDescription)
     : RemoteDecoderParent(aParent, aManagerTaskQueue, aDecodeTaskQueue),
-      mAudioInfo(aAudioInfo),
-      mDecodedFramePool(4) {
+      mAudioInfo(aAudioInfo) {
   CreateDecoderParams params(mAudioInfo);
   params.mTaskQueue = mDecodeTaskQueue;
   params.mOptions = aOptions;
   MediaResult error(NS_OK);
   params.mError = &error;
 
   if (VorbisDataDecoder::IsVorbis(params.mConfig.mMimeType)) {
     mDecoder = new VorbisDataDecoder(params);
@@ -104,70 +103,44 @@ RemoteAudioDecoderParent::RemoteAudioDec
   *aSuccess = !!mDecoder;
 }
 
 MediaResult RemoteAudioDecoderParent::ProcessDecodedData(
     const MediaDataDecoder::DecodedData& aData,
     DecodedOutputIPDL& aDecodedData) {
   MOZ_ASSERT(OnManagerThread());
 
-  // If we are here, we know all previously returned RemoteAudioDataIPDL got
-  // used by the child. We can mark all previously sent ShmemBuffer as available
-  // again.
-  ReleaseUsedShmems();
-
   nsTArray<RemoteAudioDataIPDL> array;
 
   for (const auto& data : aData) {
     MOZ_ASSERT(data->mType == MediaData::Type::AUDIO_DATA,
                "Can only decode audio using RemoteAudioDecoderParent!");
     AudioData* audio = static_cast<AudioData*>(data.get());
 
     MOZ_ASSERT(audio->Data().Elements(),
                "Decoded audio must output an AlignedAudioBuffer "
                "to be used with RemoteAudioDecoderParent");
 
-    ShmemBuffer buffer = mDecodedFramePool.Get(
-        this, audio->Data().Length() * sizeof(AudioDataValue),
-        ShmemPool::AllocationPolicy::Unsafe);
+    ShmemBuffer buffer =
+        AllocateBuffer(audio->Data().Length() * sizeof(AudioDataValue));
     if (!buffer.Valid()) {
       return MediaResult(NS_ERROR_OUT_OF_MEMORY,
                          "ShmemBuffer::Get failed in "
                          "RemoteAudioDecoderParent::ProcessDecodedData");
     }
-    if (audio->Data().Length() > buffer.Get().Size<AudioDataValue>()) {
-      mDecodedFramePool.Put(std::move(buffer));
-      return MediaResult(NS_ERROR_OUT_OF_MEMORY,
-                         "ShmemBuffer::Get returned less than requested in "
-                         "RemoteAudioDecoderParent::ProcessDecodedData");
-    }
 
     PodCopy(buffer.Get().get<AudioDataValue>(), audio->Data().Elements(),
             audio->Data().Length());
 
-    mUsedShmems.AppendElement(buffer.Get());
-
     RemoteAudioDataIPDL output(
         MediaDataIPDL(data->mOffset, data->mTime, data->mTimecode,
                       data->mDuration, data->mKeyframe),
         audio->mChannels, audio->mRate, audio->mChannelMap,
         audio->Data().Length(), std::move(buffer.Get()));
     array.AppendElement(output);
   }
 
   aDecodedData = std::move(array);
 
   return NS_OK;
 }
 
-void RemoteAudioDecoderParent::CleanupOnActorDestroy() {
-  ReleaseUsedShmems();
-  mDecodedFramePool.Cleanup(this);
-}
-
-void RemoteAudioDecoderParent::ReleaseUsedShmems() {
-  for (ShmemBuffer& mem : mUsedShmems) {
-    mDecodedFramePool.Put(ShmemBuffer(mem.Get()));
-  }
-  mUsedShmems.Clear();
-}
-
 }  // namespace mozilla

--- a/dom/media/ipc/RemoteAudioDecoder.h
+++ b/dom/media/ipc/RemoteAudioDecoder.h
@@ -30,25 +30,20 @@ class RemoteAudioDecoderParent final : p
                            const CreateDecoderParams::OptionSet& aOptions,
                            TaskQueue* aManagerTaskQueue,
                            TaskQueue* aDecodeTaskQueue, bool* aSuccess,
                            nsCString* aErrorDescription);
 
  protected:
   MediaResult ProcessDecodedData(const MediaDataDecoder::DecodedData& aData,
                                  DecodedOutputIPDL& aDecodedData) override;
-  void CleanupOnActorDestroy() override;
-
  private:
-  void ReleaseUsedShmems();
   // Can only be accessed from the manager thread
   // Note: we can't keep a reference to the original AudioInfo here
   // because unlike in typical MediaDataDecoder situations, we're being
   // passed a deserialized AudioInfo from RecvPRemoteDecoderConstructor
   // which is destroyed when RecvPRemoteDecoderConstructor returns.
   const AudioInfo mAudioInfo;
-  ShmemPool mDecodedFramePool;
-  AutoTArray<ShmemBuffer, 4> mUsedShmems;
 };
 
 }  // namespace mozilla
 
 #endif  // include_dom_media_ipc_RemoteAudioDecoderChild_h

--- a/dom/media/ipc/RemoteDecoderParent.cpp
+++ b/dom/media/ipc/RemoteDecoderParent.cpp
@@ -12,18 +12,19 @@
 namespace mozilla {
 
 using media::TimeUnit;
 
 RemoteDecoderParent::RemoteDecoderParent(RemoteDecoderManagerParent* aParent,
                                          TaskQueue* aManagerTaskQueue,
                                          TaskQueue* aDecodeTaskQueue)
     : mParent(aParent),
+      mDecodeTaskQueue(aDecodeTaskQueue),
       mManagerTaskQueue(aManagerTaskQueue),
-      mDecodeTaskQueue(aDecodeTaskQueue) {
+      mDecodedFramePool(1, ShmemPool::PoolType::DynamicPool) {
   MOZ_COUNT_CTOR(RemoteDecoderParent);
   MOZ_ASSERT(OnManagerThread());
   // We hold a reference to ourselves to keep us alive until IPDL
   // explictly destroys us. There may still be refs held by
   // tasks, but no new ones should be added after we're
   // destroyed.
   mIPDLSelfRef = this;
 }
@@ -92,16 +93,21 @@ mozilla::ipc::IPCResult RemoteDecoderPar
   data->mEOS = aData.eos();
   data->mDiscardPadding = aData.discardPadding();
 
   RefPtr<RemoteDecoderParent> self = this;
   mDecoder->Decode(data)->Then(
       mManagerTaskQueue, __func__,
       [self, this, resolver = std::move(aResolver)](
           MediaDataDecoder::DecodePromise::ResolveOrRejectValue&& aValue) {
+        // If we are here, we know all previously returned DecodedOutputIPDL got
+        // used by the child. We can mark all previously sent ShmemBuffer as
+        // available again.
+        ReleaseUsedShmems();
+
         if (!self->CanRecv()) {
           // Avoid unnecessarily creating shmem objects later.
           return;
         }
         if (aValue.IsReject()) {
           resolver(aValue.RejectValue());
           return;
         }
@@ -186,16 +192,42 @@ mozilla::ipc::IPCResult RemoteDecoderPar
 }
 
 void RemoteDecoderParent::ActorDestroy(ActorDestroyReason aWhy) {
   MOZ_ASSERT(OnManagerThread());
   if (mDecoder) {
     mDecoder->Shutdown();
     mDecoder = nullptr;
   }
-  CleanupOnActorDestroy();
+  ReleaseUsedShmems();
+  mDecodedFramePool.Cleanup(this);
+}
+
+ShmemBuffer RemoteDecoderParent::AllocateBuffer(size_t aSize) {
+  ShmemBuffer buffer =
+      mDecodedFramePool.Get(this, aSize, ShmemPool::AllocationPolicy::Unsafe);
+  if (!buffer.Valid()) {
+    return buffer;
+  }
+  if (aSize > buffer.Get().Size<uint8_t>()) {
+    ReleaseBuffer(std::move(buffer));
+    return ShmemBuffer();
+  }
+  mUsedShmems.AppendElement(buffer.Get());
+  return buffer;
+}
+
+void RemoteDecoderParent::ReleaseBuffer(ShmemBuffer&& aBuffer) {
+  mDecodedFramePool.Put(std::move(aBuffer));
+}
+
+void RemoteDecoderParent::ReleaseUsedShmems() {
+  for (ShmemBuffer& mem : mUsedShmems) {
+    ReleaseBuffer(ShmemBuffer(mem.Get()));
+  }
+  mUsedShmems.Clear();
 }
 
 bool RemoteDecoderParent::OnManagerThread() {
   return mParent->OnManagerThread();
 }
 
 }  // namespace mozilla

--- a/dom/media/ipc/RemoteDecoderParent.h
+++ b/dom/media/ipc/RemoteDecoderParent.h
@@ -43,20 +43,26 @@ class RemoteDecoderParent : public PRemo
   virtual ~RemoteDecoderParent();
 
   bool OnManagerThread();
   void Error(const MediaResult& aError);
 
   virtual MediaResult ProcessDecodedData(
       const MediaDataDecoder::DecodedData& aDatam,
       DecodedOutputIPDL& aDecodedData) = 0;
-  virtual void CleanupOnActorDestroy() {}
+  ShmemBuffer AllocateBuffer(size_t aLength);
 
-  RefPtr<RemoteDecoderManagerParent> mParent;
+  const RefPtr<RemoteDecoderManagerParent> mParent;
+  const RefPtr<TaskQueue> mDecodeTaskQueue;
+  RefPtr<MediaDataDecoder> mDecoder;
+
+ private:
+  void ReleaseBuffer(ShmemBuffer&& aBuffer);
+  void ReleaseUsedShmems();
   RefPtr<RemoteDecoderParent> mIPDLSelfRef;
-  RefPtr<TaskQueue> mManagerTaskQueue;
-  RefPtr<TaskQueue> mDecodeTaskQueue;
-  RefPtr<MediaDataDecoder> mDecoder;
+  const RefPtr<TaskQueue> mManagerTaskQueue;
+  ShmemPool mDecodedFramePool;
+  AutoTArray<ShmemBuffer, 4> mUsedShmems;
 };
 
 }  // namespace mozilla
 
 #endif  // include_dom_media_ipc_RemoteDecoderParent_h

--- a/dom/media/ipc/RemoteVideoDecoder.cpp
+++ b/dom/media/ipc/RemoteVideoDecoder.cpp
@@ -123,17 +123,17 @@ RefPtr<mozilla::layers::Image> RemoteVid
   bool setData = image->CopyData(pData);
   MOZ_ASSERT(setData);
 
   switch (memOrShmem.type()) {
     case MemoryOrShmem::Tuintptr_t:
       delete[] reinterpret_cast<uint8_t*>(memOrShmem.get_uintptr_t());
       break;
     case MemoryOrShmem::TShmem:
-      DeallocShmem(memOrShmem.get_Shmem());
+      // Memory buffer will be recycled by the parent automatically.
       break;
     default:
       MOZ_ASSERT(false, "Unknown MemoryOrShmem type");
   }
 
   if (!setData) {
     return nullptr;
   }
@@ -350,30 +350,24 @@ MediaResult RemoteVideoDecoderParent::Pr
         sd = mParent->StoreImage(video->mImage, texture);
         size = texture->GetSize();
       }
     } else {
       PlanarYCbCrImage* image =
           static_cast<PlanarYCbCrImage*>(video->mImage.get());
 
       SurfaceDescriptorBuffer sdBuffer;
-      Shmem buffer;
-      if (!AllocShmem(image->GetDataSize(), Shmem::SharedMemory::TYPE_BASIC,
-                      &buffer)) {
+      ShmemBuffer buffer = AllocateBuffer(image->GetDataSize());
+      if (!buffer.Valid()) {
         return MediaResult(NS_ERROR_OUT_OF_MEMORY,
                            "AllocShmem failed in "
                            "RemoteVideoDecoderParent::ProcessDecodedData");
       }
-      if (image->GetDataSize() > buffer.Size<uint8_t>()) {
-        return MediaResult(NS_ERROR_OUT_OF_MEMORY,
-                           "AllocShmem returned less than requested in "
-                           "RemoteVideoDecoderParent::ProcessDecodedData");
-      }
 
-      sdBuffer.data() = std::move(buffer);
+      sdBuffer.data() = std::move(buffer.Get());
       image->BuildSurfaceDescriptorBuffer(sdBuffer);
 
       sd = sdBuffer;
       size = image->GetSize();
     }
 
     RemoteVideoDataIPDL output(
         MediaDataIPDL(data->mOffset, data->mTime, data->mTimecode,