Bug 1316476 - Limit the amount of device resets we allow with the gpu process. r=dvander
authorRyan Hunt <rhunt@eqrion.net>
Wed, 09 Nov 2016 20:57:04 -0600
changeset 352149 473160a4dbaf251b37d2a02c9aa9b0bae5c0e2b2
parent 352148 28dc1d71a791a56cb27ace782d4f4fa758323d28
child 352150 dbed090b39a49e46086507444c9cdf5984976d18
push id6795
push userjlund@mozilla.com
push dateMon, 23 Jan 2017 14:19:46 +0000
treeherdermozilla-esr52@76101b503191 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdvander
bugs1316476
milestone52.0a1
Bug 1316476 - Limit the amount of device resets we allow with the gpu process. r=dvander
gfx/ipc/GPUProcessManager.cpp
gfx/ipc/GPUProcessManager.h
gfx/thebes/gfxPrefs.h
--- a/gfx/ipc/GPUProcessManager.cpp
+++ b/gfx/ipc/GPUProcessManager.cpp
@@ -56,24 +56,27 @@ GPUProcessManager::Shutdown()
 {
   sSingleton = nullptr;
 }
 
 GPUProcessManager::GPUProcessManager()
  : mTaskFactory(this),
    mNextLayerTreeId(0),
    mNumProcessAttempts(0),
+   mDeviceResetCount(0),
    mProcess(nullptr),
    mGPUChild(nullptr)
 {
   MOZ_COUNT_CTOR(GPUProcessManager);
 
   mObserver = new Observer(this);
   nsContentUtils::RegisterShutdownObserver(mObserver);
 
+  mDeviceResetLastTime = TimeStamp::Now();
+
   LayerTreeOwnerTracker::Initialize();
 }
 
 GPUProcessManager::~GPUProcessManager()
 {
   MOZ_COUNT_DTOR(GPUProcessManager);
 
   LayerTreeOwnerTracker::Shutdown();
@@ -253,34 +256,85 @@ GPUProcessManager::OnProcessLaunchComple
 
   nsTArray<LayerTreeIdMapping> mappings;
   LayerTreeOwnerTracker::Get()->Iterate([&](uint64_t aLayersId, base::ProcessId aProcessId) {
     mappings.AppendElement(LayerTreeIdMapping(aLayersId, aProcessId));
   });
   mGPUChild->SendAddLayerTreeIdMapping(mappings);
 }
 
+static bool
+ShouldLimitDeviceResets(uint32_t count, int32_t deltaMilliseconds)
+{
+  // We decide to limit by comparing the amount of resets that have happened
+  // and time since the last reset to two prefs. 
+  int32_t timeLimit = gfxPrefs::DeviceResetThresholdMilliseconds();
+  int32_t countLimit = gfxPrefs::DeviceResetLimitCount();
+
+  bool hasTimeLimit = timeLimit != -1;
+  bool hasCountLimit = countLimit != -1;
+
+  bool triggeredTime = deltaMilliseconds < timeLimit;
+  bool triggeredCount = count > (uint32_t)countLimit;
+
+  // If we have both prefs set then it needs to trigger both limits,
+  // otherwise we only test the pref that is set or none
+  if (hasTimeLimit && hasCountLimit) {
+    return triggeredTime && triggeredCount;
+  } else if (hasTimeLimit) {
+    return triggeredTime;
+  } else if (hasCountLimit) {
+    return triggeredCount;
+  }
+
+  return false;
+}
+
 void
 GPUProcessManager::OnProcessDeviceReset(GPUProcessHost* aHost)
 {
+  // Detect whether the device is resetting too quickly or too much
+  // indicating that we should give up and use software
+  mDeviceResetCount++;
+
+  auto newTime = TimeStamp::Now();
+  auto delta = (int32_t)(newTime - mDeviceResetLastTime).ToMilliseconds();
+  mDeviceResetLastTime = newTime;
+
+  if (ShouldLimitDeviceResets(mDeviceResetCount, delta)) {
+    DestroyProcess();
+    DisableGPUProcess("GPU processed experienced too many device resets");
+
+    HandleProcessLost();
+    return;
+  }
+
+  // We're good, do a reset like normal
   for (auto& session : mRemoteSessions) {
     session->NotifyDeviceReset();
   }
 }
 
 void
 GPUProcessManager::OnProcessUnexpectedShutdown(GPUProcessHost* aHost)
 {
   MOZ_ASSERT(mProcess && mProcess == aHost);
 
   DestroyProcess();
 
   if (mNumProcessAttempts > uint32_t(gfxPrefs::GPUProcessDevMaxRestarts())) {
     DisableGPUProcess("GPU processed crashed too many times");
   }
+
+  HandleProcessLost();
+}
+
+void
+GPUProcessManager::HandleProcessLost()
+{
   if (gfxConfig::IsEnabled(Feature::GPU_PROCESS)) {
     LaunchGPUProcess();
   }
 
   // The shutdown and restart sequence for the GPU process is as follows:
   //
   //  (1) The GPU process dies. IPDL will enqueue an ActorDestroy message on
   //      each channel owning a bridge to the GPU process, on the thread
--- a/gfx/ipc/GPUProcessManager.h
+++ b/gfx/ipc/GPUProcessManager.h
@@ -171,16 +171,18 @@ private:
 
   // Permanently disable the GPU process and record a message why.
   void DisableGPUProcess(const char* aMessage);
 
   // Shutdown the GPU process.
   void CleanShutdown();
   void DestroyProcess();
 
+  void HandleProcessLost();
+
   void EnsureVsyncIOThread();
   void ShutdownVsyncIOThread();
 
   void EnsureImageBridgeChild();
   void EnsureVRManager();
 
   RefPtr<CompositorSession> CreateRemoteSession(
     nsBaseWidget* aWidget,
@@ -211,16 +213,19 @@ private:
   ipc::TaskFactory<GPUProcessManager> mTaskFactory;
   RefPtr<VsyncIOThreadHolder> mVsyncIOThread;
   uint64_t mNextLayerTreeId;
   uint32_t mNumProcessAttempts;
 
   nsTArray<RefPtr<RemoteCompositorSession>> mRemoteSessions;
   nsTArray<GPUProcessListener*> mListeners;
 
+  uint32_t mDeviceResetCount;
+  TimeStamp mDeviceResetLastTime;
+
   // Fields that are associated with the current GPU process.
   GPUProcessHost* mProcess;
   MOZ_INIT_OUTSIDE_CTOR uint64_t mProcessToken;
   GPUChild* mGPUChild;
   RefPtr<VsyncBridgeChild> mVsyncBridge;
 };
 
 } // namespace gfx
--- a/gfx/thebes/gfxPrefs.h
+++ b/gfx/thebes/gfxPrefs.h
@@ -367,16 +367,19 @@ private:
   DECL_GFX_PREF(Once, "gfx.canvas.skiagl.cache-size",          CanvasSkiaGLCacheSize, int32_t, 96);
   DECL_GFX_PREF(Once, "gfx.canvas.skiagl.dynamic-cache",       CanvasSkiaGLDynamicCache, bool, false);
 
   DECL_GFX_PREF(Live, "gfx.color_management.enablev4",         CMSEnableV4, bool, false);
   DECL_GFX_PREF(Live, "gfx.color_management.mode",             CMSMode, int32_t,-1);
   // The zero default here should match QCMS_INTENT_DEFAULT from qcms.h
   DECL_GFX_PREF(Live, "gfx.color_management.rendering_intent", CMSRenderingIntent, int32_t, 0);
 
+  DECL_GFX_PREF(Once, "gfx.device-reset.limit",                DeviceResetLimitCount, int32_t, 10);
+  DECL_GFX_PREF(Once, "gfx.device-reset.threshold-ms",         DeviceResetThresholdMilliseconds, int32_t, -1);
+
   DECL_GFX_PREF(Once, "gfx.direct2d.disabled",                 Direct2DDisabled, bool, false);
   DECL_GFX_PREF(Once, "gfx.direct2d.force-enabled",            Direct2DForceEnabled, bool, false);
   DECL_GFX_PREF(Live, "gfx.direct3d11.reuse-decoder-device",   Direct3D11ReuseDecoderDevice, int32_t, -1);
   DECL_GFX_PREF(Live, "gfx.draw-color-bars",                   CompositorDrawColorBars, bool, false);
   DECL_GFX_PREF(Once, "gfx.e10s.hide-plugins-for-scroll",      HidePluginsForScroll, bool, true);
   DECL_GFX_PREF(Live, "gfx.gralloc.fence-with-readpixels",     GrallocFenceWithReadPixels, bool, false);
   DECL_GFX_PREF(Live, "gfx.layerscope.enabled",                LayerScopeEnabled, bool, false);
   DECL_GFX_PREF(Live, "gfx.layerscope.port",                   LayerScopePort, int32_t, 23456);