Bug 1572337: Make GetRunningEventDelay handle threadpools r=froydnj
☠☠ backed out by 9671d8079106 ☠ ☠
authorRandell Jesup <rjesup@wgate.com>
Thu, 07 Nov 2019 12:53:32 +0000
changeset 501065 00da7156d3fa29a438c6ad3b7686abc7765ada35
parent 501064 4eda65e054d8ebe2247c25f4c49abb000d5e6646
child 501066 b3510afc9f796da5a0a5fb1a74eb0d4c96d5d494
push id114167
push usercsabou@mozilla.com
push dateFri, 08 Nov 2019 00:35:25 +0000
treeherdermozilla-inbound@ac63c8962183 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1572337
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1572337: Make GetRunningEventDelay handle threadpools r=froydnj Threadpools run an event that then runs other events, so we need to tweak things for GetRunningEventDelay() Differential Revision: https://phabricator.services.mozilla.com/D44058
xpcom/threads/LazyIdleThread.cpp
xpcom/threads/PrioritizedEventQueue.cpp
xpcom/threads/nsIThread.idl
xpcom/threads/nsThread.cpp
xpcom/threads/nsThread.h
xpcom/threads/nsThreadPool.cpp
xpcom/threads/nsThreadPool.h
--- a/xpcom/threads/LazyIdleThread.cpp
+++ b/xpcom/threads/LazyIdleThread.cpp
@@ -401,16 +401,24 @@ LazyIdleThread::GetRunningEventDelay(Tim
     return mThread->GetRunningEventDelay(aDelay, aStart);
   }
   *aDelay = TimeDuration();
   *aStart = TimeStamp();
   return NS_OK;
 }
 
 NS_IMETHODIMP
+LazyIdleThread::SetRunningEventDelay(TimeDuration aDelay, TimeStamp aStart) {
+  if (mThread) {
+    return mThread->SetRunningEventDelay(aDelay, aStart);
+  }
+  return NS_OK;
+}
+
+NS_IMETHODIMP
 LazyIdleThread::IsOnCurrentThread(bool* aIsOnCurrentThread) {
   if (mThread) {
     return mThread->IsOnCurrentThread(aIsOnCurrentThread);
   }
 
   *aIsOnCurrentThread = false;
   return NS_OK;
 }
--- a/xpcom/threads/PrioritizedEventQueue.cpp
+++ b/xpcom/threads/PrioritizedEventQueue.cpp
@@ -142,19 +142,24 @@ EventQueuePriority PrioritizedEventQueue
 
   if (aUpdateState) {
     mProcessHighPriorityQueue = highPending;
   }
 
   return queue;
 }
 
+// The delay returned is the queuing delay a hypothetical Input event would
+// see due to the current running event if it had arrived while the current
+// event was queued.  This means that any event running at  priority below
+// Input doesn't cause queuing delay for Input events, and we return
+// TimeDuration() for those cases.
 already_AddRefed<nsIRunnable> PrioritizedEventQueue::GetEvent(
     EventQueuePriority* aPriority, const MutexAutoLock& aProofOfLock,
-    mozilla::TimeDuration* aLastEventDelay) {
+    mozilla::TimeDuration* aHypotheticalInputEventDelay) {
 #ifndef RELEASE_OR_BETA
   // Clear mNextIdleDeadline so that it is possible to determine that
   // we're running an idle runnable in ProcessNextEvent.
   *mNextIdleDeadline = TimeStamp();
 #endif
 
   EventQueuePriority queue = SelectQueue(true, aProofOfLock);
   auto guard = MakeScopeExit([&] {
@@ -178,76 +183,81 @@ already_AddRefed<nsIRunnable> Prioritize
   // EventQueue).
   nsCOMPtr<nsIRunnable> event;
   switch (queue) {
     default:
       MOZ_CRASH();
       break;
 
     case EventQueuePriority::High:
-      event = mHighQueue->GetEvent(aPriority, aProofOfLock, aLastEventDelay);
+      event = mHighQueue->GetEvent(aPriority, aProofOfLock,
+                                   aHypotheticalInputEventDelay);
       MOZ_ASSERT(event);
       mInputHandlingStartTime = TimeStamp();
       mProcessHighPriorityQueue = false;
       break;
 
     case EventQueuePriority::Input:
-      event = mInputQueue->GetEvent(aPriority, aProofOfLock, aLastEventDelay);
+      event = mInputQueue->GetEvent(aPriority, aProofOfLock,
+                                    aHypotheticalInputEventDelay);
       MOZ_ASSERT(event);
       break;
 
       // All queue priorities below Input don't add their queuing time to the
       // time an input event will be delayed, so report 0 for time-in-queue
       // if we're below Input; input events will only be delayed by the time
       // an event actually runs (if the event is below Input event's priority)
     case EventQueuePriority::MediumHigh:
       event = mMediumHighQueue->GetEvent(aPriority, aProofOfLock);
-      *aLastEventDelay = TimeDuration();
+      *aHypotheticalInputEventDelay = TimeDuration();
       break;
 
     case EventQueuePriority::Normal:
       event = mNormalQueue->GetEvent(aPriority, aProofOfLock);
-      *aLastEventDelay = TimeDuration();
+      *aHypotheticalInputEventDelay = TimeDuration();
       break;
 
     case EventQueuePriority::Idle:
     case EventQueuePriority::DeferredTimers:
-      *aLastEventDelay = TimeDuration();
+      *aHypotheticalInputEventDelay = TimeDuration();
       // If we get here, then all queues except deferredtimers and idle are
       // empty.
 
       if (mIdleQueue->IsEmpty(aProofOfLock) &&
           mDeferredTimersQueue->IsEmpty(aProofOfLock)) {
         mIdlePeriodState.RanOutOfTasks(*mMutex);
         return nullptr;
       }
 
       TimeStamp idleDeadline = mIdlePeriodState.GetDeadlineForIdleTask(*mMutex);
       if (!idleDeadline) {
         return nullptr;
       }
 
-      nsCOMPtr<nsIRunnable> event =
-          mDeferredTimersQueue->GetEvent(aPriority, aProofOfLock);
+      event = mDeferredTimersQueue->GetEvent(aPriority, aProofOfLock);
       if (!event) {
         event = mIdleQueue->GetEvent(aPriority, aProofOfLock);
       }
       if (event) {
         nsCOMPtr<nsIIdleRunnable> idleEvent = do_QueryInterface(event);
         if (idleEvent) {
           idleEvent->SetDeadline(idleDeadline);
         }
 
 #ifndef RELEASE_OR_BETA
         // Store the next idle deadline to be able to determine budget use
         // in ProcessNextEvent.
         *mNextIdleDeadline = idleDeadline;
 #endif
       }
       break;
+  }  // switch (queue)
+
+  if (!event) {
+    *aHypotheticalInputEventDelay = TimeDuration();
   }
 
   return event.forget();
 }
 
 void PrioritizedEventQueue::DidRunEvent(const MutexAutoLock& aProofOfLock) {
   if (IsEmpty(aProofOfLock)) {
     // Certainly no more idle tasks.
--- a/xpcom/threads/nsIThread.idl
+++ b/xpcom/threads/nsIThread.idl
@@ -191,10 +191,23 @@ interface nsIThread : nsISerialEventTarg
    *   TimeDuration() if this thread uses a PrioritizedEventQueue (i.e. MainThread)
    *   and the event priority is below Input.
    * @param start
    *   The time the currently running event began to run, or TimeStamp() if no
    *   event is running.
    */
   [noscript] void getRunningEventDelay(out TimeDuration delay, out TimeStamp start);
 
+  /**
+   * Set information on the timing of the currently-running event.
+   * Overrides the values returned by getRunningEventDelay
+   *
+   * @param delay
+   *   Delay the running event spent in queues, or TimeDuration() if
+   *   there's no running event.
+   * @param start
+   *   The time the currently running event began to run, or TimeStamp() if no
+   *   event is running.
+   */
+  [noscript] void setRunningEventDelay(in TimeDuration delay, in TimeStamp start);
+
   [noscript] void setNameForWakeupTelemetry(in ACString name);
 };
--- a/xpcom/threads/nsThread.cpp
+++ b/xpcom/threads/nsThread.cpp
@@ -596,16 +596,17 @@ nsThread::nsThread(NotNull<SynchronizedE
       mScriptObserver(nullptr),
       mThread(nullptr),
       mStackSize(aStackSize),
       mNestedEventLoopDepth(0),
       mCurrentEventLoopDepth(MaxValue<uint32_t>::value),
       mShutdownRequired(false),
       mPriority(PRIORITY_NORMAL),
       mIsMainThread(aMainThread == MAIN_THREAD),
+      mIsAPoolThreadFree(nullptr),
       mCanInvokeJS(false),
       mCurrentEvent(nullptr),
       mCurrentEventStart(TimeStamp::Now()),
 #ifdef EARLY_BETA_OR_EARLIER
       mLastWakeupCheckTime(mCurrentEventStart),
 #endif
       mCurrentPerformanceCounter(nullptr) {
   mLastLongTaskEnd = mCurrentEventStart;
@@ -728,18 +729,32 @@ nsThread::DelayedDispatch(already_AddRef
   MOZ_ASSERT(mEventTarget);
   NS_ENSURE_TRUE(mEventTarget, NS_ERROR_NOT_IMPLEMENTED);
 
   return mEventTarget->DelayedDispatch(std::move(aEvent), aDelayMs);
 }
 
 NS_IMETHODIMP
 nsThread::GetRunningEventDelay(TimeDuration* aDelay, TimeStamp* aStart) {
-  *aDelay = mLastEventDelay;
-  *aStart = mLastEventStart;
+  if (mIsAPoolThreadFree && *mIsAPoolThreadFree) {
+    // if there are unstarted threads in the pool, a new event to the
+    // pool would not be delayed at all (beyond thread start time)
+    *aDelay = TimeDuration();
+    *aStart = TimeStamp();
+  } else {
+    *aDelay = mLastEventDelay;
+    *aStart = mLastEventStart;
+  }
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsThread::SetRunningEventDelay(TimeDuration aDelay, TimeStamp aStart) {
+  mLastEventDelay = aDelay;
+  mLastEventStart = aStart;
   return NS_OK;
 }
 
 NS_IMETHODIMP
 nsThread::IsOnCurrentThread(bool* aResult) {
   if (mEventTarget) {
     return mEventTarget->IsOnCurrentThread(aResult);
   }
--- a/xpcom/threads/nsThread.h
+++ b/xpcom/threads/nsThread.h
@@ -9,16 +9,17 @@
 
 #include "mozilla/Mutex.h"
 #include "nsIIdlePeriod.h"
 #include "nsIThreadInternal.h"
 #include "nsISupportsPriority.h"
 #include "nsThreadUtils.h"
 #include "nsString.h"
 #include "nsTObserverArray.h"
+#include "mozilla/Atomics.h"
 #include "mozilla/Attributes.h"
 #include "mozilla/LinkedList.h"
 #include "mozilla/MemoryReporting.h"
 #include "mozilla/SynchronizedEventQueue.h"
 #include "mozilla/NotNull.h"
 #include "mozilla/TimeStamp.h"
 #include "nsAutoPtr.h"
 #include "mozilla/AlreadyAddRefed.h"
@@ -84,16 +85,22 @@ class nsThread : public nsIThreadInterna
   size_t StackSize() const { return mStackSize; }
 
   uint32_t ThreadId() const { return mThreadId; }
 
   // If this flag is true, then the nsThread was created using
   // nsIThreadManager::NewThread.
   bool ShutdownRequired() { return mShutdownRequired; }
 
+  // Lets GetRunningEventDelay() determine if the pool this is part
+  // of has an unstarted thread
+  void SetPoolThreadFreePtr(mozilla::Atomic<bool, mozilla::Relaxed>* aPtr) {
+    mIsAPoolThreadFree = aPtr;
+  }
+
   void SetScriptObserver(mozilla::CycleCollectedJSContext* aScriptObserver);
 
   uint32_t RecursionDepth() const;
 
   void ShutdownComplete(NotNull<struct nsThreadShutdownContext*> aContext);
 
   void WaitForAllAsynchronousShutdowns();
 
@@ -223,16 +230,17 @@ class nsThread : public nsIThreadInterna
   mozilla::TimeStamp mLastLongTaskEnd;
   mozilla::TimeStamp mLastLongNonIdleTaskEnd;
 
   mozilla::Atomic<bool> mShutdownRequired;
 
   int8_t mPriority;
 
   bool mIsMainThread;
+  mozilla::Atomic<bool, mozilla::Relaxed>* mIsAPoolThreadFree;
 
   // Set to true if this thread creates a JSRuntime.
   bool mCanInvokeJS;
 
   bool mHasTLSEntry = false;
 
   // Used to track which event is being executed by ProcessNextEvent
   nsCOMPtr<nsIRunnable> mCurrentEvent;
--- a/xpcom/threads/nsThreadPool.cpp
+++ b/xpcom/threads/nsThreadPool.cpp
@@ -51,17 +51,18 @@ nsThreadPool::nsThreadPool()
     : mMutex("[nsThreadPool.mMutex]"),
       mEventsAvailable(mMutex, "[nsThreadPool.mEventsAvailable]"),
       mThreadLimit(DEFAULT_THREAD_LIMIT),
       mIdleThreadLimit(DEFAULT_IDLE_THREAD_LIMIT),
       mIdleThreadTimeout(DEFAULT_IDLE_THREAD_TIMEOUT),
       mIdleCount(0),
       mStackSize(nsIThreadManager::DEFAULT_STACK_SIZE),
       mShutdown(false),
-      mRegressiveMaxIdleTime(false) {
+      mRegressiveMaxIdleTime(false),
+      mIsAPoolThreadFree(true) {
   static std::once_flag flag;
   std::call_once(flag, [] { gCurrentThreadPool.infallibleInit(); });
 
   LOG(("THRD-P(%p) constructor!!!\n", this));
 }
 
 nsThreadPool::~nsThreadPool() {
   // Threads keep a reference to the nsThreadPool until they return from Run()
@@ -123,17 +124,21 @@ nsresult nsThreadPool::PutEvent(already_
 
   bool killThread = false;
   {
     MutexAutoLock lock(mMutex);
     if (mShutdown) {
       killThread = true;
     } else if (mThreads.Count() < (int32_t)mThreadLimit) {
       mThreads.AppendObject(thread);
+      if (mThreads.Count() >= (int32_t)mThreadLimit) {
+        mIsAPoolThreadFree = false;
+      }
     } else {
+      // Someone else may have also been starting a thread
       killThread = true;  // okay, we don't need this thread anymore
     }
   }
   LOG(("THRD-P(%p) put [%p kill=%d]\n", this, thread.get(), killThread));
   if (killThread) {
     // We never dispatched any events to the thread, so we can shut it down
     // asynchronously without worrying about anything.
     ShutdownThread(thread);
@@ -157,47 +162,81 @@ void nsThreadPool::ShutdownThread(nsIThr
   // shutdown requires this thread have an event loop (and it may not, see bug
   // 10204784).  The simplest way to cover all cases is to asynchronously
   // shutdown aThread from the main thread.
   SystemGroup::Dispatch(TaskCategory::Other,
                         NewRunnableMethod("nsIThread::AsyncShutdown", aThread,
                                           &nsIThread::AsyncShutdown));
 }
 
+// This event 'runs' for the lifetime of the worker thread.  The actual
+// eventqueue is mEvents, and is shared by all the worker threads.  This
+// means that the set of threads together define the delay seen by a new
+// event sent to the pool.
+//
+// To model the delay experienced by the pool, we can have each thread in
+// the pool report 0 if it's idle OR if the pool is below the threadlimit;
+// or otherwise the current event's queuing delay plus current running
+// time.
+//
+// To reconstruct the delays for the pool, the profiler can look at all the
+// threads that are part of a pool (pools have defined naming patterns that
+// can be user to connect them).  If all threads have delays at time X,
+// that means that all threads saturated at that point and any event
+// dispatched to the pool would get a delay.
+//
+// The delay experienced by an event dispatched when all pool threads are
+// busy is based on the calculations shown in platform.cpp.  Run that
+// algorithm for each thread in the pool, and the delay at time X is the
+// longest value for time X of any of the threads, OR the time from X until
+// any one of the threads reports 0 (i.e. it's not busy), whichever is
+// shorter.
+
+// In order to record this when the profiler samples threads in the pool,
+// each thread must (effectively) override GetRunnningEventDelay, by
+// resetting the mLastEventDelay/Start values in the nsThread when we start
+// to run an event (or when we run out of events to run).  Note that handling
+// the shutdown of a thread may be a little tricky.
+
 NS_IMETHODIMP
 nsThreadPool::Run() {
   LOG(("THRD-P(%p) enter %s\n", this, mName.BeginReading()));
 
   nsCOMPtr<nsIThread> current;
   nsThreadManager::get().GetCurrentThread(getter_AddRefs(current));
 
   bool shutdownThreadOnExit = false;
   bool exitThread = false;
   bool wasIdle = false;
   TimeStamp idleSince;
 
+  // This thread is an nsThread created below with NS_NewNamedThread()
+  static_cast<nsThread*>(current.get())
+      ->SetPoolThreadFreePtr(&mIsAPoolThreadFree);
+
   nsCOMPtr<nsIThreadPoolListener> listener;
   {
     MutexAutoLock lock(mMutex);
     listener = mListener;
   }
 
   if (listener) {
     listener->OnThreadCreated();
   }
 
   MOZ_ASSERT(!gCurrentThreadPool.get());
   gCurrentThreadPool.set(this);
 
   do {
     nsCOMPtr<nsIRunnable> event;
+    TimeDuration delay;
     {
       MutexAutoLock lock(mMutex);
 
-      event = mEvents.GetEvent(nullptr, lock);
+      event = mEvents.GetEvent(nullptr, lock, &delay);
       if (!event) {
         TimeStamp now = TimeStamp::Now();
         uint32_t idleTimeoutDivider =
             (mIdleCount && mRegressiveMaxIdleTime) ? mIdleCount : 1;
         TimeDuration timeout = TimeDuration::FromMilliseconds(
             static_cast<double>(mIdleThreadTimeout) / idleTimeoutDivider);
 
         // If we are shutting down, then don't keep any idle threads
@@ -223,17 +262,22 @@ nsThreadPool::Run() {
           }
         }
 
         if (exitThread) {
           if (wasIdle) {
             --mIdleCount;
           }
           shutdownThreadOnExit = mThreads.RemoveObject(current);
+
+          // keep track if there are threads available to start
+          mIsAPoolThreadFree = (mThreads.Count() < (int32_t)mThreadLimit);
         } else {
+          current->SetRunningEventDelay(TimeDuration(), TimeStamp());
+
           AUTO_PROFILER_LABEL("nsThreadPool::Run::Wait", IDLE);
 
           TimeDuration delta = timeout - (now - idleSince);
           LOG(("THRD-P(%p) %s waiting [%f]\n", this, mName.BeginReading(),
                delta.ToMilliseconds()));
           {
             AUTO_PROFILER_THREAD_SLEEP;
             mEventsAvailable.Wait(delta);
@@ -248,16 +292,20 @@ nsThreadPool::Run() {
     if (event) {
       LOG(("THRD-P(%p) %s running [%p]\n", this, mName.BeginReading(),
            event.get()));
 
       // Delay event processing to encourage whoever dispatched this event
       // to run.
       DelayForChaosMode(ChaosFeature::TaskRunning, 1000);
 
+      // We'll handle the case of unstarted threads available
+      // when we sample.
+      current->SetRunningEventDelay(delay, TimeStamp::Now());
+
       event->Run();
     }
   } while (!exitThread);
 
   if (listener) {
     listener->OnThreadShuttingDown();
   }
 
--- a/xpcom/threads/nsThreadPool.h
+++ b/xpcom/threads/nsThreadPool.h
@@ -8,16 +8,17 @@
 #define nsThreadPool_h__
 
 #include "nsIThreadPool.h"
 #include "nsIThread.h"
 #include "nsIRunnable.h"
 #include "nsCOMArray.h"
 #include "nsCOMPtr.h"
 #include "nsThreadUtils.h"
+#include "mozilla/Atomics.h"
 #include "mozilla/Attributes.h"
 #include "mozilla/AlreadyAddRefed.h"
 #include "mozilla/EventQueue.h"
 #include "mozilla/Mutex.h"
 #include "mozilla/Monitor.h"
 
 class nsThreadPool final : public nsIThreadPool, public nsIRunnable {
  public:
@@ -42,16 +43,17 @@ class nsThreadPool final : public nsIThr
   uint32_t mThreadLimit;
   uint32_t mIdleThreadLimit;
   uint32_t mIdleThreadTimeout;
   uint32_t mIdleCount;
   uint32_t mStackSize;
   nsCOMPtr<nsIThreadPoolListener> mListener;
   bool mShutdown;
   bool mRegressiveMaxIdleTime;
+  mozilla::Atomic<bool, mozilla::Relaxed> mIsAPoolThreadFree;
   nsCString mName;
   nsThreadPoolNaming mThreadNaming;
 };
 
 #define NS_THREADPOOL_CID                            \
   { /* 547ec2a8-315e-4ec4-888e-6e4264fe90eb */       \
     0x547ec2a8, 0x315e, 0x4ec4, {                    \
       0x88, 0x8e, 0x6e, 0x42, 0x64, 0xfe, 0x90, 0xeb \