Bug 1518193 - Use new free task to also free JIT LIFO data freed after minor GC r=sfink
authorJon Coppeard <jcoppeard@mozilla.com>
Thu, 10 Jan 2019 11:00:20 +0000
changeset 453217 7d9e12dcfe7f70e64ea9126eeee3e3627d80a796
parent 453216 1bdbec17ea7c84775814586a6b21536c6f12d7c2
child 453218 d2a84a3dcae072a3ea765345c35aaaa92842860b
push id111060
push userjcoppeard@mozilla.com
push dateThu, 10 Jan 2019 11:00:42 +0000
treeherdermozilla-inbound@3922da7f8c51 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfink
bugs1518193
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1518193 - Use new free task to also free JIT LIFO data freed after minor GC r=sfink
js/src/gc/GC.cpp
js/src/gc/GCRuntime.h
js/src/gc/Zone.cpp
js/src/jit/BaselineJIT.cpp
js/src/vm/Stack-inl.h
js/src/vm/TypeInference.cpp
--- a/js/src/gc/GC.cpp
+++ b/js/src/gc/GC.cpp
@@ -923,17 +923,18 @@ GCRuntime::GCRuntime(JSRuntime* rt)
       incrementalState(gc::State::NotActive),
       initialState(gc::State::NotActive),
 #ifdef JS_GC_ZEAL
       useZeal(false),
 #endif
       lastMarkSlice(false),
       safeToYield(true),
       sweepOnBackgroundThread(false),
-      blocksToFreeAfterSweeping(
+      lifoBlocksToFree((size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
+      lifoBlocksToFreeAfterMinorGC(
           (size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
       sweepGroupIndex(0),
       sweepGroups(nullptr),
       currentSweepGroup(nullptr),
       sweepZone(nullptr),
       hasMarkedGrayRoots(false),
       abortSweepAfterCurrentGroup(false),
       startedCompacting(false),
@@ -959,19 +960,17 @@ GCRuntime::GCRuntime(JSRuntime* rt)
       arenasEmptyAtShutdown(true),
 #endif
       lock(mutexid::GCLock),
       allocTask(rt, emptyChunks_.ref()),
       sweepTask(rt),
       freeTask(rt),
       decommitTask(rt),
       nursery_(rt),
-      storeBuffer_(rt, nursery()),
-      blocksToFreeAfterMinorGC(
-          (size_t)JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE) {
+      storeBuffer_(rt, nursery()) {
   setGCMode(JSGC_MODE_GLOBAL);
 }
 
 #ifdef JS_GC_ZEAL
 
 void GCRuntime::getZealBits(uint32_t* zealBits, uint32_t* frequency,
                             uint32_t* scheduled) {
   *zealBits = zealModeBits;
@@ -2901,17 +2900,17 @@ void GCRuntime::updateRuntimePointersToR
   jit::JitRuntime::SweepJitcodeGlobalTable(rt);
   for (JS::detail::WeakCacheBase* cache : rt->weakCaches()) {
     cache->sweep();
   }
 
   // Type inference may put more blocks here to free.
   {
     AutoLockHelperThreadState lock;
-    blocksToFreeAfterSweeping.ref().freeAll();
+    lifoBlocksToFree.ref().freeAll();
   }
 
   // Call callbacks to get the rest of the system to fixup other untraced
   // pointers.
   callWeakPointerZonesCallbacks();
 }
 
 void GCRuntime::protectAndHoldArenas(Arena* arenaList) {
@@ -3537,18 +3536,16 @@ void GCRuntime::sweepBackgroundThings(Zo
   }
 }
 
 void GCRuntime::assertBackgroundSweepingFinished() {
 #ifdef DEBUG
   {
     AutoLockHelperThreadState lock;
     MOZ_ASSERT(backgroundSweepZones.ref().isEmpty());
-    MOZ_ASSERT(blocksToFreeAfterSweeping.ref().computedSizeOfExcludingThis() ==
-               0);
   }
 
   for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
     for (auto i : AllAllocKinds()) {
       MOZ_ASSERT(!zone->arenas.arenaListsToSweep(i));
       MOZ_ASSERT(zone->arenas.doneBackgroundFinalize(i));
     }
   }
@@ -3582,17 +3579,17 @@ void BackgroundSweepTask::run() {
   setFinishing(lock);
 }
 
 void GCRuntime::sweepFromBackgroundThread(AutoLockHelperThreadState& lock) {
   do {
     ZoneList zones;
     zones.transferFrom(backgroundSweepZones.ref());
     LifoAlloc freeLifoAlloc(JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE);
-    freeLifoAlloc.transferFrom(&blocksToFreeAfterSweeping.ref());
+    freeLifoAlloc.transferFrom(&lifoBlocksToFree.ref());
 
     AutoUnlockHelperThreadState unlock(lock);
     sweepBackgroundThings(zones, freeLifoAlloc);
 
     // The main thread may call queueZonesAndStartBackgroundSweep() while this is
     // running so we must check there is no more work after releasing the
     // lock.
   } while (!backgroundSweepZones.ref().isEmpty());
@@ -3602,30 +3599,30 @@ void GCRuntime::waitBackgroundSweepEnd()
   sweepTask.join();
 
   // TODO: Improve assertion to work in incremental GC?
   if (!isIncrementalGCInProgress()) {
     assertBackgroundSweepingFinished();
   }
 }
 
-void GCRuntime::freeUnusedLifoBlocksAfterSweeping(LifoAlloc* lifo) {
+void GCRuntime::queueUnusedLifoBlocksForFree(LifoAlloc* lifo) {
   MOZ_ASSERT(JS::RuntimeHeapIsBusy());
   AutoLockHelperThreadState lock;
-  blocksToFreeAfterSweeping.ref().transferUnusedFrom(lifo);
-}
-
-void GCRuntime::freeAllLifoBlocksAfterSweeping(LifoAlloc* lifo) {
+  lifoBlocksToFree.ref().transferUnusedFrom(lifo);
+}
+
+void GCRuntime::queueAllLifoBlocksForFree(LifoAlloc* lifo) {
   MOZ_ASSERT(JS::RuntimeHeapIsBusy());
   AutoLockHelperThreadState lock;
-  blocksToFreeAfterSweeping.ref().transferFrom(lifo);
-}
-
-void GCRuntime::freeAllLifoBlocksAfterMinorGC(LifoAlloc* lifo) {
-  blocksToFreeAfterMinorGC.ref().transferFrom(lifo);
+  lifoBlocksToFree.ref().transferFrom(lifo);
+}
+
+void GCRuntime::queueAllLifoBlocksForFreeAfterMinorGC(LifoAlloc* lifo) {
+  lifoBlocksToFreeAfterMinorGC.ref().transferFrom(lifo);
 }
 
 void GCRuntime::startBackgroundFree() {
   if (CanUseExtraThreads()) {
     AutoLockHelperThreadState lock;
     freeTask.startOrRunIfIdle(lock);
   } else {
     freeTask.joinAndRunFromMainThread(rt);
@@ -3643,22 +3640,22 @@ void BackgroundFreeTask::run() {
   // Signal to the main thread that we're about to finish, because we release
   // the lock again before GCParallelTask's state is changed to finished.
   setFinishing(lock);
 }
 
 void GCRuntime::freeFromBackgroundThread(AutoLockHelperThreadState& lock) {
   do {
     LifoAlloc lifoBlocks(JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE);
-    lifoBlocks.transferFrom(&blocksToFreeAfterSweeping.ref());
+    lifoBlocks.transferFrom(&lifoBlocksToFree.ref());
 
     AutoUnlockHelperThreadState unlock(lock);
 
     lifoBlocks.freeAll();
-  } while (!blocksToFreeAfterSweeping.ref().isEmpty());
+  } while (!lifoBlocksToFree.ref().isEmpty());
 }
 
 struct IsAboutToBeFinalizedFunctor {
   template <typename T>
   bool operator()(Cell** t) {
     mozilla::DebugOnly<const Cell*> prior = *t;
     bool result = IsAboutToBeFinalizedUnbarriered(reinterpret_cast<T**>(t));
     // Sweep should not have to deal with moved pointers, since moving GC
@@ -3916,17 +3913,17 @@ void GCRuntime::purgeRuntime() {
 
   for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
     zone->purgeAtomCacheOrDefer();
     zone->externalStringCache().purge();
     zone->functionToStringCache().purge();
   }
 
   JSContext* cx = rt->mainContextFromOwnThread();
-  freeUnusedLifoBlocksAfterSweeping(&cx->tempLifoAlloc());
+  queueUnusedLifoBlocksForFree(&cx->tempLifoAlloc());
   cx->interpreterStack().purge(rt);
   cx->frontendCollectionPool().purge();
 
   rt->caches().purge();
 
   if (auto cache = rt->maybeThisRuntimeSharedImmutableStrings()) {
     cache->purge();
   }
@@ -4304,16 +4301,17 @@ bool GCRuntime::beginMarkPhase(JS::gcrea
                               gcstats::PhaseKind::BUFFER_GRAY_ROOTS,
                               helperLock);
     }
     AutoUnlockHelperThreadState unlock(helperLock);
 
     // Discard JIT code. For incremental collections, the sweep phase will
     // also discard JIT code.
     DiscardJITCodeForGC(rt);
+    freeQueuedLifoBlocksAfterMinorGC();
 
     /*
      * Relazify functions after discarding JIT code (we can't relazify
      * functions with JIT code) and before the actual mark phase, so that
      * the current GC can collect the JSScripts we're unlinking here.  We do
      * this only when we're performing a shrinking GC, as too much
      * relazification can cause performance issues when we have to reparse
      * the same functions over and over.
@@ -5805,19 +5803,16 @@ void GCRuntime::beginSweepPhase(JS::gcre
   AutoSetThreadIsSweeping threadIsSweeping;
 
   releaseHeldRelocatedArenas();
 
   computeNonIncrementalMarkingForValidation(session);
 
   gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP);
 
-  sweepOnBackgroundThread = reason != JS::gcreason::DESTROY_RUNTIME &&
-                            !gcTracer.traceEnabled() && CanUseExtraThreads();
-
   hasMarkedGrayRoots = false;
 
   AssertNoWrappersInGrayList(rt);
   DropStringWrappers(rt);
 
   groupZonesForSweeping(reason);
 
   sweepActions->assertFinished();
@@ -6619,16 +6614,17 @@ void GCRuntime::endSweepPhase(bool destr
   }
 #endif
 
   AssertNoWrappersInGrayList(rt);
 }
 
 void GCRuntime::beginCompactPhase() {
   MOZ_ASSERT(!isBackgroundSweeping());
+  assertBackgroundSweepingFinished();
 
   gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::COMPACT);
 
   MOZ_ASSERT(zonesToMaybeCompact.ref().isEmpty());
   for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
     if (CanRelocateZone(zone)) {
       zonesToMaybeCompact.ref().append(zone);
     }
@@ -6797,17 +6793,17 @@ GCRuntime::IncrementalResult GCRuntime::
       for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
         zone->setNeedsIncrementalBarrier(false);
         zone->changeGCState(Zone::MarkBlackOnly, Zone::NoGC);
         zone->arenas.unmarkPreMarkedFreeCells();
       }
 
       {
         AutoLockHelperThreadState lock;
-        blocksToFreeAfterSweeping.ref().freeAll();
+        lifoBlocksToFree.ref().freeAll();
       }
 
       lastMarkSlice = false;
       incrementalState = State::Finish;
 
       MOZ_ASSERT(!marker.shouldCheckCompartments());
 
       break;
@@ -6909,16 +6905,21 @@ static bool IsShutdownGC(JS::gcreason::R
 static bool ShouldCleanUpEverything(JS::gcreason::Reason reason,
                                     JSGCInvocationKind gckind) {
   // During shutdown, we must clean everything up, for the sake of leak
   // detection. When a runtime has no contexts, or we're doing a GC before a
   // shutdown CC, those are strong indications that we're shutting down.
   return IsShutdownGC(reason) || gckind == GC_SHRINK;
 }
 
+static bool ShouldSweepOnBackgroundThread(JS::gcreason::Reason reason) {
+  return reason != JS::gcreason::DESTROY_RUNTIME &&
+         !gcTracer.traceEnabled() && CanUseExtraThreads();
+}
+
 void GCRuntime::incrementalSlice(SliceBudget& budget,
                                  JS::gcreason::Reason reason,
                                  AutoGCSession& session) {
   AutoDisableBarriers disableBarriers(rt);
 
   bool destroyingRuntime = (reason == JS::gcreason::DESTROY_RUNTIME);
 
   number++;
@@ -6966,16 +6967,17 @@ void GCRuntime::incrementalSlice(SliceBu
     budget.makeUnlimited();
   }
 
   switch (incrementalState) {
     case State::NotActive:
       incMajorGcNumber();
       initialReason = reason;
       cleanUpEverything = ShouldCleanUpEverything(reason, invocationKind);
+      sweepOnBackgroundThread = ShouldSweepOnBackgroundThread(reason);
       isCompacting = shouldCompact();
       MOZ_ASSERT(!lastMarkSlice);
       rootsRemoved = false;
 
       incrementalState = State::MarkRoots;
 
       MOZ_FALLTHROUGH;
 
@@ -7779,32 +7781,46 @@ void GCRuntime::minorGC(JS::gcreason::Re
   gcstats::AutoPhase ap(stats(), phase);
 
   nursery().clearMinorGCRequest();
   TraceLoggerThread* logger = TraceLoggerForCurrentThread();
   AutoTraceLog logMinorGC(logger, TraceLogger_MinorGC);
   nursery().collect(reason);
   MOZ_ASSERT(nursery().isEmpty());
 
-  blocksToFreeAfterMinorGC.ref().freeAll();
+  freeQueuedLifoBlocksAfterMinorGC();
 
 #ifdef JS_GC_ZEAL
   if (hasZealMode(ZealMode::CheckHeapAfterGC)) {
     CheckHeapAfterGC(rt);
   }
 #endif
 
   {
     AutoLockGC lock(rt);
     for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
       maybeAllocTriggerZoneGC(zone, lock);
     }
   }
 }
 
+void GCRuntime::freeQueuedLifoBlocksAfterMinorGC() {
+  MOZ_ASSERT(nursery().isEmpty());
+  if (lifoBlocksToFreeAfterMinorGC.ref().isEmpty()) {
+    return;
+  }
+
+  {
+    AutoLockHelperThreadState lock;
+    lifoBlocksToFree.ref().transferFrom(&lifoBlocksToFreeAfterMinorGC.ref());
+  }
+
+  startBackgroundFree();
+}
+
 JS::AutoDisableGenerationalGC::AutoDisableGenerationalGC(JSContext* cx)
     : cx(cx) {
   if (!cx->generationalDisabled) {
     cx->runtime()->gc.evictNursery(JS::gcreason::API);
     cx->nursery().disable();
   }
   ++cx->generationalDisabled;
 }
--- a/js/src/gc/GCRuntime.h
+++ b/js/src/gc/GCRuntime.h
@@ -465,19 +465,20 @@ class GCRuntime {
   void endVerifyPreBarriers();
   void finishVerifier();
   bool isVerifyPreBarriersEnabled() const { return !!verifyPreData; }
   bool shouldYieldForZeal(ZealMode mode);
 #else
   bool isVerifyPreBarriersEnabled() const { return false; }
 #endif
 
-  // Free certain LifoAlloc blocks when it is safe to do so.
-  void freeUnusedLifoBlocksAfterSweeping(LifoAlloc* lifo);
-  void freeAllLifoBlocksAfterSweeping(LifoAlloc* lifo);
+  // Free certain LifoAlloc blocks on a background thread if possible.
+  void queueUnusedLifoBlocksForFree(LifoAlloc* lifo);
+  void queueAllLifoBlocksForFree(LifoAlloc* lifo);
+  void queueAllLifoBlocksForFreeAfterMinorGC(LifoAlloc* lifo);
 
   // Public here for ReleaseArenaLists and FinalizeTypedArenas.
   void releaseArena(Arena* arena, const AutoLockGC& lock);
 
   void releaseHeldRelocatedArenas();
   void releaseHeldRelocatedArenasWithoutUnlocking(const AutoLockGC& lock);
 
   // Allocator
@@ -591,16 +592,17 @@ class GCRuntime {
   void purgeRuntime();
   MOZ_MUST_USE bool beginMarkPhase(JS::gcreason::Reason reason,
                                    AutoGCSession& session);
   bool prepareZonesForCollection(JS::gcreason::Reason reason, bool* isFullOut);
   bool shouldPreserveJITCode(JS::Realm* realm,
                              const mozilla::TimeStamp& currentTime,
                              JS::gcreason::Reason reason,
                              bool canAllocateMoreCode);
+  void freeQueuedLifoBlocksAfterMinorGC();
   void traceRuntimeForMajorGC(JSTracer* trc, AutoGCSession& session);
   void traceRuntimeAtoms(JSTracer* trc, const AutoAccessAtomsZone& atomsAccess);
   void traceKeptAtoms(JSTracer* trc);
   void traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrMark);
   void maybeDoCycleCollection();
   void markCompartments();
   IncrementalProgress markUntilBudgetExhausted(SliceBudget& sliceBudget,
                                                gcstats::PhaseKind phase);
@@ -854,20 +856,21 @@ class GCRuntime {
 
   /* Whether any sweeping will take place in the separate GC helper thread. */
   MainThreadData<bool> sweepOnBackgroundThread;
 
   /* Singly linked list of zones to be swept in the background. */
   HelperThreadLockData<ZoneList> backgroundSweepZones;
 
   /*
-   * Free LIFO blocks are transferred to this allocator before being freed on
-   * the background GC thread after sweeping.
+   * Free LIFO blocks are transferred to these allocators before being freed on
+   * a background thread.
    */
-  HelperThreadLockData<LifoAlloc> blocksToFreeAfterSweeping;
+  HelperThreadLockData<LifoAlloc> lifoBlocksToFree;
+  MainThreadData<LifoAlloc> lifoBlocksToFreeAfterMinorGC;
 
   /* Index of current sweep group (for stats). */
   MainThreadData<unsigned> sweepGroupIndex;
 
   /*
    * Incremental sweep state.
    */
 
@@ -1017,20 +1020,16 @@ class GCRuntime {
  private:
   MainThreadData<Nursery> nursery_;
   MainThreadData<gc::StoreBuffer> storeBuffer_;
 
  public:
   Nursery& nursery() { return nursery_.ref(); }
   gc::StoreBuffer& storeBuffer() { return storeBuffer_.ref(); }
 
-  // Free LIFO blocks are transferred to this allocator before being freed
-  // after minor GC.
-  MainThreadData<LifoAlloc> blocksToFreeAfterMinorGC;
-
   void* addressOfNurseryPosition() {
     return nursery_.refNoCheck().addressOfPosition();
   }
   const void* addressOfNurseryCurrentEnd() {
     return nursery_.refNoCheck().addressOfCurrentEnd();
   }
   const void* addressOfStringNurseryCurrentEnd() {
     return nursery_.refNoCheck().addressOfCurrentStringEnd();
@@ -1040,17 +1039,16 @@ class GCRuntime {
   }
 
   void minorGC(JS::gcreason::Reason reason,
                gcstats::PhaseKind phase = gcstats::PhaseKind::MINOR_GC)
       JS_HAZ_GC_CALL;
   void evictNursery(JS::gcreason::Reason reason = JS::gcreason::EVICT_NURSERY) {
     minorGC(reason, gcstats::PhaseKind::EVICT_NURSERY);
   }
-  void freeAllLifoBlocksAfterMinorGC(LifoAlloc* lifo);
 
   friend class MarkingValidator;
   friend class AutoEnterIteration;
 };
 
 /* Prevent compartments and zones from being collected during iteration. */
 class MOZ_RAII AutoEnterIteration {
   GCRuntime* gc;
--- a/js/src/gc/Zone.cpp
+++ b/js/src/gc/Zone.cpp
@@ -533,16 +533,20 @@ Zone* ZoneList::front() const {
 void ZoneList::append(Zone* zone) {
   ZoneList singleZone(zone);
   transferFrom(singleZone);
 }
 
 void ZoneList::transferFrom(ZoneList& other) {
   check();
   other.check();
+  if (!other.head) {
+    return;
+  }
+
   MOZ_ASSERT(tail != other.tail);
 
   if (tail) {
     tail->listNext_ = other.head;
   } else {
     head = other.head;
   }
   tail = other.tail;
--- a/js/src/jit/BaselineJIT.cpp
+++ b/js/src/jit/BaselineJIT.cpp
@@ -50,18 +50,18 @@ using namespace js::jit;
   MOZ_ASSERT(stackVal->kind() != StackValue::Stack);
   return SlotIgnore;
 }
 
 void ICStubSpace::freeAllAfterMinorGC(Zone* zone) {
   if (zone->isAtomsZone()) {
     MOZ_ASSERT(allocator_.isEmpty());
   } else {
-    zone->runtimeFromMainThread()->gc.freeAllLifoBlocksAfterMinorGC(
-        &allocator_);
+    JSRuntime* rt = zone->runtimeFromMainThread();
+    rt->gc.queueAllLifoBlocksForFreeAfterMinorGC(&allocator_);
   }
 }
 
 static bool CheckFrame(InterpreterFrame* fp) {
   if (fp->isDebuggerEvalFrame()) {
     // Debugger eval-in-frame. These are likely short-running scripts so
     // don't bother compiling them for now.
     JitSpew(JitSpew_BaselineAbort, "debugger frame");
--- a/js/src/vm/Stack-inl.h
+++ b/js/src/vm/Stack-inl.h
@@ -190,17 +190,17 @@ inline CallObject& InterpreterFrame::cal
 inline void InterpreterFrame::unsetIsDebuggee() {
   MOZ_ASSERT(!script()->isDebuggee());
   flags_ &= ~DEBUGGEE;
 }
 
 /*****************************************************************************/
 
 inline void InterpreterStack::purge(JSRuntime* rt) {
-  rt->gc.freeUnusedLifoBlocksAfterSweeping(&allocator_);
+  rt->gc.queueUnusedLifoBlocksForFree(&allocator_);
 }
 
 uint8_t* InterpreterStack::allocateFrame(JSContext* cx, size_t size) {
   size_t maxFrames;
   if (cx->realm()->principals() == cx->runtime()->trustedPrincipals()) {
     maxFrames = MAX_FRAMES_TRUSTED;
   } else {
     maxFrames = MAX_FRAMES;
--- a/js/src/vm/TypeInference.cpp
+++ b/js/src/vm/TypeInference.cpp
@@ -4766,17 +4766,17 @@ void TypeZone::beginSweep() {
   // Clear the analysis pool, but don't release its data yet. While sweeping
   // types any live data will be allocated into the pool.
   sweepTypeLifoAlloc.ref().steal(&typeLifoAlloc());
 
   generation = !generation;
 }
 
 void TypeZone::endSweep(JSRuntime* rt) {
-  rt->gc.freeAllLifoBlocksAfterSweeping(&sweepTypeLifoAlloc.ref());
+  rt->gc.queueAllLifoBlocksForFree(&sweepTypeLifoAlloc.ref());
 }
 
 void TypeZone::clearAllNewScriptsOnOOM() {
   for (auto iter = zone()->cellIter<ObjectGroup>(); !iter.done(); iter.next()) {
     ObjectGroup* group = iter;
     if (!IsAboutToBeFinalizedUnbarriered(&group)) {
       group->maybeClearNewScriptOnOOM();
     }