Bug 1109336 - Track granular timings within Mark Roots, r=terrence
authorSteve Fink <sfink@mozilla.com>
Wed, 31 Dec 2014 14:45:56 -0800
changeset 221905 2efc03ac92f93ded5e78feca09b3e9188616ab61
parent 221904 3cf774a0954d0d096655256f2f598f34ca912ac2
child 221906 9096b9b4aa73d58742e88ced38f1d2018464f295
push id28050
push userphilringnalda@gmail.com
push dateSun, 04 Jan 2015 04:02:52 +0000
treeherdermozilla-central@55f3224d7513 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersterrence
bugs1109336
milestone37.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1109336 - Track granular timings within Mark Roots, r=terrence
js/src/gc/GCInternals.h
js/src/gc/GCRuntime.h
js/src/gc/Iteration.cpp
js/src/gc/Nursery.cpp
js/src/gc/RootMarking.cpp
js/src/gc/Statistics.cpp
js/src/gc/Statistics.h
js/src/gc/Verifier.cpp
js/src/jsgc.cpp
js/src/jsgcinlines.h
--- a/js/src/gc/GCInternals.h
+++ b/js/src/gc/GCInternals.h
@@ -113,20 +113,32 @@ class AutoStopVerifyingBarriers
     {
         restartPreVerifier = gc->endVerifyPreBarriers() && !isShutdown;
         restartPostVerifier = gc->endVerifyPostBarriers() && !isShutdown &&
             JS::IsGenerationalGCEnabled(rt);
         MOZ_GUARD_OBJECT_NOTIFIER_INIT;
     }
 
     ~AutoStopVerifyingBarriers() {
+        // Nasty special case: verification runs a minor GC, which *may* nest
+        // inside of an outer minor GC. This is not allowed by the
+        // gc::Statistics phase tree. So we pause the "real" GC, if in fact one
+        // is in progress.
+        gcstats::Phase outer = gc->stats.currentPhase();
+        if (outer != gcstats::PHASE_NONE)
+            gc->stats.endPhase(outer);
+        MOZ_ASSERT(gc->stats.currentPhase() == gcstats::PHASE_NONE);
+
         if (restartPreVerifier)
             gc->startVerifyPreBarriers();
         if (restartPostVerifier)
             gc->startVerifyPostBarriers();
+
+        if (outer != gcstats::PHASE_NONE)
+            gc->stats.beginPhase(outer);
     }
 };
 #else
 struct AutoStopVerifyingBarriers
 {
     AutoStopVerifyingBarriers(JSRuntime *, bool) {}
 };
 #endif /* JS_GC_ZEAL */
--- a/js/src/gc/GCRuntime.h
+++ b/js/src/gc/GCRuntime.h
@@ -310,19 +310,25 @@ class GCRuntime
     void incFJMinorCollecting() { fjCollectionCounter++; }
     void decFJMinorCollecting() { fjCollectionCounter--; }
 
     bool triggerGC(JS::gcreason::Reason reason);
     void maybeAllocTriggerZoneGC(Zone *zone, const AutoLockGC &lock);
     bool triggerZoneGC(Zone *zone, JS::gcreason::Reason reason);
     bool maybeGC(Zone *zone);
     void maybePeriodicFullGC();
-    void minorGC(JS::gcreason::Reason reason);
+    void minorGC(JS::gcreason::Reason reason) {
+        gcstats::AutoPhase ap(stats, gcstats::PHASE_MINOR_GC);
+        minorGCImpl(reason, nullptr);
+    }
     void minorGC(JSContext *cx, JS::gcreason::Reason reason);
-    void evictNursery(JS::gcreason::Reason reason = JS::gcreason::EVICT_NURSERY) { minorGC(reason); }
+    void evictNursery(JS::gcreason::Reason reason = JS::gcreason::EVICT_NURSERY) {
+        gcstats::AutoPhase ap(stats, gcstats::PHASE_EVICT_NURSERY);
+        minorGCImpl(reason, nullptr);
+    }
     bool gcIfNeeded(JSContext *cx = nullptr);
     void gc(JSGCInvocationKind gckind, JS::gcreason::Reason reason);
     void gcSlice(JSGCInvocationKind gckind, JS::gcreason::Reason reason, int64_t millis = 0);
     void gcFinalSlice(JSGCInvocationKind gckind, JS::gcreason::Reason reason);
     void gcDebugSlice(SliceBudget &budget);
 
     void runDebugGC();
     inline void poke();
@@ -526,16 +532,18 @@ class GCRuntime
     // Free certain LifoAlloc blocks from the background sweep thread.
     void freeUnusedLifoBlocksAfterSweeping(LifoAlloc *lifo);
     void freeAllLifoBlocksAfterSweeping(LifoAlloc *lifo);
 
     // Public here for ReleaseArenaLists and FinalizeTypedArenas.
     void releaseArena(ArenaHeader *aheader, const AutoLockGC &lock);
 
   private:
+    void minorGCImpl(JS::gcreason::Reason reason, Nursery::TypeObjectList *pretenureTypes);
+
     // For ArenaLists::allocateFromArena()
     friend class ArenaLists;
     Chunk *pickChunk(const AutoLockGC &lock,
                      AutoMaybeStartBackgroundAllocation &maybeStartBGAlloc);
     ArenaHeader *allocateArena(Chunk *chunk, Zone *zone, AllocKind kind, const AutoLockGC &lock);
     inline void arenaAllocatedDuringGC(JS::Zone *zone, ArenaHeader *arena);
 
     template <AllowGC allowGC>
--- a/js/src/gc/Iteration.cpp
+++ b/js/src/gc/Iteration.cpp
@@ -20,16 +20,17 @@ using namespace js::gc;
 void
 js::TraceRuntime(JSTracer *trc)
 {
     MOZ_ASSERT(!IS_GC_MARKING_TRACER(trc));
 
     JSRuntime *rt = trc->runtime();
     rt->gc.evictNursery();
     AutoPrepareForTracing prep(rt, WithAtoms);
+    gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_TRACE_HEAP);
     rt->gc.markRuntime(trc);
 }
 
 static void
 IterateCompartmentsArenasCells(JSRuntime *rt, Zone *zone, void *data,
                                JSIterateCompartmentCallback compartmentCallback,
                                IterateArenaCallback arenaCallback,
                                IterateCellCallback cellCallback)
--- a/js/src/gc/Nursery.cpp
+++ b/js/src/gc/Nursery.cpp
@@ -747,18 +747,16 @@ js::Nursery::collect(JSRuntime *rt, JS::
     rt->gc.stats.count(gcstats::STAT_MINOR_GC);
 
     TraceMinorGCStart();
 
     TIME_START(total);
 
     AutoStopVerifyingBarriers av(rt, false);
 
-    gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_MINOR_GC);
-
     // Move objects pointed to by roots from the nursery to the major heap.
     MinorCollectionTracer trc(rt, this);
 
     // Mark the store buffer. This must happen first.
     TIME_START(markValues);
     sb.markValues(&trc);
     TIME_END(markValues);
 
@@ -793,17 +791,20 @@ js::Nursery::collect(JSRuntime *rt, JS::
 #endif
     TIME_END(checkHashTables);
 
     TIME_START(markRuntime);
     rt->gc.markRuntime(&trc);
     TIME_END(markRuntime);
 
     TIME_START(markDebugger);
-    Debugger::markAll(&trc);
+    {
+        gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_MARK_ROOTS);
+        Debugger::markAll(&trc);
+    }
     TIME_END(markDebugger);
 
     TIME_START(clearNewObjectCache);
     rt->newObjectCache.clearNurseryObjects(rt);
     TIME_END(clearNewObjectCache);
 
     // Most of the work is done here. This loop iterates over objects that have
     // been moved to the major heap. If these objects have any outgoing pointers
--- a/js/src/gc/RootMarking.cpp
+++ b/js/src/gc/RootMarking.cpp
@@ -438,65 +438,75 @@ js::gc::MarkForkJoinStack(ForkJoinNurser
 }
 #endif  // JSGC_FJGENERATIONAL
 
 void
 js::gc::GCRuntime::markRuntime(JSTracer *trc,
                                TraceOrMarkRuntime traceOrMark,
                                TraceRootsOrUsedSaved rootsSource)
 {
+    gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_ROOTS);
+
     MOZ_ASSERT(trc->callback != GCMarker::GrayCallback);
     MOZ_ASSERT(traceOrMark == TraceRuntime || traceOrMark == MarkRuntime);
     MOZ_ASSERT(rootsSource == TraceRoots || rootsSource == UseSavedRoots);
 
     MOZ_ASSERT(!rt->mainThread.suppressGC);
 
     if (traceOrMark == MarkRuntime) {
+        gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_CCWS);
+
         for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
             if (!c->zone()->isCollecting())
                 c->markCrossCompartmentWrappers(trc);
         }
         Debugger::markAllCrossCompartmentEdges(trc);
     }
 
-    AutoGCRooter::traceAll(trc);
+    {
+        gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_ROOTERS);
+
+        AutoGCRooter::traceAll(trc);
 
-    if (!rt->isBeingDestroyed()) {
-        MarkExactStackRoots(rt, trc);
-        rt->markSelfHostingGlobal(trc);
-    }
+        if (!rt->isBeingDestroyed()) {
+            MarkExactStackRoots(rt, trc);
+            rt->markSelfHostingGlobal(trc);
+        }
 
-    for (RootRange r = rootsHash.all(); !r.empty(); r.popFront()) {
-        const RootEntry &entry = r.front();
-        const char *name = entry.value().name ? entry.value().name : "root";
-        JSGCRootType type = entry.value().type;
-        void *key = entry.key();
-        if (type == JS_GC_ROOT_VALUE_PTR) {
-            MarkValueRoot(trc, reinterpret_cast<Value *>(key), name);
-        } else if (*reinterpret_cast<void **>(key)){
-            if (type == JS_GC_ROOT_STRING_PTR)
-                MarkStringRoot(trc, reinterpret_cast<JSString **>(key), name);
-            else if (type == JS_GC_ROOT_OBJECT_PTR)
-                MarkObjectRoot(trc, reinterpret_cast<JSObject **>(key), name);
-            else if (type == JS_GC_ROOT_SCRIPT_PTR)
-                MarkScriptRoot(trc, reinterpret_cast<JSScript **>(key), name);
-            else
-                MOZ_CRASH("unexpected js::RootInfo::type value");
+        for (RootRange r = rootsHash.all(); !r.empty(); r.popFront()) {
+            const RootEntry &entry = r.front();
+            const char *name = entry.value().name ? entry.value().name : "root";
+            JSGCRootType type = entry.value().type;
+            void *key = entry.key();
+            if (type == JS_GC_ROOT_VALUE_PTR) {
+                MarkValueRoot(trc, reinterpret_cast<Value *>(key), name);
+            } else if (*reinterpret_cast<void **>(key)){
+                if (type == JS_GC_ROOT_STRING_PTR)
+                    MarkStringRoot(trc, reinterpret_cast<JSString **>(key), name);
+                else if (type == JS_GC_ROOT_OBJECT_PTR)
+                    MarkObjectRoot(trc, reinterpret_cast<JSObject **>(key), name);
+                else if (type == JS_GC_ROOT_SCRIPT_PTR)
+                    MarkScriptRoot(trc, reinterpret_cast<JSScript **>(key), name);
+                else
+                    MOZ_CRASH("unexpected js::RootInfo::type value");
+            }
         }
+
+        MarkPersistentRootedChains(trc);
     }
 
-    MarkPersistentRootedChains(trc);
-
     if (rt->scriptAndCountsVector) {
         ScriptAndCountsVector &vec = *rt->scriptAndCountsVector;
         for (size_t i = 0; i < vec.length(); i++)
             MarkScriptRoot(trc, &vec[i].script, "scriptAndCountsVector");
     }
 
     if (!rt->isBeingDestroyed() && !trc->runtime()->isHeapMinorCollecting()) {
+        gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_RUNTIME_DATA);
+
         if (traceOrMark == TraceRuntime || rt->atomsCompartment()->zone()->isCollecting()) {
             MarkPermanentAtoms(trc);
             MarkAtoms(trc);
             MarkWellKnownSymbols(trc);
             jit::JitRuntime::Mark(trc);
         }
     }
 
@@ -541,16 +551,18 @@ js::gc::GCRuntime::markRuntime(JSTracer 
             c->lazyArrayBuffers->trace(trc);
     }
 
     MarkInterpreterActivations(&rt->mainThread, trc);
 
     jit::MarkJitActivations(&rt->mainThread, trc);
 
     if (!isHeapMinorCollecting()) {
+        gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_EMBEDDING);
+
         /*
          * All JSCompartment::markRoots() does is mark the globals for
          * compartments which have been entered. Globals aren't nursery
          * allocated so there's no need to do this for minor GCs.
          */
         for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next())
             c->markRoots(trc);
 
--- a/js/src/gc/Statistics.cpp
+++ b/js/src/gc/Statistics.cpp
@@ -1,16 +1,17 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
  * vim: set ts=8 sts=4 et sw=4 tw=99:
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "gc/Statistics.h"
 
+#include "mozilla/ArrayUtils.h"
 #include "mozilla/PodOperations.h"
 #include "mozilla/UniquePtr.h"
 
 #include <ctype.h>
 #include <stdarg.h>
 #include <stdio.h>
 
 #include "jscrashreport.h"
@@ -276,33 +277,63 @@ t(int64_t t)
 
 struct PhaseInfo
 {
     Phase index;
     const char *name;
     Phase parent;
 };
 
+// The zeroth entry in the timing arrays is used for phases that have a
+// unique lineage.
+static const size_t PHASE_DAG_NONE = 0;
+
+// These are really just fields of PhaseInfo, but I have to initialize them
+// programmatically, which prevents making phases[] const. (And marking these
+// fields mutable does not work on Windows; the whole thing gets created in
+// read-only memory anyway.)
+struct ExtraPhaseInfo
+{
+    // Depth in the tree of each phase type
+    size_t depth;
+
+    // Index into the set of parallel arrays of timing data, for parents with
+    // at least one multi-parented child
+    int dagSlot;
+};
+
 static const Phase PHASE_NO_PARENT = PHASE_LIMIT;
 
+struct DagChildEdge {
+    Phase parent;
+    Phase child;
+} dagChildEdges[] = {
+    { PHASE_MARK, PHASE_MARK_ROOTS },
+    { PHASE_MINOR_GC, PHASE_MARK_ROOTS },
+    { PHASE_TRACE_HEAP, PHASE_MARK_ROOTS },
+    { PHASE_EVICT_NURSERY, PHASE_MARK_ROOTS },
+    { PHASE_COMPACT_UPDATE, PHASE_MARK_ROOTS }
+};
+
 /*
  * Note that PHASE_MUTATOR, PHASE_GC_BEGIN, and PHASE_GC_END never have any
  * child phases. If beginPhase is called while one of these is active, they
  * will automatically be suspended and resumed when the phase stack is next
  * empty. Timings for these phases are thus exclusive of any other phase.
  */
 
 static const PhaseInfo phases[] = {
     { PHASE_MUTATOR, "Mutator Running", PHASE_NO_PARENT },
     { PHASE_GC_BEGIN, "Begin Callback", PHASE_NO_PARENT },
     { PHASE_WAIT_BACKGROUND_THREAD, "Wait Background Thread", PHASE_NO_PARENT },
     { PHASE_MARK_DISCARD_CODE, "Mark Discard Code", PHASE_NO_PARENT },
     { PHASE_PURGE, "Purge", PHASE_NO_PARENT },
     { PHASE_MARK, "Mark", PHASE_NO_PARENT },
-        { PHASE_MARK_ROOTS, "Mark Roots", PHASE_MARK },
+        { PHASE_UNMARK, "Unmark", PHASE_MARK },
+        /* PHASE_MARK_ROOTS */
         { PHASE_MARK_DELAYED, "Mark Delayed", PHASE_MARK },
     { PHASE_SWEEP, "Sweep", PHASE_NO_PARENT },
         { PHASE_SWEEP_MARK, "Mark During Sweeping", PHASE_SWEEP },
             { PHASE_SWEEP_MARK_TYPES, "Mark Types During Sweeping", PHASE_SWEEP_MARK },
             { PHASE_SWEEP_MARK_INCOMING_BLACK, "Mark Incoming Black Pointers", PHASE_SWEEP_MARK },
             { PHASE_SWEEP_MARK_WEAK, "Mark Weak", PHASE_SWEEP_MARK },
             { PHASE_SWEEP_MARK_INCOMING_GRAY, "Mark Incoming Gray Pointers", PHASE_SWEEP_MARK },
             { PHASE_SWEEP_MARK_GRAY, "Mark Gray", PHASE_SWEEP_MARK },
@@ -327,29 +358,105 @@ static const PhaseInfo phases[] = {
         { PHASE_SWEEP_STRING, "Sweep String", PHASE_SWEEP },
         { PHASE_SWEEP_SCRIPT, "Sweep Script", PHASE_SWEEP },
         { PHASE_SWEEP_SHAPE, "Sweep Shape", PHASE_SWEEP },
         { PHASE_SWEEP_JITCODE, "Sweep JIT code", PHASE_SWEEP },
         { PHASE_FINALIZE_END, "Finalize End Callback", PHASE_SWEEP },
         { PHASE_DESTROY, "Deallocate", PHASE_SWEEP },
     { PHASE_COMPACT, "Compact", PHASE_NO_PARENT },
         { PHASE_COMPACT_MOVE, "Compact Move", PHASE_COMPACT },
-        { PHASE_COMPACT_UPDATE, "Compact Update", PHASE_COMPACT, },
+        { PHASE_COMPACT_UPDATE, "Compact Update", PHASE_COMPACT },
+            /* PHASE_MARK_ROOTS */
             { PHASE_COMPACT_UPDATE_CELLS, "Compact Update Cells", PHASE_COMPACT_UPDATE, },
     { PHASE_GC_END, "End Callback", PHASE_NO_PARENT },
-    { PHASE_MINOR_GC, "Minor GC", PHASE_NO_PARENT },
+    { PHASE_MINOR_GC, "All Minor GCs", PHASE_NO_PARENT },
+        /* PHASE_MARK_ROOTS */
+    { PHASE_EVICT_NURSERY, "Minor GCs to Evict Nursery", PHASE_NO_PARENT },
+        /* PHASE_MARK_ROOTS */
+    { PHASE_TRACE_HEAP, "Trace Heap", PHASE_NO_PARENT },
+        /* PHASE_MARK_ROOTS */
+    { PHASE_MARK_ROOTS, "Mark Roots", PHASE_MULTI_PARENTS },
+        { PHASE_MARK_CCWS, "Mark Cross Compartment Wrappers", PHASE_MARK_ROOTS },
+        { PHASE_MARK_ROOTERS, "Mark Rooters", PHASE_MARK_ROOTS },
+        { PHASE_MARK_RUNTIME_DATA, "Mark Runtime-wide Data", PHASE_MARK_ROOTS },
+        { PHASE_MARK_EMBEDDING, "Mark Embedding", PHASE_MARK_ROOTS },
+        { PHASE_MARK_COMPARTMENTS, "Mark Compartments", PHASE_MARK_ROOTS },
     { PHASE_LIMIT, nullptr, PHASE_NO_PARENT }
 };
 
+ExtraPhaseInfo phaseExtra[PHASE_LIMIT] = { { 0, 0 } };
+
+// Mapping from all nodes with a multi-parented child to a Vector of all
+// multi-parented children and their descendants. (Single-parented children will
+// not show up in this list.)
+static mozilla::Vector<Phase> dagDescendants[Statistics::MAX_MULTIPARENT_PHASES + 1];
+
+struct AllPhaseIterator {
+    int current;
+    int baseLevel;
+    size_t activeSlot;
+    mozilla::Vector<Phase>::Range descendants;
+
+    explicit AllPhaseIterator(int64_t (*table)[PHASE_LIMIT])
+      : current(0)
+      , baseLevel(0)
+      , activeSlot(PHASE_DAG_NONE)
+      , descendants(dagDescendants[PHASE_DAG_NONE].all()) /* empty range */
+    {
+    }
+
+    void get(Phase *phase, size_t *dagSlot, size_t *level = nullptr) {
+        MOZ_ASSERT(!done());
+        *dagSlot = activeSlot;
+        *phase = descendants.empty() ? Phase(current) : descendants.front();
+        if (level)
+            *level = phaseExtra[*phase].depth + baseLevel;
+    }
+
+    void advance() {
+        MOZ_ASSERT(!done());
+
+        if (!descendants.empty()) {
+            descendants.popFront();
+            if (!descendants.empty())
+                return;
+
+            ++current;
+            activeSlot = PHASE_DAG_NONE;
+            baseLevel = 0;
+            return;
+        }
+
+        if (phaseExtra[current].dagSlot != PHASE_DAG_NONE) {
+            activeSlot = phaseExtra[current].dagSlot;
+            descendants = dagDescendants[activeSlot].all();
+            MOZ_ASSERT(!descendants.empty());
+            baseLevel += phaseExtra[current].depth + 1;
+            return;
+        }
+
+        ++current;
+    }
+
+    bool done() const {
+        return phases[current].parent == PHASE_MULTI_PARENTS;
+    }
+};
+
 static void
-FormatPhaseTimes(StatisticsSerializer &ss, const char *name, int64_t *times)
+FormatPhaseTimes(StatisticsSerializer &ss, const char *name, int64_t (*times)[PHASE_LIMIT])
 {
     ss.beginObject(name);
-    for (unsigned i = 0; phases[i].name; i++)
-        ss.appendIfNonzeroMS(phases[i].name, t(times[phases[i].index]));
+
+    for (AllPhaseIterator iter(times); !iter.done(); iter.advance()) {
+        Phase phase;
+        size_t dagSlot;
+        iter.get(&phase, &dagSlot);
+        ss.appendIfNonzeroMS(phases[phase].name, t(times[dagSlot][phase]));
+    }
     ss.endObject();
 }
 
 void
 Statistics::gcDuration(int64_t *total, int64_t *maxPause)
 {
     *total = *maxPause = 0;
     for (SliceData *slice = slices.begin(); slice != slices.end(); slice++) {
@@ -537,58 +644,68 @@ UniqueChars
 Statistics::formatTotals()
 {
     int64_t total, longest;
     gcDuration(&total, &longest);
 
     const char *format =
 "\
   ---- Totals ----\n\
-    Total Time: %.3f\n\
-    Max Pause: %.3f\n\
+    Total Time: %.3fms\n\
+    Max Pause: %.3fms\n\
 ";
     char buffer[1024];
     memset(buffer, 0, sizeof(buffer));
     JS_snprintf(buffer, sizeof(buffer), format, t(total), t(longest));
     return make_string_copy(buffer);
 }
 
 static int64_t
-SumChildTimes(Phase phase, int64_t *phaseTimes)
+SumChildTimes(size_t phaseSlot, Phase phase, int64_t (*phaseTimes)[PHASE_LIMIT])
 {
+    // Sum the contributions from single-parented children.
     int64_t total = 0;
-    for (unsigned i = 0; phases[i].name; i++) {
+    for (unsigned i = 0; i < PHASE_LIMIT; i++) {
         if (phases[i].parent == phase)
-            total += phaseTimes[phases[i].index];
+            total += phaseTimes[phaseSlot][i];
+    }
+
+    // Sum the contributions from multi-parented children.
+    size_t dagSlot = phaseExtra[phase].dagSlot;
+    if (dagSlot != PHASE_DAG_NONE) {
+        for (size_t i = 0; i < mozilla::ArrayLength(dagChildEdges); i++) {
+            if (dagChildEdges[i].parent == phase) {
+                Phase child = dagChildEdges[i].child;
+                total += phaseTimes[dagSlot][child];
+            }
+        }
     }
     return total;
 }
 
 UniqueChars
-Statistics::formatPhaseTimes(int64_t *phaseTimes)
+Statistics::formatPhaseTimes(int64_t (*phaseTimes)[PHASE_LIMIT])
 {
     static const char *LevelToIndent[] = { "", "  ", "    ", "      " };
     static const int64_t MaxUnaccountedChildTimeUS = 50;
 
     FragmentVector fragments;
     char buffer[128];
-    for (unsigned i = 0; phases[i].name; i++) {
-        unsigned level = 0;
-        unsigned current = i;
-        while (phases[current].parent != PHASE_NO_PARENT) {
-            current = phases[current].parent;
-            level++;
-        }
+    for (AllPhaseIterator iter(phaseTimes); !iter.done(); iter.advance()) {
+        Phase phase;
+        size_t dagSlot;
+        size_t level;
+        iter.get(&phase, &dagSlot, &level);
         MOZ_ASSERT(level < 4);
 
-        int64_t ownTime = phaseTimes[phases[i].index];
-        int64_t childTime = SumChildTimes(Phase(i), phaseTimes);
+        int64_t ownTime = phaseTimes[dagSlot][phase];
+        int64_t childTime = SumChildTimes(dagSlot, phase, phaseTimes);
         if (ownTime > 0) {
             JS_snprintf(buffer, sizeof(buffer), "      %s%s: %.3fms\n",
-                        LevelToIndent[level], phases[i].name, t(ownTime));
+                        LevelToIndent[level], phases[phase].name, t(ownTime));
             if (!fragments.append(make_string_copy(buffer)))
                 return UniqueChars(nullptr);
 
             if (childTime && (ownTime - childTime) > MaxUnaccountedChildTimeUS) {
                 MOZ_ASSERT(level < 3);
                 JS_snprintf(buffer, sizeof(buffer), "      %s%s: %.3fms\n",
                             LevelToIndent[level + 1], "Other", t(ownTime - childTime));
                 if (!fragments.append(make_string_copy(buffer)))
@@ -645,23 +762,72 @@ Statistics::Statistics(JSRuntime *rt)
     fp(nullptr),
     fullFormat(false),
     gcDepth(0),
     nonincrementalReason(nullptr),
     timedGCStart(0),
     preBytes(0),
     maxPauseInInterval(0),
     phaseNestingDepth(0),
+    activeDagSlot(PHASE_DAG_NONE),
     suspendedPhaseNestingDepth(0),
-    sliceCallback(nullptr)
+    sliceCallback(nullptr),
+    abortSlices(false)
 {
     PodArrayZero(phaseTotals);
     PodArrayZero(counts);
     PodArrayZero(phaseStartTimes);
-    PodArrayZero(phaseTimes);
+    for (size_t d = 0; d < MAX_MULTIPARENT_PHASES + 1; d++)
+        PodArrayZero(phaseTimes[d]);
+
+    static bool initialized = false;
+    if (!initialized) {
+        initialized = true;
+
+        for (size_t i = 0; i < PHASE_LIMIT; i++)
+            MOZ_ASSERT(phases[i].index == i);
+
+        // Create a static table of descendants for every phase with multiple
+        // children. This assumes that all descendants come linearly in the
+        // list, which is reasonable since full dags are not supported; any
+        // path from the leaf to the root must encounter at most one node with
+        // multiple parents.
+        size_t dagSlot = 0;
+        for (size_t i = 0; i < mozilla::ArrayLength(dagChildEdges); i++) {
+            Phase parent = dagChildEdges[i].parent;
+            if (!phaseExtra[parent].dagSlot)
+                phaseExtra[parent].dagSlot = ++dagSlot;
+
+            Phase child = dagChildEdges[i].child;
+            MOZ_ASSERT(phases[child].parent == PHASE_MULTI_PARENTS);
+            int j = child;
+            do {
+                dagDescendants[phaseExtra[parent].dagSlot].append(Phase(j));
+                j++;
+            } while (j != PHASE_LIMIT && phases[j].parent != PHASE_MULTI_PARENTS);
+        }
+        MOZ_ASSERT(dagSlot <= MAX_MULTIPARENT_PHASES);
+
+        // Fill in the depth of each node in the tree. Multi-parented nodes
+        // have depth 0.
+        mozilla::Vector<Phase> stack;
+        stack.append(PHASE_LIMIT); // Dummy entry to avoid special-casing the first node
+        for (int i = 0; i < PHASE_LIMIT; i++) {
+            if (phases[i].parent == PHASE_NO_PARENT ||
+                phases[i].parent == PHASE_MULTI_PARENTS)
+            {
+                stack.clear();
+            } else {
+                while (stack.back() != phases[i].parent)
+                    stack.popBack();
+            }
+            phaseExtra[i].depth = stack.length();
+            stack.append(Phase(i));
+        }
+    }
 
     char *env = getenv("MOZ_GCTIMER");
     if (!env || strcmp(env, "none") == 0) {
         fp = nullptr;
         return;
     }
 
     if (strcmp(env, "stdout") == 0) {
@@ -713,31 +879,42 @@ Statistics::clearMaxGCPauseAccumulator()
 }
 
 int64_t
 Statistics::getMaxGCPauseSinceClear()
 {
     return maxPauseInInterval;
 }
 
+static int64_t
+SumPhase(Phase phase, int64_t (*times)[PHASE_LIMIT])
+{
+    int64_t sum = 0;
+    for (size_t i = 0; i < Statistics::MAX_MULTIPARENT_PHASES + 1; i++)
+        sum += times[i][phase];
+    return sum;
+}
+
 void
 Statistics::printStats()
 {
     if (fullFormat) {
         UniqueChars msg = formatDetailedMessage();
         if (msg)
             fprintf(fp, "GC(T+%.3fs) %s\n", t(slices[0].start - startupTime) / 1000.0, msg.get());
     } else {
         int64_t total, longest;
         gcDuration(&total, &longest);
 
+        int64_t markTotal = SumPhase(PHASE_MARK, phaseTimes);
         fprintf(fp, "%f %f %f\n",
                 t(total),
-                t(phaseTimes[PHASE_MARK]),
-                t(phaseTimes[PHASE_SWEEP]));
+                t(markTotal),
+                t(phaseTimes[PHASE_DAG_NONE][PHASE_SWEEP]));
+        MOZ_ASSERT(phaseExtra[PHASE_SWEEP].dagSlot == PHASE_DAG_NONE);
     }
     fflush(fp);
 }
 
 void
 Statistics::beginGC(JSGCInvocationKind kind)
 {
     slices.clearAndFree();
@@ -748,82 +925,94 @@ Statistics::beginGC(JSGCInvocationKind k
     preBytes = runtime->gc.usage.gcBytes();
 }
 
 void
 Statistics::endGC()
 {
     crash::SnapshotGCStack();
 
-    for (int i = 0; i < PHASE_LIMIT; i++)
-        phaseTotals[i] += phaseTimes[i];
+    for (size_t j = 0; j < MAX_MULTIPARENT_PHASES + 1; j++)
+        for (int i = 0; i < PHASE_LIMIT; i++)
+            phaseTotals[j][i] += phaseTimes[j][i];
 
     int64_t total, longest;
     gcDuration(&total, &longest);
 
     int64_t sccTotal, sccLongest;
     sccDurations(&sccTotal, &sccLongest);
 
     runtime->addTelemetry(JS_TELEMETRY_GC_IS_COMPARTMENTAL, !zoneStats.isCollectingAllZones());
     runtime->addTelemetry(JS_TELEMETRY_GC_MS, t(total));
     runtime->addTelemetry(JS_TELEMETRY_GC_MAX_PAUSE_MS, t(longest));
-    runtime->addTelemetry(JS_TELEMETRY_GC_MARK_MS, t(phaseTimes[PHASE_MARK]));
-    runtime->addTelemetry(JS_TELEMETRY_GC_SWEEP_MS, t(phaseTimes[PHASE_SWEEP]));
-    runtime->addTelemetry(JS_TELEMETRY_GC_MARK_ROOTS_MS, t(phaseTimes[PHASE_MARK_ROOTS]));
-    runtime->addTelemetry(JS_TELEMETRY_GC_MARK_GRAY_MS, t(phaseTimes[PHASE_SWEEP_MARK_GRAY]));
+    int64_t markTotal = SumPhase(PHASE_MARK, phaseTimes);
+    int64_t markRootsTotal = SumPhase(PHASE_MARK_ROOTS, phaseTimes);
+    runtime->addTelemetry(JS_TELEMETRY_GC_MARK_MS, t(markTotal));
+    runtime->addTelemetry(JS_TELEMETRY_GC_SWEEP_MS, t(phaseTimes[PHASE_DAG_NONE][PHASE_SWEEP]));
+    runtime->addTelemetry(JS_TELEMETRY_GC_MARK_ROOTS_MS, t(markRootsTotal));
+    runtime->addTelemetry(JS_TELEMETRY_GC_MARK_GRAY_MS, t(phaseTimes[PHASE_DAG_NONE][PHASE_SWEEP_MARK_GRAY]));
     runtime->addTelemetry(JS_TELEMETRY_GC_NON_INCREMENTAL, !!nonincrementalReason);
     runtime->addTelemetry(JS_TELEMETRY_GC_INCREMENTAL_DISABLED, !runtime->gc.isIncrementalGCAllowed());
     runtime->addTelemetry(JS_TELEMETRY_GC_SCC_SWEEP_TOTAL_MS, t(sccTotal));
     runtime->addTelemetry(JS_TELEMETRY_GC_SCC_SWEEP_MAX_PAUSE_MS, t(sccLongest));
 
     double mmu50 = computeMMU(50 * PRMJ_USEC_PER_MSEC);
     runtime->addTelemetry(JS_TELEMETRY_GC_MMU_50, mmu50 * 100);
 
     if (fp)
         printStats();
 
     // Clear the timers at the end of a GC because we accumulate time in
     // between GCs for some (which come before PHASE_GC_BEGIN in the list.)
     PodZero(&phaseStartTimes[PHASE_GC_BEGIN], PHASE_LIMIT - PHASE_GC_BEGIN);
-    PodZero(&phaseTimes[PHASE_GC_BEGIN], PHASE_LIMIT - PHASE_GC_BEGIN);
+    for (size_t d = PHASE_DAG_NONE; d < MAX_MULTIPARENT_PHASES + 1; d++)
+        PodZero(&phaseTimes[d][PHASE_GC_BEGIN], PHASE_LIMIT - PHASE_GC_BEGIN);
+
+    abortSlices = false;
 }
 
 void
 Statistics::beginSlice(const ZoneGCStats &zoneStats, JSGCInvocationKind gckind,
                        JS::gcreason::Reason reason)
 {
     this->zoneStats = zoneStats;
 
     bool first = !runtime->gc.isIncrementalGCInProgress();
     if (first)
         beginGC(gckind);
 
     SliceData data(reason, PRMJ_Now(), GetPageFaultCount());
-    if (!slices.append(data))
-        CrashAtUnhandlableOOM("Failed to allocate statistics slice.");
+    if (!slices.append(data)) {
+        // OOM testing fails if we CrashAtUnhandlableOOM here.
+        abortSlices = true;
+        slices.clear();
+        return;
+    }
 
     runtime->addTelemetry(JS_TELEMETRY_GC_REASON, reason);
 
     // Slice callbacks should only fire for the outermost level
     if (++gcDepth == 1) {
         bool wasFullGC = zoneStats.isCollectingAllZones();
         if (sliceCallback)
             (*sliceCallback)(runtime, first ? JS::GC_CYCLE_BEGIN : JS::GC_SLICE_BEGIN,
                              JS::GCDescription(!wasFullGC));
     }
 }
 
 void
 Statistics::endSlice()
 {
-    slices.back().end = PRMJ_Now();
-    slices.back().endFaults = GetPageFaultCount();
+    if (!abortSlices) {
+        slices.back().end = PRMJ_Now();
+        slices.back().endFaults = GetPageFaultCount();
 
-    runtime->addTelemetry(JS_TELEMETRY_GC_SLICE_MS, t(slices.back().end - slices.back().start));
-    runtime->addTelemetry(JS_TELEMETRY_GC_RESET, !!slices.back().resetReason);
+        runtime->addTelemetry(JS_TELEMETRY_GC_SLICE_MS, t(slices.back().end - slices.back().start));
+        runtime->addTelemetry(JS_TELEMETRY_GC_RESET, !!slices.back().resetReason);
+    }
 
     bool last = !runtime->gc.isIncrementalGCInProgress();
     if (last)
         endGC();
 
     // Slice callbacks should only fire for the outermost level
     if (--gcDepth == 0) {
         bool wasFullGC = zoneStats.isCollectingAllZones();
@@ -841,31 +1030,31 @@ void
 Statistics::startTimingMutator()
 {
     // Should only be called from outside of GC
     MOZ_ASSERT(phaseNestingDepth == 0);
     MOZ_ASSERT(suspendedPhaseNestingDepth == 0);
 
     timedGCTime = 0;
     phaseStartTimes[PHASE_MUTATOR] = 0;
-    phaseTimes[PHASE_MUTATOR] = 0;
+    phaseTimes[PHASE_DAG_NONE][PHASE_MUTATOR] = 0;
     timedGCStart = 0;
 
     beginPhase(PHASE_MUTATOR);
 }
 
 bool
 Statistics::stopTimingMutator(double &mutator_ms, double &gc_ms)
 {
     // This should only be called from outside of GC, while timing the mutator.
     if (phaseNestingDepth != 1 || phaseNesting[0] != PHASE_MUTATOR)
         return false;
 
     endPhase(PHASE_MUTATOR);
-    mutator_ms = t(phaseTimes[PHASE_MUTATOR]);
+    mutator_ms = t(phaseTimes[PHASE_DAG_NONE][PHASE_MUTATOR]);
     gc_ms = t(timedGCTime);
 
     return true;
 }
 
 void
 Statistics::beginPhase(Phase phase)
 {
@@ -883,63 +1072,70 @@ Statistics::beginPhase(Phase phase)
         parent = phaseNestingDepth ? phaseNesting[phaseNestingDepth - 1] : PHASE_NO_PARENT;
     }
 
     // Guard against any other re-entry.
     MOZ_ASSERT(!phaseStartTimes[phase]);
 
     MOZ_ASSERT(phases[phase].index == phase);
     MOZ_ASSERT(phaseNestingDepth < MAX_NESTING);
-    MOZ_ASSERT_IF(gcDepth == 1 && phase != PHASE_MINOR_GC, phases[phase].parent == parent);
+    MOZ_ASSERT(phases[phase].parent == parent || phases[phase].parent == PHASE_MULTI_PARENTS);
 
     phaseNesting[phaseNestingDepth] = phase;
     phaseNestingDepth++;
 
+    if (phases[phase].parent == PHASE_MULTI_PARENTS)
+        activeDagSlot = phaseExtra[parent].dagSlot;
+
     phaseStartTimes[phase] = PRMJ_Now();
 }
 
 void
 Statistics::recordPhaseEnd(Phase phase)
 {
     int64_t now = PRMJ_Now();
 
     if (phase == PHASE_MUTATOR)
         timedGCStart = now;
 
     phaseNestingDepth--;
 
     int64_t t = now - phaseStartTimes[phase];
     if (!slices.empty())
-        slices.back().phaseTimes[phase] += t;
-    phaseTimes[phase] += t;
+        slices.back().phaseTimes[activeDagSlot][phase] += t;
+    phaseTimes[activeDagSlot][phase] += t;
     phaseStartTimes[phase] = 0;
 }
 
 void
 Statistics::endPhase(Phase phase)
 {
     recordPhaseEnd(phase);
 
+    if (phases[phase].parent == PHASE_MULTI_PARENTS)
+        activeDagSlot = PHASE_DAG_NONE;
+
     // When emptying the stack, we may need to resume a callback phase
     // (PHASE_GC_BEGIN/END) or return to timing the mutator (PHASE_MUTATOR).
     if (phaseNestingDepth == 0 && suspendedPhaseNestingDepth > 0) {
         Phase resumePhase = suspendedPhases[--suspendedPhaseNestingDepth];
         if (resumePhase == PHASE_MUTATOR)
             timedGCTime += PRMJ_Now() - timedGCStart;
         beginPhase(resumePhase);
     }
 }
 
 void
 Statistics::endParallelPhase(Phase phase, const GCParallelTask *task)
 {
     phaseNestingDepth--;
 
-    slices.back().phaseTimes[phase] += task->duration();
-    phaseTimes[phase] += task->duration();
+    if (!slices.empty())
+        slices.back().phaseTimes[PHASE_DAG_NONE][phase] += task->duration();
+    phaseTimes[PHASE_DAG_NONE][phase] += task->duration();
     phaseStartTimes[phase] = 0;
 }
 
 int64_t
 Statistics::beginSCC()
 {
     return PRMJ_Now();
 }
@@ -960,16 +1156,19 @@ Statistics::endSCC(unsigned scc, int64_t
  * that means that, for any 50ms window of time, at least 80% of the window is
  * devoted to the mutator. In other words, the GC is running for at most 20% of
  * the window, or 10ms. The GC can run multiple slices during the 50ms window
  * as long as the total time it spends is at most 10ms.
  */
 double
 Statistics::computeMMU(int64_t window)
 {
+    if (abortSlices)
+        return 0.0;
+
     MOZ_ASSERT(!slices.empty());
 
     int64_t gc = slices[0].end - slices[0].start;
     int64_t gcMax = gc;
 
     if (gc >= window)
         return 0.0;
 
--- a/js/src/gc/Statistics.h
+++ b/js/src/gc/Statistics.h
@@ -28,17 +28,17 @@ namespace gcstats {
 
 enum Phase {
     PHASE_MUTATOR,
     PHASE_GC_BEGIN,
     PHASE_WAIT_BACKGROUND_THREAD,
     PHASE_MARK_DISCARD_CODE,
     PHASE_PURGE,
     PHASE_MARK,
-    PHASE_MARK_ROOTS,
+    PHASE_UNMARK,
     PHASE_MARK_DELAYED,
     PHASE_SWEEP,
     PHASE_SWEEP_MARK,
     PHASE_SWEEP_MARK_TYPES,
     PHASE_SWEEP_MARK_INCOMING_BLACK,
     PHASE_SWEEP_MARK_WEAK,
     PHASE_SWEEP_MARK_INCOMING_GRAY,
     PHASE_SWEEP_MARK_GRAY,
@@ -67,18 +67,28 @@ enum Phase {
     PHASE_FINALIZE_END,
     PHASE_DESTROY,
     PHASE_COMPACT,
     PHASE_COMPACT_MOVE,
     PHASE_COMPACT_UPDATE,
     PHASE_COMPACT_UPDATE_CELLS,
     PHASE_GC_END,
     PHASE_MINOR_GC,
+    PHASE_EVICT_NURSERY,
+    PHASE_TRACE_HEAP,
+    PHASE_MARK_ROOTS,
+    PHASE_MARK_CCWS,
+    PHASE_MARK_ROOTERS,
+    PHASE_MARK_RUNTIME_DATA,
+    PHASE_MARK_EMBEDDING,
+    PHASE_MARK_COMPARTMENTS,
 
-    PHASE_LIMIT
+    PHASE_LIMIT,
+    PHASE_NONE = PHASE_LIMIT,
+    PHASE_MULTI_PARENTS
 };
 
 enum Stat {
     STAT_NEW_CHUNK,
     STAT_DESTROY_CHUNK,
     STAT_MINOR_GC,
 
     // Number of times a 'put' into a storebuffer overflowed, triggering a
@@ -106,18 +116,47 @@ struct ZoneGCStats
 
     bool isCollectingAllZones() const { return collectedZoneCount == zoneCount; }
 
     ZoneGCStats()
       : collectedZoneCount(0), zoneCount(0), collectedCompartmentCount(0), compartmentCount(0)
     {}
 };
 
+/*
+ * Struct for collecting timing statistics on a "phase tree". The tree is
+ * specified as a limited DAG, but the timings are collected for the whole tree
+ * that you would get by expanding out the DAG by duplicating subtrees rooted
+ * at nodes with multiple parents.
+ *
+ * During execution, a child phase can be activated multiple times, and the
+ * total time will be accumulated. (So for example, you can start and end
+ * PHASE_MARK_ROOTS multiple times before completing the parent phase.)
+ *
+ * Incremental GC is represented by recording separate timing results for each
+ * slice within the overall GC.
+ */
 struct Statistics
 {
+    /*
+     * Phases are allowed to have multiple parents, though any path from root
+     * to leaf is allowed at most one multi-parented phase. We keep a full set
+     * of timings for each of the multi-parented phases, to be able to record
+     * all the timings in the expanded tree induced by our dag.
+     *
+     * Note that this wastes quite a bit of space, since we have a whole
+     * separate array of timing data containing all the phases. We could be
+     * more clever and keep an array of pointers biased by the offset of the
+     * multi-parented phase, and thereby preserve the simple
+     * timings[slot][PHASE_*] indexing. But the complexity doesn't seem worth
+     * the few hundred bytes of savings. If we want to extend things to full
+     * DAGs, this decision should be reconsidered.
+     */
+    static const size_t MAX_MULTIPARENT_PHASES = 6;
+
     explicit Statistics(JSRuntime *rt);
     ~Statistics();
 
     void beginPhase(Phase phase);
     void endPhase(Phase phase);
     void endParallelPhase(Phase phase, const GCParallelTask *task);
 
     void beginSlice(const ZoneGCStats &zoneStats, JSGCInvocationKind gckind,
@@ -142,16 +181,25 @@ struct Statistics
     char16_t *formatJSON(uint64_t timestamp);
     UniqueChars formatDetailedMessage();
 
     JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback);
 
     int64_t clearMaxGCPauseAccumulator();
     int64_t getMaxGCPauseSinceClear();
 
+    // Return the current phase, suppressing the synthetic PHASE_MUTATOR phase.
+    Phase currentPhase() {
+        if (phaseNestingDepth == 0)
+            return PHASE_NONE;
+        if (phaseNestingDepth == 1)
+            return phaseNesting[0] == PHASE_MUTATOR ? PHASE_NONE : phaseNesting[0];
+        return phaseNesting[phaseNestingDepth - 1];
+    }
+
   private:
     JSRuntime *runtime;
 
     int64_t startupTime;
 
     FILE *fp;
     bool fullFormat;
 
@@ -166,85 +214,93 @@ struct Statistics
     JSGCInvocationKind gckind;
 
     const char *nonincrementalReason;
 
     struct SliceData {
         SliceData(JS::gcreason::Reason reason, int64_t start, size_t startFaults)
           : reason(reason), resetReason(nullptr), start(start), startFaults(startFaults)
         {
-            mozilla::PodArrayZero(phaseTimes);
+            for (size_t i = 0; i < MAX_MULTIPARENT_PHASES + 1; i++)
+                mozilla::PodArrayZero(phaseTimes[i]);
         }
 
         JS::gcreason::Reason reason;
         const char *resetReason;
         int64_t start, end;
         size_t startFaults, endFaults;
-        int64_t phaseTimes[PHASE_LIMIT];
+        int64_t phaseTimes[MAX_MULTIPARENT_PHASES + 1][PHASE_LIMIT];
 
         int64_t duration() const { return end - start; }
     };
 
     Vector<SliceData, 8, SystemAllocPolicy> slices;
 
     /* Most recent time when the given phase started. */
     int64_t phaseStartTimes[PHASE_LIMIT];
 
     /* Bookkeeping for GC timings when timingMutator is true */
     int64_t timedGCStart;
     int64_t timedGCTime;
 
     /* Total time in a given phase for this GC. */
-    int64_t phaseTimes[PHASE_LIMIT];
+    int64_t phaseTimes[MAX_MULTIPARENT_PHASES + 1][PHASE_LIMIT];
 
     /* Total time in a given phase over all GCs. */
-    int64_t phaseTotals[PHASE_LIMIT];
+    int64_t phaseTotals[MAX_MULTIPARENT_PHASES + 1][PHASE_LIMIT];
 
     /* Number of events of this type for this GC. */
     unsigned int counts[STAT_LIMIT];
 
     /* Allocated space before the GC started. */
     size_t preBytes;
 
     /* Records the maximum GC pause in an API-controlled interval (in us). */
     int64_t maxPauseInInterval;
 
     /* Phases that are currently on stack. */
     static const size_t MAX_NESTING = 8;
     Phase phaseNesting[MAX_NESTING];
     size_t phaseNestingDepth;
+    size_t activeDagSlot;
 
     /*
      * To avoid recursive nesting, we discontinue a callback phase when any
      * other phases are started. Remember what phase to resume when the inner
      * phases are complete. (And because GCs can nest within the callbacks any
      * number of times, we need a whole stack of of phases to resume.)
      */
     Phase suspendedPhases[MAX_NESTING];
     size_t suspendedPhaseNestingDepth;
 
     /* Sweep times for SCCs of compartments. */
     Vector<int64_t, 0, SystemAllocPolicy> sccTimes;
 
     JS::GCSliceCallback sliceCallback;
 
+    /*
+     * True if we saw an OOM while allocating slices. Slices will not be
+     * individually recorded for the remainder of this GC.
+     */
+    bool abortSlices;
+
     void beginGC(JSGCInvocationKind kind);
     void endGC();
 
     void recordPhaseEnd(Phase phase);
 
     void gcDuration(int64_t *total, int64_t *maxPause);
     void sccDurations(int64_t *total, int64_t *maxPause);
     void printStats();
     bool formatData(StatisticsSerializer &ss, uint64_t timestamp);
 
     UniqueChars formatDescription();
     UniqueChars formatSliceDescription(unsigned i, const SliceData &slice);
     UniqueChars formatTotals();
-    UniqueChars formatPhaseTimes(int64_t *phaseTimes);
+    UniqueChars formatPhaseTimes(int64_t (*phaseTimes)[PHASE_LIMIT]);
 
     double computeMMU(int64_t resolution);
 };
 
 struct AutoGCSlice
 {
     AutoGCSlice(Statistics &stats, const ZoneGCStats &zoneStats, JSGCInvocationKind gckind,
                 JS::gcreason::Reason reason
--- a/js/src/gc/Verifier.cpp
+++ b/js/src/gc/Verifier.cpp
@@ -193,16 +193,18 @@ gc::GCRuntime::startVerifyPreBarriers()
         chunk->bitmap.clear();
 
     number++;
 
     VerifyPreTracer *trc = js_new<VerifyPreTracer>(rt, JSTraceCallback(nullptr));
     if (!trc)
         return;
 
+    gcstats::AutoPhase ap(stats, gcstats::PHASE_TRACE_HEAP);
+
     /*
      * Passing a function pointer directly to js_new trips a compiler bug in
      * MSVC. Work around by filling the pointer after allocating with nullptr.
      */
     trc->setTraceCallback(AccumulateEdge);
 
     const size_t size = 64 * 1024 * 1024;
     trc->root = (VerifyNode *)js_malloc(size);
@@ -490,20 +492,17 @@ js::gc::GCRuntime::endVerifyPostBarriers
     VerifyPostTracer::EdgeSet edges;
     AutoPrepareForTracing prep(rt, SkipAtoms);
 
     /* Visit every entry in the store buffer and put the edges in a hash set. */
     trc->setTraceCallback(PostVerifierCollectStoreBufferEdges);
     if (!edges.init())
         goto oom;
     trc->edges = &edges;
-    {
-        gcstats::AutoPhase ap(stats, gcstats::PHASE_MINOR_GC);
-        storeBuffer.markAll(trc);
-    }
+    storeBuffer.markAll(trc);
 
     /* Walk the heap to find any edges not the the |edges| set. */
     trc->setTraceCallback(PostVerifierVisitEdge);
     for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
         for (size_t kind = 0; kind < FINALIZE_LIMIT; ++kind) {
             for (ZoneCellIterUnderGC cells(zone, AllocKind(kind)); !cells.done(); cells.next()) {
                 Cell *src = cells.getCell();
                 JS_TraceChildren(trc, src, MapAllocToTraceKind(AllocKind(kind)));
--- a/js/src/jsgc.cpp
+++ b/js/src/jsgc.cpp
@@ -2645,30 +2645,34 @@ GCRuntime::updatePointersToRelocatedCell
     // as much as possible.
     ArenasToUpdate source(rt);
     if (CanUseExtraThreads())
         updateAllCellPointersParallel(source);
     else
         updateAllCellPointersSerial(&trc, source);
 
     // Mark roots to update them.
-    markRuntime(&trc, MarkRuntime);
-    Debugger::markAll(&trc);
-    Debugger::markAllCrossCompartmentEdges(&trc);
-
-    for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
-        WeakMapBase::markAll(c, &trc);
-        if (c->watchpointMap)
-            c->watchpointMap->markAll(&trc);
-    }
-
-    // Mark all gray roots, making sure we call the trace callback to get the
-    // current set.
-    if (JSTraceDataOp op = grayRootTracer.op)
-        (*op)(&trc, grayRootTracer.data);
+    {
+        markRuntime(&trc, MarkRuntime);
+
+        gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_ROOTS);
+        Debugger::markAll(&trc);
+        Debugger::markAllCrossCompartmentEdges(&trc);
+
+        for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
+            WeakMapBase::markAll(c, &trc);
+            if (c->watchpointMap)
+                c->watchpointMap->markAll(&trc);
+        }
+
+        // Mark all gray roots, making sure we call the trace callback to get the
+        // current set.
+        if (JSTraceDataOp op = grayRootTracer.op)
+            (*op)(&trc, grayRootTracer.data);
+    }
 
     // Sweep everything to fix up weak pointers
     WatchpointMap::sweepAll(rt);
     Debugger::sweepAll(rt->defaultFreeOp());
     for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
         if (CanRelocateZone(rt, zone))
             rt->gc.sweepZoneAfterCompacting(zone);
     }
@@ -4062,32 +4066,38 @@ GCRuntime::beginMarkPhase(JS::gcreason::
         gcstats::AutoPhase ap(stats, gcstats::PHASE_PURGE);
         purgeRuntime();
     }
 
     /*
      * Mark phase.
      */
     gcstats::AutoPhase ap1(stats, gcstats::PHASE_MARK);
-    gcstats::AutoPhase ap2(stats, gcstats::PHASE_MARK_ROOTS);
-
-    for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
-        /* Unmark everything in the zones being collected. */
-        zone->allocator.arenas.unmarkAll();
-    }
-
-    for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
-        /* Unmark all weak maps in the compartments being collected. */
-        WeakMapBase::unmarkCompartment(c);
-    }
-
-    if (isFull)
-        UnmarkScriptData(rt);
+
+    {
+        gcstats::AutoPhase ap(stats, gcstats::PHASE_UNMARK);
+
+        for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+            /* Unmark everything in the zones being collected. */
+            zone->allocator.arenas.unmarkAll();
+        }
+
+        for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
+            /* Unmark all weak maps in the compartments being collected. */
+            WeakMapBase::unmarkCompartment(c);
+        }
+
+        if (isFull)
+            UnmarkScriptData(rt);
+    }
 
     markRuntime(gcmarker, MarkRuntime);
+
+    gcstats::AutoPhase ap2(stats, gcstats::PHASE_MARK_ROOTS);
+
     if (isIncremental)
         bufferGrayRoots();
 
     /*
      * This code ensures that if a compartment is "dead", then it will be
      * collected in this GC. A compartment is considered dead if its maybeAlive
      * flag is false. The maybeAlive flag is set if:
      *   (1) the compartment has incoming cross-compartment edges, or
@@ -4105,37 +4115,41 @@ GCRuntime::beginMarkPhase(JS::gcreason::
      *
      * Read barriers and allocations can also cause revival. This might happen
      * during a function like JS_TransplantObject, which iterates over all
      * compartments, live or dead, and operates on their objects. See bug 803376
      * for details on this problem. To avoid the problem, we try to avoid
      * allocation and read barriers during JS_TransplantObject and the like.
      */
 
-    /* Set the maybeAlive flag based on cross-compartment edges. */
-    for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
-        for (JSCompartment::WrapperEnum e(c); !e.empty(); e.popFront()) {
-            const CrossCompartmentKey &key = e.front().key();
-            JSCompartment *dest;
-            switch (key.kind) {
-              case CrossCompartmentKey::ObjectWrapper:
-              case CrossCompartmentKey::DebuggerObject:
-              case CrossCompartmentKey::DebuggerSource:
-              case CrossCompartmentKey::DebuggerEnvironment:
-                dest = static_cast<JSObject *>(key.wrapped)->compartment();
-                break;
-              case CrossCompartmentKey::DebuggerScript:
-                dest = static_cast<JSScript *>(key.wrapped)->compartment();
-                break;
-              default:
-                dest = nullptr;
-                break;
+    {
+        gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_COMPARTMENTS);
+
+        /* Set the maybeAlive flag based on cross-compartment edges. */
+        for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
+            for (JSCompartment::WrapperEnum e(c); !e.empty(); e.popFront()) {
+                const CrossCompartmentKey &key = e.front().key();
+                JSCompartment *dest;
+                switch (key.kind) {
+                  case CrossCompartmentKey::ObjectWrapper:
+                  case CrossCompartmentKey::DebuggerObject:
+                  case CrossCompartmentKey::DebuggerSource:
+                  case CrossCompartmentKey::DebuggerEnvironment:
+                    dest = static_cast<JSObject *>(key.wrapped)->compartment();
+                    break;
+                  case CrossCompartmentKey::DebuggerScript:
+                    dest = static_cast<JSScript *>(key.wrapped)->compartment();
+                    break;
+                  default:
+                    dest = nullptr;
+                    break;
+                }
+                if (dest)
+                    dest->maybeAlive = true;
             }
-            if (dest)
-                dest->maybeAlive = true;
         }
     }
 
     /*
      * For black roots, code in gc/Marking.cpp will already have set maybeAlive
      * during MarkRuntime.
      */
 
@@ -4295,37 +4309,38 @@ js::gc::MarkingValidator::nonIncremental
     }
 
     /*
      * After this point, the function should run to completion, so we shouldn't
      * do anything fallible.
      */
     initialized = true;
 
-    for (GCCompartmentsIter c(runtime); !c.done(); c.next())
-        WeakMapBase::unmarkCompartment(c);
-
     /* Re-do all the marking, but non-incrementally. */
     js::gc::State state = gc->incrementalState;
     gc->incrementalState = MARK_ROOTS;
 
-    MOZ_ASSERT(gcmarker->isDrained());
-    gcmarker->reset();
-
-    for (auto chunk = gc->allNonEmptyChunks(); !chunk.done(); chunk.next())
-        chunk->bitmap.clear();
-
     {
-        gcstats::AutoPhase ap1(gc->stats, gcstats::PHASE_MARK);
-        gcstats::AutoPhase ap2(gc->stats, gcstats::PHASE_MARK_ROOTS);
+        gcstats::AutoPhase ap(gc->stats, gcstats::PHASE_MARK);
+
+        {
+            gcstats::AutoPhase ap(gc->stats, gcstats::PHASE_UNMARK);
+
+            for (GCCompartmentsIter c(runtime); !c.done(); c.next())
+                WeakMapBase::unmarkCompartment(c);
+
+            MOZ_ASSERT(gcmarker->isDrained());
+            gcmarker->reset();
+
+            for (auto chunk = gc->allNonEmptyChunks(); !chunk.done(); chunk.next())
+                chunk->bitmap.clear();
+        }
+
         gc->markRuntime(gcmarker, GCRuntime::MarkRuntime, GCRuntime::UseSavedRoots);
-    }
-
-    {
-        gcstats::AutoPhase ap1(gc->stats, gcstats::PHASE_MARK);
+
         SliceBudget budget;
         gc->incrementalState = MARK;
         gc->marker.drainMarkStack(budget);
     }
 
     gc->incrementalState = SWEEP;
     {
         gcstats::AutoPhase ap1(gc->stats, gcstats::PHASE_SWEEP);
@@ -6086,17 +6101,17 @@ class AutoDisableStoreBuffer
  *
  * Returns true if we "reset" an existing incremental GC, which would force us
  * to run another cycle.
  */
 MOZ_NEVER_INLINE bool
 GCRuntime::gcCycle(bool incremental, SliceBudget &budget, JSGCInvocationKind gckind,
                    JS::gcreason::Reason reason)
 {
-    minorGC(reason);
+    evictNursery(reason);
 
     /*
      * Marking can trigger many incidental post barriers, some of them for
      * objects which are not going to be live after the GC.
      */
     AutoDisableStoreBuffer adsb(this);
 
     AutoTraceSession session(rt, MajorCollecting);
@@ -6460,40 +6475,38 @@ GCRuntime::onOutOfMallocMemory(const Aut
 
     // Immediately decommit as many arenas as possible in the hopes that this
     // might let the OS scrape together enough pages to satisfy the failing
     // malloc request.
     decommitAllWithoutUnlocking(lock);
 }
 
 void
-GCRuntime::minorGC(JS::gcreason::Reason reason)
+GCRuntime::minorGCImpl(JS::gcreason::Reason reason, Nursery::TypeObjectList *pretenureTypes)
 {
     minorGCRequested = false;
     TraceLoggerThread *logger = TraceLoggerForMainThread(rt);
     AutoTraceLog logMinorGC(logger, TraceLogger_MinorGC);
-    nursery.collect(rt, reason, nullptr);
+    nursery.collect(rt, reason, pretenureTypes);
     MOZ_ASSERT_IF(!rt->mainThread.suppressGC, nursery.isEmpty());
 }
 
+// Alternate to the runtime-taking form that allows marking type objects as
+// needing pretenuring.
 void
 GCRuntime::minorGC(JSContext *cx, JS::gcreason::Reason reason)
 {
-    // Alternate to the runtime-taking form above which allows marking type
-    // objects as needing pretenuring.
-    minorGCRequested = false;
-    TraceLoggerThread *logger = TraceLoggerForMainThread(rt);
-    AutoTraceLog logMinorGC(logger, TraceLogger_MinorGC);
+    gcstats::AutoPhase ap(stats, gcstats::PHASE_MINOR_GC);
+
     Nursery::TypeObjectList pretenureTypes;
-    nursery.collect(rt, reason, &pretenureTypes);
+    minorGCImpl(reason, &pretenureTypes);
     for (size_t i = 0; i < pretenureTypes.length(); i++) {
         if (pretenureTypes[i]->canPreTenure())
             pretenureTypes[i]->setShouldPreTenure(cx);
     }
-    MOZ_ASSERT_IF(!rt->mainThread.suppressGC, nursery.isEmpty());
 }
 
 void
 GCRuntime::disableGenerationalGC()
 {
     if (isGenerationalGCEnabled()) {
         minorGC(JS::gcreason::API);
         nursery.disable();
--- a/js/src/jsgcinlines.h
+++ b/js/src/jsgcinlines.h
@@ -325,41 +325,42 @@ class ZoneCellIter : public ZoneCellIter
     ArenaLists *lists;
     AllocKind kind;
 
   public:
     ZoneCellIter(JS::Zone *zone, AllocKind kind)
       : lists(&zone->allocator.arenas),
         kind(kind)
     {
+        JSRuntime *rt = zone->runtimeFromMainThread();
+
         /*
          * We have a single-threaded runtime, so there's no need to protect
          * against other threads iterating or allocating. However, we do have
          * background finalization; we have to wait for this to finish if it's
          * currently active.
          */
         if (IsBackgroundFinalized(kind) &&
             zone->allocator.arenas.needBackgroundFinalizeWait(kind))
         {
-            zone->runtimeFromMainThread()->gc.waitBackgroundSweepEnd();
+            rt->gc.waitBackgroundSweepEnd();
         }
 
         /* Evict the nursery before iterating so we can see all things. */
-        JSRuntime *rt = zone->runtimeFromMainThread();
         rt->gc.evictNursery();
 
         if (lists->isSynchronizedFreeList(kind)) {
             lists = nullptr;
         } else {
-            MOZ_ASSERT(!zone->runtimeFromMainThread()->isHeapBusy());
+            MOZ_ASSERT(!rt->isHeapBusy());
             lists->copyFreeListToArena(kind);
         }
 
         /* Assert that no GCs can occur while a ZoneCellIter is live. */
-        noAlloc.disallowAlloc(zone->runtimeFromMainThread());
+        noAlloc.disallowAlloc(rt);
 
         init(zone, kind);
     }
 
     ~ZoneCellIter() {
         if (lists)
             lists->clearFreeListInArena(kind);
     }