Bug 650161 - Add new stats phases for compacting GC r=terrence
☠☠ backed out by 66edbd1cf41f ☠ ☠
authorJon Coppeard <jcoppeard@mozilla.com>
Thu, 14 Aug 2014 11:52:31 +0100
changeset 199512 bdcd6002052392cc8a73a8ef3d2c2de05c89eef7
parent 199511 edc768336c80b4d41e7a6961a9187063c8eac197
child 199513 8d1e96d1eb31a7d1dfd107764ac32e75f913a52a
push id27309
push userryanvm@gmail.com
push dateThu, 14 Aug 2014 20:21:31 +0000
treeherdermozilla-central@31db71673c6e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersterrence
bugs650161
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 650161 - Add new stats phases for compacting GC r=terrence
js/src/gc/GCRuntime.h
js/src/gc/Statistics.cpp
js/src/gc/Statistics.h
js/src/gc/Tracer.cpp
js/src/gc/Zone.cpp
js/src/jscompartment.cpp
js/src/jsgc.cpp
js/src/jsinfer.cpp
js/src/vm/Shape.cpp
--- a/js/src/gc/GCRuntime.h
+++ b/js/src/gc/GCRuntime.h
@@ -494,20 +494,20 @@ class GCRuntime
     void pushZealSelectedObjects();
     bool beginMarkPhase(JS::gcreason::Reason reason);
     bool shouldPreserveJITCode(JSCompartment *comp, int64_t currentTime,
                                JS::gcreason::Reason reason);
     void bufferGrayRoots();
     bool drainMarkStack(SliceBudget &sliceBudget, gcstats::Phase phase);
     template <class CompartmentIterT> void markWeakReferences(gcstats::Phase phase);
     void markWeakReferencesInCurrentGroup(gcstats::Phase phase);
-    template <class ZoneIterT, class CompartmentIterT> void markGrayReferences();
-    void markGrayReferencesInCurrentGroup();
+    template <class ZoneIterT, class CompartmentIterT> void markGrayReferences(gcstats::Phase phase);
+    void markGrayReferencesInCurrentGroup(gcstats::Phase phase);
     void markAllWeakReferences(gcstats::Phase phase);
-    void markAllGrayReferences();
+    void markAllGrayReferences(gcstats::Phase phase);
 
     void beginSweepPhase(bool lastGC);
     void findZoneGroups();
     bool findZoneEdgesForWeakMaps();
     void getNextZoneGroup();
     void endMarkingZoneGroup();
     void beginSweepingZoneGroup();
     bool shouldReleaseObservedTypes();
@@ -518,16 +518,17 @@ class GCRuntime
     void decommitArenasFromAvailableList(Chunk **availableListHeadp);
     void decommitArenas();
     void expireChunksAndArenas(bool shouldShrink);
     void sweepBackgroundThings(bool onBackgroundThread);
     void assertBackgroundSweepingFinished();
     bool shouldCompact();
 #ifdef JSGC_COMPACTING
     void compactPhase();
+    ArenaHeader *relocateArenas();
     void updatePointersToRelocatedCells();
     void releaseRelocatedArenas(ArenaHeader *relocatedList);
 #endif
     void finishCollection();
 
     void computeNonIncrementalMarkingForValidation();
     void validateIncrementalMarking();
     void finishMarkingValidation();
--- a/js/src/gc/Statistics.cpp
+++ b/js/src/gc/Statistics.cpp
@@ -302,16 +302,20 @@ static const PhaseInfo phases[] = {
     { PHASE_DISCARD_TI, "Discard TI", PHASE_DISCARD_ANALYSIS },
     { PHASE_FREE_TI_ARENA, "Free TI Arena", PHASE_DISCARD_ANALYSIS },
     { PHASE_SWEEP_TYPES, "Sweep Types", PHASE_DISCARD_ANALYSIS },
     { PHASE_SWEEP_OBJECT, "Sweep Object", PHASE_SWEEP },
     { PHASE_SWEEP_STRING, "Sweep String", PHASE_SWEEP },
     { PHASE_SWEEP_SCRIPT, "Sweep Script", PHASE_SWEEP },
     { PHASE_SWEEP_SHAPE, "Sweep Shape", PHASE_SWEEP },
     { PHASE_SWEEP_JITCODE, "Sweep JIT code", PHASE_SWEEP },
+    { PHASE_COMPACT, "Compact", PHASE_NO_PARENT },
+    { PHASE_COMPACT_MOVE, "Compact Move", PHASE_COMPACT },
+    { PHASE_COMPACT_UPDATE, "Compact Update", PHASE_COMPACT, },
+    { PHASE_COMPACT_UPDATE_GRAY, "Compact Update Gray", PHASE_COMPACT_UPDATE, },
     { PHASE_FINALIZE_END, "Finalize End Callback", PHASE_SWEEP },
     { PHASE_DESTROY, "Deallocate", PHASE_SWEEP },
     { PHASE_GC_END, "End Callback", PHASE_NO_PARENT },
     { PHASE_LIMIT, nullptr, PHASE_NO_PARENT }
 };
 
 static void
 FormatPhaseTimes(StatisticsSerializer &ss, const char *name, int64_t *times)
--- a/js/src/gc/Statistics.h
+++ b/js/src/gc/Statistics.h
@@ -53,16 +53,20 @@ enum Phase {
     PHASE_DISCARD_TI,
     PHASE_FREE_TI_ARENA,
     PHASE_SWEEP_TYPES,
     PHASE_SWEEP_OBJECT,
     PHASE_SWEEP_STRING,
     PHASE_SWEEP_SCRIPT,
     PHASE_SWEEP_SHAPE,
     PHASE_SWEEP_JITCODE,
+    PHASE_COMPACT,
+    PHASE_COMPACT_MOVE,
+    PHASE_COMPACT_UPDATE,
+    PHASE_COMPACT_UPDATE_GRAY,
     PHASE_FINALIZE_END,
     PHASE_DESTROY,
     PHASE_GC_END,
 
     PHASE_LIMIT
 };
 
 enum Stat {
@@ -222,27 +226,26 @@ struct AutoPhase
 
     Statistics &stats;
     Phase phase;
     MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER
 };
 
 struct MaybeAutoPhase
 {
-    explicit MaybeAutoPhase(MOZ_GUARD_OBJECT_NOTIFIER_ONLY_PARAM)
+    explicit MaybeAutoPhase(Statistics &statsArg, bool condition, Phase phaseArg
+                            MOZ_GUARD_OBJECT_NOTIFIER_PARAM)
       : stats(nullptr)
     {
         MOZ_GUARD_OBJECT_NOTIFIER_INIT;
-    }
-    void construct(Statistics &statsArg, Phase phaseArg)
-    {
-        JS_ASSERT(!stats);
-        stats = &statsArg;
-        phase = phaseArg;
-        stats->beginPhase(phase);
+        if (condition) {
+            stats = &statsArg;
+            phase = phaseArg;
+            stats->beginPhase(phase);
+        }
     }
     ~MaybeAutoPhase() {
         if (stats)
             stats->endPhase(phase);
     }
 
     Statistics *stats;
     Phase phase;
--- a/js/src/gc/Tracer.cpp
+++ b/js/src/gc/Tracer.cpp
@@ -549,19 +549,18 @@ GCMarker::markDelayedChildren(ArenaHeade
      * aheader. However, prepareForIncrementalGC sets the
      * allocatedDuringIncremental flag if we continue marking.
      */
 }
 
 bool
 GCMarker::markDelayedChildren(SliceBudget &budget)
 {
-    gcstats::MaybeAutoPhase ap;
-    if (runtime()->gc.state() == MARK)
-        ap.construct(runtime()->gc.stats, gcstats::PHASE_MARK_DELAYED);
+    GCRuntime &gc = runtime()->gc;
+    gcstats::MaybeAutoPhase ap(gc.stats, gc.state() == MARK, gcstats::PHASE_MARK_DELAYED);
 
     JS_ASSERT(unmarkedArenaStackTop);
     do {
         /*
          * If marking gets delayed at the same arena again, we must repeat
          * marking of its things. For that we pop arena from the stack and
          * clear its hasDelayedMarking flag before we begin the marking.
          */
--- a/js/src/gc/Zone.cpp
+++ b/js/src/gc/Zone.cpp
@@ -108,36 +108,41 @@ Zone::sweep(FreeOp *fop, bool releaseTyp
 {
     /*
      * Periodically release observed types for all scripts. This is safe to
      * do when there are no frames for the zone on the stack.
      */
     if (active)
         releaseTypes = false;
 
+    GCRuntime &gc = fop->runtime()->gc;
+
     {
-        gcstats::AutoPhase ap(fop->runtime()->gc.stats, gcstats::PHASE_DISCARD_ANALYSIS);
+        gcstats::MaybeAutoPhase ap(gc.stats, !gc.isHeapCompacting(),
+                                   gcstats::PHASE_DISCARD_ANALYSIS);
         types.sweep(fop, releaseTypes, oom);
     }
 
-    if (!fop->runtime()->debuggerList.isEmpty())
+    if (!fop->runtime()->debuggerList.isEmpty()) {
+        gcstats::MaybeAutoPhase ap1(gc.stats, !gc.isHeapCompacting(),
+                                    gcstats::PHASE_SWEEP_TABLES);
+        gcstats::MaybeAutoPhase ap2(gc.stats, !gc.isHeapCompacting(),
+                                    gcstats::PHASE_SWEEP_TABLES_BREAKPOINT);
         sweepBreakpoints(fop);
+    }
 }
 
 void
 Zone::sweepBreakpoints(FreeOp *fop)
 {
     /*
      * Sweep all compartments in a zone at the same time, since there is no way
      * to iterate over the scripts belonging to a single compartment in a zone.
      */
 
-    gcstats::AutoPhase ap1(fop->runtime()->gc.stats, gcstats::PHASE_SWEEP_TABLES);
-    gcstats::AutoPhase ap2(fop->runtime()->gc.stats, gcstats::PHASE_SWEEP_TABLES_BREAKPOINT);
-
     JS_ASSERT(isGCSweepingOrCompacting());
     for (ZoneCellIterUnderGC i(this, FINALIZE_SCRIPT); !i.done(); i.next()) {
         JSScript *script = i.get<JSScript>();
         JS_ASSERT_IF(isGCSweeping(), script->zone()->isGCSweeping());
         if (!script->hasAnyBreakpointsOrStepMode())
             continue;
 
         bool scriptGone = IsScriptAboutToBeFinalized(&script);
--- a/js/src/jscompartment.cpp
+++ b/js/src/jscompartment.cpp
@@ -567,64 +567,63 @@ JSCompartment::markRoots(JSTracer *trc)
     if (enterCompartmentDepth && global_)
         MarkObjectRoot(trc, global_.unsafeGet(), "on-stack compartment global");
 }
 
 void
 JSCompartment::sweep(FreeOp *fop, bool releaseTypes)
 {
     JS_ASSERT(!activeAnalysis);
-
-    /* This function includes itself in PHASE_SWEEP_TABLES. */
-    sweepCrossCompartmentWrappers();
-
     JSRuntime *rt = runtimeFromMainThread();
 
     {
-        gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_SWEEP_TABLES);
-
-        /* Remove dead references held weakly by the compartment. */
-
-        sweepBaseShapeTable();
-        sweepInitialShapeTable();
-        {
-            gcstats::AutoPhase ap(runtimeFromMainThread()->gc.stats,
-                                  gcstats::PHASE_SWEEP_TABLES_TYPE_OBJECT);
-            sweepNewTypeObjectTable(newTypeObjects);
-            sweepNewTypeObjectTable(lazyTypeObjects);
-        }
-        sweepCallsiteClones();
-        savedStacks_.sweep(rt);
-
-        if (global_ && IsObjectAboutToBeFinalized(global_.unsafeGet()))
-            global_.set(nullptr);
-
-        if (selfHostingScriptSource &&
-            IsObjectAboutToBeFinalized((JSObject **) selfHostingScriptSource.unsafeGet()))
-        {
-            selfHostingScriptSource.set(nullptr);
-        }
-
-        if (jitCompartment_)
-            jitCompartment_->sweep(fop, this);
-
-        /*
-         * JIT code increments activeUseCount for any RegExpShared used by jit
-         * code for the lifetime of the JIT script. Thus, we must perform
-         * sweeping after clearing jit code.
-         */
-        regExps.sweep(rt);
-
-        if (debugScopes)
-            debugScopes->sweep(rt);
-
-        /* Finalize unreachable (key,value) pairs in all weak maps. */
-        WeakMapBase::sweepCompartment(this);
+        gcstats::MaybeAutoPhase ap(rt->gc.stats, !rt->isHeapCompacting(),
+                                   gcstats::PHASE_SWEEP_TABLES_WRAPPER);
+        sweepCrossCompartmentWrappers();
     }
 
+    /* Remove dead references held weakly by the compartment. */
+
+    sweepBaseShapeTable();
+    sweepInitialShapeTable();
+    {
+        gcstats::MaybeAutoPhase ap(rt->gc.stats, !rt->isHeapCompacting(),
+                                   gcstats::PHASE_SWEEP_TABLES_TYPE_OBJECT);
+        sweepNewTypeObjectTable(newTypeObjects);
+        sweepNewTypeObjectTable(lazyTypeObjects);
+    }
+    sweepCallsiteClones();
+    savedStacks_.sweep(rt);
+
+    if (global_ && IsObjectAboutToBeFinalized(global_.unsafeGet()))
+        global_.set(nullptr);
+
+    if (selfHostingScriptSource &&
+        IsObjectAboutToBeFinalized((JSObject **) selfHostingScriptSource.unsafeGet()))
+    {
+        selfHostingScriptSource.set(nullptr);
+    }
+
+    if (jitCompartment_)
+        jitCompartment_->sweep(fop, this);
+
+    /*
+     * JIT code increments activeUseCount for any RegExpShared used by jit
+     * code for the lifetime of the JIT script. Thus, we must perform
+     * sweeping after clearing jit code.
+     */
+    regExps.sweep(rt);
+
+    if (debugScopes)
+        debugScopes->sweep(rt);
+
+    /* Finalize unreachable (key,value) pairs in all weak maps. */
+    WeakMapBase::sweepCompartment(this);
+
+    /* Sweep list of native iterators. */
     NativeIterator *ni = enumerators->next();
     while (ni != enumerators) {
         JSObject *iterObj = ni->iterObj();
         NativeIterator *next = ni->next();
         if (gc::IsObjectAboutToBeFinalized(&iterObj))
             ni->unlink();
         ni = next;
     }
@@ -633,21 +632,16 @@ JSCompartment::sweep(FreeOp *fop, bool r
 /*
  * Remove dead wrappers from the table. We must sweep all compartments, since
  * string entries in the crossCompartmentWrappers table are not marked during
  * markCrossCompartmentWrappers.
  */
 void
 JSCompartment::sweepCrossCompartmentWrappers()
 {
-    JSRuntime *rt = runtimeFromMainThread();
-
-    gcstats::AutoPhase ap1(rt->gc.stats, gcstats::PHASE_SWEEP_TABLES);
-    gcstats::AutoPhase ap2(rt->gc.stats, gcstats::PHASE_SWEEP_TABLES_WRAPPER);
-
     /* Remove dead wrappers from the table. */
     for (WrapperMap::Enum e(crossCompartmentWrappers); !e.empty(); e.popFront()) {
         CrossCompartmentKey key = e.front().key();
         bool keyDying = IsCellAboutToBeFinalized(&key.wrapped);
         bool valDying = IsValueAboutToBeFinalized(e.front().value().unsafeGet());
         bool dbgDying = key.debugger && IsObjectAboutToBeFinalized(&key.debugger);
         if (keyDying || valDying || dbgDying) {
             JS_ASSERT(key.kind != CrossCompartmentKey::StringWrapper);
--- a/js/src/jsgc.cpp
+++ b/js/src/jsgc.cpp
@@ -2215,16 +2215,36 @@ ArenaLists::relocateArenas(ArenaHeader *
      * are different now.
      */
     purge();
     checkEmptyFreeLists();
 
     return relocatedList;
 }
 
+ArenaHeader *
+GCRuntime::relocateArenas()
+{
+    gcstats::AutoPhase ap(stats, gcstats::PHASE_COMPACT_MOVE);
+
+    ArenaHeader *relocatedList = nullptr;
+    for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+        JS_ASSERT(zone->isGCFinished());
+        JS_ASSERT(!zone->isPreservingCode());
+
+        // We cannot move atoms as we depend on their addresses being constant.
+        if (!rt->isAtomsZone(zone)) {
+            zone->setGCState(Zone::Compact);
+            relocatedList = zone->allocator.arenas.relocateArenas(relocatedList);
+        }
+    }
+
+    return relocatedList;
+}
+
 struct MovingTracer : JSTracer {
     MovingTracer(JSRuntime *rt) : JSTracer(rt, Visit, TraceWeakMapValues) {}
 
     static void Visit(JSTracer *jstrc, void **thingp, JSGCTraceKind kind);
     static void Sweep(JSTracer *jstrc);
 };
 
 void
@@ -2249,18 +2269,16 @@ MovingTracer::Sweep(JSTracer *jstrc)
     FreeOp *fop = rt->defaultFreeOp();
 
     WatchpointMap::sweepAll(rt);
 
     Debugger::sweepAll(fop);
 
     for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) {
         if (zone->isCollecting()) {
-            gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_SWEEP_COMPARTMENTS);
-
             bool oom = false;
             zone->sweep(fop, false, &oom);
             JS_ASSERT(!oom);
 
             for (CompartmentsInZoneIter c(zone); !c.done(); c.next()) {
                 c->sweep(fop, false);
                 ArrayBufferObject::sweep(c);
             }
@@ -2296,70 +2314,64 @@ UpdateCellPointers(MovingTracer *trc, Ce
  *
  * The latter is necessary to update weak references which are not marked as
  * part of the traversal.
  */
 void
 GCRuntime::updatePointersToRelocatedCells()
 {
     JS_ASSERT(rt->currentThreadHasExclusiveAccess());
+
+    gcstats::AutoPhase ap(stats, gcstats::PHASE_COMPACT_UPDATE);
     MovingTracer trc(rt);
 
-    {
-        // TODO: Maybe give compaction its own set of phases.
-        gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK);
-
-        // TODO: We may need to fix up other weak pointers here.
-
-        // Fixup compartment global pointers as these get accessed during marking.
-        for (GCCompartmentsIter comp(rt); !comp.done(); comp.next())
-            comp->fixupAfterMovingGC();
-
-        // Fixup cross compartment wrappers as we assert the existence of wrappers in the map.
-        for (CompartmentsIter comp(rt, SkipAtoms); !comp.done(); comp.next())
-            comp->fixupCrossCompartmentWrappers(&trc);
-
-        // Fixup generators as these are not normally traced.
-        for (ContextIter i(rt); !i.done(); i.next()) {
-            for (JSGenerator *gen = i.get()->innermostGenerator(); gen; gen = gen->prevGenerator)
-                gen->obj = MaybeForwarded(gen->obj.get());
-        }
-
-        // Iterate through all allocated cells to update internal pointers.
-        for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
-            ArenaLists &al = zone->allocator.arenas;
-            for (unsigned i = 0; i < FINALIZE_LIMIT; ++i) {
-                AllocKind thingKind = static_cast<AllocKind>(i);
-                JSGCTraceKind traceKind = MapAllocToTraceKind(thingKind);
-                for (ArenaHeader *arena = al.getFirstArena(thingKind); arena; arena = arena->next) {
-                    for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) {
-                        UpdateCellPointers(&trc, i.getCell(), traceKind);
-                    }
+    // TODO: We may need to fix up other weak pointers here.
+
+    // Fixup compartment global pointers as these get accessed during marking.
+    for (GCCompartmentsIter comp(rt); !comp.done(); comp.next())
+        comp->fixupAfterMovingGC();
+
+    // Fixup cross compartment wrappers as we assert the existence of wrappers in the map.
+    for (CompartmentsIter comp(rt, SkipAtoms); !comp.done(); comp.next())
+        comp->fixupCrossCompartmentWrappers(&trc);
+
+    // Fixup generators as these are not normally traced.
+    for (ContextIter i(rt); !i.done(); i.next()) {
+        for (JSGenerator *gen = i.get()->innermostGenerator(); gen; gen = gen->prevGenerator)
+            gen->obj = MaybeForwarded(gen->obj.get());
+    }
+
+    // Iterate through all allocated cells to update internal pointers.
+    for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+        ArenaLists &al = zone->allocator.arenas;
+        for (unsigned i = 0; i < FINALIZE_LIMIT; ++i) {
+            AllocKind thingKind = static_cast<AllocKind>(i);
+            JSGCTraceKind traceKind = MapAllocToTraceKind(thingKind);
+            for (ArenaHeader *arena = al.getFirstArena(thingKind); arena; arena = arena->next) {
+                for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) {
+                    UpdateCellPointers(&trc, i.getCell(), traceKind);
                 }
             }
         }
-
-        // Mark roots to update them.
-        markRuntime(&trc, MarkRuntime);
-        Debugger::markAll(&trc);
-        Debugger::markCrossCompartmentDebuggerObjectReferents(&trc);
-
-        for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
-            if (c->watchpointMap)
-                c->watchpointMap->markAll(&trc);
-        }
-    }
-
-    {
-        gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_SWEEP);
-
-        markAllGrayReferences();
-
-        MovingTracer::Sweep(&trc);
-    }
+    }
+
+    // Mark roots to update them.
+    markRuntime(&trc, MarkRuntime);
+    Debugger::markAll(&trc);
+    Debugger::markCrossCompartmentDebuggerObjectReferents(&trc);
+
+    for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
+        if (c->watchpointMap)
+            c->watchpointMap->markAll(&trc);
+    }
+
+    markAllGrayReferences(gcstats::PHASE_COMPACT_UPDATE_GRAY);
+    markAllWeakReferences(gcstats::PHASE_COMPACT_UPDATE_GRAY);
+
+    MovingTracer::Sweep(&trc);
 }
 
 void
 GCRuntime::releaseRelocatedArenas(ArenaHeader *relocatedList)
 {
     // Release the relocated arenas, now containing only forwarding pointers
 
 #ifdef DEBUG
@@ -3737,17 +3749,16 @@ GCRuntime::beginMarkPhase(JS::gcreason::
 }
 
 template <class CompartmentIterT>
 void
 GCRuntime::markWeakReferences(gcstats::Phase phase)
 {
     JS_ASSERT(marker.isDrained());
 
-    gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_MARK);
     gcstats::AutoPhase ap1(stats, phase);
 
     for (;;) {
         bool markedAny = false;
         for (CompartmentIterT c(rt); !c.done(); c.next()) {
             markedAny |= WatchpointMap::markCompartmentIteratively(c, &marker);
             markedAny |= WeakMapBase::markCompartmentIteratively(c, &marker);
         }
@@ -3765,57 +3776,47 @@ GCRuntime::markWeakReferences(gcstats::P
 void
 GCRuntime::markWeakReferencesInCurrentGroup(gcstats::Phase phase)
 {
     markWeakReferences<GCCompartmentGroupIter>(phase);
 }
 
 template <class ZoneIterT, class CompartmentIterT>
 void
-GCRuntime::markGrayReferences()
-{
-    {
-        gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_MARK);
-        gcstats::AutoPhase ap1(stats, gcstats::PHASE_SWEEP_MARK_GRAY);
-        marker.setMarkColorGray();
-        if (marker.hasBufferedGrayRoots()) {
-            for (ZoneIterT zone(rt); !zone.done(); zone.next())
-                marker.markBufferedGrayRoots(zone);
-        } else {
-            JS_ASSERT(!isIncremental);
-            if (JSTraceDataOp op = grayRootTracer.op)
-                (*op)(&marker, grayRootTracer.data);
-        }
-        SliceBudget budget;
-        marker.drainMarkStack(budget);
-    }
-
-    markWeakReferences<CompartmentIterT>(gcstats::PHASE_SWEEP_MARK_GRAY_WEAK);
-
-    JS_ASSERT(marker.isDrained());
-
-    marker.setMarkColorBlack();
+GCRuntime::markGrayReferences(gcstats::Phase phase)
+{
+    gcstats::AutoPhase ap(stats, phase);
+    if (marker.hasBufferedGrayRoots()) {
+        for (ZoneIterT zone(rt); !zone.done(); zone.next())
+            marker.markBufferedGrayRoots(zone);
+    } else {
+        JS_ASSERT(!isIncremental);
+        if (JSTraceDataOp op = grayRootTracer.op)
+            (*op)(&marker, grayRootTracer.data);
+    }
+    SliceBudget budget;
+    marker.drainMarkStack(budget);
 }
 
 void
-GCRuntime::markGrayReferencesInCurrentGroup()
-{
-    markGrayReferences<GCZoneGroupIter, GCCompartmentGroupIter>();
+GCRuntime::markGrayReferencesInCurrentGroup(gcstats::Phase phase)
+{
+    markGrayReferences<GCZoneGroupIter, GCCompartmentGroupIter>(phase);
 }
 
 void
 GCRuntime::markAllWeakReferences(gcstats::Phase phase)
 {
     markWeakReferences<GCCompartmentsIter>(phase);
 }
 
 void
-GCRuntime::markAllGrayReferences()
-{
-    markGrayReferences<GCZonesIter, GCCompartmentsIter>();
+GCRuntime::markAllGrayReferences(gcstats::Phase phase)
+{
+    markGrayReferences<GCZonesIter, GCCompartmentsIter>(phase);
 }
 
 #ifdef DEBUG
 
 class js::gc::MarkingValidator
 {
   public:
     explicit MarkingValidator(GCRuntime *gc);
@@ -3926,32 +3927,37 @@ js::gc::MarkingValidator::nonIncremental
         gcstats::AutoPhase ap1(gc->stats, gcstats::PHASE_MARK);
         SliceBudget budget;
         gc->incrementalState = MARK;
         gc->marker.drainMarkStack(budget);
     }
 
     gc->incrementalState = SWEEP;
     {
-        gcstats::AutoPhase ap(gc->stats, gcstats::PHASE_SWEEP);
+        gcstats::AutoPhase ap1(gc->stats, gcstats::PHASE_SWEEP);
+        gcstats::AutoPhase ap2(gc->stats, gcstats::PHASE_SWEEP_MARK);
         gc->markAllWeakReferences(gcstats::PHASE_SWEEP_MARK_WEAK);
 
         /* Update zone state for gray marking. */
         for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
             JS_ASSERT(zone->isGCMarkingBlack());
             zone->setGCState(Zone::MarkGray);
         }
-
-        gc->markAllGrayReferences();
+        gc->marker.setMarkColorGray();
+
+        gc->markAllGrayReferences(gcstats::PHASE_SWEEP_MARK_GRAY);
+        gc->markAllWeakReferences(gcstats::PHASE_SWEEP_MARK_GRAY_WEAK);
 
         /* Restore zone state. */
         for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
             JS_ASSERT(zone->isGCMarkingGray());
             zone->setGCState(Zone::Mark);
         }
+        JS_ASSERT(gc->marker.isDrained());
+        gc->marker.setMarkColorBlack();
     }
 
     /* Take a copy of the non-incremental mark state and restore the original. */
     for (GCChunkSet::Range r(gc->chunkSet.all()); !r.empty(); r.popFront()) {
         Chunk *chunk = r.front();
         ChunkBitmap *bitmap = &chunk->bitmap;
         ChunkBitmap *entry = map.lookup(chunk)->value();
         Swap(*entry, *bitmap);
@@ -4342,17 +4348,16 @@ js::DelayCrossCompartmentGrayMarking(JSO
 #endif
 }
 
 static void
 MarkIncomingCrossCompartmentPointers(JSRuntime *rt, const uint32_t color)
 {
     JS_ASSERT(color == BLACK || color == GRAY);
 
-    gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_SWEEP_MARK);
     static const gcstats::Phase statsPhases[] = {
         gcstats::PHASE_SWEEP_MARK_INCOMING_BLACK,
         gcstats::PHASE_SWEEP_MARK_INCOMING_GRAY
     };
     gcstats::AutoPhase ap1(rt->gc.stats, statsPhases[color]);
 
     bool unlinkList = color == GRAY;
 
@@ -4467,16 +4472,18 @@ js::NotifyGCPostSwap(JSObject *a, JSObje
         DelayCrossCompartmentGrayMarking(b);
     if (removedFlags & JS_GC_SWAP_OBJECT_B_REMOVED)
         DelayCrossCompartmentGrayMarking(a);
 }
 
 void
 GCRuntime::endMarkingZoneGroup()
 {
+    gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_MARK);
+
     /*
      * Mark any incoming black pointers from previously swept compartments
      * whose referents are not marked. This can occur when gray cells become
      * black by the action of UnmarkGray.
      */
     MarkIncomingCrossCompartmentPointers(rt, BLACK);
 
     markWeakReferencesInCurrentGroup(gcstats::PHASE_SWEEP_MARK_WEAK);
@@ -4486,32 +4493,32 @@ GCRuntime::endMarkingZoneGroup()
      * group.  Note that there may be pointers to the atoms compartment, and
      * these will be marked through, as they are not marked with
      * MarkCrossCompartmentXXX.
      */
     for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
         JS_ASSERT(zone->isGCMarkingBlack());
         zone->setGCState(Zone::MarkGray);
     }
+    marker.setMarkColorGray();
 
     /* Mark incoming gray pointers from previously swept compartments. */
-    marker.setMarkColorGray();
     MarkIncomingCrossCompartmentPointers(rt, GRAY);
-    marker.setMarkColorBlack();
 
     /* Mark gray roots and mark transitively inside the current compartment group. */
-    markGrayReferencesInCurrentGroup();
+    markGrayReferencesInCurrentGroup(gcstats::PHASE_SWEEP_MARK_GRAY);
+    markWeakReferencesInCurrentGroup(gcstats::PHASE_SWEEP_MARK_GRAY_WEAK);
 
     /* Restore marking state. */
     for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
         JS_ASSERT(zone->isGCMarkingGray());
         zone->setGCState(Zone::Mark);
     }
-
-    JS_ASSERT(marker.isDrained());
+    MOZ_ASSERT(marker.isDrained());
+    marker.setMarkColorBlack();
 }
 
 void
 GCRuntime::beginSweepingZoneGroup()
 {
     /*
      * Begin sweeping the group of zones in gcCurrentZoneGroup,
      * performing actions that must be done before yielding to caller.
@@ -4574,16 +4581,18 @@ GCRuntime::beginSweepingZoneGroup()
 
         for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
             gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_DISCARD_CODE);
             zone->discardJitCode(&fop);
         }
 
         for (GCCompartmentGroupIter c(rt); !c.done(); c.next()) {
             gcstats::AutoSCC scc(stats, zoneGroupIndex);
+            gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_TABLES);
+
             c->sweep(&fop, releaseObservedTypes && !c->zone()->isPreservingCode());
         }
 
         for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
             gcstats::AutoSCC scc(stats, zoneGroupIndex);
 
             // If there is an OOM while sweeping types, the type information
             // will be deoptimized so that it is still correct (i.e.
@@ -4935,28 +4944,19 @@ GCRuntime::endSweepPhase(bool lastGC)
 
 #ifdef JSGC_COMPACTING
 void
 GCRuntime::compactPhase()
 {
     JS_ASSERT(rt->gc.nursery.isEmpty());
     JS_ASSERT(!sweepOnBackgroundThread);
 
-    ArenaHeader *relocatedList = nullptr;
-    for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
-        JS_ASSERT(zone->isGCFinished());
-        JS_ASSERT(!zone->isPreservingCode());
-
-        // We cannot move atoms as we depend on their addresses being constant.
-        if (!rt->isAtomsZone(zone)) {
-            zone->setGCState(Zone::Compact);
-            relocatedList = zone->allocator.arenas.relocateArenas(relocatedList);
-        }
-    }
-
+    gcstats::AutoPhase ap(stats, gcstats::PHASE_COMPACT);
+
+    ArenaHeader *relocatedList = relocateArenas();
     updatePointersToRelocatedCells();
     releaseRelocatedArenas(relocatedList);
 
 #ifdef DEBUG
     CheckHashTablesAfterMovingGC(rt);
     for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
         if (!rt->isAtomsZone(zone) && !zone->isPreservingCode())
             zone->allocator.arenas.checkEmptyFreeLists();
--- a/js/src/jsinfer.cpp
+++ b/js/src/jsinfer.cpp
@@ -4385,17 +4385,18 @@ TypeZone::sweep(FreeOp *fop, bool releas
                 } else {
                     output.setSweepIndex(newCompilerOutputCount++);
                 }
             }
         }
     }
 
     {
-        gcstats::AutoPhase ap2(rt->gc.stats, gcstats::PHASE_DISCARD_TI);
+        gcstats::MaybeAutoPhase ap2(rt->gc.stats, !rt->isHeapCompacting(),
+                                    gcstats::PHASE_DISCARD_TI);
 
         for (ZoneCellIterUnderGC i(zone(), FINALIZE_SCRIPT); !i.done(); i.next()) {
             JSScript *script = i.get<JSScript>();
             if (script->types) {
                 types::TypeScript::Sweep(fop, script, oom);
 
                 if (releaseTypes) {
                     script->types->destroy();
@@ -4416,17 +4417,18 @@ TypeZone::sweep(FreeOp *fop, bool releas
                     if (script->hasParallelIonScript())
                         script->parallelIonScript()->recompileInfoRef().shouldSweep(*this);
                 }
             }
         }
     }
 
     {
-        gcstats::AutoPhase ap2(rt->gc.stats, gcstats::PHASE_SWEEP_TYPES);
+        gcstats::MaybeAutoPhase ap2(rt->gc.stats, !rt->isHeapCompacting(),
+                                    gcstats::PHASE_SWEEP_TYPES);
 
         for (gc::ZoneCellIterUnderGC iter(zone(), gc::FINALIZE_TYPE_OBJECT);
              !iter.done(); iter.next())
         {
             TypeObject *object = iter.get<TypeObject>();
             object->sweep(fop, oom);
         }
 
@@ -4444,17 +4446,18 @@ TypeZone::sweep(FreeOp *fop, bool releas
                 (*compilerOutputs)[sweepIndex++] = output;
             }
         }
         JS_ASSERT(sweepIndex == newCompilerOutputCount);
         JS_ALWAYS_TRUE(compilerOutputs->resize(newCompilerOutputCount));
     }
 
     {
-        gcstats::AutoPhase ap2(rt->gc.stats, gcstats::PHASE_FREE_TI_ARENA);
+        gcstats::MaybeAutoPhase ap2(rt->gc.stats, !rt->isHeapCompacting(),
+                                    gcstats::PHASE_FREE_TI_ARENA);
         rt->freeLifoAlloc.transferFrom(&oldAlloc);
     }
 }
 
 void
 TypeZone::clearAllNewScriptsOnOOM()
 {
     for (gc::ZoneCellIterUnderGC iter(zone(), gc::FINALIZE_TYPE_OBJECT);
--- a/js/src/vm/Shape.cpp
+++ b/js/src/vm/Shape.cpp
@@ -1537,18 +1537,19 @@ BaseShape::assertConsistency()
         JS_ASSERT(getObjectFlags() == unowned->getObjectFlags());
     }
 #endif
 }
 
 void
 JSCompartment::sweepBaseShapeTable()
 {
-    gcstats::AutoPhase ap(runtimeFromMainThread()->gc.stats,
-                          gcstats::PHASE_SWEEP_TABLES_BASE_SHAPE);
+    GCRuntime &gc = runtimeFromMainThread()->gc;
+    gcstats::MaybeAutoPhase ap(gc.stats, !gc.isHeapCompacting(),
+                               gcstats::PHASE_SWEEP_TABLES_BASE_SHAPE);
 
     if (baseShapes.initialized()) {
         for (BaseShapeSet::Enum e(baseShapes); !e.empty(); e.popFront()) {
             UnownedBaseShape *base = e.front().unbarrieredGet();
             if (IsBaseShapeAboutToBeFinalized(&base)) {
                 e.removeFront();
             } else if (base != e.front()) {
                 StackBaseShape sbase(base);
@@ -1834,18 +1835,19 @@ EmptyShape::insertInitialShape(Exclusive
         JSContext *ncx = cx->asJSContext();
         ncx->runtime()->newObjectCache.invalidateEntriesForShape(ncx, shape, proto);
     }
 }
 
 void
 JSCompartment::sweepInitialShapeTable()
 {
-    gcstats::AutoPhase ap(runtimeFromMainThread()->gc.stats,
-                          gcstats::PHASE_SWEEP_TABLES_INITIAL_SHAPE);
+    GCRuntime &gc = runtimeFromMainThread()->gc;
+    gcstats::MaybeAutoPhase ap(gc.stats, !gc.isHeapCompacting(),
+                               gcstats::PHASE_SWEEP_TABLES_INITIAL_SHAPE);
 
     if (initialShapes.initialized()) {
         for (InitialShapeSet::Enum e(initialShapes); !e.empty(); e.popFront()) {
             const InitialShapeEntry &entry = e.front();
             Shape *shape = entry.shape.unbarrieredGet();
             JSObject *proto = entry.proto.raw();
             if (IsShapeAboutToBeFinalized(&shape) ||
                 (entry.proto.isObject() && IsObjectAboutToBeFinalized(&proto)))