Bug 1376096 - Add per-sweep-group actions in addition to per-zone actions r=sfink
authorJon Coppeard <jcoppeard@mozilla.com>
Wed, 28 Jun 2017 15:17:53 -0700
changeset 601717 8b1346172862351217927a431114e81465259a24
parent 601716 d365e0c892e30ea94209454ed26f57d1ad48c926
child 601718 af66c0b948814ad0283c57533861d8c63cb14352
push id66200
push userhchang@mozilla.com
push dateThu, 29 Jun 2017 03:53:43 +0000
reviewerssfink
bugs1376096
milestone56.0a1
Bug 1376096 - Add per-sweep-group actions in addition to per-zone actions r=sfink
js/src/gc/GCRuntime.h
js/src/jsgc.cpp
--- a/js/src/gc/GCRuntime.h
+++ b/js/src/gc/GCRuntime.h
@@ -40,16 +40,23 @@ class MarkingValidator;
 struct MovingTracer;
 
 enum IncrementalProgress
 {
     NotFinished = 0,
     Finished
 };
 
+enum SweepActionList
+{
+    PerSweepGroupActionList,
+    PerZoneActionList,
+    SweepActionListCount
+};
+
 class ChunkPool
 {
     Chunk* head_;
     size_t count_;
 
   public:
     ChunkPool() : head_(nullptr), count_(0) {}
 
@@ -1001,18 +1008,17 @@ class GCRuntime
     void sweepJitDataOnMainThread(FreeOp* fop);
     void endSweepingSweepGroup();
     IncrementalProgress performSweepActions(SliceBudget& sliceBudget,
                                             AutoLockForExclusiveAccess& lock);
     static IncrementalProgress sweepTypeInformation(GCRuntime* gc, FreeOp* fop, Zone* zone,
                                                     SliceBudget& budget, AllocKind kind);
     static IncrementalProgress mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, Zone* zone,
                                                       SliceBudget& budget, AllocKind kind);
-    static IncrementalProgress sweepAtomsTable(GCRuntime* gc, FreeOp* fop, Zone* zone,
-                                               SliceBudget& budget, AllocKind kind);
+    static IncrementalProgress sweepAtomsTable(GCRuntime* gc, SliceBudget& budget);
     void startSweepingAtomsTable();
     IncrementalProgress sweepAtomsTable(SliceBudget& budget);
     static IncrementalProgress finalizeAllocKind(GCRuntime* gc, FreeOp* fop, Zone* zone,
                                                  SliceBudget& budget, AllocKind kind);
     static IncrementalProgress sweepShapeTree(GCRuntime* gc, FreeOp* fop, Zone* zone,
                                               SliceBudget& budget, AllocKind kind);
     void endSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock);
     bool allCCVisibleZonesWereCollected() const;
@@ -1222,18 +1228,20 @@ class GCRuntime
 
   private:
     /* Index of current sweep group (for stats). */
     ActiveThreadData<unsigned> sweepGroupIndex;
 
     /*
      * Incremental sweep state.
      */
+
     ActiveThreadData<JS::Zone*> sweepGroups;
     ActiveThreadOrGCTaskData<JS::Zone*> currentSweepGroup;
+    ActiveThreadData<SweepActionList> sweepActionList;
     ActiveThreadData<size_t> sweepPhaseIndex;
     ActiveThreadData<JS::Zone*> sweepZone;
     ActiveThreadData<size_t> sweepActionIndex;
     ActiveThreadData<mozilla::Maybe<AtomSet::Enum>> maybeAtomsToSweep;
     ActiveThreadData<bool> abortSweepAfterCurrentGroup;
 
     /*
      * Concurrent sweep infrastructure.
--- a/js/src/jsgc.cpp
+++ b/js/src/jsgc.cpp
@@ -406,42 +406,59 @@ static const FinalizePhase BackgroundFin
             AllocKind::OBJECT_GROUP
         }
     }
 };
 
 // Incremental sweeping is controlled by a list of actions that describe what
 // happens and in what order. Due to the incremental nature of sweeping an
 // action does not necessarily run to completion so the current state is tracked
-// in the GCRuntime by the performSweepActions() method.
+// in the GCRuntime by the performSweepActions() method. We may yield to the
+// mutator after running part of any action.
+//
+// There are two types of action: per-sweep-group and per-zone.
 //
-// Actions are performed in phases run per sweep group, and each action is run
-// for every zone in the group, i.e. as if by the following pseudocode:
+// Per-sweep-group actions are run first. Per-zone actions are grouped into
+// phases, with each phase run once per sweep group, and each action in it run
+// for every zone in the group.
+//
+// This is illustrated by the following pseudocode:
 //
 //   for each sweep group:
-//     for each phase:
+//     for each per-sweep-group action:
+//       run part or all of action
+//       maybe yield to the mutator
+//     for each per-zone phase:
 //       for each zone in sweep group:
 //         for each action in phase:
-//           perform_action
-
-struct SweepAction
+//           run part or all of action
+//           maybe yield to the mutator
+//
+// Progress through the loops is stored in GCRuntime, e.g. |sweepActionIndex|
+// for looping through the sweep actions.
+
+using PerSweepGroupSweepAction = IncrementalProgress (*)(GCRuntime* gc, SliceBudget& budget);
+
+struct PerZoneSweepAction
 {
     using Func = IncrementalProgress (*)(GCRuntime* gc, FreeOp* fop, Zone* zone,
                                          SliceBudget& budget, AllocKind kind);
 
     Func func;
     AllocKind kind;
 
-    SweepAction(Func func, AllocKind kind) : func(func), kind(kind) {}
+    PerZoneSweepAction(Func func, AllocKind kind) : func(func), kind(kind) {}
 };
 
-using SweepActionVector = Vector<SweepAction, 0, SystemAllocPolicy>;
-using SweepPhaseVector = Vector<SweepActionVector, 0, SystemAllocPolicy>;
-
-static SweepPhaseVector SweepPhases;
+using PerSweepGroupActionVector = Vector<PerSweepGroupSweepAction, 0, SystemAllocPolicy>;
+using PerZoneSweepActionVector = Vector<PerZoneSweepAction, 0, SystemAllocPolicy>;
+using PerZoneSweepPhaseVector = Vector<PerZoneSweepActionVector, 0, SystemAllocPolicy>;
+
+static PerSweepGroupActionVector PerSweepGroupSweepActions;
+static PerZoneSweepPhaseVector PerZoneSweepPhases;
 
 bool
 js::gc::InitializeStaticData()
 {
     return GCRuntime::initializeSweepActions();
 }
 
 template<>
@@ -5343,19 +5360,20 @@ GCRuntime::beginSweepingSweepGroup()
             zone->arenas.queueForForegroundSweep(&fop, IncrementalFinalizePhases[i]);
 
         for (unsigned i = 0; i < ArrayLength(BackgroundFinalizePhases); ++i)
             zone->arenas.queueForBackgroundSweep(&fop, BackgroundFinalizePhases[i]);
 
         zone->arenas.queueForegroundThingsForSweep(&fop);
     }
 
+    sweepActionList = PerSweepGroupActionList;
+    sweepActionIndex = 0;
     sweepPhaseIndex = 0;
     sweepZone = currentSweepGroup;
-    sweepActionIndex = 0;
 }
 
 void
 GCRuntime::endSweepingSweepGroup()
 {
     {
         gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::FINALIZE_END);
         FreeOp fop(rt);
@@ -5567,20 +5585,19 @@ GCRuntime::startSweepingAtomsTable()
         return;
     }
 
     // Initialize remaining atoms to sweep.
     maybeAtoms.emplace(*atomsTable);
 }
 
 /* static */ IncrementalProgress
-GCRuntime::sweepAtomsTable(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget,
-                           AllocKind kind)
-{
-    if (!zone->isAtomsZone())
+GCRuntime::sweepAtomsTable(GCRuntime* gc, SliceBudget& budget)
+{
+    if (!gc->atomsZone->isGCSweeping())
         return Finished;
 
     return gc->sweepAtomsTable(budget);
 }
 
 
 IncrementalProgress
 GCRuntime::sweepAtomsTable(SliceBudget& budget)
@@ -5655,80 +5672,118 @@ GCRuntime::sweepShapeTree(GCRuntime* gc,
 
     if (!SweepArenaList<AccessorShape>(&al.gcAccessorShapeArenasToUpdate.ref(), budget))
         return NotFinished;
 
     return Finished;
 }
 
 static void
-AddSweepPhase(bool* ok)
+AddPerSweepGroupSweepAction(bool* ok, PerSweepGroupSweepAction action)
 {
     if (*ok)
-        *ok = SweepPhases.emplaceBack();
+        *ok = PerSweepGroupSweepActions.emplaceBack(action);
 }
 
 static void
-AddSweepAction(bool* ok, SweepAction::Func func, AllocKind kind = AllocKind::LIMIT)
+AddPerZoneSweepPhase(bool* ok)
 {
     if (*ok)
-        *ok = SweepPhases.back().emplaceBack(func, kind);
+        *ok = PerZoneSweepPhases.emplaceBack();
+}
+
+static void
+AddPerZoneSweepAction(bool* ok, PerZoneSweepAction::Func func, AllocKind kind = AllocKind::LIMIT)
+{
+    if (*ok)
+        *ok = PerZoneSweepPhases.back().emplaceBack(func, kind);
 }
 
 /* static */ bool
 GCRuntime::initializeSweepActions()
 {
     bool ok = true;
 
-    AddSweepPhase(&ok);
-    AddSweepAction(&ok, GCRuntime::sweepAtomsTable);
+    AddPerSweepGroupSweepAction(&ok, GCRuntime::sweepAtomsTable);
+
+    AddPerZoneSweepPhase(&ok);
     for (auto kind : ForegroundObjectFinalizePhase.kinds)
-        AddSweepAction(&ok, GCRuntime::finalizeAllocKind, kind);
-
-    AddSweepPhase(&ok);
-    AddSweepAction(&ok, GCRuntime::sweepTypeInformation);
-    AddSweepAction(&ok, GCRuntime::mergeSweptObjectArenas);
+        AddPerZoneSweepAction(&ok, GCRuntime::finalizeAllocKind, kind);
+
+    AddPerZoneSweepPhase(&ok);
+    AddPerZoneSweepAction(&ok, GCRuntime::sweepTypeInformation);
+    AddPerZoneSweepAction(&ok, GCRuntime::mergeSweptObjectArenas);
 
     for (const auto& finalizePhase : IncrementalFinalizePhases) {
-        AddSweepPhase(&ok);
+        AddPerZoneSweepPhase(&ok);
         for (auto kind : finalizePhase.kinds)
-            AddSweepAction(&ok, GCRuntime::finalizeAllocKind, kind);
-    }
-
-    AddSweepPhase(&ok);
-    AddSweepAction(&ok, GCRuntime::sweepShapeTree);
+            AddPerZoneSweepAction(&ok, GCRuntime::finalizeAllocKind, kind);
+    }
+
+    AddPerZoneSweepPhase(&ok);
+    AddPerZoneSweepAction(&ok, GCRuntime::sweepShapeTree);
 
     return ok;
 }
 
+static inline SweepActionList
+NextSweepActionList(SweepActionList list)
+{
+    MOZ_ASSERT(list < SweepActionListCount);
+    return SweepActionList(unsigned(list) + 1);
+}
+
 IncrementalProgress
 GCRuntime::performSweepActions(SliceBudget& budget, AutoLockForExclusiveAccess& lock)
 {
     AutoSetThreadIsSweeping threadIsSweeping;
 
     gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP);
     FreeOp fop(rt);
 
     if (drainMarkStack(budget, gcstats::PhaseKind::SWEEP_MARK) == NotFinished)
         return NotFinished;
 
     for (;;) {
-        for (; sweepPhaseIndex < SweepPhases.length(); sweepPhaseIndex++) {
-            const auto& actions = SweepPhases[sweepPhaseIndex];
-            for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) {
+        for (; sweepActionList < SweepActionListCount;
+             sweepActionList = NextSweepActionList(sweepActionList))
+        {
+            switch (sweepActionList) {
+              case PerSweepGroupActionList: {
+                const auto& actions = PerSweepGroupSweepActions;
                 for (; sweepActionIndex < actions.length(); sweepActionIndex++) {
-                    const auto& action = actions[sweepActionIndex];
-                    if (action.func(this, &fop, sweepZone, budget, action.kind) == NotFinished)
+                    auto action = actions[sweepActionIndex];
+                    if (action(this, budget) == NotFinished)
                         return NotFinished;
                 }
                 sweepActionIndex = 0;
+                break;
+              }
+
+              case PerZoneActionList:
+                for (; sweepPhaseIndex < PerZoneSweepPhases.length(); sweepPhaseIndex++) {
+                    const auto& actions = PerZoneSweepPhases[sweepPhaseIndex];
+                    for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) {
+                        for (; sweepActionIndex < actions.length(); sweepActionIndex++) {
+                            const auto& action = actions[sweepActionIndex];
+                            if (action.func(this, &fop, sweepZone, budget, action.kind) == NotFinished)
+                                return NotFinished;
+                        }
+                        sweepActionIndex = 0;
+                    }
+                    sweepZone = currentSweepGroup;
+                }
+                sweepPhaseIndex = 0;
+                break;
+
+              default:
+                MOZ_CRASH("Unexpected sweepActionList value");
             }
-            sweepZone = currentSweepGroup;
         }
-        sweepPhaseIndex = 0;
+        sweepActionList = PerSweepGroupActionList;
 
         endSweepingSweepGroup();
         getNextSweepGroup();
         if (!currentSweepGroup)
             return Finished;
 
         endMarkingSweepGroup();
         beginSweepingSweepGroup();