Bug 1382275 - Refactor incremental sweeping to more clearly express the control flow r=sfink
authorJon Coppeard <jcoppeard@mozilla.com>
Wed, 26 Jul 2017 14:19:11 +0100
changeset 371297 0c66a9949ff91d545f1a8a0ab7e0b1dc470a3f21
parent 371296 78859e22ce1770da7993efd8c2604f58efa23632
child 371298 14755efcbf65e408fd122d027af8a81ddd2b7b08
push id32241
push usercbook@mozilla.com
push dateThu, 27 Jul 2017 08:57:54 +0000
treeherdermozilla-central@e5693cea1ec9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfink
bugs1382275
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1382275 - Refactor incremental sweeping to more clearly express the control flow r=sfink
js/src/gc/GCRuntime.h
js/src/jsgc.cpp
js/src/jsgcinlines.h
js/src/vm/Initialization.cpp
mfbt/EnumSet.h
--- a/js/src/gc/GCRuntime.h
+++ b/js/src/gc/GCRuntime.h
@@ -16,16 +16,17 @@
 
 #include "gc/AtomMarking.h"
 #include "gc/Heap.h"
 #include "gc/Nursery.h"
 #include "gc/Statistics.h"
 #include "gc/StoreBuffer.h"
 #include "gc/Tracer.h"
 #include "js/GCAnnotations.h"
+#include "js/UniquePtr.h"
 
 namespace js {
 
 class AutoLockGC;
 class AutoLockHelperThreadState;
 class VerifyPreTracer;
 
 namespace gc {
@@ -41,21 +42,27 @@ struct MovingTracer;
 class WeakCacheSweepIterator;
 
 enum IncrementalProgress
 {
     NotFinished = 0,
     Finished
 };
 
-enum SweepActionList
+// Interface to a sweep action.
+//
+// Note that we don't need perfect forwarding for args here because the
+// types are not deduced but come ultimately from the type of a function pointer
+// passed to SweepFunc.
+template <typename... Args>
+struct SweepAction
 {
-    PerSweepGroupActionList,
-    PerZoneActionList,
-    SweepActionListCount
+    virtual ~SweepAction() {}
+    virtual IncrementalProgress run(Args... args) = 0;
+    virtual void assertFinished() const = 0;
 };
 
 class ChunkPool
 {
     Chunk* head_;
     size_t count_;
 
   public:
@@ -792,17 +799,17 @@ class GCRuntime
 
     bool isIncrementalGCEnabled() const { return mode == JSGC_MODE_INCREMENTAL && incrementalAllowed; }
     bool isIncrementalGCInProgress() const { return state() != State::NotActive; }
 
     bool isCompactingGCEnabled() const;
 
     bool isShrinkingGC() const { return invocationKind == GC_SHRINK; }
 
-    static bool initializeSweepActions();
+    bool initSweepActions();
 
     void setGrayRootsTracer(JSTraceDataOp traceOp, void* data);
     MOZ_MUST_USE bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data);
     void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data);
 
     bool triggerGCForTooMuchMalloc() {
         if (!triggerGC(JS::gcreason::TOO_MUCH_MALLOC))
             return false;
@@ -1030,29 +1037,29 @@ class GCRuntime
     void endMarkingSweepGroup();
     void beginSweepingSweepGroup();
     bool shouldReleaseObservedTypes();
     void sweepDebuggerOnMainThread(FreeOp* fop);
     void sweepJitDataOnMainThread(FreeOp* fop);
     void endSweepingSweepGroup();
     IncrementalProgress performSweepActions(SliceBudget& sliceBudget,
                                             AutoLockForExclusiveAccess& lock);
-    static IncrementalProgress sweepTypeInformation(GCRuntime* gc, FreeOp* fop, Zone* zone,
-                                                    SliceBudget& budget, AllocKind kind);
-    static IncrementalProgress mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, Zone* zone,
-                                                      SliceBudget& budget, AllocKind kind);
-    static IncrementalProgress sweepAtomsTable(GCRuntime* gc, SliceBudget& budget);
+    static IncrementalProgress sweepTypeInformation(GCRuntime* gc, FreeOp* fop, SliceBudget& budget,
+                                                    Zone* zone);
+    static IncrementalProgress mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, SliceBudget& budget,
+                                                      Zone* zone);
+    static IncrementalProgress sweepAtomsTable(GCRuntime* gc, FreeOp* fop, SliceBudget& budget);
     void startSweepingAtomsTable();
     IncrementalProgress sweepAtomsTable(SliceBudget& budget);
-    static IncrementalProgress sweepWeakCaches(GCRuntime* gc, SliceBudget& budget);
+    static IncrementalProgress sweepWeakCaches(GCRuntime* gc, FreeOp* fop, SliceBudget& budget);
     IncrementalProgress sweepWeakCaches(SliceBudget& budget);
-    static IncrementalProgress finalizeAllocKind(GCRuntime* gc, FreeOp* fop, Zone* zone,
-                                                 SliceBudget& budget, AllocKind kind);
-    static IncrementalProgress sweepShapeTree(GCRuntime* gc, FreeOp* fop, Zone* zone,
-                                              SliceBudget& budget, AllocKind kind);
+    static IncrementalProgress finalizeAllocKind(GCRuntime* gc, FreeOp* fop, SliceBudget& budget,
+                                                 Zone* zone, AllocKind kind);
+    static IncrementalProgress sweepShapeTree(GCRuntime* gc, FreeOp* fop, SliceBudget& budget,
+                                              Zone* zone);
     void endSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock);
     bool allCCVisibleZonesWereCollected() const;
     void sweepZones(FreeOp* fop, ZoneGroup* group, bool lastGC);
     void sweepZoneGroups(FreeOp* fop, bool destroyingRuntime);
     void decommitAllWithoutUnlocking(const AutoLockGC& lock);
     void startDecommit();
     void queueZonesForBackgroundSweep(ZoneList& zones);
     void sweepBackgroundThings(ZoneList& zones, LifoAlloc& freeBlocks);
@@ -1262,20 +1269,18 @@ class GCRuntime
     ActiveThreadData<unsigned> sweepGroupIndex;
 
     /*
      * Incremental sweep state.
      */
 
     ActiveThreadData<JS::Zone*> sweepGroups;
     ActiveThreadOrGCTaskData<JS::Zone*> currentSweepGroup;
-    ActiveThreadData<SweepActionList> sweepActionList;
-    ActiveThreadData<size_t> sweepPhaseIndex;
+    ActiveThreadData<UniquePtr<SweepAction<GCRuntime*, FreeOp*, SliceBudget&>>> sweepActions;
     ActiveThreadOrGCTaskData<JS::Zone*> sweepZone;
-    ActiveThreadData<size_t> sweepActionIndex;
     ActiveThreadData<mozilla::Maybe<AtomSet::Enum>> maybeAtomsToSweep;
     ActiveThreadOrGCTaskData<JS::detail::WeakCacheBase*> sweepCache;
     ActiveThreadData<bool> abortSweepAfterCurrentGroup;
 
     friend class WeakCacheSweepIterator;
 
     /*
      * List head of arenas allocated during the sweep phase.
--- a/js/src/jsgc.cpp
+++ b/js/src/jsgc.cpp
@@ -190,19 +190,21 @@
 #include "mozilla/ArrayUtils.h"
 #include "mozilla/DebugOnly.h"
 #include "mozilla/MacroForEach.h"
 #include "mozilla/MemoryReporting.h"
 #include "mozilla/Move.h"
 #include "mozilla/ScopeExit.h"
 #include "mozilla/SizePrintfMacros.h"
 #include "mozilla/TimeStamp.h"
+#include "mozilla/TypeTraits.h"
 #include "mozilla/Unused.h"
 
 #include <ctype.h>
+#include <initializer_list>
 #include <string.h>
 #ifndef XP_WIN
 # include <sys/mman.h>
 # include <unistd.h>
 #endif
 
 #include "jsapi.h"
 #include "jsatom.h"
@@ -252,16 +254,17 @@
 
 using namespace js;
 using namespace js::gc;
 
 using mozilla::ArrayLength;
 using mozilla::Get;
 using mozilla::HashCodeScrambler;
 using mozilla::Maybe;
+using mozilla::Move;
 using mozilla::Swap;
 using mozilla::TimeStamp;
 
 using JS::AutoGCRooter;
 
 /* Increase the IGC marking slice time if we are in highFrequencyGC mode. */
 static const int IGC_MARK_SLICE_MULTIPLIER = 2;
 
@@ -404,69 +407,16 @@ static const FinalizePhase BackgroundFin
             AllocKind::SHAPE,
             AllocKind::ACCESSOR_SHAPE,
             AllocKind::BASE_SHAPE,
             AllocKind::OBJECT_GROUP
         }
     }
 };
 
-// Incremental sweeping is controlled by a list of actions that describe what
-// happens and in what order. Due to the incremental nature of sweeping an
-// action does not necessarily run to completion so the current state is tracked
-// in the GCRuntime by the performSweepActions() method. We may yield to the
-// mutator after running part of any action.
-//
-// There are two types of action: per-sweep-group and per-zone.
-//
-// Per-sweep-group actions are run first. Per-zone actions are grouped into
-// phases, with each phase run once per sweep group, and each action in it run
-// for every zone in the group.
-//
-// This is illustrated by the following pseudocode:
-//
-//   for each sweep group:
-//     for each per-sweep-group action:
-//       run part or all of action
-//       maybe yield to the mutator
-//     for each per-zone phase:
-//       for each zone in sweep group:
-//         for each action in phase:
-//           run part or all of action
-//           maybe yield to the mutator
-//
-// Progress through the loops is stored in GCRuntime, e.g. |sweepActionIndex|
-// for looping through the sweep actions.
-
-using PerSweepGroupSweepAction = IncrementalProgress (*)(GCRuntime* gc, SliceBudget& budget);
-
-struct PerZoneSweepAction
-{
-    using Func = IncrementalProgress (*)(GCRuntime* gc, FreeOp* fop, Zone* zone,
-                                         SliceBudget& budget, AllocKind kind);
-
-    Func func;
-    AllocKind kind;
-
-    PerZoneSweepAction(Func func, AllocKind kind) : func(func), kind(kind) {}
-};
-
-using PerSweepGroupActionVector = Vector<PerSweepGroupSweepAction, 0, SystemAllocPolicy>;
-using PerZoneSweepActionVector = Vector<PerZoneSweepAction, 0, SystemAllocPolicy>;
-using PerZoneSweepPhaseVector = Vector<PerZoneSweepActionVector, 0, SystemAllocPolicy>;
-
-static PerSweepGroupActionVector PerSweepGroupSweepActions;
-static PerZoneSweepPhaseVector PerZoneSweepPhases;
-
-bool
-js::gc::InitializeStaticData()
-{
-    return GCRuntime::initializeSweepActions();
-}
-
 template<>
 JSObject*
 ArenaCellIterImpl::get<JSObject>() const
 {
     MOZ_ASSERT(!done());
     return reinterpret_cast<JSObject*>(getCell());
 }
 
@@ -909,19 +859,17 @@ GCRuntime::GCRuntime(JSRuntime* rt) :
     isFull(false),
     incrementalState(gc::State::NotActive),
     lastMarkSlice(false),
     sweepOnBackgroundThread(false),
     blocksToFreeAfterSweeping((size_t) JSContext::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
     sweepGroupIndex(0),
     sweepGroups(nullptr),
     currentSweepGroup(nullptr),
-    sweepPhaseIndex(0),
     sweepZone(nullptr),
-    sweepActionIndex(0),
     abortSweepAfterCurrentGroup(false),
     arenasAllocatedDuringSweep(nullptr),
     startedCompacting(false),
     relocatedArenasToRelease(nullptr),
 #ifdef JS_GC_ZEAL
     markingValidator(nullptr),
 #endif
     interFrameGC(false),
@@ -1183,16 +1131,19 @@ GCRuntime::init(uint32_t maxbytes, uint3
 #endif
 
     if (!InitTrace(*this))
         return false;
 
     if (!marker.init(mode))
         return false;
 
+    if (!initSweepActions())
+        return false;
+
     return true;
 }
 
 void
 GCRuntime::finish()
 {
     /* Wait for nursery background free to end and disable it to release memory. */
     if (nursery().isEnabled()) {
@@ -5037,17 +4988,17 @@ class ImmediateSweepWeakCacheTask : publ
     ImmediateSweepWeakCacheTask(const ImmediateSweepWeakCacheTask&) = delete;
 
   public:
     ImmediateSweepWeakCacheTask(JSRuntime* rt, JS::detail::WeakCacheBase& wc)
       : GCParallelTask(rt), cache(wc)
     {}
 
     ImmediateSweepWeakCacheTask(ImmediateSweepWeakCacheTask&& other)
-      : GCParallelTask(mozilla::Move(other)), cache(other.cache)
+      : GCParallelTask(Move(other)), cache(other.cache)
     {}
 
     void run() override {
         cache.sweep();
     }
 };
 
 static void
@@ -5411,26 +5362,25 @@ GCRuntime::beginSweepingSweepGroup()
             zone->arenas.queueForForegroundSweep(&fop, IncrementalFinalizePhases[i]);
 
         for (unsigned i = 0; i < ArrayLength(BackgroundFinalizePhases); ++i)
             zone->arenas.queueForBackgroundSweep(&fop, BackgroundFinalizePhases[i]);
 
         zone->arenas.queueForegroundThingsForSweep(&fop);
     }
 
-    sweepActionList = PerSweepGroupActionList;
-    sweepActionIndex = 0;
-    sweepPhaseIndex = 0;
-    sweepZone = nullptr;
     sweepCache = nullptr;
+    sweepActions->assertFinished();
 }
 
 void
 GCRuntime::endSweepingSweepGroup()
 {
+    sweepActions->assertFinished();
+
     {
         gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::FINALIZE_END);
         FreeOp fop(rt);
         callFinalizeCallbacks(&fop, JSFINALIZE_GROUP_END);
     }
 
     /* Update the GC state for zones we have swept. */
     for (GCSweepGroupIter zone(rt); !zone.done(); zone.next()) {
@@ -5566,29 +5516,26 @@ SweepArenaList(Arena** arenasToSweep, Sl
         if (sliceBudget.isOverBudget())
             return false;
     }
 
     return true;
 }
 
 /* static */ IncrementalProgress
-GCRuntime::sweepTypeInformation(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget,
-                                AllocKind kind)
+GCRuntime::sweepTypeInformation(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, Zone* zone)
 {
     // Sweep dead type information stored in scripts and object groups, but
     // don't finalize them yet. We have to sweep dead information from both live
     // and dead scripts and object groups, so that no dead references remain in
     // them. Type inference can end up crawling these zones again, such as for
     // TypeCompartment::markSetsUnknown, and if this happens after sweeping for
     // the sweep group finishes we won't be able to determine which things in
     // the zone are live.
 
-    MOZ_ASSERT(kind == AllocKind::LIMIT);
-
     gcstats::AutoPhase ap1(gc->stats(), gcstats::PhaseKind::SWEEP_COMPARTMENTS);
     gcstats::AutoPhase ap2(gc->stats(), gcstats::PhaseKind::SWEEP_TYPES);
 
     ArenaLists& al = zone->arenas;
 
     AutoClearTypeInferenceStateOnOOM oom(zone);
 
     if (!SweepArenaList<JSScript>(&al.gcScriptArenasToUpdate.ref(), budget, &oom))
@@ -5602,24 +5549,22 @@ GCRuntime::sweepTypeInformation(GCRuntim
         gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::SWEEP_TYPES_END);
         zone->types.endSweep(gc->rt);
     }
 
     return Finished;
 }
 
 /* static */ IncrementalProgress
-GCRuntime::mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget,
-                                  AllocKind kind)
+GCRuntime::mergeSweptObjectArenas(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, Zone* zone)
 {
     // Foreground finalized objects have already been finalized, and now their
     // arenas can be reclaimed by freeing empty ones and making non-empty ones
     // available for allocation.
 
-    MOZ_ASSERT(kind == AllocKind::LIMIT);
     zone->arenas.mergeForegroundSweptObjectArenas();
     return Finished;
 }
 
 void
 GCRuntime::startSweepingAtomsTable()
 {
     auto& maybeAtoms = maybeAtomsToSweep.ref();
@@ -5636,25 +5581,24 @@ GCRuntime::startSweepingAtomsTable()
         return;
     }
 
     // Initialize remaining atoms to sweep.
     maybeAtoms.emplace(*atomsTable);
 }
 
 /* static */ IncrementalProgress
-GCRuntime::sweepAtomsTable(GCRuntime* gc, SliceBudget& budget)
+GCRuntime::sweepAtomsTable(GCRuntime* gc, FreeOp* fop, SliceBudget& budget)
 {
     if (!gc->atomsZone->isGCSweeping())
         return Finished;
 
     return gc->sweepAtomsTable(budget);
 }
 
-
 IncrementalProgress
 GCRuntime::sweepAtomsTable(SliceBudget& budget)
 {
     gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP_ATOMS_TABLE);
 
     auto& maybeAtoms = maybeAtomsToSweep.ref();
     if (!maybeAtoms)
         return Finished;
@@ -5778,17 +5722,17 @@ class IncrementalSweepWeakCacheTask : pu
                 break;
 
             cache_ = work_.next(lock);
         } while(cache_);
     }
 };
 
 /* static */ IncrementalProgress
-GCRuntime::sweepWeakCaches(GCRuntime* gc, SliceBudget& budget)
+GCRuntime::sweepWeakCaches(GCRuntime* gc, FreeOp* fop, SliceBudget& budget)
 {
     return gc->sweepWeakCaches(budget);
 }
 
 static const size_t MaxWeakCacheSweepTasks = 8;
 
 static size_t
 WeakCacheSweepTaskCount()
@@ -5813,17 +5757,17 @@ GCRuntime::sweepWeakCaches(SliceBudget& 
         // Tasks run until budget or work is exhausted.
     }
 
     AutoLockHelperThreadState lock;
     return work.empty(lock) ? Finished : NotFinished;
 }
 
 /* static */ IncrementalProgress
-GCRuntime::finalizeAllocKind(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget,
+GCRuntime::finalizeAllocKind(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, Zone* zone,
                              AllocKind kind)
 {
     // Set the number of things per arena for this AllocKind.
     size_t thingsPerArena = Arena::thingsPerArena(kind);
     auto& sweepList = gc->incrementalSweepList.ref();
     sweepList.setThingsPerArena(thingsPerArena);
 
     if (!zone->arenas.foregroundFinalize(fop, kind, budget, sweepList))
@@ -5831,142 +5775,306 @@ GCRuntime::finalizeAllocKind(GCRuntime* 
 
     // Reset the slots of the sweep list that we used.
     sweepList.reset(thingsPerArena);
 
     return Finished;
 }
 
 /* static */ IncrementalProgress
-GCRuntime::sweepShapeTree(GCRuntime* gc, FreeOp* fop, Zone* zone, SliceBudget& budget,
-                          AllocKind kind)
+GCRuntime::sweepShapeTree(GCRuntime* gc, FreeOp* fop, SliceBudget& budget, Zone* zone)
 {
     // Remove dead shapes from the shape tree, but don't finalize them yet.
 
-    MOZ_ASSERT(kind == AllocKind::LIMIT);
-
     gcstats::AutoPhase ap(gc->stats(), gcstats::PhaseKind::SWEEP_SHAPE);
 
     ArenaLists& al = zone->arenas;
 
     if (!SweepArenaList<Shape>(&al.gcShapeArenasToUpdate.ref(), budget))
         return NotFinished;
 
     if (!SweepArenaList<AccessorShape>(&al.gcAccessorShapeArenasToUpdate.ref(), budget))
         return NotFinished;
 
     return Finished;
 }
 
-static void
-AddPerSweepGroupSweepAction(bool* ok, PerSweepGroupSweepAction action)
-{
-    if (*ok)
-        *ok = PerSweepGroupSweepActions.emplaceBack(action);
-}
-
-static void
-AddPerZoneSweepPhase(bool* ok)
-{
-    if (*ok)
-        *ok = PerZoneSweepPhases.emplaceBack();
-}
-
-static void
-AddPerZoneSweepAction(bool* ok, PerZoneSweepAction::Func func, AllocKind kind = AllocKind::LIMIT)
-{
-    if (*ok)
-        *ok = PerZoneSweepPhases.back().emplaceBack(func, kind);
-}
-
-/* static */ bool
-GCRuntime::initializeSweepActions()
-{
-    bool ok = true;
-
-    AddPerSweepGroupSweepAction(&ok, GCRuntime::sweepAtomsTable);
-    AddPerSweepGroupSweepAction(&ok, GCRuntime::sweepWeakCaches);
-
-    AddPerZoneSweepPhase(&ok);
-    for (auto kind : ForegroundObjectFinalizePhase.kinds)
-        AddPerZoneSweepAction(&ok, GCRuntime::finalizeAllocKind, kind);
-
-    AddPerZoneSweepPhase(&ok);
-    AddPerZoneSweepAction(&ok, GCRuntime::sweepTypeInformation);
-    AddPerZoneSweepAction(&ok, GCRuntime::mergeSweptObjectArenas);
-
-    for (const auto& finalizePhase : IncrementalFinalizePhases) {
-        AddPerZoneSweepPhase(&ok);
-        for (auto kind : finalizePhase.kinds)
-            AddPerZoneSweepAction(&ok, GCRuntime::finalizeAllocKind, kind);
-    }
-
-    AddPerZoneSweepPhase(&ok);
-    AddPerZoneSweepAction(&ok, GCRuntime::sweepShapeTree);
-
-    return ok;
-}
-
-static inline SweepActionList
-NextSweepActionList(SweepActionList list)
-{
-    MOZ_ASSERT(list < SweepActionListCount);
-    return SweepActionList(unsigned(list) + 1);
+// An iterator for a standard container that provides an STL-like begin()/end()
+// interface. This iterator provides a done()/get()/next() style interface.
+template <typename Container>
+class ContainerIter
+{
+    using Iter = decltype(mozilla::DeclVal<const Container>().begin());
+    using Elem = decltype(*mozilla::DeclVal<Iter>());
+
+    Iter iter;
+    const Iter end;
+
+  public:
+    explicit ContainerIter(const Container& container)
+      : iter(container.begin()), end(container.end())
+    {}
+
+    bool done() const {
+        return iter == end;
+    }
+
+    Elem get() const {
+        return *iter;
+    }
+
+    void next() {
+        MOZ_ASSERT(!done());
+        ++iter;
+    }
+};
+
+// IncrementalIter is a template class that makes a normal iterator into one
+// that can be used to perform incremental work by using external state that
+// persists between instantiations. The state is only initialised on the first
+// use and subsequent uses carry on from the previous state.
+template <typename Iter>
+struct IncrementalIter
+{
+    using State = Maybe<Iter>;
+    using Elem = decltype(mozilla::DeclVal<Iter>().get());
+
+  private:
+    State& maybeIter;
+
+  public:
+    template <typename... Args>
+    IncrementalIter(State& maybeIter, Args&&... args)
+      : maybeIter(maybeIter)
+    {
+        if (maybeIter.isNothing())
+            maybeIter.emplace(mozilla::Forward<Args>(args)...);
+    }
+
+    ~IncrementalIter() {
+        if (done())
+            maybeIter.reset();
+    }
+
+    bool done() const {
+        return maybeIter.ref().done();
+    }
+
+    Elem get() const {
+        return maybeIter.ref().get();
+    }
+
+    void next() {
+        maybeIter.ref().next();
+    }
+};
+
+// Implementation of the SweepAction interface that calls a function.
+template <typename... Args>
+class SweepActionFunc : public SweepAction<Args...>
+{
+    using Func = IncrementalProgress (*)(Args...);
+
+    Func func;
+
+  public:
+    SweepActionFunc(Func f) : func(f) {}
+    IncrementalProgress run(Args... args) override {
+        return func(args...);
+    }
+    void assertFinished() const override { }
+};
+
+// Implementation of the SweepAction interface that calls a list of actions in
+// sequence.
+template <typename... Args>
+class SweepActionSequence : public SweepAction<Args...>
+{
+    using Action = SweepAction<Args...>;
+    using ActionVector = Vector<UniquePtr<Action>, 0, SystemAllocPolicy>;
+    using Iter = IncrementalIter<ContainerIter<ActionVector>>;
+
+    ActionVector actions;
+    typename Iter::State iterState;
+
+  public:
+    bool init(UniquePtr<Action>* acts, size_t count) {
+        for (size_t i = 0; i < count; i++) {
+            if (!actions.emplaceBack(Move(acts[i])))
+                return false;
+        }
+        return true;
+    }
+
+    IncrementalProgress run(Args... args) override {
+        for (Iter iter(iterState, actions); !iter.done(); iter.next()) {
+            if (iter.get()->run(args...) == NotFinished)
+                return NotFinished;
+        }
+        return Finished;
+    }
+
+    void assertFinished() const override {
+        MOZ_ASSERT(iterState.isNothing());
+        for (const auto& action : actions)
+            action->assertFinished();
+    }
+};
+
+template <typename Iter, typename Init, typename... Args>
+class SweepActionForEach : public SweepAction<Args...>
+{
+    using Elem = decltype(mozilla::DeclVal<Iter>().get());
+    using Action = SweepAction<Args..., Elem>;
+    using IncrIter = IncrementalIter<Iter>;
+
+    Init iterInit;
+    UniquePtr<Action> action;
+    typename IncrIter::State iterState;
+
+  public:
+    SweepActionForEach(const Init& init, UniquePtr<Action> action)
+      : iterInit(init), action(Move(action))
+    {}
+
+    IncrementalProgress run(Args... args) override {
+        for (IncrIter iter(iterState, iterInit); !iter.done(); iter.next()) {
+            if (action->run(args..., iter.get()) == NotFinished)
+                return NotFinished;
+        }
+        return Finished;
+    }
+
+    void assertFinished() const override {
+        MOZ_ASSERT(iterState.isNothing());
+        action->assertFinished();
+    }
+};
+
+// Helper class to remove the last template parameter from the instantiation of
+// a variadic template. For example:
+//
+//   RemoveLastTemplateParameter<Foo<X, Y, Z>>::Type ==> Foo<X, Y>
+//
+// This works by recursively instantiating the Impl template with the contents
+// of the parameter pack so long as there are at least two parameters. The
+// specialization that matches when only one parameter remains discards it and
+// instantiates the target template with parameters previously processed.
+template <typename T>
+class RemoveLastTemplateParameter {};
+
+template <template <typename...> class Target, typename... Args>
+class RemoveLastTemplateParameter<Target<Args...>>
+{
+    template <typename... Ts>
+    struct List {};
+
+    template <typename R, typename... Ts>
+    struct Impl {};
+
+    template <typename... Rs, typename T>
+    struct Impl<List<Rs...>, T>
+    {
+        using Type = Target<Rs...>;
+    };
+
+    template <typename... Rs, typename H, typename T, typename... Ts>
+    struct Impl<List<Rs...>, H, T, Ts...>
+    {
+        using Type = typename Impl<List<Rs..., H>, T, Ts...>::Type;
+    };
+
+  public:
+    using Type = typename Impl<List<>, Args...>::Type;
+};
+
+template <typename... Args>
+static UniquePtr<SweepAction<Args...>>
+SweepFunc(IncrementalProgress (*func)(Args...)) {
+    return MakeUnique<SweepActionFunc<Args...>>(func);
+}
+
+template <typename... Args, typename... Rest>
+static UniquePtr<SweepAction<Args...>>
+SweepSequence(UniquePtr<SweepAction<Args...>> first, Rest... rest)
+{
+    UniquePtr<SweepAction<Args...>> actions[] = { Move(first), Move(rest)... };
+    auto seq = MakeUnique<SweepActionSequence<Args...>>();
+    if (!seq || !seq->init(actions, ArrayLength(actions)))
+        return nullptr;
+
+    return UniquePtr<SweepAction<Args...>>(Move(seq));
+}
+
+template <typename... Args>
+static UniquePtr<typename RemoveLastTemplateParameter<SweepAction<Args...>>::Type>
+SweepForEachZone(JSRuntime* rt, UniquePtr<SweepAction<Args...>> action)
+{
+    if (!action)
+        return nullptr;
+
+    using Action = typename RemoveLastTemplateParameter<
+        SweepActionForEach<GCSweepGroupIter, JSRuntime*, Args...>>::Type;
+    return js::MakeUnique<Action>(rt, Move(action));
+}
+
+template <typename... Args>
+static UniquePtr<typename RemoveLastTemplateParameter<SweepAction<Args...>>::Type>
+SweepForEachAllocKind(AllocKinds kinds, UniquePtr<SweepAction<Args...>> action)
+{
+    if (!action)
+        return nullptr;
+
+    using Action = typename RemoveLastTemplateParameter<
+        SweepActionForEach<ContainerIter<AllocKinds>, AllocKinds, Args...>>::Type;
+    return js::MakeUnique<Action>(kinds, Move(action));
+}
+
+bool
+GCRuntime::initSweepActions()
+{
+    sweepActions.ref() = SweepSequence(
+        SweepFunc(sweepAtomsTable),
+        SweepFunc(sweepWeakCaches),
+        SweepForEachZone(rt,
+            SweepForEachAllocKind(ForegroundObjectFinalizePhase.kinds,
+                SweepFunc(finalizeAllocKind))),
+        SweepForEachZone(rt,
+            SweepSequence(
+                SweepFunc(sweepTypeInformation),
+                SweepFunc(mergeSweptObjectArenas))),
+        SweepForEachZone(rt,
+            SweepForEachAllocKind(IncrementalFinalizePhases[0].kinds,
+                SweepFunc(finalizeAllocKind))),
+        SweepForEachZone(rt,
+            SweepForEachAllocKind(IncrementalFinalizePhases[1].kinds,
+                SweepFunc(finalizeAllocKind))),
+        SweepForEachZone(rt,
+            SweepFunc(sweepShapeTree)));
+
+    static_assert(ArrayLength(IncrementalFinalizePhases) == 2,
+                  "We must have a phase for each element in IncrementalFinalizePhases");
+
+    return sweepActions != nullptr;
 }
 
 IncrementalProgress
 GCRuntime::performSweepActions(SliceBudget& budget, AutoLockForExclusiveAccess& lock)
 {
     AutoSetThreadIsSweeping threadIsSweeping;
 
     gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::SWEEP);
     FreeOp fop(rt);
 
     if (drainMarkStack(budget, gcstats::PhaseKind::SWEEP_MARK) == NotFinished)
         return NotFinished;
 
     for (;;) {
-        for (; sweepActionList < SweepActionListCount;
-             sweepActionList = NextSweepActionList(sweepActionList))
-        {
-            switch (sweepActionList) {
-              case PerSweepGroupActionList: {
-                const auto& actions = PerSweepGroupSweepActions;
-                for (; sweepActionIndex < actions.length(); sweepActionIndex++) {
-                    auto action = actions[sweepActionIndex];
-                    if (action(this, budget) == NotFinished)
-                        return NotFinished;
-                }
-                sweepActionIndex = 0;
-                break;
-              }
-
-              case PerZoneActionList:
-                for (; sweepPhaseIndex < PerZoneSweepPhases.length(); sweepPhaseIndex++) {
-                    const auto& actions = PerZoneSweepPhases[sweepPhaseIndex];
-                    if (!sweepZone)
-                        sweepZone = currentSweepGroup;
-                    for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) {
-                        for (; sweepActionIndex < actions.length(); sweepActionIndex++) {
-                            const auto& action = actions[sweepActionIndex];
-                            if (action.func(this, &fop, sweepZone, budget, action.kind) == NotFinished)
-                                return NotFinished;
-                        }
-                        sweepActionIndex = 0;
-                    }
-                    sweepZone = nullptr;
-                }
-                sweepPhaseIndex = 0;
-                break;
-
-              default:
-                MOZ_CRASH("Unexpected sweepActionList value");
-            }
-        }
-        sweepActionList = PerSweepGroupActionList;
+        if (sweepActions->run(this, &fop, budget) == NotFinished)
+           return NotFinished;
 
         endSweepingSweepGroup();
         getNextSweepGroup();
         if (!currentSweepGroup)
             return Finished;
 
         endMarkingSweepGroup();
         beginSweepingSweepGroup();
--- a/js/src/jsgcinlines.h
+++ b/js/src/jsgcinlines.h
@@ -433,17 +433,16 @@ class GCZonesIter
     operator JS::Zone*() const { return get(); }
     JS::Zone* operator->() const { return get(); }
 };
 
 typedef CompartmentsIterT<GCZonesIter> GCCompartmentsIter;
 
 /* Iterates over all zones in the current sweep group. */
 class GCSweepGroupIter {
-  private:
     JS::Zone* current;
 
   public:
     explicit GCSweepGroupIter(JSRuntime* rt) {
         MOZ_ASSERT(CurrentThreadIsPerformingGC());
         current = rt->gc.getCurrentSweepGroup();
     }
 
--- a/js/src/vm/Initialization.cpp
+++ b/js/src/vm/Initialization.cpp
@@ -101,17 +101,16 @@ JS::detail::InitWithFailureDiagnostic(bo
     js::oom::SetThreadType(js::oom::THREAD_TYPE_COOPERATING);
 #endif
 
     RETURN_IF_FAIL(js::Mutex::Init());
 
     RETURN_IF_FAIL(js::wasm::InitInstanceStaticData());
 
     js::gc::InitMemorySubsystem(); // Ensure gc::SystemPageSize() works.
-    RETURN_IF_FAIL(js::gc::InitializeStaticData());
 
     RETURN_IF_FAIL(js::jit::InitProcessExecutableMemory());
 
     MOZ_ALWAYS_TRUE(js::MemoryProtectionExceptionHandler::install());
 
     RETURN_IF_FAIL(js::jit::InitializeIon());
 
     RETURN_IF_FAIL(js::InitDateTimeState());
--- a/mfbt/EnumSet.h
+++ b/mfbt/EnumSet.h
@@ -229,17 +229,17 @@ public:
   class ConstIterator
   {
     const EnumSet<T>* mSet;
     uint32_t mPos;
 #ifdef DEBUG
     uint64_t mVersion;
 #endif
 
-    void checkVersion() {
+    void checkVersion() const {
       // Check that the set has not been modified while being iterated.
       MOZ_ASSERT_IF(mSet, mSet->mVersion == mVersion);
     }
 
    public:
     ConstIterator(const EnumSet<T>& aSet, uint32_t aPos)
      : mSet(&aSet), mPos(aPos)
     {
@@ -269,27 +269,27 @@ public:
 #endif
       aOther.mSet = nullptr;
     }
 
     ~ConstIterator() {
       checkVersion();
     }
 
-    bool operator==(const ConstIterator& other) {
+    bool operator==(const ConstIterator& other) const {
       MOZ_ASSERT(mSet == other.mSet);
       checkVersion();
       return mPos == other.mPos;
     }
 
-    bool operator!=(const ConstIterator& other) {
+    bool operator!=(const ConstIterator& other) const {
       return !(*this == other);
     }
 
-    T operator*() {
+    T operator*() const {
       MOZ_ASSERT(mSet);
       MOZ_ASSERT(mPos < kMaxBits);
       MOZ_ASSERT(mSet->contains(T(mPos)));
       checkVersion();
       return T(mPos);
     }
 
     ConstIterator& operator++() {