Bug 1248949 - Optimize Arena::thingsPerArena. r=terrence
authorJan de Mooij <jdemooij@mozilla.com>
Thu, 18 Feb 2016 14:53:40 +0100
changeset 331897 1354c6e4ac2fd1af0126a9354cb99fd73e8c14dc
parent 331896 b34a59d6ffe8af9449bcac1eb660f86279d80ad7
child 331898 c2aeece5eb1067f08cc5e43d0e53b6b694f27e28
push id11113
push userrjesup@wgate.com
push dateThu, 18 Feb 2016 19:00:12 +0000
reviewersterrence
bugs1248949
milestone47.0a1
Bug 1248949 - Optimize Arena::thingsPerArena. r=terrence
js/src/gc/Heap.h
js/src/jsgc.cpp
js/src/vm/MemoryMetrics.cpp
--- a/js/src/gc/Heap.h
+++ b/js/src/gc/Heap.h
@@ -739,39 +739,35 @@ struct Arena
      * <-------------------> = first thing offset
      */
     ArenaHeader aheader;
     uint8_t     data[ArenaSize - sizeof(ArenaHeader)];
 
   private:
     static JS_FRIEND_DATA(const uint32_t) ThingSizes[];
     static JS_FRIEND_DATA(const uint32_t) FirstThingOffsets[];
+    static const uint32_t ThingsPerArena[];
 
   public:
     static void staticAsserts();
 
     static size_t thingSize(AllocKind kind) {
         return ThingSizes[size_t(kind)];
     }
 
     static size_t firstThingOffset(AllocKind kind) {
         return FirstThingOffsets[size_t(kind)];
     }
 
-    static size_t thingsPerArena(size_t thingSize) {
-        MOZ_ASSERT(thingSize % CellSize == 0);
-
-        /* We should be able to fit FreeSpan in any GC thing. */
-        MOZ_ASSERT(thingSize >= sizeof(FreeSpan));
-
-        return (ArenaSize - sizeof(ArenaHeader)) / thingSize;
+    static size_t thingsPerArena(AllocKind kind) {
+        return ThingsPerArena[size_t(kind)];
     }
 
-    static size_t thingsSpan(size_t thingSize) {
-        return thingsPerArena(thingSize) * thingSize;
+    static size_t thingsSpan(AllocKind kind) {
+        return thingsPerArena(kind) * thingSize(kind);
     }
 
     static bool isAligned(uintptr_t thing, size_t thingSize) {
         /* Things ends at the arena end. */
         uintptr_t tailOffset = (ArenaSize - thing) & ArenaMask;
         return tailOffset % thingSize == 0;
     }
 
--- a/js/src/jsgc.cpp
+++ b/js/src/jsgc.cpp
@@ -256,24 +256,30 @@ const AllocKind gc::slotsToThingKind[] =
     /*  8 */ AllocKind::OBJECT8,  AllocKind::OBJECT12, AllocKind::OBJECT12, AllocKind::OBJECT12,
     /* 12 */ AllocKind::OBJECT12, AllocKind::OBJECT16, AllocKind::OBJECT16, AllocKind::OBJECT16,
     /* 16 */ AllocKind::OBJECT16
 };
 
 static_assert(JS_ARRAY_LENGTH(slotsToThingKind) == SLOTS_TO_THING_KIND_LIMIT,
               "We have defined a slot count for each kind.");
 
-// Assert that SortedArenaList::MinThingSize is <= the real minimum thing size.
-#define CHECK_MIN_THING_SIZE_INNER(x_)                                         \
+// Assert that SortedArenaList::MinThingSize and sizeof(FreeSpan) are <= the
+// real minimum thing size. Also assert each size is a multiple of CellSize.
+#define CHECK_THING_SIZE_INNER(x_)                                             \
     static_assert(x_ >= SortedArenaList::MinThingSize,                         \
-    #x_ " is less than SortedArenaList::MinThingSize!");
-#define CHECK_MIN_THING_SIZE(...) { __VA_ARGS__ }; /* Define the array. */     \
-    MOZ_FOR_EACH(CHECK_MIN_THING_SIZE_INNER, (), (__VA_ARGS__ UINT32_MAX))
-
-const uint32_t Arena::ThingSizes[] = CHECK_MIN_THING_SIZE(
+                  #x_ " is less than SortedArenaList::MinThingSize!");         \
+    static_assert(x_ >= sizeof(FreeSpan),                                      \
+                  #x_ " is less than sizeof(FreeSpan)");                       \
+    static_assert(x_ % CellSize == 0,                                          \
+                  #x_ " not a multiple of CellSize");
+
+#define CHECK_THING_SIZE(...) { __VA_ARGS__ }; /* Define the array. */         \
+    MOZ_FOR_EACH(CHECK_THING_SIZE_INNER, (), (__VA_ARGS__ 0x20))
+
+const uint32_t Arena::ThingSizes[] = CHECK_THING_SIZE(
     sizeof(JSFunction),         /* AllocKind::FUNCTION            */
     sizeof(FunctionExtended),   /* AllocKind::FUNCTION_EXTENDED   */
     sizeof(JSObject_Slots0),    /* AllocKind::OBJECT0             */
     sizeof(JSObject_Slots0),    /* AllocKind::OBJECT0_BACKGROUND  */
     sizeof(JSObject_Slots2),    /* AllocKind::OBJECT2             */
     sizeof(JSObject_Slots2),    /* AllocKind::OBJECT2_BACKGROUND  */
     sizeof(JSObject_Slots4),    /* AllocKind::OBJECT4             */
     sizeof(JSObject_Slots4),    /* AllocKind::OBJECT4_BACKGROUND  */
@@ -291,18 +297,18 @@ const uint32_t Arena::ThingSizes[] = CHE
     sizeof(ObjectGroup),        /* AllocKind::OBJECT_GROUP        */
     sizeof(JSFatInlineString),  /* AllocKind::FAT_INLINE_STRING   */
     sizeof(JSString),           /* AllocKind::STRING              */
     sizeof(JSExternalString),   /* AllocKind::EXTERNAL_STRING     */
     sizeof(JS::Symbol),         /* AllocKind::SYMBOL              */
     sizeof(jit::JitCode),       /* AllocKind::JITCODE             */
 );
 
-#undef CHECK_MIN_THING_SIZE_INNER
-#undef CHECK_MIN_THING_SIZE
+#undef CHECK_THING_SIZE_INNER
+#undef CHECK_THING_SIZE
 
 #define OFFSET(type) uint32_t(sizeof(ArenaHeader) + (ArenaSize - sizeof(ArenaHeader)) % sizeof(type))
 
 const uint32_t Arena::FirstThingOffsets[] = {
     OFFSET(JSFunction),         /* AllocKind::FUNCTION            */
     OFFSET(FunctionExtended),   /* AllocKind::FUNCTION_EXTENDED   */
     OFFSET(JSObject_Slots0),    /* AllocKind::OBJECT0             */
     OFFSET(JSObject_Slots0),    /* AllocKind::OBJECT0_BACKGROUND  */
@@ -326,16 +332,48 @@ const uint32_t Arena::FirstThingOffsets[
     OFFSET(JSString),           /* AllocKind::STRING              */
     OFFSET(JSExternalString),   /* AllocKind::EXTERNAL_STRING     */
     OFFSET(JS::Symbol),         /* AllocKind::SYMBOL              */
     OFFSET(jit::JitCode),       /* AllocKind::JITCODE             */
 };
 
 #undef OFFSET
 
+#define COUNT(type) uint32_t((ArenaSize - sizeof(ArenaHeader)) / sizeof(type))
+
+const uint32_t Arena::ThingsPerArena[] = {
+    COUNT(JSFunction),          /* AllocKind::FUNCTION            */
+    COUNT(FunctionExtended),    /* AllocKind::FUNCTION_EXTENDED   */
+    COUNT(JSObject_Slots0),     /* AllocKind::OBJECT0             */
+    COUNT(JSObject_Slots0),     /* AllocKind::OBJECT0_BACKGROUND  */
+    COUNT(JSObject_Slots2),     /* AllocKind::OBJECT2             */
+    COUNT(JSObject_Slots2),     /* AllocKind::OBJECT2_BACKGROUND  */
+    COUNT(JSObject_Slots4),     /* AllocKind::OBJECT4             */
+    COUNT(JSObject_Slots4),     /* AllocKind::OBJECT4_BACKGROUND  */
+    COUNT(JSObject_Slots8),     /* AllocKind::OBJECT8             */
+    COUNT(JSObject_Slots8),     /* AllocKind::OBJECT8_BACKGROUND  */
+    COUNT(JSObject_Slots12),    /* AllocKind::OBJECT12            */
+    COUNT(JSObject_Slots12),    /* AllocKind::OBJECT12_BACKGROUND */
+    COUNT(JSObject_Slots16),    /* AllocKind::OBJECT16            */
+    COUNT(JSObject_Slots16),    /* AllocKind::OBJECT16_BACKGROUND */
+    COUNT(JSScript),            /* AllocKind::SCRIPT              */
+    COUNT(LazyScript),          /* AllocKind::LAZY_SCRIPT         */
+    COUNT(Shape),               /* AllocKind::SHAPE               */
+    COUNT(AccessorShape),       /* AllocKind::ACCESSOR_SHAPE      */
+    COUNT(BaseShape),           /* AllocKind::BASE_SHAPE          */
+    COUNT(ObjectGroup),         /* AllocKind::OBJECT_GROUP        */
+    COUNT(JSFatInlineString),   /* AllocKind::FAT_INLINE_STRING   */
+    COUNT(JSString),            /* AllocKind::STRING              */
+    COUNT(JSExternalString),    /* AllocKind::EXTERNAL_STRING     */
+    COUNT(JS::Symbol),          /* AllocKind::SYMBOL              */
+    COUNT(jit::JitCode),        /* AllocKind::JITCODE             */
+};
+
+#undef COUNT
+
 struct js::gc::FinalizePhase
 {
     size_t length;
     const AllocKind* kinds;
     gcstats::Phase statsPhase;
 };
 
 #define PHASE(x, p) { ArrayLength(x), x, p }
@@ -450,19 +488,21 @@ ArenaHeader::unmarkAll()
     uintptr_t* word = chunk()->bitmap.arenaBits(this);
     memset(word, 0, ArenaBitmapWords * sizeof(uintptr_t));
 }
 
 /* static */ void
 Arena::staticAsserts()
 {
     static_assert(JS_ARRAY_LENGTH(ThingSizes) == size_t(AllocKind::LIMIT),
-        "We haven't defined all thing sizes.");
+                  "We haven't defined all thing sizes.");
     static_assert(JS_ARRAY_LENGTH(FirstThingOffsets) == size_t(AllocKind::LIMIT),
-        "We haven't defined all offsets.");
+                  "We haven't defined all offsets.");
+    static_assert(JS_ARRAY_LENGTH(ThingsPerArena) == size_t(AllocKind::LIMIT),
+                  "We haven't defined all counts.");
 }
 
 void
 Arena::setAsFullyUnused(AllocKind thingKind)
 {
     FreeSpan fullSpan;
     size_t thingSize = Arena::thingSize(thingKind);
     fullSpan.initFinal(thingsStart(thingKind), thingsEnd() - thingSize, thingSize);
@@ -537,17 +577,17 @@ Arena::finalize(FreeOp* fop, AllocKind t
         // Otherwise, end the list with a span that covers the final stretch of free things.
         newListTail->initFinal(firstThingOrSuccessorOfLastMarkedThing, lastThing, thingSize);
     }
 
 #ifdef DEBUG
     size_t nfree = 0;
     for (const FreeSpan* span = &newListHead; !span->isEmpty(); span = span->nextSpan())
         nfree += span->length(thingSize);
-    MOZ_ASSERT(nfree + nmarked == thingsPerArena(thingSize));
+    MOZ_ASSERT(nfree + nmarked == thingsPerArena(thingKind));
 #endif
     aheader.setFirstFreeSpan(&newListHead);
     return nmarked;
 }
 
 // Finalize arenas from src list, releasing empty arenas if keepArenas wasn't
 // specified and inserting the others into the appropriate destination size
 // bins.
@@ -565,17 +605,17 @@ FinalizeTypedArenas(FreeOp* fop,
     if (!fop->onBackgroundThread())
         maybeLock.emplace(fop->runtime());
 
     // During background sweeping free arenas are released later on in
     // sweepBackgroundThings().
     MOZ_ASSERT_IF(fop->onBackgroundThread(), keepArenas == ArenaLists::KEEP_ARENAS);
 
     size_t thingSize = Arena::thingSize(thingKind);
-    size_t thingsPerArena = Arena::thingsPerArena(thingSize);
+    size_t thingsPerArena = Arena::thingsPerArena(thingKind);
 
     while (ArenaHeader* aheader = *src) {
         *src = aheader->next;
         size_t nmarked = aheader->getArena()->finalize<T>(fop, thingKind, thingSize);
         size_t nfree = thingsPerArena - nmarked;
 
         if (nmarked)
             dest.insertAt(aheader, nfree);
@@ -2028,17 +2068,17 @@ size_t ArenaHeader::countFreeCells()
     FreeSpan firstSpan(getFirstFreeSpan());
     for (const FreeSpan* span = &firstSpan; !span->isEmpty(); span = span->nextSpan())
         count += span->length(thingSize);
     return count;
 }
 
 size_t ArenaHeader::countUsedCells()
 {
-    return Arena::thingsPerArena(getThingSize()) - countFreeCells();
+    return Arena::thingsPerArena(getAllocKind()) - countFreeCells();
 }
 
 ArenaHeader*
 ArenaList::removeRemainingArenas(ArenaHeader** arenap)
 {
     // This is only ever called to remove arenas that are after the cursor, so
     // we don't need to update it.
 #ifdef DEBUG
@@ -2090,17 +2130,17 @@ ArenaList::pickArenasToRelocate(size_t& 
         fullArenaCount++;
 
     for (ArenaHeader* arena = *cursorp_; arena; arena = arena->next) {
         followingUsedCells += arena->countUsedCells();
         nonFullArenaCount++;
     }
 
     mozilla::DebugOnly<size_t> lastFreeCells(0);
-    size_t cellsPerArena = Arena::thingsPerArena((*arenap)->getThingSize());
+    size_t cellsPerArena = Arena::thingsPerArena((*arenap)->getAllocKind());
 
     while (*arenap) {
         ArenaHeader* arena = *arenap;
         if (followingUsedCells <= previousFreeCells)
             break;
 
         size_t freeCells = arena->countFreeCells();
         size_t usedCells = cellsPerArena - freeCells;
@@ -2361,17 +2401,17 @@ GCRuntime::relocateArenas(Zone* zone, JS
 
     if (!zone->arenas.relocateArenas(zone, relocatedListOut, reason, sliceBudget, stats))
         return false;
 
 #ifdef DEBUG
     // Check that we did as much compaction as we should have. There
     // should always be less than one arena's worth of free cells.
     for (auto i : AllAllocKinds()) {
-        size_t thingsPerArena = Arena::thingsPerArena(Arena::thingSize(i));
+        size_t thingsPerArena = Arena::thingsPerArena(i);
         if (CanRelocateAllocKind(i)) {
             ArenaList& al = zone->arenas.arenaLists[i];
             size_t freeCells = 0;
             for (ArenaHeader* arena = al.arenaAfterCursor(); arena; arena = arena->next)
                 freeCells += arena->countFreeCells();
             MOZ_ASSERT(freeCells < thingsPerArena);
         }
     }
@@ -2849,17 +2889,17 @@ GCRuntime::releaseRelocatedArenasWithout
         size_t thingSize = aheader->getThingSize();
         Arena* arena = aheader->getArena();
         FreeSpan fullSpan;
         fullSpan.initFinal(arena->thingsStart(thingKind), arena->thingsEnd() - thingSize, thingSize);
         aheader->setFirstFreeSpan(&fullSpan);
 
 #if defined(JS_CRASH_DIAGNOSTICS) || defined(JS_GC_ZEAL)
         JS_POISON(reinterpret_cast<void*>(arena->thingsStart(thingKind)),
-                  JS_MOVED_TENURED_PATTERN, Arena::thingsSpan(thingSize));
+                  JS_MOVED_TENURED_PATTERN, Arena::thingsSpan(thingKind));
 #endif
 
         releaseArena(aheader, lock);
         ++count;
     }
 }
 
 // In debug mode we don't always release relocated arenas straight away.
@@ -2935,17 +2975,17 @@ ArenaLists::forceFinalizeNow(FreeOp* fop
 {
     MOZ_ASSERT(backgroundFinalizeState[thingKind] == BFS_DONE);
 
     ArenaHeader* arenas = arenaLists[thingKind].head();
     if (!arenas)
         return;
     arenaLists[thingKind].clear();
 
-    size_t thingsPerArena = Arena::thingsPerArena(Arena::thingSize(thingKind));
+    size_t thingsPerArena = Arena::thingsPerArena(thingKind);
     SortedArenaList finalizedSorted(thingsPerArena);
 
     auto unlimited = SliceBudget::unlimited();
     FinalizeArenas(fop, &arenas, finalizedSorted, thingKind, unlimited, keepArenas);
     MOZ_ASSERT(!arenas);
 
     if (empty) {
         MOZ_ASSERT(keepArenas == KEEP_ARENAS);
@@ -3004,17 +3044,17 @@ ArenaLists::queueForBackgroundSweep(Free
 ArenaLists::backgroundFinalize(FreeOp* fop, ArenaHeader* listHead, ArenaHeader** empty)
 {
     MOZ_ASSERT(listHead);
     MOZ_ASSERT(empty);
 
     AllocKind thingKind = listHead->getAllocKind();
     Zone* zone = listHead->zone;
 
-    size_t thingsPerArena = Arena::thingsPerArena(Arena::thingSize(thingKind));
+    size_t thingsPerArena = Arena::thingsPerArena(thingKind);
     SortedArenaList finalizedSorted(thingsPerArena);
 
     auto unlimited = SliceBudget::unlimited();
     FinalizeArenas(fop, &listHead, finalizedSorted, thingKind, unlimited, KEEP_ARENAS);
     MOZ_ASSERT(!listHead);
 
     finalizedSorted.extractEmpty(empty);
 
@@ -5442,17 +5482,17 @@ static bool
 SweepArenaList(ArenaHeader** arenasToSweep, SliceBudget& sliceBudget, Args... args)
 {
     while (ArenaHeader* arena = *arenasToSweep) {
         for (ArenaCellIterUnderGC i(arena); !i.done(); i.next())
             SweepThing(i.get<T>(), args...);
 
         *arenasToSweep = (*arenasToSweep)->next;
         AllocKind kind = MapTypeToFinalizeKind<T>::kind;
-        sliceBudget.step(Arena::thingsPerArena(Arena::thingSize(kind)));
+        sliceBudget.step(Arena::thingsPerArena(kind));
         if (sliceBudget.isOverBudget())
             return false;
     }
 
     return true;
 }
 
 GCRuntime::IncrementalProgress
@@ -5515,17 +5555,17 @@ GCRuntime::sweepPhase(SliceBudget& slice
 
             for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) {
                 Zone* zone = sweepZone;
 
                 while (sweepKindIndex < IncrementalFinalizePhases[finalizePhase].length) {
                     AllocKind kind = IncrementalFinalizePhases[finalizePhase].kinds[sweepKindIndex];
 
                     /* Set the number of things per arena for this AllocKind. */
-                    size_t thingsPerArena = Arena::thingsPerArena(Arena::thingSize(kind));
+                    size_t thingsPerArena = Arena::thingsPerArena(kind);
                     incrementalSweepList.setThingsPerArena(thingsPerArena);
 
                     if (!zone->arenas.foregroundFinalize(&fop, kind, sliceBudget,
                                                          incrementalSweepList))
                         return NotFinished;
 
                     /* Reset the slots of the sweep list that we used. */
                     incrementalSweepList.reset(thingsPerArena);
--- a/js/src/vm/MemoryMetrics.cpp
+++ b/js/src/vm/MemoryMetrics.cpp
@@ -351,17 +351,17 @@ StatsCompartmentCallback(JSRuntime* rt, 
 static void
 StatsArenaCallback(JSRuntime* rt, void* data, gc::Arena* arena,
                    JS::TraceKind traceKind, size_t thingSize)
 {
     RuntimeStats* rtStats = static_cast<StatsClosure*>(data)->rtStats;
 
     // The admin space includes (a) the header and (b) the padding between the
     // end of the header and the start of the first GC thing.
-    size_t allocationSpace = arena->thingsSpan(thingSize);
+    size_t allocationSpace = Arena::thingsSpan(arena->aheader.getAllocKind());
     rtStats->currZoneStats->gcHeapArenaAdmin += gc::ArenaSize - allocationSpace;
 
     // We don't call the callback on unused things.  So we compute the
     // unused space like this:  arenaUnused = maxArenaUnused - arenaUsed.
     // We do this by setting arenaUnused to maxArenaUnused here, and then
     // subtracting thingSize for every used cell, in StatsCellCallback().
     rtStats->currZoneStats->unusedGCThings.addToKind(traceKind, allocationSpace);
 }