Bug 1298018 - Part 4: Lazily allocate nursery chunks. r=jonco
authorPaul Bone <pbone@mozilla.com>
Mon, 30 Oct 2017 14:44:12 +1100
changeset 443522 d8dbac587181077425996af8a29fb3df03765614
parent 443521 bf065ffab5f7649ec7724ad226e97a853715497a
child 443523 7bd9208b8c7c92521abfc8e4cae34bfbf419547a
push id1618
push userCallek@gmail.com
push dateThu, 11 Jan 2018 17:45:48 +0000
treeherdermozilla-release@882ca853e05a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjonco
bugs1298018
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1298018 - Part 4: Lazily allocate nursery chunks. r=jonco
js/src/gc/Nursery.cpp
js/src/gc/Nursery.h
--- a/js/src/gc/Nursery.cpp
+++ b/js/src/gc/Nursery.cpp
@@ -111,17 +111,18 @@ js::NurseryChunk::toChunk(JSRuntime* rt)
 
 js::Nursery::Nursery(JSRuntime* rt)
   : runtime_(rt)
   , position_(0)
   , currentStartChunk_(0)
   , currentStartPosition_(0)
   , currentEnd_(0)
   , currentChunk_(0)
-  , maxNurseryChunks_(0)
+  , maxChunkCount_(0)
+  , chunkCountLimit_(0)
   , previousPromotionRate_(0)
   , profileThreshold_(0)
   , enableProfiling_(false)
   , reportTenurings_(0)
   , minorGCTriggerReason_(JS::gcreason::NO_REASON)
   , minorGcCount_(0)
   , freeMallocedBuffersTask(nullptr)
 #ifdef JS_GC_ZEAL
@@ -135,20 +136,20 @@ js::Nursery::init(uint32_t maxNurseryByt
     if (!mallocedBuffers.init())
         return false;
 
     freeMallocedBuffersTask = js_new<FreeMallocedBuffersTask>(runtime()->defaultFreeOp());
     if (!freeMallocedBuffersTask || !freeMallocedBuffersTask->init())
         return false;
 
     /* maxNurseryBytes parameter is rounded down to a multiple of chunk size. */
-    maxNurseryChunks_ = maxNurseryBytes >> ChunkShift;
+    chunkCountLimit_ = maxNurseryBytes >> ChunkShift;
 
     /* If no chunks are specified then the nursery is permanently disabled. */
-    if (maxNurseryChunks_ == 0)
+    if (chunkCountLimit_ == 0)
         return true;
 
     if (!allocateFirstChunk(lock))
         return false;
 
     setCurrentChunk(0);
     setStartPosition();
 
@@ -186,17 +187,17 @@ js::Nursery::~Nursery()
     js_delete(freeMallocedBuffersTask);
 }
 
 void
 js::Nursery::enable()
 {
     MOZ_ASSERT(isEmpty());
     MOZ_ASSERT(!runtime()->gc.isVerifyPreBarriersEnabled());
-    if (isEnabled() || !maxChunks())
+    if (isEnabled() || !chunkCountLimit())
         return;
 
     {
         AutoLockGCBgAlloc lock(runtime());
         if (!allocateFirstChunk(lock))
             return;
     }
 
@@ -213,17 +214,20 @@ js::Nursery::enable()
 void
 js::Nursery::disable()
 {
     MOZ_ASSERT(isEmpty());
     if (!isEnabled())
         return;
 
     freeChunksFrom(0);
+    maxChunkCount_ = 0;
+
     currentEnd_ = 0;
+
     runtime()->gc.storeBuffer().disable();
 }
 
 bool
 js::Nursery::isEmpty() const
 {
     if (!isEnabled())
         return true;
@@ -234,17 +238,17 @@ js::Nursery::isEmpty() const
     }
     return position() == currentStartPosition_;
 }
 
 #ifdef JS_GC_ZEAL
 void
 js::Nursery::enterZealMode() {
     if (isEnabled())
-        growAllocableSpace(maxChunks());
+        maxChunkCount_ = chunkCountLimit();
 }
 
 void
 js::Nursery::leaveZealMode() {
     if (isEnabled()) {
         MOZ_ASSERT(isEmpty());
         setCurrentChunk(0);
         setStartPosition();
@@ -300,19 +304,28 @@ js::Nursery::allocate(size_t size)
 
 #ifdef JS_GC_ZEAL
     static const size_t CanarySize = (sizeof(Nursery::Canary) + CellAlignBytes - 1) & ~CellAlignMask;
     if (runtime()->gc.hasZealMode(ZealMode::CheckNursery))
         size += CanarySize;
 #endif
 
     if (currentEnd() < position() + size) {
-        if (currentChunk_ + 1 == numChunks())
+        unsigned chunkno = currentChunk_ + 1;
+        MOZ_ASSERT(chunkno <= chunkCountLimit());
+        MOZ_ASSERT(chunkno <= maxChunkCount());
+        MOZ_ASSERT(chunkno <= allocatedChunkCount());
+        if (chunkno == maxChunkCount())
             return nullptr;
-        setCurrentChunk(currentChunk_ + 1);
+        if (MOZ_UNLIKELY(chunkno == allocatedChunkCount())) {
+            if (!allocateNextChunk(chunkno))
+                return nullptr;
+            MOZ_ASSERT(chunkno < allocatedChunkCount());
+        }
+        setCurrentChunk(chunkno);
     }
 
     void* thing = (void*)position();
     position_ = position() + size;
 
     JS_EXTRA_POISON(thing, JS_ALLOCATED_NURSERY_PATTERN, size);
 
 #ifdef JS_GC_ZEAL
@@ -521,16 +534,17 @@ js::Nursery::renderProfileJSON(JSONPrint
 
     json.property("status", "complete");
 
     json.property("reason", JS::gcreason::ExplainReason(previousGC.reason));
     json.property("bytes_tenured", previousGC.tenuredBytes);
     json.property("bytes_used", previousGC.nurseryUsedBytes);
     json.property("cur_capacity", previousGC.nurseryCapacity);
     json.property("new_capacity", spaceToEnd());
+    json.property("lazy_capacity", allocatedChunkCount() * ChunkSize);
 
     json.beginObjectProperty("phase_times");
 
 #define EXTRACT_NAME(name, text) #name,
     static const char* names[] = {
 FOR_EACH_NURSERY_PROFILE_TIME(EXTRACT_NAME)
 #undef EXTRACT_NAME
     "" };
@@ -683,17 +697,17 @@ js::Nursery::collect(JS::gcreason::Reaso
     // We ignore gcMaxBytes when allocating for minor collection. However, if we
     // overflowed, we disable the nursery. The next time we allocate, we'll fail
     // because gcBytes >= gcMaxBytes.
     if (rt->gc.usage.gcBytes() >= rt->gc.tunables.gcMaxBytes())
         disable();
     // Disable the nursery if the user changed the configuration setting.  The
     // nursery can only be re-enabled by resetting the configurationa and
     // restarting firefox.
-    if (maxNurseryChunks_ == 0)
+    if (chunkCountLimit_ == 0)
         disable();
 
     endProfile(ProfileKey::Total);
     minorGcCount_++;
 
     TimeDuration totalTime = profileDurations_[ProfileKey::Total];
     rt->addTelemetry(JS_TELEMETRY_GC_MINOR_US, totalTime.ToMicroseconds());
     rt->addTelemetry(JS_TELEMETRY_GC_MINOR_REASON, reason);
@@ -706,17 +720,17 @@ js::Nursery::collect(JS::gcreason::Reaso
     TraceMinorGCEnd();
 
     if (enableProfiling_ && totalTime >= profileThreshold_) {
         rt->gc.stats().maybePrintProfileHeaders();
 
         fprintf(stderr, "MinorGC: %20s %5.1f%% %4u        ",
                 JS::gcreason::ExplainReason(reason),
                 promotionRate * 100,
-                numChunks());
+                maxChunkCount());
         printProfileDurations(profileDurations_);
 
         if (reportTenurings_) {
             for (auto& entry : tenureCounts.entries) {
                 if (entry.count >= reportTenurings_) {
                     fprintf(stderr, "  %d x ", entry.count);
                     entry.group->print();
                 }
@@ -911,64 +925,92 @@ js::Nursery::sweep(JSTracer* trc)
     sweepDictionaryModeObjects();
 }
 
 void
 js::Nursery::clear()
 {
 #ifdef JS_GC_ZEAL
     /* Poison the nursery contents so touching a freed object will crash. */
-    for (unsigned i = 0; i < numChunks(); i++)
+    for (unsigned i = 0; i < allocatedChunkCount(); i++)
         chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN);
 
     if (runtime()->hasZealMode(ZealMode::GenerationalGC)) {
         /* Only reset the alloc point when we are close to the end. */
-        if (currentChunk_ + 1 == numChunks())
+        if (currentChunk_ + 1 == maxChunkCount())
             setCurrentChunk(0);
     } else
 #endif
     {
 #ifdef JS_CRASH_DIAGNOSTICS
-        for (unsigned i = 0; i < numChunks(); ++i)
+        for (unsigned i = 0; i < allocatedChunkCount(); ++i)
             chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN);
 #endif
         setCurrentChunk(0);
     }
 
     /* Set current start position for isEmpty checks. */
     setStartPosition();
 }
 
 size_t
 js::Nursery::spaceToEnd() const
 {
-    unsigned lastChunk = numChunks() - 1;
+    unsigned lastChunk = maxChunkCount() - 1;
 
     MOZ_ASSERT(lastChunk >= currentStartChunk_);
     MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <= NurseryChunkUsableSize);
 
     size_t bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) +
                    ((lastChunk - currentStartChunk_) * NurseryChunkUsableSize);
 
-    MOZ_ASSERT(bytes <= numChunks() * NurseryChunkUsableSize);
+    MOZ_ASSERT(bytes <= maxChunkCount() * NurseryChunkUsableSize);
 
     return bytes;
 }
 
 MOZ_ALWAYS_INLINE void
 js::Nursery::setCurrentChunk(unsigned chunkno)
 {
-    MOZ_ASSERT(chunkno < maxChunks());
-    MOZ_ASSERT(chunkno < numChunks());
+    MOZ_ASSERT(chunkno < chunkCountLimit());
+    MOZ_ASSERT(chunkno < allocatedChunkCount());
     currentChunk_ = chunkno;
     position_ = chunk(chunkno).start();
     currentEnd_ = chunk(chunkno).end();
     chunk(chunkno).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN);
 }
 
+bool
+js::Nursery::allocateNextChunk(const unsigned chunkno)
+{
+    const unsigned priorCount = allocatedChunkCount();
+    const unsigned newCount = priorCount + 1;
+
+    MOZ_ASSERT(chunkno == currentChunk_ + 1);
+    MOZ_ASSERT(chunkno == allocatedChunkCount());
+    MOZ_ASSERT(chunkno < chunkCountLimit());
+    MOZ_ASSERT(chunkno < maxChunkCount());
+
+    if (!chunks_.resize(newCount))
+        return false;
+
+    Chunk* newChunk;
+    {
+        AutoLockGCBgAlloc lock(runtime());
+        newChunk = runtime()->gc.getOrAllocChunk(lock);
+    }
+    if (!newChunk) {
+        chunks_.shrinkTo(priorCount);
+        return false;
+    }
+
+    chunks_[chunkno] = NurseryChunk::fromChunk(newChunk);
+    return true;
+}
+
 MOZ_ALWAYS_INLINE void
 js::Nursery::setStartPosition()
 {
     currentStartChunk_ = currentChunk_;
     currentStartPosition_ = position();
 }
 
 void
@@ -995,73 +1037,43 @@ js::Nursery::maybeResizeNursery(JS::gcre
      * This incorrect promotion rate results in better nursery sizing
      * decisions, however we should to better tuning based on the real
      * promotion rate in the future.
      */
     const float promotionRate =
         float(previousGC.tenuredBytes) / float(previousGC.nurseryCapacity);
 
     newMaxNurseryChunks = runtime()->gc.tunables.gcMaxNurseryBytes() >> ChunkShift;
-    if (newMaxNurseryChunks != maxNurseryChunks_) {
-        maxNurseryChunks_ = newMaxNurseryChunks;
+    if (newMaxNurseryChunks != chunkCountLimit_) {
+        chunkCountLimit_ = newMaxNurseryChunks;
         /* The configured maximum nursery size is changing */
-        if (numChunks() > newMaxNurseryChunks) {
+        if (maxChunkCount() > newMaxNurseryChunks) {
             /* We need to shrink the nursery */
             shrinkAllocableSpace(newMaxNurseryChunks);
 
             previousPromotionRate_ = promotionRate;
             return;
         }
     }
 
     if (promotionRate > GrowThreshold) {
         // The GC nursery is an optimization and so if we fail to allocate
         // nursery chunks we do not report an error.
         growAllocableSpace();
     } else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) {
-        shrinkAllocableSpace(numChunks() - 1);
+        shrinkAllocableSpace(maxChunkCount() - 1);
     }
 
     previousPromotionRate_ = promotionRate;
 }
 
-bool
+void
 js::Nursery::growAllocableSpace()
 {
-    return growAllocableSpace(Min(numChunks() * 2, maxChunks()));
-}
-
-bool
-js::Nursery::growAllocableSpace(unsigned newCount)
-{
-    unsigned priorCount = numChunks();
-
-    if (newCount == priorCount)
-        return false;
-
-    MOZ_ASSERT(newCount > priorCount);
-    MOZ_ASSERT(newCount <= maxChunks());
-    MOZ_ASSERT(priorCount >= 1);
-
-    if (!chunks_.resize(newCount))
-        return false;
-
-    AutoLockGCBgAlloc lock(runtime());
-    for (unsigned i = priorCount; i < newCount; i++) {
-        auto newChunk = runtime()->gc.getOrAllocChunk(lock);
-        if (!newChunk) {
-            chunks_.shrinkTo(i);
-            return false;
-        }
-
-        chunks_[i] = NurseryChunk::fromChunk(newChunk);
-        chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN);
-    }
-
-    return true;
+    maxChunkCount_ = Min(maxChunkCount() * 2, chunkCountLimit());
 }
 
 void
 js::Nursery::freeChunksFrom(unsigned firstFreeChunk)
 {
     MOZ_ASSERT(firstFreeChunk < chunks_.length());
     {
         AutoLockGC lock(runtime());
@@ -1076,49 +1088,54 @@ js::Nursery::shrinkAllocableSpace(unsign
 {
 #ifdef JS_GC_ZEAL
     if (runtime()->hasZealMode(ZealMode::GenerationalGC))
         return;
 #endif
 
     // Don't shrink the nursery to zero (use Nursery::disable() instead) and
     // don't attempt to shrink it to the same size.
-    if ((newCount == 0) || (newCount == numChunks()))
+    if ((newCount == 0) || (newCount == maxChunkCount()))
         return;
 
-    MOZ_ASSERT(newCount < numChunks());
+    MOZ_ASSERT(newCount < maxChunkCount());
 
-    freeChunksFrom(newCount);
+    if (newCount < allocatedChunkCount())
+        freeChunksFrom(newCount);
+
+    maxChunkCount_ = newCount;
 }
 
 void
 js::Nursery::minimizeAllocableSpace()
 {
     shrinkAllocableSpace(1);
 }
 
 bool
 js::Nursery::allocateFirstChunk(AutoLockGCBgAlloc& lock)
 {
-    // This assertion isn't required for correctness, but we do assume this
+    // These assertions aren't required for correctness, but we do assume this
     // is only called to initialize or re-enable the nursery.
-    MOZ_ASSERT(numChunks() == 0);
+    MOZ_ASSERT(allocatedChunkCount() == 0);
+    MOZ_ASSERT(maxChunkCount() == 0);
 
-    MOZ_ASSERT(maxChunks() > 0);
+    MOZ_ASSERT(chunkCountLimit() > 0);
 
     if (!chunks_.resize(1))
         return false;
 
     auto chunk = runtime()->gc.getOrAllocChunk(lock);
     if (!chunk) {
         chunks_.shrinkTo(0);
         return false;
     }
 
     chunks_[0] = NurseryChunk::fromChunk(chunk);
+    maxChunkCount_ = 1;
 
     return true;
 }
 
 bool
 js::Nursery::queueDictionaryModeObjectToSweep(NativeObject* obj)
 {
     MOZ_ASSERT(IsInsideNursery(obj));
--- a/js/src/gc/Nursery.h
+++ b/js/src/gc/Nursery.h
@@ -136,24 +136,32 @@ class Nursery
     static const size_t Alignment = gc::ChunkSize;
     static const size_t ChunkShift = gc::ChunkShift;
 
     explicit Nursery(JSRuntime* rt);
     ~Nursery();
 
     MOZ_MUST_USE bool init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock);
 
-    unsigned maxChunks() const { return maxNurseryChunks_; }
-    unsigned numChunks() const { return chunks_.length(); }
+    unsigned chunkCountLimit() const { return chunkCountLimit_; }
+
+    // Number of allocated (ready to use) chunks.
+    unsigned allocatedChunkCount() const { return chunks_.length(); }
 
-    bool exists() const { return maxChunks() != 0; }
+    // Total number of chunks and the capacity of the nursery. Chunks will be
+    // lazilly allocated and added to the chunks array up to this limit, after
+    // that the nursery must be collected, this limit may be raised during
+    // collection.
+    unsigned maxChunkCount() const { return maxChunkCount_; }
+
+    bool exists() const { return chunkCountLimit() != 0; }
 
     void enable();
     void disable();
-    bool isEnabled() const { return numChunks() != 0; }
+    bool isEnabled() const { return maxChunkCount() != 0; }
 
     /* Return true if no allocations have been made since the last collection. */
     bool isEmpty() const;
 
     /*
      * Check whether an arbitrary pointer is within the nursery. This is
      * slower than IsInsideNursery(Cell*), but works on all types of pointers.
      */
@@ -228,17 +236,17 @@ class Nursery
         MOZ_ASSERT(IsInsideNursery(cell));
         MOZ_ASSERT(isEnabled());
         return cellsWithUid_.append(cell);
     }
 
     MOZ_MUST_USE bool queueDictionaryModeObjectToSweep(NativeObject* obj);
 
     size_t sizeOfHeapCommitted() const {
-        return numChunks() * gc::ChunkSize;
+        return allocatedChunkCount() * gc::ChunkSize;
     }
     size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const {
         if (!mallocedBuffers.initialized())
             return 0;
         size_t total = 0;
         for (MallocedBuffersSet::Range r = mallocedBuffers.all(); !r.empty(); r.popFront())
             total += mallocSizeOf(r.front());
         total += mallocedBuffers.sizeOfExcludingThis(mallocSizeOf);
@@ -247,17 +255,17 @@ class Nursery
 
     // The number of bytes from the start position to the end of the nursery.
     size_t spaceToEnd() const;
 
     // Free space remaining, not counting chunk trailers.
     MOZ_ALWAYS_INLINE size_t freeSpace() const {
         MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize);
         return (currentEnd_ - position_) +
-               (numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize;
+               (maxChunkCount() - currentChunk_ - 1) * NurseryChunkUsableSize;
     }
 
 #ifdef JS_GC_ZEAL
     void enterZealMode();
     void leaveZealMode();
 #endif
 
     /* Write profile time JSON on JSONPrinter. */
@@ -305,18 +313,28 @@ class Nursery
     uintptr_t currentStartPosition_;
 
     /* Pointer to the last byte of space in the current chunk. */
     uintptr_t currentEnd_;
 
     /* The index of the chunk that is currently being allocated from. */
     unsigned currentChunk_;
 
-    /* Maximum number of chunks to allocate for the nursery. */
-    unsigned maxNurseryChunks_;
+    /*
+     * The nursery may grow the chunks_ vector up to this size without a
+     * collection.  This allows the nursery to grow lazilly.  This limit may
+     * change during maybeResizeNursery() each collection.
+     */
+    unsigned maxChunkCount_;
+
+    /*
+     * This limit is fixed by configuration.  It represents the maximum size
+     * the nursery is permitted to tune itself to in maybeResizeNursery();
+     */
+    unsigned chunkCountLimit_;
 
     /* Promotion rate for the previous minor collection. */
     float previousPromotionRate_;
 
     /* Report minor collections taking at least this long, if enabled. */
     mozilla::TimeDuration profileThreshold_;
     bool enableProfiling_;
 
@@ -426,16 +444,18 @@ class Nursery
     void setStartPosition();
 
     /*
      * Ensure that the first chunk has been allocated. Callers will probably
      * want to call setCurrentChunk(0) next.
      */
     MOZ_MUST_USE bool allocateFirstChunk(AutoLockGCBgAlloc& lock);
 
+    MOZ_MUST_USE bool allocateNextChunk(unsigned chunkno);
+
     MOZ_ALWAYS_INLINE uintptr_t currentEnd() const;
 
     uintptr_t position() const { return position_; }
 
     JSRuntime* runtime() const { return runtime_; }
 
     /* Allocates a new GC thing from the tenured generation during minor GC. */
     gc::TenuredCell* allocateFromTenured(JS::Zone* zone, gc::AllocKind thingKind);
@@ -476,23 +496,22 @@ class Nursery
      * collection.
      */
     void clear();
 
     void sweepDictionaryModeObjects();
 
     /* Change the allocable space provided by the nursery. */
     void maybeResizeNursery(JS::gcreason::Reason reason);
-    bool growAllocableSpace();
-    bool growAllocableSpace(unsigned newSize);
+    void growAllocableSpace();
     void shrinkAllocableSpace(unsigned newCount);
     void minimizeAllocableSpace();
 
     // Free the chunks starting at firstFreeChunk until the end of the chunks
-    // vector. Shrinks the vector but does not update maxChunksAlloc().
+    // vector. Shrinks the vector but does not update maxChunkCount().
     void freeChunksFrom(unsigned firstFreeChunk);
 
     /* Profile recording and printing. */
     void maybeClearProfileDurations();
     void startProfile(ProfileKey key);
     void endProfile(ProfileKey key);
     static void printProfileDurations(const ProfileDurations& times);