Bug 1433007 - (part 5) Add a sub-chunk limit to the nursery r=sfink
authorPaul Bone <pbone@mozilla.com>
Fri, 15 Feb 2019 04:30:21 +0000
changeset 459496 cd3427c916a4d424e46f999089db29f3825fbec9
parent 459495 d6ef554aa023f5fd53f64ddac770d4d332552566
child 459497 6cec6714210566cd5a156c40b1759a3c46d9d69f
push id111964
push usercsabou@mozilla.com
push dateFri, 15 Feb 2019 18:54:44 +0000
treeherdermozilla-inbound@db3c4f905082 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfink
bugs1433007
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1433007 - (part 5) Add a sub-chunk limit to the nursery r=sfink + Add a limit so that the nursery can collect after using a fraction of a chunk. + Don't do idle time collection if the nursery is empty, even though the free space may now be less than the free space threshold, collecting it wont help. + Modify a test that expects a larger nursery. Differential Revision: https://phabricator.services.mozilla.com/D19348
js/src/gc/Nursery.cpp
js/src/gc/Nursery.h
js/src/jit-test/tests/gc/bug-1293127.js
--- a/js/src/gc/Nursery.cpp
+++ b/js/src/gc/Nursery.cpp
@@ -247,16 +247,17 @@ bool js::Nursery::isEmpty() const {
   }
   return position() == currentStartPosition_;
 }
 
 #ifdef JS_GC_ZEAL
 void js::Nursery::enterZealMode() {
   if (isEnabled()) {
     capacity_ = chunkCountLimit() * NurseryChunkUsableSize;
+    setCurrentEnd();
   }
 }
 
 void js::Nursery::leaveZealMode() {
   if (isEnabled()) {
     MOZ_ASSERT(isEmpty());
     setCurrentChunk(0, true);
     setStartPosition();
@@ -692,17 +693,17 @@ inline void js::Nursery::startProfile(Pr
 
 inline void js::Nursery::endProfile(ProfileKey key) {
   profileDurations_[key] = ReallyNow() - startTimes_[key];
   totalDurations_[key] += profileDurations_[key];
 }
 
 bool js::Nursery::shouldCollect() const {
   uint32_t threshold = tunables().nurseryFreeThresholdForIdleCollection();
-  return minorGCRequested() || freeSpace() < threshold;
+  return !isEmpty() && (minorGCRequested() || freeSpace() < threshold);
 }
 
 static inline bool IsFullStoreBufferReason(JS::GCReason reason) {
   return reason == JS::GCReason::FULL_WHOLE_CELL_BUFFER ||
          reason == JS::GCReason::FULL_GENERIC_BUFFER ||
          reason == JS::GCReason::FULL_VALUE_BUFFER ||
          reason == JS::GCReason::FULL_CELL_PTR_BUFFER ||
          reason == JS::GCReason::FULL_SLOT_BUFFER ||
@@ -1046,18 +1047,32 @@ void js::Nursery::clear() {
 
 size_t js::Nursery::spaceToEnd(unsigned chunkCount) const {
   unsigned lastChunk = chunkCount - 1;
 
   MOZ_ASSERT(lastChunk >= currentStartChunk_);
   MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <=
              NurseryChunkUsableSize);
 
-  size_t bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) +
-                 ((lastChunk - currentStartChunk_) * NurseryChunkUsableSize);
+  size_t bytes;
+
+  if (chunkCount != 1) {
+    // In the general case we have to add:
+    //  + the bytes used in the first
+    //    chunk which may be less than the total size of a chunk since in some
+    //    zeal modes we start the first chunk at some later position
+    //    (currentStartPosition_).
+    //  + the size of all the other chunks.
+    bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) +
+            ((lastChunk - currentStartChunk_) * NurseryChunkUsableSize);
+  } else {
+    // In sub-chunk mode, but it also works whenever chunkCount == 1, we need to
+    // use currentEnd_ since it may not refer to a full chunk.
+    bytes = currentEnd_ - currentStartPosition_;
+  }
 
   MOZ_ASSERT(bytes <= maxChunkCount() * NurseryChunkUsableSize);
 
   return bytes;
 }
 
 MOZ_ALWAYS_INLINE void js::Nursery::setCurrentChunk(unsigned chunkno,
                                                     bool fullPoison) {
@@ -1065,27 +1080,32 @@ MOZ_ALWAYS_INLINE void js::Nursery::setC
   MOZ_ASSERT(chunkno < allocatedChunkCount());
 
   if (!fullPoison && chunkno == currentChunk_ &&
       position_ < chunk(chunkno).end() && position_ >= chunk(chunkno).start()) {
     // When we setup a new chunk the whole chunk must be poisoned with the
     // correct value (JS_FRESH_NURSERY_PATTERN).
     //  1. The first time it was used it was fully poisoned with the
     //     correct value.
-    //  2. When it is swept, only the used part is poisoned with the sept
+    //  2. When it is swept, only the used part is poisoned with the swept
     //     value.
     //  3. We repoison the swept part here, with the correct value.
     chunk(chunkno).poisonAndInit(runtime(), position_ - chunk(chunkno).start());
   } else {
     chunk(chunkno).poisonAndInit(runtime());
   }
 
   currentChunk_ = chunkno;
   position_ = chunk(chunkno).start();
-  currentEnd_ = chunk(chunkno).end();
+  setCurrentEnd();
+}
+
+MOZ_ALWAYS_INLINE void js::Nursery::setCurrentEnd() {
+  MOZ_ASSERT_IF(isSubChunkMode(), currentChunk_ == 0 && currentEnd_ <= chunk(0).end());
+  currentEnd_ = chunk(currentChunk_).start() + Min(capacity_, NurseryChunkUsableSize);
   if (canAllocateStrings_) {
     currentStringEnd_ = currentEnd_;
   }
 }
 
 bool js::Nursery::allocateNextChunk(const unsigned chunkno,
                                     AutoLockGCBgAlloc& lock) {
   const unsigned priorCount = allocatedChunkCount();
@@ -1165,33 +1185,42 @@ void js::Nursery::maybeResizeNursery(JS:
    */
   const float factor = promotionRate / PromotionGoal;
   const unsigned newCapacity = unsigned(float(capacity()) * factor);
 
   // If one of these conditions is true then we always shrink or grow the
   // nursery.  This way the thresholds still have an effect even if the goal
   // seeking says the current size is ideal.
   if (maxChunkCount() < chunkCountLimit() && promotionRate > GrowThreshold) {
-    unsigned lowLimit = (maxChunkCount() + 1) * NurseryChunkUsableSize;
+    unsigned lowLimit = capacity() + SubChunkStep;
     unsigned highLimit = Min(chunkCountLimit() * NurseryChunkUsableSize, capacity() * 2);
 
     growAllocableSpace(mozilla::Clamp(newCapacity, lowLimit, highLimit));
-  } else if (maxChunkCount() > 1 && promotionRate < ShrinkThreshold) {
-    unsigned lowLimit = Max(NurseryChunkUsableSize, capacity() / 2);
-    unsigned highLimit = (maxChunkCount() - 1) * NurseryChunkUsableSize;
+  } else if (capacity() >= SubChunkLimit + SubChunkStep && promotionRate < ShrinkThreshold) {
+    unsigned lowLimit = Max(SubChunkLimit, capacity() / 2);
+    unsigned highLimit = capacity() - SubChunkStep;
 
     shrinkAllocableSpace(mozilla::Clamp(newCapacity, lowLimit, highLimit));
   }
+
+  // Assert that the limits are set such that we can shrink the nursery below
+  // one chunk.
+  static_assert(SubChunkLimit + SubChunkStep < NurseryChunkUsableSize,
+      "Nursery limit must be at least one step from the full chunk size");
 }
 
 void js::Nursery::growAllocableSpace(unsigned newCapacity) {
-  MOZ_ASSERT(newCapacity > currentChunk_ * NurseryChunkUsableSize);
-  // round up to whole chunk.
-  capacity_ = JS_ROUNDUP(newCapacity, NurseryChunkUsableSize);
+  MOZ_ASSERT_IF(!isSubChunkMode(), newCapacity > currentChunk_ * NurseryChunkUsableSize);
+  if (isSubChunkMode()) {
+    capacity_ = Min(JS_ROUNDUP(newCapacity, SubChunkStep), NurseryChunkUsableSize);
+  } else {
+    capacity_ = JS_ROUNDUP(newCapacity, NurseryChunkUsableSize);
+  }
   MOZ_ASSERT(capacity_ <= chunkCountLimit_ * NurseryChunkUsableSize);
+  setCurrentEnd();
 }
 
 void js::Nursery::freeChunksFrom(unsigned firstFreeChunk) {
   MOZ_ASSERT(firstFreeChunk < chunks_.length());
   {
     AutoLockGC lock(runtime());
     for (unsigned i = firstFreeChunk; i < chunks_.length(); i++) {
       runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock);
@@ -1202,56 +1231,71 @@ void js::Nursery::freeChunksFrom(unsigne
 
 void js::Nursery::shrinkAllocableSpace(unsigned newCapacity) {
 #ifdef JS_GC_ZEAL
   if (runtime()->hasZealMode(ZealMode::GenerationalGC)) {
     return;
   }
 #endif
 
-  unsigned newCount = newCapacity / NurseryChunkUsableSize;
-
+  unsigned stepSize = newCapacity < NurseryChunkUsableSize ? SubChunkStep :
+    NurseryChunkUsableSize;
+  newCapacity -= newCapacity % stepSize;
   // Don't shrink the nursery to zero (use Nursery::disable() instead)
-  MOZ_ASSERT(newCount != 0);
-
+  // This can't happen due to the rounding-down performed above because of the
+  // clamping in maybeResizeNursery().
+  MOZ_ASSERT(newCapacity != 0);
   // Don't attempt to shrink it to the same size.
-  if (newCount == maxChunkCount()) {
+  if (newCapacity == capacity()) {
     return;
   }
+  MOZ_ASSERT(newCapacity < capacity());
 
-  MOZ_ASSERT(newCount < maxChunkCount());
-
+  unsigned newCount =
+      (newCapacity + NurseryChunkUsableSize - 1) / NurseryChunkUsableSize;
   if (newCount < allocatedChunkCount()) {
     freeChunksFrom(newCount);
   }
 
-  capacity_ = newCount * NurseryChunkUsableSize;
+  capacity_ = newCapacity;
+  setCurrentEnd();
 }
 
-void js::Nursery::minimizeAllocableSpace() { shrinkAllocableSpace(NurseryChunkUsableSize); }
+void js::Nursery::minimizeAllocableSpace() {
+  shrinkAllocableSpace(SubChunkLimit);
+}
 
 bool js::Nursery::queueDictionaryModeObjectToSweep(NativeObject* obj) {
   MOZ_ASSERT(IsInsideNursery(obj));
   return dictionaryModeObjects_.append(obj);
 }
 
 uintptr_t js::Nursery::currentEnd() const {
-  MOZ_ASSERT(currentEnd_ == chunk(currentChunk_).end());
+  // These are separate asserts because it can be useful to see which one
+  // failed.
+  MOZ_ASSERT_IF(isSubChunkMode(), currentChunk_ == 0);
+  MOZ_ASSERT_IF(isSubChunkMode(), currentEnd_ < chunk(currentChunk_).end());
+  MOZ_ASSERT_IF(!isSubChunkMode(), currentEnd_ == chunk(currentChunk_).end());
+  MOZ_ASSERT(currentEnd_ != chunk(currentChunk_).start());
   return currentEnd_;
 }
 
 gcstats::Statistics& js::Nursery::stats() const {
   return runtime()->gc.stats();
 }
 
 MOZ_ALWAYS_INLINE const js::gc::GCSchedulingTunables& js::Nursery::tunables()
     const {
   return runtime()->gc.tunables;
 }
 
+bool js::Nursery::isSubChunkMode() const {
+  return capacity() < NurseryChunkUsableSize;
+}
+
 void js::Nursery::sweepDictionaryModeObjects() {
   for (auto obj : dictionaryModeObjects_) {
     if (!IsForwarded(obj)) {
       obj->sweepDictionaryListPointer();
     } else {
       Forwarded(obj)->updateDictionaryListPointerAfterMinorGC(obj);
     }
   }
--- a/js/src/gc/Nursery.h
+++ b/js/src/gc/Nursery.h
@@ -136,16 +136,38 @@ inline bool CanNurseryAllocateFinalizedC
   return clasp->flags & JSCLASS_SKIP_NURSERY_FINALIZE;
 }
 
 class Nursery {
  public:
   static const size_t Alignment = gc::ChunkSize;
   static const size_t ChunkShift = gc::ChunkShift;
 
+  /*
+   * SubChunkLimit is the lower limit of the nursery's capacity.
+   * SubChunkStep is the minimum amount to adjust the nursery's size by (to
+   * avoid too many size adjustments and allow quicker changes in size than eg:
+   * 4k).
+   */
+#ifndef JS_GC_SMALL_CHUNK_SIZE
+  /*
+   * 192K is conservative, not too low that root marking dominates.  The Limit
+   * should be a multiple of the Step.
+   */
+  static const size_t SubChunkLimit = 192 * 1024;
+  static const size_t SubChunkStep = 64 * 1024;
+#else
+  /*
+   * With small chunk sizes (256K) we need to use smaller sub chunk limits and
+   * steps so that a full chunk minus one step is still larger than the limit.
+   */
+  static const size_t SubChunkLimit = 64 * 1024;
+  static const size_t SubChunkStep = 16 * 1024;
+#endif
+
   struct alignas(gc::CellAlignBytes) CellAlignedByte {
     char byte;
   };
 
   struct StringLayout {
     JS::Zone* zone;
     CellAlignedByte cell;
   };
@@ -323,17 +345,21 @@ class Nursery {
   }
 
   // The number of bytes from the start position to the end of the nursery.
   // pass maxChunkCount(), allocatedChunkCount() or chunkCountLimit()
   // to calculate the nursery size, current lazy-allocated size or nursery
   // limit respectively.
   size_t spaceToEnd(unsigned chunkCount) const;
 
-  size_t capacity() const { return capacity_; }
+  size_t capacity() const {
+    MOZ_ASSERT(capacity_ >= SubChunkLimit || capacity_ == 0);
+    MOZ_ASSERT(capacity_ <= chunkCountLimit() * NurseryChunkUsableSize);
+    return capacity_;
+  }
   size_t lazyCapacity() const { return spaceToEnd(allocatedChunkCount()); }
 
   // Used and free space, not counting chunk trailers.
   //
   // usedSpace() + freeSpace() == capacity()
   //
   MOZ_ALWAYS_INLINE size_t usedSpace() const {
     return capacity() - freeSpace();
@@ -548,29 +574,32 @@ class Nursery {
   /*
    * Set the current chunk. This updates the currentChunk_, position_
    * currentEnd_ and currentStringEnd_ values as approprite. It'll also
    * poison the chunk, either a portion of the chunk if it is already the
    * current chunk, or the whole chunk if fullPoison is true or it is not
    * the current chunk.
    */
   void setCurrentChunk(unsigned chunkno, bool fullPoison = false);
+  void setCurrentEnd();
   void setStartPosition();
 
   /*
    * Allocate the next chunk, or the first chunk for initialization.
    * Callers will probably want to call setCurrentChunk(0) next.
    */
   MOZ_MUST_USE bool allocateNextChunk(unsigned chunkno,
                                       AutoLockGCBgAlloc& lock);
 
   MOZ_ALWAYS_INLINE uintptr_t currentEnd() const;
 
   uintptr_t position() const { return position_; }
 
+  MOZ_ALWAYS_INLINE bool isSubChunkMode() const;
+
   JSRuntime* runtime() const { return runtime_; }
   gcstats::Statistics& stats() const;
 
   const js::gc::GCSchedulingTunables& tunables() const;
 
   /* Common internal allocator function. */
   void* allocate(size_t size);
 
--- a/js/src/jit-test/tests/gc/bug-1293127.js
+++ b/js/src/jit-test/tests/gc/bug-1293127.js
@@ -1,11 +1,11 @@
-// Test that we can create 1000 cross compartment wrappers to nursery objects
-// without trigger a minor GC.
+// Test that we can create 700 cross compartment wrappers to nursery objects
+// without triggering a minor GC.
 gczeal(0);
 let g = newGlobal({newCompartment: true});
 evalcx("function f(x) { return {x: x}; }", g);
 gc();
 let initial = gcparam("gcNumber");
-for (let i = 0; i < 1000; i++)
+for (let i = 0; i < 700; i++)
     g.f(i);
 let final = gcparam("gcNumber");
 assertEq(final, initial);