Bug 1074961 - Part 4: move refillFreeList into GCRuntime; r=sfink
authorTerrence Cole <terrence@mozilla.com>
Wed, 08 Oct 2014 17:46:33 -0700
changeset 209876 bdd40e1efa4f6e956581c408ab8789622e1fcd2b
parent 209875 d5cd609c75fc6fb7226a43c9018652afeae1f458
child 209877 313123f12e959a9113a2e922d37727593f364d7d
push id27628
push userkwierso@gmail.com
push dateSat, 11 Oct 2014 02:00:16 +0000
treeherdermozilla-central@f74ad36bb97b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfink
bugs1074961
milestone35.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1074961 - Part 4: move refillFreeList into GCRuntime; r=sfink
js/src/gc/GCRuntime.h
js/src/gc/Heap.h
js/src/gc/Nursery.cpp
js/src/gc/Nursery.h
js/src/gc/Zone.h
js/src/jsgc.cpp
js/src/jsgc.h
js/src/jsgcinlines.h
--- a/js/src/gc/GCRuntime.h
+++ b/js/src/gc/GCRuntime.h
@@ -2,16 +2,18 @@
  * vim: set ts=8 sts=4 et sw=4 tw=99:
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef gc_GCRuntime_h
 #define gc_GCRuntime_h
 
+#include <setjmp.h>
+
 #include "jsgc.h"
 
 #include "gc/Heap.h"
 #ifdef JSGC_GENERATIONAL
 # include "gc/Nursery.h"
 #endif
 #include "gc/Statistics.h"
 #ifdef JSGC_GENERATIONAL
@@ -463,22 +465,31 @@ class GCRuntime
     void startVerifyPostBarriers();
     bool endVerifyPostBarriers();
     void finishVerifier();
     bool isVerifyPreBarriersEnabled() const { return !!verifyPreData; }
 #else
     bool isVerifyPreBarriersEnabled() const { return false; }
 #endif
 
+    template <AllowGC allowGC>
+    static void *refillFreeListFromAnyThread(ThreadSafeContext *cx, AllocKind thingKind);
+
   private:
-    // For ArenaLists::allocateFromArenaInline()
+    // For ArenaLists::allocateFromArena()
     friend class ArenaLists;
-    Chunk *pickChunk(Zone *zone, AutoMaybeStartBackgroundAllocation &maybeStartBackgroundAllocation);
+    Chunk *pickChunk(Zone *zone, AutoMaybeStartBackgroundAllocation &maybeStartBGAlloc);
     inline void arenaAllocatedDuringGC(JS::Zone *zone, ArenaHeader *arena);
 
+    template <AllowGC allowGC>
+    static void *refillFreeListFromMainThread(JSContext *cx, AllocKind thingKind);
+    static void *refillFreeListOffMainThread(ExclusiveContext *cx, AllocKind thingKind);
+    static void *refillFreeListPJS(ForkJoinContext *cx, AllocKind thingKind);
+    static void *refillFreeListInGC(Zone *zone, AllocKind thingKind);
+
     /*
      * Return the list of chunks that can be released outside the GC lock.
      * Must be called either during the GC or with the GC lock taken.
      */
     Chunk *expireChunkPool(bool shrinkBuffers, bool releaseAll);
     void expireAndFreeChunkPool(bool releaseAll);
     void freeChunkList(Chunk *chunkListHead);
     void prepareToFreeChunk(ChunkInfo &info);
--- a/js/src/gc/Heap.h
+++ b/js/src/gc/Heap.h
@@ -468,17 +468,17 @@ class FreeList
 #ifdef DEBUG
     bool isSameNonEmptySpan(const FreeSpan &another) const {
         MOZ_ASSERT(!isEmpty());
         MOZ_ASSERT(!another.isEmpty());
         return head.first == another.first && head.last == another.last;
     }
 #endif
 
-    MOZ_ALWAYS_INLINE void *allocate(size_t thingSize) {
+    MOZ_ALWAYS_INLINE TenuredCell *allocate(size_t thingSize) {
         MOZ_ASSERT(thingSize % CellSize == 0);
         head.checkSpan(thingSize);
         uintptr_t thing = head.first;
         if (thing < head.last) {
             // We have two or more things in the free list head, so we can do a
             // simple bump-allocate.
             head.first = thing + thingSize;
         } else if (MOZ_LIKELY(thing)) {
@@ -487,17 +487,17 @@ class FreeList
             // may be empty).
             setHead(reinterpret_cast<FreeSpan *>(thing));
         } else {
             // The free list head is empty.
             return nullptr;
         }
         head.checkSpan(thingSize);
         JS_EXTRA_POISON(reinterpret_cast<void *>(thing), JS_ALLOCATED_TENURED_PATTERN, thingSize);
-        return reinterpret_cast<void *>(thing);
+        return reinterpret_cast<TenuredCell *>(thing);
     }
 };
 
 /* Every arena has a header. */
 struct ArenaHeader : public JS::shadow::ArenaHeader
 {
     friend struct FreeLists;
 
--- a/js/src/gc/Nursery.cpp
+++ b/js/src/gc/Nursery.cpp
@@ -401,20 +401,21 @@ GetObjectAllocKindForCopy(const Nursery 
     }
 
     AllocKind kind = GetGCObjectFixedSlotsKind(obj->fakeNativeNumFixedSlots());
     MOZ_ASSERT(!IsBackgroundFinalized(kind));
     MOZ_ASSERT(CanBeFinalizedInBackground(kind, obj->getClass()));
     return GetBackgroundAllocKind(kind);
 }
 
-MOZ_ALWAYS_INLINE void *
+MOZ_ALWAYS_INLINE TenuredCell *
 js::Nursery::allocateFromTenured(Zone *zone, AllocKind thingKind)
 {
-    void *t = zone->allocator.arenas.allocateFromFreeList(thingKind, Arena::thingSize(thingKind));
+    TenuredCell *t =
+        zone->allocator.arenas.allocateFromFreeList(thingKind, Arena::thingSize(thingKind));
     if (t)
         return t;
     zone->allocator.arenas.checkEmptyFreeList(thingKind);
     return zone->allocator.arenas.allocateFromArena(zone, thingKind);
 }
 
 void
 js::Nursery::setSlotsForwardingPointer(HeapSlot *oldSlots, HeapSlot *newSlots, uint32_t nslots)
@@ -564,17 +565,17 @@ js::Nursery::markSlot(MinorCollectionTra
     slotp->unsafeGet()->setObject(*tenured);
 }
 
 void *
 js::Nursery::moveToTenured(MinorCollectionTracer *trc, JSObject *src)
 {
     AllocKind dstKind = GetObjectAllocKindForCopy(*this, src);
     Zone *zone = src->zone();
-    JSObject *dst = static_cast<JSObject *>(allocateFromTenured(zone, dstKind));
+    JSObject *dst = reinterpret_cast<JSObject *>(allocateFromTenured(zone, dstKind));
     if (!dst)
         CrashAtUnhandlableOOM("Failed to allocate object while tenuring.");
 
     trc->tenuredSize += moveObjectToTenured(dst, src, dstKind);
 
     RelocationOverlay *overlay = RelocationOverlay::fromCell(src);
     overlay->forwardTo(dst);
     trc->insertIntoFixupList(overlay);
--- a/js/src/gc/Nursery.h
+++ b/js/src/gc/Nursery.h
@@ -262,17 +262,17 @@ class Nursery
     void *addressOfPosition() const { return (void*)&position_; }
 
     JSRuntime *runtime() const { return runtime_; }
 
     /* Allocates and registers external slots with the nursery. */
     HeapSlot *allocateHugeSlots(JS::Zone *zone, size_t nslots);
 
     /* Allocates a new GC thing from the tenured generation during minor GC. */
-    void *allocateFromTenured(JS::Zone *zone, gc::AllocKind thingKind);
+    gc::TenuredCell *allocateFromTenured(JS::Zone *zone, gc::AllocKind thingKind);
 
     struct TenureCountCache;
 
     /* Common internal allocator function. */
     void *allocate(size_t size);
 
     /*
      * Move the object at |src| in the Nursery to an already-allocated cell
--- a/js/src/gc/Zone.h
+++ b/js/src/gc/Zone.h
@@ -7,20 +7,20 @@
 #ifndef gc_Zone_h
 #define gc_Zone_h
 
 #include "mozilla/Atomics.h"
 #include "mozilla/DebugOnly.h"
 #include "mozilla/MemoryReporting.h"
 
 #include "jscntxt.h"
-#include "jsgc.h"
 #include "jsinfer.h"
 
 #include "gc/FindSCCs.h"
+#include "gc/GCRuntime.h"
 
 namespace js {
 
 namespace jit {
 class JitZone;
 }
 
 // Encapsulates the data needed to perform allocation. Typically there is
@@ -30,19 +30,19 @@ class Allocator
 {
   public:
     explicit Allocator(JS::Zone *zone);
 
     js::gc::ArenaLists arenas;
 
   private:
     // Since allocators can be accessed from worker threads, the parent zone_
-    // should not be accessed in general. ArenaLists is allowed to actually do
+    // should not be accessed in general. GCRuntime is allowed to actually do
     // the allocation, however.
-    friend class gc::ArenaLists;
+    friend class js::gc::GCRuntime;
 
     JS::Zone *zone_;
 };
 
 namespace gc {
 
 // This class encapsulates the data that determines when we need to do a zone GC.
 class ZoneHeapThreshold
--- a/js/src/jsgc.cpp
+++ b/js/src/jsgc.cpp
@@ -1916,19 +1916,19 @@ GCRuntime::arenaAllocatedDuringGC(JS::Zo
         arena->allocatedDuringIncremental = true;
         marker.delayMarkingArena(arena);
     } else if (zone->isGCSweeping()) {
         arena->setNextAllocDuringSweep(arenasAllocatedDuringSweep);
         arenasAllocatedDuringSweep = arena;
     }
 }
 
-inline void *
-ArenaLists::allocateFromArenaInline(Zone *zone, AllocKind thingKind,
-                                    AutoMaybeStartBackgroundAllocation &maybeStartBackgroundAllocation)
+TenuredCell *
+ArenaLists::allocateFromArena(Zone *zone, AllocKind thingKind,
+                              AutoMaybeStartBackgroundAllocation &maybeStartBGAlloc)
 {
     /*
      * Parallel JS Note:
      *
      * This function can be called from parallel threads all of which
      * are associated with the same compartment. In that case, each
      * thread will have a distinct ArenaLists.  Therefore, whenever we
      * fall through to pickChunk() we must be sure that we are holding
@@ -1974,26 +1974,26 @@ ArenaLists::allocateFromArenaInline(Zone
          * Move the free span stored in the arena to the free list and
          * allocate from it.
          */
         FreeSpan firstFreeSpan = aheader->getFirstFreeSpan();
         freeLists[thingKind].setHead(&firstFreeSpan);
         aheader->setAsFullyUsed();
         if (MOZ_UNLIKELY(zone->wasGCStarted()))
             zone->runtimeFromMainThread()->gc.arenaAllocatedDuringGC(zone, aheader);
-        void *thing = freeLists[thingKind].allocate(Arena::thingSize(thingKind));
+        TenuredCell *thing = freeLists[thingKind].allocate(Arena::thingSize(thingKind));
         MOZ_ASSERT(thing);   // This allocation is infallible.
         return thing;
     }
 
     /* Make sure we hold the GC lock before we call pickChunk. */
     JSRuntime *rt = zone->runtimeFromAnyThread();
     if (!maybeLock.locked())
         maybeLock.lock(rt);
-    Chunk *chunk = rt->gc.pickChunk(zone, maybeStartBackgroundAllocation);
+    Chunk *chunk = rt->gc.pickChunk(zone, maybeStartBGAlloc);
     if (!chunk)
         return nullptr;
 
     /*
      * While we still hold the GC lock get an arena from some chunk, mark it
      * as full as its single free span is moved to the free lists, and insert
      * it to the list as a fully allocated arena.
      */
@@ -2015,21 +2015,21 @@ ArenaLists::allocateFromArenaInline(Zone
     Arena *arena = aheader->getArena();
     size_t thingSize = Arena::thingSize(thingKind);
     FreeSpan fullSpan;
     fullSpan.initFinal(arena->thingsStart(thingKind), arena->thingsEnd() - thingSize, thingSize);
     freeLists[thingKind].setHead(&fullSpan);
     return freeLists[thingKind].allocate(thingSize);
 }
 
-void *
+TenuredCell *
 ArenaLists::allocateFromArena(JS::Zone *zone, AllocKind thingKind)
 {
     AutoMaybeStartBackgroundAllocation maybeStartBackgroundAllocation;
-    return allocateFromArenaInline(zone, thingKind, maybeStartBackgroundAllocation);
+    return allocateFromArena(zone, thingKind, maybeStartBackgroundAllocation);
 }
 
 void
 ArenaLists::wipeDuringParallelExecution(JSRuntime *rt)
 {
     MOZ_ASSERT(InParallelSection());
 
     // First, check that we all objects we have allocated are eligible
@@ -2717,17 +2717,17 @@ RunLastDitchGC(JSContext *cx, JS::Zone *
      * return that list head.
      */
     size_t thingSize = Arena::thingSize(thingKind);
     return zone->allocator.arenas.allocateFromFreeList(thingKind, thingSize);
 }
 
 template <AllowGC allowGC>
 /* static */ void *
-ArenaLists::refillFreeListFromMainThread(JSContext *cx, AllocKind thingKind)
+GCRuntime::refillFreeListFromMainThread(JSContext *cx, AllocKind thingKind)
 {
     MOZ_ASSERT(!cx->runtime()->isHeapBusy(), "allocating while under GC");
     MOZ_ASSERT_IF(allowGC, !cx->runtime()->currentThreadHasExclusiveAccess());
 
     Allocator *allocator = cx->allocator();
     Zone *zone = allocator->zone_;
 
     // If we have grown past our GC heap threshold while in the middle of an
@@ -2749,91 +2749,91 @@ ArenaLists::refillFreeListFromMainThread
             }
 
             if (void *thing = RunLastDitchGC(cx, zone, thingKind))
                 return thing;
             ranGC = true;
         }
 
         AutoMaybeStartBackgroundAllocation maybeStartBGAlloc;
-        void *thing = allocator->arenas.allocateFromArenaInline(zone, thingKind, maybeStartBGAlloc);
+        void *thing = allocator->arenas.allocateFromArena(zone, thingKind, maybeStartBGAlloc);
         if (MOZ_LIKELY(thing))
             return thing;
 
-        // Even if allocateFromArenaInline failed due to OOM, a background
+        // Even if allocateFromArena failed due to OOM, a background
         // finalization task may be running (freeing more memory); wait for it
         // to finish, then try to allocate again in case it freed up the memory
         // we need.
         cx->runtime()->gc.waitBackgroundSweepEnd();
 
-        thing = allocator->arenas.allocateFromArenaInline(zone, thingKind, maybeStartBGAlloc);
+        thing = allocator->arenas.allocateFromArena(zone, thingKind, maybeStartBGAlloc);
         if (MOZ_LIKELY(thing))
             return thing;
 
         // Retry after a last-ditch GC, unless we've already tried that.
         outOfMemory = true;
     } while (!ranGC);
 
     MOZ_ASSERT(allowGC, "A fallible allocation must not report OOM on failure.");
     js_ReportOutOfMemory(cx);
     return nullptr;
 }
 
 /* static */ void *
-ArenaLists::refillFreeListOffMainThread(ExclusiveContext *cx, AllocKind thingKind)
+GCRuntime::refillFreeListOffMainThread(ExclusiveContext *cx, AllocKind thingKind)
 {
     Allocator *allocator = cx->allocator();
     Zone *zone = allocator->zone_;
     JSRuntime *rt = zone->runtimeFromAnyThread();
 
     AutoMaybeStartBackgroundAllocation maybeStartBGAlloc;
 
     // If we're off the main thread, we try to allocate once and return
     // whatever value we get. We need to first ensure the main thread is not in
     // a GC session.
     AutoLockHelperThreadState lock;
     while (rt->isHeapBusy())
         HelperThreadState().wait(GlobalHelperThreadState::PRODUCER);
 
-    return allocator->arenas.allocateFromArenaInline(zone, thingKind, maybeStartBGAlloc);
+    return allocator->arenas.allocateFromArena(zone, thingKind, maybeStartBGAlloc);
 }
 
 /* static */ void *
-ArenaLists::refillFreeListPJS(ForkJoinContext *cx, AllocKind thingKind)
+GCRuntime::refillFreeListPJS(ForkJoinContext *cx, AllocKind thingKind)
 {
     Allocator *allocator = cx->allocator();
     Zone *zone = allocator->zone_;
 
     AutoMaybeStartBackgroundAllocation maybeStartBGAlloc;
-    return allocator->arenas.allocateFromArenaInline(zone, thingKind, maybeStartBGAlloc);
+    return allocator->arenas.allocateFromArena(zone, thingKind, maybeStartBGAlloc);
 }
 
 template <AllowGC allowGC>
 /* static */ void *
-ArenaLists::refillFreeList(ThreadSafeContext *cx, AllocKind thingKind)
+GCRuntime::refillFreeListFromAnyThread(ThreadSafeContext *cx, AllocKind thingKind)
 {
     MOZ_ASSERT(cx->allocator()->arenas.freeLists[thingKind].isEmpty());
 
     if (cx->isJSContext())
         return refillFreeListFromMainThread<allowGC>(cx->asJSContext(), thingKind);
 
     if (cx->allocator()->zone_->runtimeFromAnyThread()->exclusiveThreadsPresent())
         return refillFreeListOffMainThread(cx->asExclusiveContext(), thingKind);
 
     return refillFreeListPJS(cx->asForkJoinContext(), thingKind);
 }
 
 template void *
-ArenaLists::refillFreeList<NoGC>(ThreadSafeContext *cx, AllocKind thingKind);
+GCRuntime::refillFreeListFromAnyThread<NoGC>(ThreadSafeContext *cx, AllocKind thingKind);
 
 template void *
-ArenaLists::refillFreeList<CanGC>(ThreadSafeContext *cx, AllocKind thingKind);
+GCRuntime::refillFreeListFromAnyThread<CanGC>(ThreadSafeContext *cx, AllocKind thingKind);
 
 /* static */ void *
-ArenaLists::refillFreeListInGC(Zone *zone, AllocKind thingKind)
+GCRuntime::refillFreeListInGC(Zone *zone, AllocKind thingKind)
 {
     /*
      * Called by compacting GC to refill a free list while we are in a GC.
      */
 
     Allocator &allocator = zone->allocator;
     MOZ_ASSERT(allocator.arenas.freeLists[thingKind].isEmpty());
     mozilla::DebugOnly<JSRuntime *> rt = zone->runtimeFromMainThread();
--- a/js/src/jsgc.h
+++ b/js/src/jsgc.h
@@ -797,29 +797,20 @@ class ArenaLists
     bool arenaIsInUse(ArenaHeader *aheader, AllocKind kind) const {
         MOZ_ASSERT(aheader);
         const FreeList &freeList = freeLists[kind];
         if (freeList.isEmpty())
             return false;
         return aheader == freeList.arenaHeader();
     }
 
-    MOZ_ALWAYS_INLINE void *allocateFromFreeList(AllocKind thingKind, size_t thingSize) {
+    MOZ_ALWAYS_INLINE TenuredCell *allocateFromFreeList(AllocKind thingKind, size_t thingSize) {
         return freeLists[thingKind].allocate(thingSize);
     }
 
-    template <AllowGC allowGC>
-    static void *refillFreeList(ThreadSafeContext *cx, AllocKind thingKind);
-    template <AllowGC allowGC>
-    static void *refillFreeListFromMainThread(JSContext *cx, AllocKind thingKind);
-    static void *refillFreeListOffMainThread(ExclusiveContext *cx, AllocKind thingKind);
-    static void *refillFreeListPJS(ForkJoinContext *cx, AllocKind thingKind);
-
-    static void *refillFreeListInGC(Zone *zone, AllocKind thingKind);
-
     /*
      * Moves all arenas from |fromArenaLists| into |this|.  In
      * parallel blocks, we temporarily create one ArenaLists per
      * parallel thread.  When the parallel block ends, we move
      * whatever allocations may have been performed back into the
      * compartment's main arena list using this function.
      */
     void adoptArenas(JSRuntime *runtime, ArenaLists *fromArenaLists);
@@ -855,24 +846,28 @@ class ArenaLists
     void wipeDuringParallelExecution(JSRuntime *rt);
 
   private:
     inline void finalizeNow(FreeOp *fop, AllocKind thingKind);
     inline void forceFinalizeNow(FreeOp *fop, AllocKind thingKind);
     inline void queueForForegroundSweep(FreeOp *fop, AllocKind thingKind);
     inline void queueForBackgroundSweep(FreeOp *fop, AllocKind thingKind);
 
-    void *allocateFromArena(JS::Zone *zone, AllocKind thingKind);
-    inline void *allocateFromArenaInline(JS::Zone *zone, AllocKind thingKind,
-                                         AutoMaybeStartBackgroundAllocation &maybeStartBackgroundAllocation);
+    // Returns false on Out-Of-Memory. This method makes no attempt to
+    // synchronize with background finalization, so may miss available memory
+    // that is waiting to be finalized.
+    TenuredCell *allocateFromArena(JS::Zone *zone, AllocKind thingKind);
+    TenuredCell *allocateFromArena(JS::Zone *zone, AllocKind thingKind,
+                                   AutoMaybeStartBackgroundAllocation &maybeStartBGAlloc);
 
     inline void normalizeBackgroundFinalizeState(AllocKind thingKind);
 
+    friend class GCRuntime;
     friend class js::Nursery;
-    friend class js::gc::ForkJoinNursery;
+    friend class ForkJoinNursery;
 };
 
 /*
  * Initial allocation size for data structures holding chunks is set to hold
  * chunks with total capacity of 16MB to avoid buffer resizes during browser
  * startup.
  */
 const size_t INITIAL_CHUNK_CAPACITY = 16 * 1024 * 1024 / ChunkSize;
--- a/js/src/jsgcinlines.h
+++ b/js/src/jsgcinlines.h
@@ -596,19 +596,20 @@ AllocateObject(ThreadSafeContext *cx, Al
             slots = cx->asExclusiveContext()->zone()->pod_malloc<HeapSlot>(nDynamicSlots);
         else
             slots = js_pod_malloc<HeapSlot>(nDynamicSlots);
         if (MOZ_UNLIKELY(!slots))
             return nullptr;
         js::Debug_SetSlotRangeToCrashOnTouch(slots, nDynamicSlots);
     }
 
-    JSObject *obj = static_cast<JSObject *>(cx->allocator()->arenas.allocateFromFreeList(kind, thingSize));
+    JSObject *obj = reinterpret_cast<JSObject *>(
+            cx->allocator()->arenas.allocateFromFreeList(kind, thingSize));
     if (!obj)
-        obj = static_cast<JSObject *>(js::gc::ArenaLists::refillFreeList<allowGC>(cx, kind));
+        obj = reinterpret_cast<JSObject *>(GCRuntime::refillFreeListFromAnyThread<allowGC>(cx, kind));
 
     if (obj)
         obj->fakeNativeSetInitialSlots(slots);
     else
         js_free(slots);
 
     CheckIncrementalZoneState(cx, obj);
     js::gc::TraceTenuredAlloc(obj, kind);
@@ -626,17 +627,17 @@ AllocateNonObject(ThreadSafeContext *cx)
     size_t thingSize = sizeof(T);
 
     MOZ_ASSERT(thingSize == Arena::thingSize(kind));
     if (!CheckAllocatorState<allowGC>(cx, kind))
         return nullptr;
 
     T *t = static_cast<T *>(cx->allocator()->arenas.allocateFromFreeList(kind, thingSize));
     if (!t)
-        t = static_cast<T *>(js::gc::ArenaLists::refillFreeList<allowGC>(cx, kind));
+        t = static_cast<T *>(GCRuntime::refillFreeListFromAnyThread<allowGC>(cx, kind));
 
     CheckIncrementalZoneState(cx, t);
     js::gc::TraceTenuredAlloc(t, kind);
     return t;
 }
 
 /*
  * When allocating for initialization from a cached object copy, we will