Bug 1464387 - Don't instantiate so many trace functions r=sfink
authorJon Coppeard <jcoppeard@mozilla.com>
Tue, 29 May 2018 11:08:09 +0100
changeset 420263 19153834935081b2ae8dce4c076291b1479e4199
parent 420262 25dfb377f644f48469566055a514e182998be4fa
child 420264 d4c518223e7f9b0b0622d651d96a9491fd776fe2
push id34069
push usernerli@mozilla.com
push dateTue, 29 May 2018 21:42:06 +0000
treeherdermozilla-central@89d79c2258be [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfink
bugs1464387
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1464387 - Don't instantiate so many trace functions r=sfink
js/src/gc/Marking.cpp
js/src/gc/Marking.h
js/src/gc/Policy.h
js/src/gc/RootMarking.cpp
js/src/gc/Tracer.h
--- a/js/src/gc/Marking.cpp
+++ b/js/src/gc/Marking.cpp
@@ -77,19 +77,19 @@ using mozilla::PodCopy;
 // The following is a rough outline of the general struture of the tracing
 // internals.
 //
 //                                                                                              //
 //   .---------.    .---------.    .--------------------------.       .----------.              //
 //   |TraceEdge|    |TraceRoot|    |TraceManuallyBarrieredEdge|  ...  |TraceRange|   ... etc.   //
 //   '---------'    '---------'    '--------------------------'       '----------'              //
 //        \              \                        /                        /                    //
-//         \              \  .----------------.  /                        /                     //
-//          o------------->o-|DispatchToTracer|-o<-----------------------o                      //
-//                           '----------------'                                                 //
+//         \              \  .-----------------. /                        /                     //
+//          o------------->o-|TraceEdgeInternal|-o<----------------------o                      //
+//                           '-----------------'                                                //
 //                              /          \                                                    //
 //                             /            \                                                   //
 //                       .---------.   .----------.         .-----------------.                 //
 //                       |DoMarking|   |DoCallback|-------> |<JSTraceCallback>|----------->     //
 //                       '---------'   '----------'         '-----------------'                 //
 //                            |                                                                 //
 //                            |                                                                 //
 //                        .--------.                                                            //
@@ -367,256 +367,108 @@ static void
 AssertShouldMarkInZone(JS::Symbol* sym)
 {
 #ifdef DEBUG
     Zone* zone = sym->asTenured().zone();
     MOZ_ASSERT(zone->shouldMarkInZone() || zone->isAtomsZone());
 #endif
 }
 
-static void
-AssertRootMarkingPhase(JSTracer* trc)
+#ifdef DEBUG
+void
+js::gc::AssertRootMarkingPhase(JSTracer* trc)
 {
     MOZ_ASSERT_IF(trc->isMarkingTracer(),
                   trc->runtime()->gc.state() == State::NotActive ||
                   trc->runtime()->gc.state() == State::MarkRoots);
 }
+#endif
 
 
 /*** Tracing Interface ***************************************************************************/
 
-// The second parameter to BaseGCType is derived automatically based on T. The
-// relation here is that for any T, the TraceKind will automatically,
-// statically select the correct Cell layout for marking. Below, we instantiate
-// each override with a declaration of the most derived layout type.
-//
-// The use of TraceKind::Null for the case where the type is not matched
-// generates a compile error as no template instantiated for that kind.
-//
-// Usage:
-//   BaseGCType<T>::type
-//
-// Examples:
-//   BaseGCType<JSFunction>::type => JSObject
-//   BaseGCType<UnownedBaseShape>::type => BaseShape
-//   etc.
-template <typename T, JS::TraceKind =
-#define EXPAND_MATCH_TYPE(name, type, _) \
-          IsBaseOf<type, T>::value ? JS::TraceKind::name :
-JS_FOR_EACH_TRACEKIND(EXPAND_MATCH_TYPE)
-#undef EXPAND_MATCH_TYPE
-          JS::TraceKind::Null>
-
-struct BaseGCType;
-#define IMPL_BASE_GC_TYPE(name, type_, _) \
-    template <typename T> struct BaseGCType<T, JS::TraceKind:: name> { typedef type_ type; };
-JS_FOR_EACH_TRACEKIND(IMPL_BASE_GC_TYPE);
-#undef IMPL_BASE_GC_TYPE
-
-// Our barrier templates are parameterized on the pointer types so that we can
-// share the definitions with Value and jsid. Thus, we need to strip the
-// pointer before sending the type to BaseGCType and re-add it on the other
-// side. As such:
-template <typename T> struct PtrBaseGCType { typedef T type; };
-template <typename T> struct PtrBaseGCType<T*> { typedef typename BaseGCType<T>::type* type; };
-
-template <typename T>
-typename PtrBaseGCType<T>::type*
-ConvertToBase(T* thingp)
-{
-    return reinterpret_cast<typename PtrBaseGCType<T>::type*>(thingp);
-}
-
-template <typename T> void DispatchToTracer(JSTracer* trc, T* thingp, const char* name);
 template <typename T> T DoCallback(JS::CallbackTracer* trc, T* thingp, const char* name);
 template <typename T> void DoMarking(GCMarker* gcmarker, T* thing);
 template <typename T> void DoMarking(GCMarker* gcmarker, const T& thing);
 template <typename T> void NoteWeakEdge(GCMarker* gcmarker, T** thingp);
 template <typename T> void NoteWeakEdge(GCMarker* gcmarker, T* thingp);
 
 template <typename T>
-void
-js::TraceEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name)
-{
-    DispatchToTracer(trc, ConvertToBase(thingp->unsafeUnbarrieredForTracing()), name);
-}
-
-template <typename T>
-void
-js::TraceEdge(JSTracer* trc, ReadBarriered<T>* thingp, const char* name)
-{
-    DispatchToTracer(trc, ConvertToBase(thingp->unsafeGet()), name);
-}
-
-template <typename T>
-void
-js::TraceNullableEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name)
-{
-    if (InternalBarrierMethods<T>::isMarkable(thingp->get()))
-        DispatchToTracer(trc, ConvertToBase(thingp->unsafeUnbarrieredForTracing()), name);
-}
-
-template <typename T>
-void
-js::TraceNullableEdge(JSTracer* trc, ReadBarriered<T>* thingp, const char* name)
-{
-    if (InternalBarrierMethods<T>::isMarkable(thingp->unbarrieredGet()))
-        DispatchToTracer(trc, ConvertToBase(thingp->unsafeGet()), name);
-}
-
-template <typename T>
 JS_PUBLIC_API(void)
 js::gc::TraceExternalEdge(JSTracer* trc, T* thingp, const char* name)
 {
     MOZ_ASSERT(InternalBarrierMethods<T>::isMarkable(*thingp));
-    DispatchToTracer(trc, ConvertToBase(thingp), name);
-}
-
-template <typename T>
-void
-js::TraceManuallyBarrieredEdge(JSTracer* trc, T* thingp, const char* name)
-{
-    DispatchToTracer(trc, ConvertToBase(thingp), name);
+    TraceEdgeInternal(trc, ConvertToBase(thingp), name);
 }
 
 template <typename T>
 JS_PUBLIC_API(void)
 js::UnsafeTraceManuallyBarrieredEdge(JSTracer* trc, T* thingp, const char* name)
 {
-    DispatchToTracer(trc, ConvertToBase(thingp), name);
-}
-
-template <typename T>
-void
-js::TraceWeakEdge(JSTracer* trc, WeakRef<T>* thingp, const char* name)
-{
-    if (!trc->isMarkingTracer()) {
-        // Non-marking tracers can select whether or not they see weak edges.
-        if (trc->traceWeakEdges())
-            DispatchToTracer(trc, ConvertToBase(thingp->unsafeUnbarrieredForTracing()), name);
-        return;
-    }
-
-    NoteWeakEdge(GCMarker::fromTracer(trc),
-                 ConvertToBase(thingp->unsafeUnbarrieredForTracing()));
-}
-
-template <typename T>
-void
-js::TraceRoot(JSTracer* trc, T* thingp, const char* name)
-{
-    AssertRootMarkingPhase(trc);
-    DispatchToTracer(trc, ConvertToBase(thingp), name);
-}
-
-template <typename T>
-void
-js::TraceRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name)
-{
-    TraceRoot(trc, thingp->unsafeGet(), name);
-}
-
-template <typename T>
-void
-js::TraceNullableRoot(JSTracer* trc, T* thingp, const char* name)
-{
-    AssertRootMarkingPhase(trc);
-    if (InternalBarrierMethods<T>::isMarkable(*thingp))
-        DispatchToTracer(trc, ConvertToBase(thingp), name);
-}
-
-template <typename T>
-void
-js::TraceNullableRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name)
-{
-    TraceNullableRoot(trc, thingp->unsafeGet(), name);
+    TraceEdgeInternal(trc, ConvertToBase(thingp), name);
 }
 
 template <typename T>
 JS_PUBLIC_API(void)
 JS::UnsafeTraceRoot(JSTracer* trc, T* thingp, const char* name)
 {
     MOZ_ASSERT(thingp);
     js::TraceNullableRoot(trc, thingp, name);
 }
 
-template <typename T>
-void
-js::TraceRange(JSTracer* trc, size_t len, WriteBarrieredBase<T>* vec, const char* name)
-{
-    JS::AutoTracingIndex index(trc);
-    for (auto i : IntegerRange(len)) {
-        if (InternalBarrierMethods<T>::isMarkable(vec[i].get()))
-            DispatchToTracer(trc, ConvertToBase(vec[i].unsafeUnbarrieredForTracing()), name);
-        ++index;
-    }
-}
-
-template <typename T>
-void
-js::TraceRootRange(JSTracer* trc, size_t len, T* vec, const char* name)
-{
-    AssertRootMarkingPhase(trc);
-    JS::AutoTracingIndex index(trc);
-    for (auto i : IntegerRange(len)) {
-        if (InternalBarrierMethods<T>::isMarkable(vec[i]))
-            DispatchToTracer(trc, ConvertToBase(&vec[i]), name);
-        ++index;
-    }
-}
-
-// Instantiate a copy of the Tracing templates for each derived type.
-#define INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS(type) \
-    template void js::TraceEdge<type>(JSTracer*, WriteBarrieredBase<type>*, const char*); \
-    template void js::TraceEdge<type>(JSTracer*, ReadBarriered<type>*, const char*); \
-    template void js::TraceNullableEdge<type>(JSTracer*, WriteBarrieredBase<type>*, const char*); \
-    template void js::TraceNullableEdge<type>(JSTracer*, ReadBarriered<type>*, const char*); \
-    template void js::TraceManuallyBarrieredEdge<type>(JSTracer*, type*, const char*); \
-    template void js::TraceWeakEdge<type>(JSTracer*, WeakRef<type>*, const char*); \
-    template void js::TraceRoot<type>(JSTracer*, type*, const char*); \
-    template void js::TraceRoot<type>(JSTracer*, ReadBarriered<type>*, const char*); \
-    template void js::TraceNullableRoot<type>(JSTracer*, type*, const char*); \
-    template void js::TraceNullableRoot<type>(JSTracer*, ReadBarriered<type>*, const char*); \
-    template void js::TraceRange<type>(JSTracer*, size_t, WriteBarrieredBase<type>*, const char*); \
-    template void js::TraceRootRange<type>(JSTracer*, size_t, type*, const char*);
-FOR_EACH_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS)
-#undef INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS
-
+// Instantiate a copy of the Tracing templates for each public GC pointer type.
 #define INSTANTIATE_PUBLIC_TRACE_FUNCTIONS(type) \
     template JS_PUBLIC_API(void) JS::UnsafeTraceRoot<type>(JSTracer*, type*, const char*); \
     template JS_PUBLIC_API(void) js::UnsafeTraceManuallyBarrieredEdge<type>(JSTracer*, type*, \
                                                                             const char*); \
     template JS_PUBLIC_API(void) js::gc::TraceExternalEdge<type>(JSTracer*, type*, const char*);
 FOR_EACH_PUBLIC_GC_POINTER_TYPE(INSTANTIATE_PUBLIC_TRACE_FUNCTIONS)
 FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(INSTANTIATE_PUBLIC_TRACE_FUNCTIONS)
 #undef INSTANTIATE_PUBLIC_TRACE_FUNCTIONS
 
+namespace js {
+namespace gc {
+
+#define INSTANTIATE_INTERNAL_TRACE_FUNCTIONS(type) \
+    template void TraceEdgeInternal<type>(JSTracer*, type*, const char*); \
+    template void TraceWeakEdgeInternal<type>(JSTracer*, type*, const char*); \
+    template void TraceRangeInternal<type>(JSTracer*, size_t len, type*, const char*);
+
+#define INSTANTIATE_INTERNAL_TRACE_FUNCTIONS_FROM_TRACEKIND(_1, type, _2) \
+    INSTANTIATE_INTERNAL_TRACE_FUNCTIONS(type*)
+
+JS_FOR_EACH_TRACEKIND(INSTANTIATE_INTERNAL_TRACE_FUNCTIONS_FROM_TRACEKIND)
+FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(INSTANTIATE_INTERNAL_TRACE_FUNCTIONS)
+
+#undef INSTANTIATE_INTERNAL_TRACE_FUNCTIONS_FROM_TRACEKIND
+#undef INSTANTIATE_INTERNAL_TRACE_FUNCTIONS
+
+} // namespace gc
+} // namespace js
+
 template <typename T>
 void
 js::TraceManuallyBarrieredCrossCompartmentEdge(JSTracer* trc, JSObject* src, T* dst,
                                                const char* name)
 {
     if (ShouldTraceCrossCompartment(trc, src, *dst))
-        DispatchToTracer(trc, dst, name);
+        TraceEdgeInternal(trc, dst, name);
 }
 template void js::TraceManuallyBarrieredCrossCompartmentEdge<JSObject*>(JSTracer*, JSObject*,
                                                                         JSObject**, const char*);
 template void js::TraceManuallyBarrieredCrossCompartmentEdge<JSScript*>(JSTracer*, JSObject*,
                                                                         JSScript**, const char*);
 
-template <typename T>
 void
-js::TraceCrossCompartmentEdge(JSTracer* trc, JSObject* src, WriteBarrieredBase<T>* dst,
+js::TraceCrossCompartmentEdge(JSTracer* trc, JSObject* src, WriteBarrieredBase<Value>* dst,
                               const char* name)
 {
     if (ShouldTraceCrossCompartment(trc, src, dst->get()))
-        DispatchToTracer(trc, dst->unsafeUnbarrieredForTracing(), name);
+        TraceEdgeInternal(trc, dst->unsafeUnbarrieredForTracing(), name);
 }
-template void js::TraceCrossCompartmentEdge<Value>(JSTracer*, JSObject*,
-                                                   WriteBarrieredBase<Value>*, const char*);
 
 template <typename T>
 void
 js::TraceProcessGlobalRoot(JSTracer* trc, T* thing, const char* name)
 {
     AssertRootMarkingPhase(trc);
     MOZ_ASSERT(ThingIsPermanentAtomOrWellKnownSymbol(thing));
 
@@ -671,17 +523,17 @@ js::TraceManuallyBarrieredGenericPointer
     DispatchTraceKindTyped(f, (*thingp)->getTraceKind(), trc, thingp, name);
 }
 
 // This method is responsible for dynamic dispatch to the real tracer
 // implementation. Consider replacing this choke point with virtual dispatch:
 // a sufficiently smart C++ compiler may be able to devirtualize some paths.
 template <typename T>
 void
-DispatchToTracer(JSTracer* trc, T* thingp, const char* name)
+js::gc::TraceEdgeInternal(JSTracer* trc, T* thingp, const char* name)
 {
 #define IS_SAME_TYPE_OR(name, type, _) mozilla::IsSame<type*, T>::value ||
     static_assert(
             JS_FOR_EACH_TRACEKIND(IS_SAME_TYPE_OR)
             mozilla::IsSame<T, JS::Value>::value ||
             mozilla::IsSame<T, jsid>::value ||
             mozilla::IsSame<T, TaggedProto>::value,
             "Only the base cell layout types are allowed into marking/tracing internals");
@@ -689,16 +541,42 @@ DispatchToTracer(JSTracer* trc, T* thing
     if (trc->isMarkingTracer())
         return DoMarking(GCMarker::fromTracer(trc), *thingp);
     if (trc->isTenuringTracer())
         return static_cast<TenuringTracer*>(trc)->traverse(thingp);
     MOZ_ASSERT(trc->isCallbackTracer());
     DoCallback(trc->asCallbackTracer(), thingp, name);
 }
 
+template <typename T>
+void
+js::gc::TraceWeakEdgeInternal(JSTracer* trc, T* thingp, const char* name)
+{
+    if (!trc->isMarkingTracer()) {
+        // Non-marking tracers can select whether or not they see weak edges.
+        if (trc->traceWeakEdges())
+            TraceEdgeInternal(trc, thingp, name);
+        return;
+    }
+
+    NoteWeakEdge(GCMarker::fromTracer(trc), thingp);
+}
+
+template <typename T>
+void
+js::gc::TraceRangeInternal(JSTracer* trc, size_t len, T* vec, const char* name)
+{
+    JS::AutoTracingIndex index(trc);
+    for (auto i : IntegerRange(len)) {
+        if (InternalBarrierMethods<T>::isMarkable(vec[i]))
+            TraceEdgeInternal(trc, &vec[i], name);
+        ++index;
+    }
+}
+
 
 /*** GC Marking Interface *************************************************************************/
 
 namespace js {
 
 typedef bool HasNoImplicitEdgesType;
 
 template <typename T>
@@ -3356,68 +3234,66 @@ IsMarkedInternalCommon(T* thingp)
         *thingp = Forwarded(*thingp);
         return true;
     }
 
     return thing.isMarkedAny();
 }
 
 template <typename T>
-static bool
-IsMarkedInternal(JSRuntime* rt, T** thingp)
+struct MightBeNurseryAllocated
+{
+    static const bool value = mozilla::IsBaseOf<JSObject, T>::value ||
+                              mozilla::IsBaseOf<JSString, T>::value;
+};
+
+template <typename T>
+bool
+js::gc::IsMarkedInternal(JSRuntime* rt, T** thingp)
 {
     if (IsOwnedByOtherRuntime(rt, *thingp))
         return true;
 
-    return IsMarkedInternalCommon(thingp);
-}
-
-template <>
-/* static */ bool
-IsMarkedInternal(JSRuntime* rt, JSObject** thingp)
-{
-    if (IsOwnedByOtherRuntime(rt, *thingp))
-        return true;
-
-    if (IsInsideNursery(*thingp)) {
+    if (MightBeNurseryAllocated<T>::value && IsInsideNursery(*thingp)) {
         MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
         Cell** cellp = reinterpret_cast<Cell**>(thingp);
         return Nursery::getForwardedPointer(cellp);
     }
+
     return IsMarkedInternalCommon(thingp);
 }
 
 template <typename S>
 struct IsMarkedFunctor : public IdentityDefaultAdaptor<S> {
     template <typename T> S operator()(T* t, JSRuntime* rt, bool* rv) {
         *rv = IsMarkedInternal(rt, &t);
         return js::gc::RewrapTaggedPointer<S, T>::wrap(t);
     }
 };
 
 template <typename T>
-static bool
-IsMarkedInternal(JSRuntime* rt, T* thingp)
+bool
+js::gc::IsMarkedInternal(JSRuntime* rt, T* thingp)
 {
     bool rv = true;
     *thingp = DispatchTyped(IsMarkedFunctor<T>(), *thingp, rt, &rv);
     return rv;
 }
 
 bool
 js::gc::IsAboutToBeFinalizedDuringSweep(TenuredCell& tenured)
 {
     MOZ_ASSERT(!IsInsideNursery(&tenured));
     MOZ_ASSERT(tenured.zoneFromAnyThread()->isGCSweeping());
     return !tenured.isMarkedAny();
 }
 
 template <typename T>
-static bool
-IsAboutToBeFinalizedInternal(T** thingp)
+bool
+js::gc::IsAboutToBeFinalizedInternal(T** thingp)
 {
     CheckIsMarkedThing(thingp);
     T* thing = *thingp;
     JSRuntime* rt = thing->runtimeFromAnyThread();
 
     /* Permanent atoms are never finalized by non-owning runtimes. */
     if (ThingIsPermanentAtomOrWellKnownSymbol(thing) && TlsContext.get()->runtime() != rt)
         return false;
@@ -3442,90 +3318,60 @@ template <typename S>
 struct IsAboutToBeFinalizedInternalFunctor : public IdentityDefaultAdaptor<S> {
     template <typename T> S operator()(T* t, bool* rv) {
         *rv = IsAboutToBeFinalizedInternal(&t);
         return js::gc::RewrapTaggedPointer<S, T>::wrap(t);
     }
 };
 
 template <typename T>
-static bool
-IsAboutToBeFinalizedInternal(T* thingp)
+bool
+js::gc::IsAboutToBeFinalizedInternal(T* thingp)
 {
     bool rv = false;
     *thingp = DispatchTyped(IsAboutToBeFinalizedInternalFunctor<T>(), *thingp, &rv);
     return rv;
 }
 
 namespace js {
 namespace gc {
 
 template <typename T>
-bool
-IsMarkedUnbarriered(JSRuntime* rt, T* thingp)
-{
-    return IsMarkedInternal(rt, ConvertToBase(thingp));
-}
-
-template <typename T>
-bool
-IsMarked(JSRuntime* rt, WriteBarrieredBase<T>* thingp)
-{
-    return IsMarkedInternal(rt, ConvertToBase(thingp->unsafeUnbarrieredForTracing()));
-}
-
-template <typename T>
-bool
-IsAboutToBeFinalizedUnbarriered(T* thingp)
-{
-    return IsAboutToBeFinalizedInternal(ConvertToBase(thingp));
-}
-
-template <typename T>
-bool
-IsAboutToBeFinalized(WriteBarrieredBase<T>* thingp)
-{
-    return IsAboutToBeFinalizedInternal(ConvertToBase(thingp->unsafeUnbarrieredForTracing()));
-}
-
-template <typename T>
-bool
-IsAboutToBeFinalized(ReadBarrieredBase<T>* thingp)
-{
-    return IsAboutToBeFinalizedInternal(ConvertToBase(thingp->unsafeUnbarrieredForTracing()));
-}
-
-template <typename T>
 JS_PUBLIC_API(bool)
 EdgeNeedsSweep(JS::Heap<T>* thingp)
 {
     return IsAboutToBeFinalizedInternal(ConvertToBase(thingp->unsafeGet()));
 }
 
 template <typename T>
 JS_PUBLIC_API(bool)
 EdgeNeedsSweepUnbarrieredSlow(T* thingp)
 {
     return IsAboutToBeFinalizedInternal(ConvertToBase(thingp));
 }
 
-// Instantiate a copy of the Tracing templates for each derived type.
-#define INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS(type) \
-    template bool IsMarkedUnbarriered<type>(JSRuntime*, type*);                \
-    template bool IsMarked<type>(JSRuntime*, WriteBarrieredBase<type>*); \
-    template bool IsAboutToBeFinalizedUnbarriered<type>(type*); \
-    template bool IsAboutToBeFinalized<type>(WriteBarrieredBase<type>*); \
-    template bool IsAboutToBeFinalized<type>(ReadBarrieredBase<type>*);
+// Instantiate a copy of the Tracing templates for each public GC type.
 #define INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS(type) \
     template JS_PUBLIC_API(bool) EdgeNeedsSweep<type>(JS::Heap<type>*); \
     template JS_PUBLIC_API(bool) EdgeNeedsSweepUnbarrieredSlow<type>(type*);
-FOR_EACH_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS)
 FOR_EACH_PUBLIC_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS)
 FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS)
-#undef INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS
+
+#define INSTANTIATE_INTERNAL_MARKING_FUNCTIONS(type) \
+    template bool IsMarkedInternal(JSRuntime* rt, type* thing); \
+    template bool IsAboutToBeFinalizedInternal(type* thingp);
+
+#define INSTANTIATE_INTERNAL_MARKING_FUNCTIONS_FROM_TRACEKIND(_1, type, _2) \
+    INSTANTIATE_INTERNAL_MARKING_FUNCTIONS(type*)
+
+JS_FOR_EACH_TRACEKIND(INSTANTIATE_INTERNAL_MARKING_FUNCTIONS_FROM_TRACEKIND)
+FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(INSTANTIATE_INTERNAL_MARKING_FUNCTIONS)
+
+#undef INSTANTIATE_INTERNAL_MARKING_FUNCTIONS_FROM_TRACEKIND
+#undef INSTANTIATE_INTERNAL_MARKING_FUNCTIONS
 
 } /* namespace gc */
 } /* namespace js */
 
 
 /*** Cycle Collector Barrier Implementation *******************************************************/
 
 /*
--- a/js/src/gc/Marking.h
+++ b/js/src/gc/Marking.h
@@ -50,41 +50,76 @@ class TenuredCell;
 
 /*** Special Cases ***/
 
 void
 PushArena(GCMarker* gcmarker, Arena* arena);
 
 /*** Liveness ***/
 
-// Report whether a thing has been marked.  Things which are in zones that are
-// not currently being collected or are owned by another runtime are always
-// reported as being marked.
+// The IsMarkedInternal and IsAboutToBeFinalizedInternal function templates are
+// used to implement the IsMarked and IsAboutToBeFinalized set of functions.
+// These internal functions are instantiated for the base GC types and should
+// not be called directly.
+//
+// Note that there are two function templates declared for each, not one
+// template and a specialization. This is necessary so that pointer arguments
+// (e.g. JSObject**) and tagged value arguments (e.g. JS::Value*) are routed to
+// separate implementations.
+
 template <typename T>
-bool
-IsMarkedUnbarriered(JSRuntime* rt, T* thingp);
+bool IsMarkedInternal(JSRuntime* rt, T* thing);
+template <typename T>
+bool IsMarkedInternal(JSRuntime* rt, T** thing);
+
+template <typename T>
+bool IsAboutToBeFinalizedInternal(T* thingp);
+template <typename T>
+bool IsAboutToBeFinalizedInternal(T** thingp);
 
 // Report whether a thing has been marked.  Things which are in zones that are
 // not currently being collected or are owned by another runtime are always
 // reported as being marked.
 template <typename T>
-bool
-IsMarked(JSRuntime* rt, WriteBarrieredBase<T>* thingp);
+inline bool
+IsMarkedUnbarriered(JSRuntime* rt, T* thingp)
+{
+    return IsMarkedInternal(rt, ConvertToBase(thingp));
+}
+
+// Report whether a thing has been marked.  Things which are in zones that are
+// not currently being collected or are owned by another runtime are always
+// reported as being marked.
+template <typename T>
+inline bool
+IsMarked(JSRuntime* rt, WriteBarrieredBase<T>* thingp)
+{
+    return IsMarkedInternal(rt, ConvertToBase(thingp->unsafeUnbarrieredForTracing()));
+}
 
 template <typename T>
-bool
-IsAboutToBeFinalizedUnbarriered(T* thingp);
+inline bool
+IsAboutToBeFinalizedUnbarriered(T* thingp)
+{
+    return IsAboutToBeFinalizedInternal(ConvertToBase(thingp));
+}
 
 template <typename T>
-bool
-IsAboutToBeFinalized(WriteBarrieredBase<T>* thingp);
+inline bool
+IsAboutToBeFinalized(WriteBarrieredBase<T>* thingp)
+{
+    return IsAboutToBeFinalizedInternal(ConvertToBase(thingp->unsafeUnbarrieredForTracing()));
+}
 
 template <typename T>
-bool
-IsAboutToBeFinalized(ReadBarrieredBase<T>* thingp);
+inline bool
+IsAboutToBeFinalized(ReadBarrieredBase<T>* thingp)
+{
+    return IsAboutToBeFinalizedInternal(ConvertToBase(thingp->unsafeUnbarrieredForTracing()));
+}
 
 bool
 IsAboutToBeFinalizedDuringSweep(TenuredCell& tenured);
 
 inline Cell*
 ToMarkable(const Value& v)
 {
     if (v.isGCThing())
--- a/js/src/gc/Policy.h
+++ b/js/src/gc/Policy.h
@@ -9,117 +9,16 @@
 #ifndef gc_Policy_h
 #define gc_Policy_h
 
 #include "mozilla/TypeTraits.h"
 #include "gc/Barrier.h"
 #include "gc/Marking.h"
 #include "js/GCPolicyAPI.h"
 
-// Forward declare the types we're defining policies for. This file is
-// included in all places that define GC things, so the real definitions
-// will be available when we do template expansion, allowing for use of
-// static members in the underlying types. We cannot, however, use
-// static_assert to verify relations between types.
-class JSLinearString;
-namespace js {
-class AccessorShape;
-class ArgumentsObject;
-class ArrayBufferObject;
-class ArrayBufferObjectMaybeShared;
-class ArrayBufferViewObject;
-class ArrayObject;
-class BaseShape;
-class DebugEnvironmentProxy;
-class DebuggerFrame;
-class ExportEntryObject;
-class EnvironmentObject;
-class GlobalObject;
-class ImportEntryObject;
-class LazyScript;
-class LexicalEnvironmentObject;
-class ModuleEnvironmentObject;
-class ModuleNamespaceObject;
-class ModuleObject;
-class NativeObject;
-class ObjectGroup;
-class PlainObject;
-class PropertyName;
-class RegExpObject;
-class SavedFrame;
-class Scope;
-class EnvironmentObject;
-class RequestedModuleObject;
-class ScriptSourceObject;
-class Shape;
-class SharedArrayBufferObject;
-class StructTypeDescr;
-class UnownedBaseShape;
-class WasmGlobalObject;
-class WasmFunctionScope;
-class WasmMemoryObject;
-namespace jit {
-class JitCode;
-} // namespace jit
-} // namespace js
-
-// Expand the given macro D for each valid GC reference type.
-#define FOR_EACH_INTERNAL_GC_POINTER_TYPE(D) \
-    D(JSFlatString*) \
-    D(JSLinearString*) \
-    D(js::AccessorShape*) \
-    D(js::ArgumentsObject*) \
-    D(js::ArrayBufferObject*) \
-    D(js::ArrayBufferObjectMaybeShared*) \
-    D(js::ArrayBufferViewObject*) \
-    D(js::ArrayObject*) \
-    D(js::BaseShape*) \
-    D(js::DebugEnvironmentProxy*) \
-    D(js::DebuggerFrame*) \
-    D(js::ExportEntryObject*) \
-    D(js::EnvironmentObject*) \
-    D(js::GlobalObject*) \
-    D(js::ImportEntryObject*) \
-    D(js::LazyScript*) \
-    D(js::LexicalEnvironmentObject*) \
-    D(js::ModuleEnvironmentObject*) \
-    D(js::ModuleNamespaceObject*) \
-    D(js::ModuleObject*) \
-    D(js::NativeObject*) \
-    D(js::ObjectGroup*) \
-    D(js::PlainObject*) \
-    D(js::PropertyName*) \
-    D(js::RegExpObject*) \
-    D(js::RegExpShared*) \
-    D(js::RequestedModuleObject*) \
-    D(js::SavedFrame*) \
-    D(js::Scope*) \
-    D(js::ScriptSourceObject*) \
-    D(js::Shape*) \
-    D(js::SharedArrayBufferObject*) \
-    D(js::StructTypeDescr*) \
-    D(js::UnownedBaseShape*) \
-    D(js::WasmFunctionScope*) \
-    D(js::WasmInstanceObject*) \
-    D(js::WasmMemoryObject*) \
-    D(js::WasmTableObject*) \
-    D(js::WasmGlobalObject*) \
-    D(js::jit::JitCode*)
-
-// Expand the given macro D for each internal tagged GC pointer type.
-#define FOR_EACH_INTERNAL_TAGGED_GC_POINTER_TYPE(D) \
-    D(js::TaggedProto)
-
-// Expand the macro D for every GC reference type that we know about.
-#define FOR_EACH_GC_POINTER_TYPE(D) \
-    FOR_EACH_PUBLIC_GC_POINTER_TYPE(D) \
-    FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(D) \
-    FOR_EACH_INTERNAL_GC_POINTER_TYPE(D) \
-    FOR_EACH_INTERNAL_TAGGED_GC_POINTER_TYPE(D)
-
 namespace js {
 
 // Define the GCPolicy for all internal pointers.
 template <typename T>
 struct InternalGCPointerPolicy : public JS::GCPointerPolicy<T> {
     using Type = typename mozilla::RemovePointer<T>::Type;
 
 #define IS_BASE_OF_OR(_1, BaseType, _2) mozilla::IsBaseOf<BaseType, Type>::value ||
--- a/js/src/gc/RootMarking.cpp
+++ b/js/src/gc/RootMarking.cpp
@@ -43,74 +43,86 @@ using TraceFunction = void (*)(JSTracer*
 // the type field. We use the following type as a compatible stand-in. No
 // actual methods from ConcreteTraceable type are actually used at runtime --
 // the real trace function has been stored inline in the DispatchWrapper.
 struct ConcreteTraceable {
     ConcreteTraceable() { MOZ_CRASH("instantiation of ConcreteTraceable"); }
     void trace(JSTracer*) {}
 };
 
-template <typename T, TraceFunction<T> TraceFn = TraceNullableRoot>
+template <typename T>
+static inline void
+TraceStackOrPersistentRoot(JSTracer* trc, T* thingp, const char* name)
+{
+    TraceNullableRoot(trc, thingp, name);
+}
+
+template <>
+inline void
+TraceStackOrPersistentRoot(JSTracer* trc, ConcreteTraceable* thingp, const char* name)
+{
+    js::DispatchWrapper<ConcreteTraceable>::TraceWrapped(trc, thingp, name);
+}
+
+template <typename T>
 static inline void
 TraceExactStackRootList(JSTracer* trc, JS::Rooted<void*>* rooter, const char* name)
 {
     while (rooter) {
         T* addr = reinterpret_cast<JS::Rooted<T>*>(rooter)->address();
-        TraceFn(trc, addr, name);
+        TraceStackOrPersistentRoot(trc, addr, name);
         rooter = rooter->previous();
     }
 }
 
 static inline void
 TraceStackRoots(JSTracer* trc, JS::RootedListHeads& stackRoots)
 {
 #define TRACE_ROOTS(name, type, _) \
     TraceExactStackRootList<type*>(trc, stackRoots[JS::RootKind::name], "exact-" #name);
 JS_FOR_EACH_TRACEKIND(TRACE_ROOTS)
 #undef TRACE_ROOTS
     TraceExactStackRootList<jsid>(trc, stackRoots[JS::RootKind::Id], "exact-id");
     TraceExactStackRootList<Value>(trc, stackRoots[JS::RootKind::Value], "exact-value");
-    TraceExactStackRootList<ConcreteTraceable,
-                           js::DispatchWrapper<ConcreteTraceable>::TraceWrapped>(
+    TraceExactStackRootList<ConcreteTraceable>(
         trc, stackRoots[JS::RootKind::Traceable], "Traceable");
 }
 
 void
 JS::RootingContext::traceStackRoots(JSTracer* trc)
 {
     TraceStackRoots(trc, stackRoots_);
 }
 
 static void
 TraceExactStackRoots(JSContext* cx, JSTracer* trc)
 {
     cx->traceStackRoots(trc);
 }
 
-template <typename T, TraceFunction<T> TraceFn = TraceNullableRoot>
+template <typename T>
 static inline void
 TracePersistentRootedList(JSTracer* trc, mozilla::LinkedList<PersistentRooted<void*>>& list,
                          const char* name)
 {
     for (PersistentRooted<void*>* r : list)
-        TraceFn(trc, reinterpret_cast<PersistentRooted<T>*>(r)->address(), name);
+        TraceStackOrPersistentRoot(trc, reinterpret_cast<PersistentRooted<T>*>(r)->address(), name);
 }
 
 void
 JSRuntime::tracePersistentRoots(JSTracer* trc)
 {
 #define TRACE_ROOTS(name, type, _) \
     TracePersistentRootedList<type*>(trc, heapRoots.ref()[JS::RootKind::name], "persistent-" #name);
 JS_FOR_EACH_TRACEKIND(TRACE_ROOTS)
 #undef TRACE_ROOTS
     TracePersistentRootedList<jsid>(trc, heapRoots.ref()[JS::RootKind::Id], "persistent-id");
     TracePersistentRootedList<Value>(trc, heapRoots.ref()[JS::RootKind::Value], "persistent-value");
-    TracePersistentRootedList<ConcreteTraceable,
-                             js::DispatchWrapper<ConcreteTraceable>::TraceWrapped>(trc,
-            heapRoots.ref()[JS::RootKind::Traceable], "persistent-traceable");
+    TracePersistentRootedList<ConcreteTraceable>(
+        trc, heapRoots.ref()[JS::RootKind::Traceable], "persistent-traceable");
 }
 
 static void
 TracePersistentRooted(JSRuntime* rt, JSTracer* trc)
 {
     rt->tracePersistentRoots(trc);
 }
 
--- a/js/src/gc/Tracer.h
+++ b/js/src/gc/Tracer.h
@@ -44,86 +44,185 @@ namespace js {
 // as part of the generic "tracing" architecture, rather than the more specific
 // marking implementation of tracing.
 //
 // 1 - In SpiderMonkey, we call this concept tracing rather than visiting
 //     because "visiting" is already used by the compiler. Also, it's been
 //     called "tracing" forever and changing it would be extremely difficult at
 //     this point.
 
+namespace gc {
+
+// Map from all trace kinds to the base GC type.
+template <JS::TraceKind kind>
+struct MapTraceKindToType {};
+
+#define DEFINE_TRACE_KIND_MAP(name, type, _) \
+    template <> struct MapTraceKindToType<JS::TraceKind::name> { \
+        using Type = type; \
+    };
+JS_FOR_EACH_TRACEKIND(DEFINE_TRACE_KIND_MAP);
+#undef DEFINE_TRACE_KIND_MAP
+
+// Map from a possibly-derived type to the base GC type.
+template <typename T>
+struct BaseGCType
+{
+    using type = typename MapTraceKindToType<JS::MapTypeToTraceKind<T>::kind>::Type;
+    static_assert(mozilla::IsBaseOf<type, T>::value, "Failed to find base type");
+};
+
+// Our barrier templates are parameterized on the pointer types so that we can
+// share the definitions with Value and jsid. Thus, we need to strip the
+// pointer before sending the type to BaseGCType and re-add it on the other
+// side. As such:
+template <typename T> struct PtrBaseGCType { using type = T; };
+template <typename T> struct PtrBaseGCType<T*> { using type = typename BaseGCType<T>::type* ; };
+
+// Cast a possibly-derived T** pointer to a base class pointer.
+template <typename T>
+typename PtrBaseGCType<T>::type*
+ConvertToBase(T* thingp)
+{
+    return reinterpret_cast<typename PtrBaseGCType<T>::type*>(thingp);
+}
+
+// Internal methods to trace edges.
+template <typename T> void TraceEdgeInternal(JSTracer* trc, T* thingp, const char* name);
+template <typename T> void TraceWeakEdgeInternal(JSTracer* trc, T* thingp, const char* name);
+template <typename T> void TraceRangeInternal(JSTracer* trc, size_t len, T* vec, const char* name);
+
+#ifdef DEBUG
+void AssertRootMarkingPhase(JSTracer* trc);
+#else
+inline void AssertRootMarkingPhase(JSTracer* trc) {}
+#endif
+
+} // namespace gc
+
 // Trace through an edge in the live object graph on behalf of tracing. The
 // effect of tracing the edge depends on the JSTracer being used. For pointer
 // types, |*thingp| must not be null.
+
 template <typename T>
-void
-TraceEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name);
+inline void
+TraceEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name)
+{
+    gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp->unsafeUnbarrieredForTracing()), name);
+}
 
 template <typename T>
-void
-TraceEdge(JSTracer* trc, ReadBarriered<T>* thingp, const char* name);
+inline void
+TraceEdge(JSTracer* trc, ReadBarriered<T>* thingp, const char* name)
+{
+    gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp->unsafeGet()), name);
+}
 
-// Trace through an edge in the live object graph on behalf of tracing.
-template <typename T>
-void
-TraceNullableEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name);
+// Trace through a possibly-null edge in the live object graph on behalf of
+// tracing.
 
 template <typename T>
-void
-TraceNullableEdge(JSTracer* trc, ReadBarriered<T>* thingp, const char* name);
+inline void
+TraceNullableEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name)
+{
+    if (InternalBarrierMethods<T>::isMarkable(thingp->get()))
+        TraceEdge(trc, thingp, name);
+}
+
+template <typename T>
+inline void
+TraceNullableEdge(JSTracer* trc, ReadBarriered<T>* thingp, const char* name)
+{
+    if (InternalBarrierMethods<T>::isMarkable(thingp->unbarrieredGet()))
+        TraceEdge(trc, thingp, name);
+}
 
 // Trace through a "root" edge. These edges are the initial edges in the object
 // graph traversal. Root edges are asserted to only be traversed in the initial
 // phase of a GC.
-template <typename T>
-void
-TraceRoot(JSTracer* trc, T* thingp, const char* name);
 
 template <typename T>
-void
-TraceRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name);
+inline void
+TraceRoot(JSTracer* trc, T* thingp, const char* name)
+{
+    gc::AssertRootMarkingPhase(trc);
+    gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name);
+}
+
+template <typename T>
+inline void
+TraceRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name)
+{
+    TraceRoot(trc, thingp->unsafeGet(), name);
+}
 
 // Idential to TraceRoot, except that this variant will not crash if |*thingp|
 // is null.
-template <typename T>
-void
-TraceNullableRoot(JSTracer* trc, T* thingp, const char* name);
 
 template <typename T>
-void
-TraceNullableRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name);
+inline void
+TraceNullableRoot(JSTracer* trc, T* thingp, const char* name)
+{
+    gc::AssertRootMarkingPhase(trc);
+    if (InternalBarrierMethods<T>::isMarkable(*thingp))
+        gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name);
+}
+
+template <typename T>
+inline void
+TraceNullableRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name)
+{
+    TraceNullableRoot(trc, thingp->unsafeGet(), name);
+}
 
 // Like TraceEdge, but for edges that do not use one of the automatic barrier
 // classes and, thus, must be treated specially for moving GC. This method is
 // separate from TraceEdge to make accidental use of such edges more obvious.
+
 template <typename T>
-void
-TraceManuallyBarrieredEdge(JSTracer* trc, T* thingp, const char* name);
+inline void
+TraceManuallyBarrieredEdge(JSTracer* trc, T* thingp, const char* name)
+{
+    gc::TraceEdgeInternal(trc, gc::ConvertToBase(thingp), name);
+}
 
 // Visits a WeakRef, but does not trace its referents. If *thingp is not marked
 // at the end of marking, it is replaced by nullptr. This method records
 // thingp, so the edge location must not change after this function is called.
+
 template <typename T>
-void
-TraceWeakEdge(JSTracer* trc, WeakRef<T>* thingp, const char* name);
+inline void
+TraceWeakEdge(JSTracer* trc, WeakRef<T>* thingp, const char* name)
+{
+    gc::TraceWeakEdgeInternal(trc, gc::ConvertToBase(thingp->unsafeUnbarrieredForTracing()), name);
+}
 
 // Trace all edges contained in the given array.
+
 template <typename T>
 void
-TraceRange(JSTracer* trc, size_t len, WriteBarrieredBase<T>* vec, const char* name);
+TraceRange(JSTracer* trc, size_t len, WriteBarrieredBase<T>* vec, const char* name)
+{
+    gc::TraceRangeInternal(trc, len, gc::ConvertToBase(vec[0].unsafeUnbarrieredForTracing()), name);
+}
 
 // Trace all root edges in the given array.
+
 template <typename T>
 void
-TraceRootRange(JSTracer* trc, size_t len, T* vec, const char* name);
+TraceRootRange(JSTracer* trc, size_t len, T* vec, const char* name)
+{
+    gc::AssertRootMarkingPhase(trc);
+    gc::TraceRangeInternal(trc, len, gc::ConvertToBase(vec), name);
+}
 
 // Trace an edge that crosses compartment boundaries. If the compartment of the
 // destination thing is not being GC'd, then the edge will not be traced.
-template <typename T>
 void
-TraceCrossCompartmentEdge(JSTracer* trc, JSObject* src, WriteBarrieredBase<T>* dst,
+TraceCrossCompartmentEdge(JSTracer* trc, JSObject* src, WriteBarrieredBase<Value>* dst,
                           const char* name);
 
 // As above but with manual barriers.
 template <typename T>
 void
 TraceManuallyBarrieredCrossCompartmentEdge(JSTracer* trc, JSObject* src, T* dst,
                                            const char* name);