Bug 1359342 - Add delayed read barriers for JitCompartment stubs r=nbp
authorJon Coppeard <jcoppeard@mozilla.com>
Tue, 27 Feb 2018 12:14:46 +0000
changeset 405528 c3d6247ece759ea353bf49815b3688406e6e37bf
parent 405527 07cab6799ff4886551c16489bf17ca2578893738
child 405529 87e6042a409d0cd33d5fbf7a2abda73c046e796a
push id100245
push userjcoppeard@mozilla.com
push dateTue, 27 Feb 2018 12:20:07 +0000
treeherdermozilla-inbound@f361cfca3755 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnbp
bugs1359342
milestone60.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1359342 - Add delayed read barriers for JitCompartment stubs r=nbp
js/src/jit/CodeGenerator.cpp
js/src/jit/CodeGenerator.h
js/src/jit/Ion.cpp
js/src/jit/JitCompartment.h
--- a/js/src/jit/CodeGenerator.cpp
+++ b/js/src/jit/CodeGenerator.cpp
@@ -372,17 +372,18 @@ MNewStringObject::templateObj() const
 {
     return &templateObj_->as<StringObject>();
 }
 
 CodeGenerator::CodeGenerator(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm)
   : CodeGeneratorSpecific(gen, graph, masm)
   , ionScriptLabels_(gen->alloc())
   , scriptCounts_(nullptr)
-  , simdRefreshTemplatesDuringLink_(0)
+  , simdTemplatesToReadBarrier_(0)
+  , compartmentStubsToReadBarrier_(0)
 {
 }
 
 CodeGenerator::~CodeGenerator()
 {
     js_delete(scriptCounts_);
 }
 
@@ -2186,17 +2187,18 @@ CodeGenerator::visitRegExpMatcher(LRegEx
     MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg);
 #endif
 
     masm.reserveStack(RegExpReservedStack);
 
     OutOfLineRegExpMatcher* ool = new(alloc()) OutOfLineRegExpMatcher(lir);
     addOutOfLineCode(ool, lir->mir());
 
-    JitCode* regExpMatcherStub = gen->compartment->jitCompartment()->regExpMatcherStubNoBarrier();
+    const JitCompartment* jitCompartment = gen->compartment->jitCompartment();
+    JitCode* regExpMatcherStub = jitCompartment->regExpMatcherStubNoBarrier(&compartmentStubsToReadBarrier_);
     masm.call(regExpMatcherStub);
     masm.branchTestUndefined(Assembler::Equal, JSReturnOperand, ool->entry());
     masm.bind(ool->rejoin());
 
     masm.freeStack(RegExpReservedStack);
 }
 
 static const int32_t RegExpSearcherResultNotFound = -1;
@@ -2334,17 +2336,18 @@ CodeGenerator::visitRegExpSearcher(LRegE
     MOZ_ASSERT(RegExpTesterStringReg != ReturnReg);
     MOZ_ASSERT(RegExpTesterLastIndexReg != ReturnReg);
 
     masm.reserveStack(RegExpReservedStack);
 
     OutOfLineRegExpSearcher* ool = new(alloc()) OutOfLineRegExpSearcher(lir);
     addOutOfLineCode(ool, lir->mir());
 
-    JitCode* regExpSearcherStub = gen->compartment->jitCompartment()->regExpSearcherStubNoBarrier();
+    const JitCompartment* jitCompartment = gen->compartment->jitCompartment();
+    JitCode* regExpSearcherStub = jitCompartment->regExpSearcherStubNoBarrier(&compartmentStubsToReadBarrier_);
     masm.call(regExpSearcherStub);
     masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpSearcherResultFailed), ool->entry());
     masm.bind(ool->rejoin());
 
     masm.freeStack(RegExpReservedStack);
 }
 
 static const int32_t RegExpTesterResultNotFound = -1;
@@ -2469,17 +2472,18 @@ CodeGenerator::visitRegExpTester(LRegExp
 
     MOZ_ASSERT(RegExpTesterRegExpReg != ReturnReg);
     MOZ_ASSERT(RegExpTesterStringReg != ReturnReg);
     MOZ_ASSERT(RegExpTesterLastIndexReg != ReturnReg);
 
     OutOfLineRegExpTester* ool = new(alloc()) OutOfLineRegExpTester(lir);
     addOutOfLineCode(ool, lir->mir());
 
-    JitCode* regExpTesterStub = gen->compartment->jitCompartment()->regExpTesterStubNoBarrier();
+    const JitCompartment* jitCompartment = gen->compartment->jitCompartment();
+    JitCode* regExpTesterStub = jitCompartment->regExpTesterStubNoBarrier(&compartmentStubsToReadBarrier_);
     masm.call(regExpTesterStub);
 
     masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpTesterResultFailed), ool->entry());
     masm.bind(ool->rejoin());
 }
 
 class OutOfLineRegExpPrototypeOptimizable : public OutOfLineCodeBase<CodeGenerator>
 {
@@ -6321,17 +6325,17 @@ CodeGenerator::visitSimdBox(LSimdBox* li
 {
     FloatRegister in = ToFloatRegister(lir->input());
     Register object = ToRegister(lir->output());
     Register temp = ToRegister(lir->temp());
     InlineTypedObject* templateObject = lir->mir()->templateObject();
     gc::InitialHeap initialHeap = lir->mir()->initialHeap();
     MIRType type = lir->mir()->input()->type();
 
-    registerSimdTemplate(lir->mir()->simdType());
+    addSimdTemplateToReadBarrier(lir->mir()->simdType());
 
     MOZ_ASSERT(lir->safepoint()->liveRegs().has(in), "Save the input register across oolCallVM");
     OutOfLineCode* ool = oolCallVM(NewTypedObjectInfo, lir,
                                    ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)),
                                    StoreRegisterTo(object));
 
     masm.createGCObject(object, temp, templateObject, initialHeap, ool->entry());
     masm.bind(ool->rejoin());
@@ -6350,35 +6354,19 @@ CodeGenerator::visitSimdBox(LSimdBox* li
         masm.storeUnalignedSimd128Float(in, objectData);
         break;
       default:
         MOZ_CRASH("Unknown SIMD kind when generating code for SimdBox.");
     }
 }
 
 void
-CodeGenerator::registerSimdTemplate(SimdType simdType)
-{
-    simdRefreshTemplatesDuringLink_ |= 1 << uint32_t(simdType);
-}
-
-void
-CodeGenerator::captureSimdTemplate(JSContext* cx)
-{
-    JitCompartment* jitCompartment = cx->compartment()->jitCompartment();
-    while (simdRefreshTemplatesDuringLink_) {
-        uint32_t typeIndex = mozilla::CountTrailingZeroes32(simdRefreshTemplatesDuringLink_);
-        simdRefreshTemplatesDuringLink_ ^= 1 << typeIndex;
-        SimdType type = SimdType(typeIndex);
-
-        // Note: the weak-reference on the template object should not have been
-        // garbage collected. It is either registered by IonBuilder, or verified
-        // before using it in the EagerSimdUnbox phase.
-        jitCompartment->registerSimdTemplateObjectFor(type);
-    }
+CodeGenerator::addSimdTemplateToReadBarrier(SimdType simdType)
+{
+    simdTemplatesToReadBarrier_ |= 1 << uint32_t(simdType);
 }
 
 void
 CodeGenerator::visitSimdUnbox(LSimdUnbox* lir)
 {
     Register object = ToRegister(lir->input());
     FloatRegister simd = ToFloatRegister(lir->output());
     Register temp = ToRegister(lir->temp());
@@ -7928,17 +7916,18 @@ static const VMFunction ConcatStringsInf
     FunctionInfo<ConcatStringsFn>(ConcatStrings<CanGC>, "ConcatStrings");
 
 void
 CodeGenerator::emitConcat(LInstruction* lir, Register lhs, Register rhs, Register output)
 {
     OutOfLineCode* ool = oolCallVM(ConcatStringsInfo, lir, ArgList(lhs, rhs),
                                    StoreRegisterTo(output));
 
-    JitCode* stringConcatStub = gen->compartment->jitCompartment()->stringConcatStubNoBarrier();
+    const JitCompartment* jitCompartment = gen->compartment->jitCompartment();
+    JitCode* stringConcatStub = jitCompartment->stringConcatStubNoBarrier(&compartmentStubsToReadBarrier_);
     masm.call(stringConcatStub);
     masm.branchTestPtr(Assembler::Zero, output, output, ool->entry());
 
     masm.bind(ool->rejoin());
 }
 
 void
 CodeGenerator::visitConcat(LConcat* lir)
@@ -10045,21 +10034,21 @@ CodeGenerator::link(JSContext* cx, Compi
     // We cancel off-thread Ion compilations in a few places during GC, but if
     // this compilation was performed off-thread it will already have been
     // removed from the relevant lists by this point. Don't allow GC here.
     JS::AutoAssertNoGC nogc(cx);
 
     RootedScript script(cx, gen->info().script());
     OptimizationLevel optimizationLevel = gen->optimizationInfo().level();
 
-    // Capture the SIMD template objects which are used during the
-    // compilation. This iterates over the template objects, using read-barriers
-    // to let the GC know that the generated code relies on these template
-    // objects.
-    captureSimdTemplate(cx);
+    // Perform any read barriers which were skipped while compiling the
+    // script, which may have happened off-thread.
+    const JitCompartment* jc = gen->compartment->jitCompartment();
+    jc->performStubReadBarriers(compartmentStubsToReadBarrier_);
+    jc->performSIMDTemplateReadBarriers(simdTemplatesToReadBarrier_);
 
     // We finished the new IonScript. Invalidate the current active IonScript,
     // so we can replace it with this new (probably higher optimized) version.
     if (script->hasIonScript()) {
         MOZ_ASSERT(script->ionScript()->isRecompiling());
         // Do a normal invalidate, except don't cancel offThread compilations,
         // since that will cancel this compilation too.
         Invalidate(cx, script, /* resetUses */ false, /* cancelOffThread*/ false);
--- a/js/src/jit/CodeGenerator.h
+++ b/js/src/jit/CodeGenerator.h
@@ -611,18 +611,20 @@ class CodeGenerator final : public CodeG
     //
     // As the template objects are weak references, the JitCompartment is using
     // Read Barriers, but such barrier cannot be used during the compilation. To
     // work around this issue, the barriers are captured during
     // CodeGenerator::link.
     //
     // Instead of saving the pointers, we just save the index of the Read
     // Barriered objects in a bit mask.
-    uint32_t simdRefreshTemplatesDuringLink_;
+    uint32_t simdTemplatesToReadBarrier_;
 
-    void registerSimdTemplate(SimdType simdType);
-    void captureSimdTemplate(JSContext* cx);
+    // Bit mask of JitCompartment stubs that are to be read-barriered.
+    uint32_t compartmentStubsToReadBarrier_;
+
+    void addSimdTemplateToReadBarrier(SimdType simdType);
 };
 
 } // namespace jit
 } // namespace js
 
 #endif /* jit_CodeGenerator_h */
--- a/js/src/jit/Ion.cpp
+++ b/js/src/jit/Ion.cpp
@@ -424,21 +424,17 @@ JitZoneGroup::patchIonBackedges(JSContex
 }
 
 JitZoneGroup::JitZoneGroup(ZoneGroup* group)
   : backedgeTarget_(group, BackedgeLoopHeader),
     backedgeList_(group)
 {}
 
 JitCompartment::JitCompartment()
-  : stubCodes_(nullptr),
-    stringConcatStub_(nullptr),
-    regExpMatcherStub_(nullptr),
-    regExpSearcherStub_(nullptr),
-    regExpTesterStub_(nullptr)
+  : stubCodes_(nullptr)
 {
 }
 
 JitCompartment::~JitCompartment()
 {
     js_delete(stubCodes_);
 }
 
@@ -454,26 +450,48 @@ JitCompartment::initialize(JSContext* cx
         return false;
     }
 
     stringsCanBeInNursery = cx->nursery().canAllocateStrings();
 
     return true;
 }
 
-bool
-JitCompartment::ensureIonStubsExist(JSContext* cx)
+template <typename T>
+static T
+PopNextBitmaskValue(uint32_t* bitmask)
+{
+    MOZ_ASSERT(*bitmask);
+    uint32_t index = mozilla::CountTrailingZeroes32(*bitmask);
+    *bitmask ^= 1 << index;
+
+    MOZ_ASSERT(index < uint32_t(T::Count));
+    return T(index);
+}
+
+void
+JitCompartment::performStubReadBarriers(uint32_t stubsToBarrier) const
 {
-    if (!stringConcatStub_) {
-        stringConcatStub_ = generateStringConcatStub(cx);
-        if (!stringConcatStub_)
-            return false;
+    while (stubsToBarrier) {
+        auto stub = PopNextBitmaskValue<StubIndex>(&stubsToBarrier);
+        const ReadBarrieredJitCode& jitCode = stubs_[stub];
+        MOZ_ASSERT(jitCode);
+        jitCode.get();
     }
-
-    return true;
+}
+
+void
+JitCompartment::performSIMDTemplateReadBarriers(uint32_t simdTemplatesToBarrier) const
+{
+    while (simdTemplatesToBarrier) {
+        auto type = PopNextBitmaskValue<SimdType>(&simdTemplatesToBarrier);
+        const ReadBarrieredObject& tpl = simdTemplateObjects_[type];
+        MOZ_ASSERT(tpl);
+        tpl.get();
+    }
 }
 
 bool
 JitZone::init(JSContext* cx)
 {
     if (!baselineCacheIRStubCodes_.init()) {
         ReportOutOfMemory(cx);
         return false;
@@ -662,27 +680,20 @@ JitCompartment::sweep(FreeOp* fop, JSCom
     stubCodes_->sweep();
 
     // If the sweep removed a bailout Fallback stub, nullptr the corresponding return addr.
     for (auto& it : bailoutReturnStubInfo_) {
         if (!stubCodes_->lookup(it.key))
            it = BailoutReturnStubInfo();
     }
 
-    if (stringConcatStub_ && IsAboutToBeFinalizedUnbarriered(&stringConcatStub_))
-        stringConcatStub_ = nullptr;
-
-    if (regExpMatcherStub_ && IsAboutToBeFinalizedUnbarriered(&regExpMatcherStub_))
-        regExpMatcherStub_ = nullptr;
-
-    if (regExpSearcherStub_ && IsAboutToBeFinalizedUnbarriered(&regExpSearcherStub_))
-        regExpSearcherStub_ = nullptr;
-
-    if (regExpTesterStub_ && IsAboutToBeFinalizedUnbarriered(&regExpTesterStub_))
-        regExpTesterStub_ = nullptr;
+    for (ReadBarrieredJitCode& stub : stubs_) {
+        if (stub && IsAboutToBeFinalized(&stub))
+            stub.set(nullptr);
+    }
 
     for (ReadBarrieredObject& obj : simdTemplateObjects_) {
         if (obj && IsAboutToBeFinalized(&obj))
             obj.set(nullptr);
     }
 }
 
 void
--- a/js/src/jit/JitCompartment.h
+++ b/js/src/jit/JitCompartment.h
@@ -525,57 +525,67 @@ class JitCompartment
 
         BailoutReturnStubInfo() : addr(nullptr), key(0) { }
         BailoutReturnStubInfo(void* addr_, uint32_t key_) : addr(addr_), key(key_) { }
     };
     mozilla::EnumeratedArray<BailoutReturnStub,
                              BailoutReturnStub::Count,
                              BailoutReturnStubInfo> bailoutReturnStubInfo_;
 
-    // Stubs to concatenate two strings inline, or perform RegExp calls inline.
-    // These bake in zone and compartment specific pointers and can't be stored
-    // in JitRuntime. These are weak pointers, but are not declared as
-    // ReadBarriered since they are only read from during Ion compilation,
-    // which may occur off thread and whose barriers are captured during
-    // CodeGenerator::link.
-    JitCode* stringConcatStub_;
-    JitCode* regExpMatcherStub_;
-    JitCode* regExpSearcherStub_;
-    JitCode* regExpTesterStub_;
+    // The JitCompartment stores stubs to concatenate strings inline and perform
+    // RegExp calls inline.  These bake in zone and compartment specific
+    // pointers and can't be stored in JitRuntime.
+    //
+    // These are weak pointers, but they can by accessed during off-thread Ion
+    // compilation and therefore can't use the usual read barrier. Instead, we
+    // record which stubs have been read and perform the appropriate barriers in
+    // CodeGenerator::link().
+
+    enum StubIndex : uint32_t
+    {
+        StringConcat = 0,
+        RegExpMatcher,
+        RegExpSearcher,
+        RegExpTester,
+        Count
+    };
+
+    mozilla::EnumeratedArray<StubIndex, StubIndex::Count, ReadBarrieredJitCode> stubs_;
+
+    // The same approach is taken for SIMD template objects.
 
     mozilla::EnumeratedArray<SimdType, SimdType::Count, ReadBarrieredObject> simdTemplateObjects_;
 
     JitCode* generateStringConcatStub(JSContext* cx);
     JitCode* generateRegExpMatcherStub(JSContext* cx);
     JitCode* generateRegExpSearcherStub(JSContext* cx);
     JitCode* generateRegExpTesterStub(JSContext* cx);
 
+    JitCode* getStubNoBarrier(StubIndex stub, uint32_t* requiredBarriersOut) const {
+        MOZ_ASSERT(CurrentThreadIsIonCompiling());
+        *requiredBarriersOut |= 1 << uint32_t(stub);
+        return stubs_[stub].unbarrieredGet();
+    }
+
   public:
     JSObject* getSimdTemplateObjectFor(JSContext* cx, Handle<SimdTypeDescr*> descr) {
         ReadBarrieredObject& tpl = simdTemplateObjects_[descr->type()];
         if (!tpl)
             tpl.set(TypedObject::createZeroed(cx, descr, 0, gc::TenuredHeap));
         return tpl.get();
     }
 
     JSObject* maybeGetSimdTemplateObjectFor(SimdType type) const {
-        const ReadBarrieredObject& tpl = simdTemplateObjects_[type];
+        // This function is used by Eager Simd Unbox phase which can run
+        // off-thread, so we cannot use the usual read barrier. For more
+        // information, see the comment above
+        // CodeGenerator::simdRefreshTemplatesDuringLink_.
 
-        // This function is used by Eager Simd Unbox phase, so we cannot use the
-        // read barrier. For more information, see the comment above
-        // CodeGenerator::simdRefreshTemplatesDuringLink_ .
-        return tpl.unbarrieredGet();
-    }
-
-    // This function is used to call the read barrier, to mark the SIMD template
-    // type as used. This function can only be called from the active thread.
-    void registerSimdTemplateObjectFor(SimdType type) {
-        ReadBarrieredObject& tpl = simdTemplateObjects_[type];
-        MOZ_ASSERT(tpl.unbarrieredGet());
-        tpl.get();
+        MOZ_ASSERT(CurrentThreadIsIonCompiling());
+        return simdTemplateObjects_[type].unbarrieredGet();
     }
 
     JitCode* getStubCode(uint32_t key) {
         ICStubCodeMap::Ptr p = stubCodes_->lookup(key);
         if (p)
             return p->value();
         return nullptr;
     }
@@ -597,57 +607,72 @@ class JitCompartment
     }
 
     JitCompartment();
     ~JitCompartment();
 
     MOZ_MUST_USE bool initialize(JSContext* cx);
 
     // Initialize code stubs only used by Ion, not Baseline.
-    MOZ_MUST_USE bool ensureIonStubsExist(JSContext* cx);
+    MOZ_MUST_USE bool ensureIonStubsExist(JSContext* cx) {
+        if (stubs_[StringConcat])
+            return true;
+        stubs_[StringConcat] = generateStringConcatStub(cx);
+        return stubs_[StringConcat];
+    }
 
     void sweep(FreeOp* fop, JSCompartment* compartment);
 
-    JitCode* stringConcatStubNoBarrier() const {
-        return stringConcatStub_;
+    JitCode* stringConcatStubNoBarrier(uint32_t* requiredBarriersOut) const {
+        return getStubNoBarrier(StringConcat, requiredBarriersOut);
     }
 
-    JitCode* regExpMatcherStubNoBarrier() const {
-        return regExpMatcherStub_;
+    JitCode* regExpMatcherStubNoBarrier(uint32_t* requiredBarriersOut) const {
+        return getStubNoBarrier(RegExpMatcher, requiredBarriersOut);
     }
 
     MOZ_MUST_USE bool ensureRegExpMatcherStubExists(JSContext* cx) {
-        if (regExpMatcherStub_)
+        if (stubs_[RegExpMatcher])
             return true;
-        regExpMatcherStub_ = generateRegExpMatcherStub(cx);
-        return regExpMatcherStub_ != nullptr;
+        stubs_[RegExpMatcher] = generateRegExpMatcherStub(cx);
+        return stubs_[RegExpMatcher];
     }
 
-    JitCode* regExpSearcherStubNoBarrier() const {
-        return regExpSearcherStub_;
+    JitCode* regExpSearcherStubNoBarrier(uint32_t* requiredBarriersOut) const {
+        return getStubNoBarrier(RegExpSearcher, requiredBarriersOut);
     }
 
     MOZ_MUST_USE bool ensureRegExpSearcherStubExists(JSContext* cx) {
-        if (regExpSearcherStub_)
+        if (stubs_[RegExpSearcher])
             return true;
-        regExpSearcherStub_ = generateRegExpSearcherStub(cx);
-        return regExpSearcherStub_ != nullptr;
+        stubs_[RegExpSearcher] = generateRegExpSearcherStub(cx);
+        return stubs_[RegExpSearcher];
     }
 
-    JitCode* regExpTesterStubNoBarrier() const {
-        return regExpTesterStub_;
+    JitCode* regExpTesterStubNoBarrier(uint32_t* requiredBarriersOut) const {
+        return getStubNoBarrier(RegExpTester, requiredBarriersOut);
     }
 
     MOZ_MUST_USE bool ensureRegExpTesterStubExists(JSContext* cx) {
-        if (regExpTesterStub_)
+        if (stubs_[RegExpTester])
             return true;
-        regExpTesterStub_ = generateRegExpTesterStub(cx);
-        return regExpTesterStub_ != nullptr;
+        stubs_[RegExpTester] = generateRegExpTesterStub(cx);
+        return stubs_[RegExpTester];
     }
 
+    // Perform the necessary read barriers on stubs and SIMD template object
+    // described by the bitmasks passed in. This function can only be called
+    // from the active thread.
+    //
+    // The stub and template object pointers must still be valid by the time
+    // these methods are called. This is arranged by cancelling off-thread Ion
+    // compilation at the start of GC and at the start of sweeping.
+    void performStubReadBarriers(uint32_t stubsToBarrier) const;
+    void performSIMDTemplateReadBarriers(uint32_t simdTemplatesToBarrier) const;
+
     size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 
     bool stringsCanBeInNursery;
 };
 
 // Called from JSCompartment::discardJitCode().
 void InvalidateAll(FreeOp* fop, JS::Zone* zone);
 void FinishInvalidation(FreeOp* fop, JSScript* script);