Unreviewed, rolling out r250002.
authorcommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 18 Sep 2019 21:13:26 +0000 (21:13 +0000)
committercommit-queue@webkit.org <commit-queue@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 18 Sep 2019 21:13:26 +0000 (21:13 +0000)
https://bugs.webkit.org/show_bug.cgi?id=201943

Patching of the callee and call is not atomic (Requested by
tadeuzagallo on #webkit).

Reverted changeset:

"Change WebAssembly calling conventions"
https://bugs.webkit.org/show_bug.cgi?id=201799
https://trac.webkit.org/changeset/250002

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@250052 268f45cc-cd09-0410-ab3c-d52691b4dbfc

30 files changed:
Source/JavaScriptCore/CMakeLists.txt
Source/JavaScriptCore/ChangeLog
Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj
Source/JavaScriptCore/wasm/WasmAirIRGenerator.cpp
Source/JavaScriptCore/wasm/WasmAirIRGenerator.h
Source/JavaScriptCore/wasm/WasmB3IRGenerator.cpp
Source/JavaScriptCore/wasm/WasmB3IRGenerator.h
Source/JavaScriptCore/wasm/WasmBBQPlan.cpp
Source/JavaScriptCore/wasm/WasmBBQPlan.h
Source/JavaScriptCore/wasm/WasmBBQPlanInlines.h
Source/JavaScriptCore/wasm/WasmBinding.cpp
Source/JavaScriptCore/wasm/WasmCallee.cpp
Source/JavaScriptCore/wasm/WasmCallee.h
Source/JavaScriptCore/wasm/WasmCallingConvention.h
Source/JavaScriptCore/wasm/WasmCodeBlock.cpp
Source/JavaScriptCore/wasm/WasmCodeBlock.h
Source/JavaScriptCore/wasm/WasmEmbedder.h
Source/JavaScriptCore/wasm/WasmFormat.h
Source/JavaScriptCore/wasm/WasmInstance.h
Source/JavaScriptCore/wasm/WasmOMGForOSREntryPlan.cpp
Source/JavaScriptCore/wasm/WasmOMGForOSREntryPlan.h
Source/JavaScriptCore/wasm/WasmOMGPlan.cpp
Source/JavaScriptCore/wasm/WasmOMGPlan.h
Source/JavaScriptCore/wasm/WasmOperations.cpp
Source/JavaScriptCore/wasm/js/JSToWasm.cpp
Source/JavaScriptCore/wasm/js/JSToWasm.h
Source/JavaScriptCore/wasm/js/WebAssemblyFunction.cpp
Source/JavaScriptCore/wasm/js/WebAssemblyFunction.h
Source/JavaScriptCore/wasm/js/WebAssemblyModuleRecord.cpp
Source/JavaScriptCore/wasm/js/WebAssemblyWrapperFunction.cpp

index 0c57125..a6c0f02 100644 (file)
@@ -393,7 +393,6 @@ set(JavaScriptCore_PUBLIC_FRAMEWORK_HEADERS
 set(JavaScriptCore_PRIVATE_FRAMEWORK_HEADERS
     ${JavaScriptCore_DERIVED_SOURCES_DIR}/Bytecodes.h
     ${JavaScriptCore_DERIVED_SOURCES_DIR}/JSCBuiltins.h
-    ${JavaScriptCore_DERIVED_SOURCES_DIR}/WasmOps.h
 
     ${JavaScriptCore_DERIVED_SOURCES_DIR}/inspector/InspectorBackendDispatchers.h
     ${JavaScriptCore_DERIVED_SOURCES_DIR}/inspector/InspectorFrontendDispatchers.h
@@ -464,10 +463,6 @@ set(JavaScriptCore_PRIVATE_FRAMEWORK_HEADERS
     assembler/X86Registers.h
     assembler/X86_64Registers.h
 
-    b3/B3Common.h
-    b3/B3Compilation.h
-    b3/B3Type.h
-
     bindings/ScriptFunctionCall.h
     bindings/ScriptObject.h
     bindings/ScriptValue.h
@@ -1006,7 +1001,6 @@ set(JavaScriptCore_PRIVATE_FRAMEWORK_HEADERS
     runtime/WriteBarrier.h
     runtime/WriteBarrierInlines.h
 
-    wasm/WasmCallee.h
     wasm/WasmCapabilities.h
     wasm/WasmCodeBlock.h
     wasm/WasmCompilationMode.h
@@ -1014,16 +1008,13 @@ set(JavaScriptCore_PRIVATE_FRAMEWORK_HEADERS
     wasm/WasmEmbedder.h
     wasm/WasmExceptionType.h
     wasm/WasmFaultSignalHandler.h
-    wasm/WasmFormat.h
     wasm/WasmIndexOrName.h
     wasm/WasmMemory.h
-    wasm/WasmMemoryInformation.h
     wasm/WasmMemoryMode.h
     wasm/WasmModule.h
     wasm/WasmName.h
     wasm/WasmNameSection.h
     wasm/WasmPageCount.h
-    wasm/WasmSignature.h
     wasm/WasmTierUpCount.h
 
     wasm/js/JSWebAssembly.h
index e12172e..61cca3e 100644 (file)
@@ -1,3 +1,17 @@
+2019-09-18  Commit Queue  <commit-queue@webkit.org>
+
+        Unreviewed, rolling out r250002.
+        https://bugs.webkit.org/show_bug.cgi?id=201943
+
+        Patching of the callee and call is not atomic (Requested by
+        tadeuzagallo on #webkit).
+
+        Reverted changeset:
+
+        "Change WebAssembly calling conventions"
+        https://bugs.webkit.org/show_bug.cgi?id=201799
+        https://trac.webkit.org/changeset/250002
+
 2019-09-17  Yusuke Suzuki  <ysuzuki@apple.com>
 
         [JSC] Generator should have internal fields
index fb40b0e..534d7d0 100644 (file)
                0F338DF61BE93D550013C88F /* B3ConstrainedValue.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F338DF41BE93D550013C88F /* B3ConstrainedValue.h */; };
                0F338DFA1BE96AA80013C88F /* B3CCallValue.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F338DF81BE96AA80013C88F /* B3CCallValue.h */; };
                0F338DFE1BED51270013C88F /* AirSimplifyCFG.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F338DFC1BED51270013C88F /* AirSimplifyCFG.h */; };
-               0F338E0C1BF0276C0013C88F /* B3Compilation.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F338E001BF0276C0013C88F /* B3Compilation.h */; settings = {ATTRIBUTES = (Private, ); }; };
+               0F338E0C1BF0276C0013C88F /* B3Compilation.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F338E001BF0276C0013C88F /* B3Compilation.h */; };
                0F338E0E1BF0276C0013C88F /* B3DataSection.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F338E021BF0276C0013C88F /* B3DataSection.h */; };
                0F338E101BF0276C0013C88F /* B3MoveConstants.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F338E041BF0276C0013C88F /* B3MoveConstants.h */; };
                0F338E111BF0276C0013C88F /* B3OpaqueByproduct.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F338E051BF0276C0013C88F /* B3OpaqueByproduct.h */; };
                0FEC85041BDACDAC0080FF74 /* B3BlockWorklist.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84BA1BDACDAC0080FF74 /* B3BlockWorklist.h */; };
                0FEC85061BDACDAC0080FF74 /* B3CheckSpecial.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84BC1BDACDAC0080FF74 /* B3CheckSpecial.h */; };
                0FEC85081BDACDAC0080FF74 /* B3CheckValue.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84BE1BDACDAC0080FF74 /* B3CheckValue.h */; };
-               0FEC850A1BDACDAC0080FF74 /* B3Common.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84C01BDACDAC0080FF74 /* B3Common.h */; settings = {ATTRIBUTES = (Private, ); }; };
+               0FEC850A1BDACDAC0080FF74 /* B3Common.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84C01BDACDAC0080FF74 /* B3Common.h */; };
                0FEC850C1BDACDAC0080FF74 /* B3Commutativity.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84C21BDACDAC0080FF74 /* B3Commutativity.h */; };
                0FEC850E1BDACDAC0080FF74 /* B3Const32Value.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84C41BDACDAC0080FF74 /* B3Const32Value.h */; };
                0FEC85101BDACDAC0080FF74 /* B3Const64Value.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84C61BDACDAC0080FF74 /* B3Const64Value.h */; };
                0FEC85361BDACDAC0080FF74 /* B3SuccessorCollection.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84EC1BDACDAC0080FF74 /* B3SuccessorCollection.h */; };
                0FEC85381BDACDAC0080FF74 /* B3SwitchCase.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84EE1BDACDAC0080FF74 /* B3SwitchCase.h */; };
                0FEC853A1BDACDAC0080FF74 /* B3SwitchValue.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84F01BDACDAC0080FF74 /* B3SwitchValue.h */; };
-               0FEC853C1BDACDAC0080FF74 /* B3Type.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84F21BDACDAC0080FF74 /* B3Type.h */; settings = {ATTRIBUTES = (Private, ); }; };
+               0FEC853C1BDACDAC0080FF74 /* B3Type.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84F21BDACDAC0080FF74 /* B3Type.h */; };
                0FEC853E1BDACDAC0080FF74 /* B3UpsilonValue.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84F41BDACDAC0080FF74 /* B3UpsilonValue.h */; };
                0FEC85401BDACDAC0080FF74 /* B3UseCounts.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84F61BDACDAC0080FF74 /* B3UseCounts.h */; };
                0FEC85421BDACDAC0080FF74 /* B3Validate.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FEC84F81BDACDAC0080FF74 /* B3Validate.h */; };
                                868921C11F9C0CB7001159F6 /* JSBigInt.h */,
                                86FA9E8F142BBB2D001773B7 /* JSBoundFunction.cpp */,
                                86FA9E90142BBB2E001773B7 /* JSBoundFunction.h */,
+                               FE48BD4323245E8700F136D0 /* JSCConfig.cpp */,
+                               FE48BD4223245E8700F136D0 /* JSCConfig.h */,
                                657CF45619BF6662004ACBF2 /* JSCallee.cpp */,
                                657CF45719BF6662004ACBF2 /* JSCallee.h */,
                                53B601EB2034B8C5006BE667 /* JSCast.h */,
-                               FE48BD4323245E8700F136D0 /* JSCConfig.cpp */,
-                               FE48BD4223245E8700F136D0 /* JSCConfig.h */,
                                BC7F8FBA0E19D1EF008632C0 /* JSCell.cpp */,
                                BC1167D80E19BCC9008066DD /* JSCell.h */,
                                0F97496F1687ADE200A4FF6A /* JSCellInlines.h */,
                                0FEA0A10170513DB00BB722C /* FTLLowerDFGToB3.h in Headers */,
                                A7D89D0217A0B90400773AD8 /* FTLLoweredNodeValue.h in Headers */,
                                0F2B9CF919D0BAC100B1D1B5 /* FTLOperations.h in Headers */,
+                               FE3842322324D51B009DD445 /* OptionEntry.h in Headers */,
                                0FD8A31C17D51F2200CA2C40 /* FTLOSREntry.h in Headers */,
                                0F235BDD17178E1C00690C7F /* FTLOSRExit.h in Headers */,
                                0F235BE017178E1C00690C7F /* FTLOSRExitCompiler.h in Headers */,
                                A1D793011B43864B004516F5 /* IntlNumberFormatPrototype.h in Headers */,
                                A125846F1B45A36000CC7F6C /* IntlNumberFormatPrototype.lut.h in Headers */,
                                A12BBFF21B044A8B00664B69 /* IntlObject.h in Headers */,
+                               FE48BD4423245E9300F136D0 /* JSCConfig.h in Headers */,
                                708EBE241CE8F35800453146 /* IntlObjectInlines.h in Headers */,
                                860BD801148EA6F200112B2F /* Intrinsic.h in Headers */,
                                534E03541E53BD2900213F64 /* IntrinsicGetterAccessCase.h in Headers */,
                                657CF45919BF6662004ACBF2 /* JSCallee.h in Headers */,
                                53B601EC2034B8C5006BE667 /* JSCast.h in Headers */,
                                A7D801A91880D6A80026C39B /* JSCBuiltins.h in Headers */,
-                               FE48BD4423245E9300F136D0 /* JSCConfig.h in Headers */,
                                BC1167DA0E19BCC9008066DD /* JSCell.h in Headers */,
                                0F9749711687ADE400A4FF6A /* JSCellInlines.h in Headers */,
                                0F1DD84A18A945BE0026F3FA /* JSCInlines.h in Headers */,
                                0F2BDC2C151FDE9100CD8910 /* Operands.h in Headers */,
                                A70447EA17A0BD4600F5898E /* OperandsInlines.h in Headers */,
                                BC18C4480E16F5CD00B34460 /* Operations.h in Headers */,
-                               FE3842322324D51B009DD445 /* OptionEntry.h in Headers */,
                                0FE228ED1436AB2700196C48 /* Options.h in Headers */,
-                               FE3842332324D51B009DD445 /* OptionsList.h in Headers */,
                                E356987222841187008CDCCB /* PackedCellPtr.h in Headers */,
                                0F9DAA0A1FD1C3D30079C5B2 /* ParallelSourceAdapter.h in Headers */,
                                E34E657520668EAA00FB81AC /* ParseHash.h in Headers */,
                                0FE050281AA9095600D33B33 /* ScopedArguments.h in Headers */,
                                0FE050291AA9095600D33B33 /* ScopedArgumentsTable.h in Headers */,
                                0FE0502B1AA9095600D33B33 /* ScopeOffset.h in Headers */,
+                               FE3842332324D51B009DD445 /* OptionsList.h in Headers */,
                                0F24E55217EE274900ABB217 /* ScratchRegisterAllocator.h in Headers */,
                                A5FD0068189AFE9C00633231 /* ScriptArguments.h in Headers */,
                                A503FA21188EFF6800110F14 /* ScriptBreakpoint.h in Headers */,
index a1f0c5d..e0c1566 100644 (file)
@@ -229,7 +229,7 @@ public:
             return fail(__VA_ARGS__);             \
     } while (0)
 
-    AirIRGenerator(const ModuleInformation&, B3::Procedure&, Vector<UnlinkedWasmToWasmCall>&, const Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>&, MemoryMode, unsigned functionIndex, TierUpCount*, ThrowWasmException, const Signature&);
+    AirIRGenerator(const ModuleInformation&, B3::Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, unsigned functionIndex, TierUpCount*, ThrowWasmException, const Signature&);
 
     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
@@ -628,8 +628,7 @@ private:
     BasicBlock* m_currentBlock { nullptr };
     BasicBlock* m_rootBlock { nullptr };
     Vector<TypedTmp> m_locals;
-    Vector<UnlinkedWasmToWasmCall>& m_outgoingCalls;
-    const Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>& m_wasmToWasmExitStubs;
+    Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
     GPRReg m_memoryBaseGPR { InvalidGPRReg };
     GPRReg m_memorySizeGPR { InvalidGPRReg };
     GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
@@ -698,15 +697,14 @@ void AirIRGenerator::restoreWasmContextInstance(BasicBlock* block, TypedTmp inst
     emitPatchpoint(block, patchpoint, Tmp(), instance);
 }
 
-AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, Vector<UnlinkedWasmToWasmCall>& outgoingCalls, const Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>& wasmToWasmExitStubs, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException, const Signature& signature)
+AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException, const Signature& signature)
     : m_info(info)
     , m_mode(mode)
     , m_functionIndex(functionIndex)
     , m_tierUp(tierUp)
     , m_proc(procedure)
     , m_code(m_proc.code())
-    , m_outgoingCalls(outgoingCalls)
-    , m_wasmToWasmExitStubs(wasmToWasmExitStubs)
+    , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
     , m_numImportFunctions(info.importFunctionCount())
 {
     m_currentBlock = m_code.addBlock();
@@ -752,7 +750,16 @@ AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& pro
     Ref<B3::Air::PrologueGenerator> prologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>([=] (CCallHelpers& jit, B3::Air::Code& code) {
         AllowMacroScratchRegisterUsage allowScratch(jit);
         code.emitDefaultPrologue(jit);
-        jit.emitPutToCallFrameHeader(nullptr, CallFrameSlot::codeBlock);
+
+        {
+            GPRReg calleeGPR = wasmCallingConventionAir().prologueScratch(0);
+            auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
+            jit.addLinkTask([compilation, moveLocation] (LinkBuffer& linkBuffer) {
+                compilation->calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
+            });
+            jit.emitPutToCallFrameHeader(calleeGPR, CallFrameSlot::callee);
+            jit.emitPutToCallFrameHeader(nullptr, CallFrameSlot::codeBlock);
+        }
 
         {
             const Checked<int32_t> wasmFrameSize = m_code.frameSize();
@@ -1918,6 +1925,8 @@ auto AirIRGenerator::addCall(uint32_t functionIndex, const Signature& signature,
     if (returnType != Type::Void)
         result = tmpForType(returnType);
 
+    Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
+
     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
         m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
 
@@ -1953,12 +1962,11 @@ auto AirIRGenerator::addCall(uint32_t functionIndex, const Signature& signature,
                 patchArgs.append({ tmp, rep });
             });
 
-            MacroAssemblerCodeRef<WasmEntryPtrTag> wasmToWasmExitStub = m_wasmToWasmExitStubs[functionIndex];
-            patchpoint->setGenerator([wasmToWasmExitStub] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
+            patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
                 AllowMacroScratchRegisterUsage allowScratch(jit);
-                MacroAssembler::Call call = jit.call(WasmEntryPtrTag);
-                jit.addLinkTask([call, wasmToWasmExitStub] (LinkBuffer& linkBuffer) {
-                    linkBuffer.link(call, CodeLocationLabel<WasmEntryPtrTag>(wasmToWasmExitStub.code()));
+                CCallHelpers::Call call = jit.threadSafePatchableNearCall();
+                jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
+                    unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
                 });
             });
 
@@ -2005,20 +2013,18 @@ auto AirIRGenerator::addCall(uint32_t functionIndex, const Signature& signature,
         auto* patchpoint = addPatchpoint(toB3Type(returnType));
         patchpoint->effects.writesPinned = true;
         patchpoint->effects.readsPinned = true;
-        patchpoint->numGPScratchRegisters++;
 
         Vector<ConstrainedTmp> patchArgs;
         wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
             patchArgs.append({ tmp, rep });
         });
 
-        Vector<UnlinkedWasmToWasmCall>* outgoingCalls = &m_outgoingCalls;
-        patchpoint->setGenerator([outgoingCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
-            GPRReg calleeGPR = params.gpScratch(0);
-            auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
-            jit.storePtr(calleeGPR, MacroAssembler::Address(MacroAssembler::stackPointerRegister, CallFrameSlot::callee * sizeof(Register) - sizeof(CallerFrameAndPC)));
-            CCallHelpers::Call callLocation = jit.threadSafePatchableNearCall();
-            outgoingCalls->append({ functionIndex, { moveLocation, callLocation }});
+        patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
+            AllowMacroScratchRegisterUsage allowScratch(jit);
+            CCallHelpers::Call call = jit.threadSafePatchableNearCall();
+            jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
+                unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
+            });
         });
 
         emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(patchArgs));
@@ -2072,7 +2078,6 @@ auto AirIRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signa
     });
 
     ExpressionType calleeCode = g64();
-    ExpressionType callee = g64();
     {
         ExpressionType calleeSignatureIndex = g64();
         // Compute the offset in the table index space we are looking for.
@@ -2081,7 +2086,6 @@ auto AirIRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signa
         append(Add64, callableFunctionBuffer, calleeSignatureIndex);
         
         append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation()), calleeCode); // Pointer to callee code.
-        append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfBoxedCalleeLoadLocation()), callee); // Pointer to boxed callee
 
         // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
         // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
@@ -2174,10 +2178,6 @@ auto AirIRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signa
 
     Vector<ConstrainedTmp> emitArgs;
     emitArgs.append(calleeCode);
-
-    append(Move, Arg::addr(callee), callee);
-    emitArgs.append({ callee, B3::ValueRep::stackArgument(CallFrameSlot::callee * sizeof(Register) - sizeof(CallerFrameAndPC)) });
-
     wasmCallingConventionAir().setupCall(m_code, returnType, patch, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
         emitArgs.append({ tmp, rep });
     });
@@ -2218,8 +2218,10 @@ auto AirIRGenerator::origin() -> B3::Origin
     return B3::Origin();
 }
 
-Expected<void, String> parseAndCompileAir(CompilationContext& compilationContext, const FunctionData& function, const Signature& signature, Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>& wasmToWasmExitStubs, const ModuleInformation& info, MemoryMode mode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
+Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileAir(CompilationContext& compilationContext, const FunctionData& function, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
 {
+    auto result = makeUnique<InternalFunction>();
+
     compilationContext.embedderEntrypointJIT = makeUnique<CCallHelpers>();
     compilationContext.wasmEntrypointJIT = makeUnique<CCallHelpers>();
 
@@ -2239,7 +2241,7 @@ Expected<void, String> parseAndCompileAir(CompilationContext& compilationContext
     
     procedure.setOptLevel(Options::webAssemblyBBQAirOptimizationLevel());
 
-    AirIRGenerator irGenerator(info, procedure, compilationContext.outgoingCalls, wasmToWasmExitStubs, mode, functionIndex, tierUp, throwWasmException, signature);
+    AirIRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, functionIndex, tierUp, throwWasmException, signature);
     FunctionParser<AirIRGenerator> parser(irGenerator, function.data.data(), function.data.size(), signature, info);
     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
 
@@ -2259,10 +2261,10 @@ Expected<void, String> parseAndCompileAir(CompilationContext& compilationContext
         B3::Air::prepareForGeneration(code);
         B3::Air::generate(code, *compilationContext.wasmEntrypointJIT);
         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
-        compilationContext.calleeSaveRegisters = code.calleeSaveRegisterAtOffsetList();
+        result->entrypoint.calleeSaveRegisters = code.calleeSaveRegisterAtOffsetList();
     }
 
-    return { };
+    return result;
 }
 
 template <typename IntType>
index 8272ce3..331b4b0 100644 (file)
@@ -31,7 +31,7 @@
 
 namespace JSC { namespace Wasm {
 
-Expected<void, String> parseAndCompileAir(CompilationContext&, const FunctionData&, const Signature&, Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>&, const ModuleInformation&, MemoryMode, uint32_t functionIndex, TierUpCount* = nullptr, ThrowWasmException = nullptr);
+Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileAir(CompilationContext&, const FunctionData&, const Signature&, Vector<UnlinkedWasmToWasmCall>&, const ModuleInformation&, MemoryMode, uint32_t functionIndex, TierUpCount* = nullptr, ThrowWasmException = nullptr);
 
 } } // namespace JSC::Wasm
 
index e253c20..470ce82 100644 (file)
@@ -172,6 +172,7 @@ public:
 
     typedef String ErrorType;
     typedef Unexpected<ErrorType> UnexpectedResult;
+    typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
     typedef Expected<void, ErrorType> PartialResult;
     template <typename ...Args>
     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
@@ -184,7 +185,7 @@ public:
             return fail(__VA_ARGS__);             \
     } while (0)
 
-    B3IRGenerator(const ModuleInformation&, Procedure&, Vector<UnlinkedWasmToWasmCall>&, Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>&, unsigned& osrEntryScratchBufferSize, MemoryMode, CompilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry, TierUpCount*, ThrowWasmException);
+    B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, unsigned& osrEntryScratchBufferSize, MemoryMode, CompilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry, TierUpCount*, ThrowWasmException);
 
     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
@@ -280,7 +281,6 @@ private:
 
     FunctionParser<B3IRGenerator>* m_parser { nullptr };
     const ModuleInformation& m_info;
-    Vector<UnlinkedWasmToWasmCall>& m_outgoingCalls;
     const MemoryMode m_mode { MemoryMode::BoundsChecking };
     const CompilationMode m_compilationMode { CompilationMode::BBQMode };
     const unsigned m_functionIndex { UINT_MAX };
@@ -292,7 +292,7 @@ private:
     BasicBlock* m_currentBlock { nullptr };
     Vector<uint32_t> m_outerLoops;
     Vector<Variable*> m_locals;
-    Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>& m_wasmToWasmExitStubs; // List each call site and the function index whose address it should be patched with.
+    Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
     unsigned& m_osrEntryScratchBufferSize;
     HashMap<ValueKey, Value*> m_constantPool;
     InsertionSet m_constantInsertionValues;
@@ -353,16 +353,15 @@ void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* bloc
     });
 }
 
-B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, Vector<UnlinkedWasmToWasmCall>& outgoingCalls, Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>& wasmToWasmExitStubs, unsigned& osrEntryScratchBufferSize, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry, TierUpCount* tierUp, ThrowWasmException throwWasmException)
+B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, unsigned& osrEntryScratchBufferSize, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry, TierUpCount* tierUp, ThrowWasmException throwWasmException)
     : m_info(info)
-    , m_outgoingCalls(outgoingCalls)
     , m_mode(mode)
     , m_compilationMode(compilationMode)
     , m_functionIndex(functionIndex)
     , m_loopIndexForOSREntry(loopIndexForOSREntry)
     , m_tierUp(tierUp)
     , m_proc(procedure)
-    , m_wasmToWasmExitStubs(wasmToWasmExitStubs)
+    , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
     , m_osrEntryScratchBufferSize(osrEntryScratchBufferSize)
     , m_constantInsertionValues(m_proc)
     , m_numImportFunctions(info.importFunctionCount())
@@ -416,7 +415,7 @@ B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure
         }
     }
 
-    wasmCallingConvention().setupFrameInPrologue(m_proc, Origin(), m_currentBlock);
+    wasmCallingConvention().setupFrameInPrologue(&compilation->calleeMoveLocation, m_proc, Origin(), m_currentBlock);
 
     {
         B3::Value* framePointer = m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin());
@@ -1401,6 +1400,8 @@ auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature,
     m_makesCalls = true;
 
     Type returnType = signature.returnType();
+    Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
+
     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
         m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
 
@@ -1414,7 +1415,6 @@ auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature,
         BasicBlock* continuation = m_proc.addBlock();
         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
 
-        MacroAssemblerCodeRef<WasmEntryPtrTag> wasmToWasmExitStub = m_wasmToWasmExitStubs[functionIndex];
         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
             [=] (PatchpointValue* patchpoint) {
                 patchpoint->effects.writesPinned = true;
@@ -1423,11 +1423,11 @@ auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature,
                 // We pessimistically assume we could be calling to something that is bounds checking.
                 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
-                patchpoint->setGenerator([wasmToWasmExitStub] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
+                patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
                     AllowMacroScratchRegisterUsage allowScratch(jit);
-                    MacroAssembler::Call call = jit.call(WasmEntryPtrTag);
-                    jit.addLinkTask([call, wasmToWasmExitStub] (LinkBuffer& linkBuffer) {
-                        linkBuffer.link(call, CodeLocationLabel<WasmEntryPtrTag>(wasmToWasmExitStub.code()));
+                    CCallHelpers::Call call = jit.threadSafePatchableNearCall();
+                    jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
+                        unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
                     });
                 });
             });
@@ -1471,18 +1471,17 @@ auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature,
         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
         restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation);
     } else {
-        Vector<UnlinkedWasmToWasmCall>* outgoingCalls = &m_outgoingCalls;
         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
             [=] (PatchpointValue* patchpoint) {
                 patchpoint->effects.writesPinned = true;
                 patchpoint->effects.readsPinned = true;
-                patchpoint->numGPScratchRegisters++;
-                patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
-                    GPRReg calleeGPR = params.gpScratch(0);
-                    auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
-                    jit.storePtr(calleeGPR, MacroAssembler::Address(MacroAssembler::stackPointerRegister, CallFrameSlot::callee * sizeof(Register) - sizeof(CallerFrameAndPC)));
-                    CCallHelpers::Call callLocation = jit.threadSafePatchableNearCall();
-                    outgoingCalls->append({ functionIndex, { moveLocation, callLocation }});
+
+                patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
+                    AllowMacroScratchRegisterUsage allowScratch(jit);
+                    CCallHelpers::Call call = jit.threadSafePatchableNearCall();
+                    jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
+                        unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
+                    });
                 });
             });
     }
@@ -1623,10 +1622,6 @@ auto B3IRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signat
         m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
             safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation())));
 
-    ExpressionType callee = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
-        m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
-            safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfBoxedCalleeLoadLocation())));
-
     Type returnType = signature.returnType();
     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
         [=] (PatchpointValue* patchpoint) {
@@ -1638,8 +1633,8 @@ auto B3IRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signat
             // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
             // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
             patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
+
             patchpoint->append(calleeCode, ValueRep::SomeRegister);
-            patchpoint->append(callee, B3::ValueRep::stackArgument(CallFrameSlot::callee * sizeof(Register) - sizeof(CallerFrameAndPC)));
             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
                 AllowMacroScratchRegisterUsage allowScratch(jit);
                 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
@@ -1700,8 +1695,10 @@ auto B3IRGenerator::origin() -> Origin
     return bitwise_cast<Origin>(origin);
 }
 
-Expected<void, String> parseAndCompile(CompilationContext& compilationContext, const FunctionData& function, const Signature& signature, Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>& wasmToWasmExitStubs, unsigned& osrEntryScratchBufferSize, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* tierUp, ThrowWasmException throwWasmException)
+Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const FunctionData& function, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, unsigned& osrEntryScratchBufferSize, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* tierUp, ThrowWasmException throwWasmException)
 {
+    auto result = makeUnique<InternalFunction>();
+
     compilationContext.embedderEntrypointJIT = makeUnique<CCallHelpers>();
     compilationContext.wasmEntrypointJIT = makeUnique<CCallHelpers>();
 
@@ -1722,7 +1719,7 @@ Expected<void, String> parseAndCompile(CompilationContext& compilationContext, c
         ? Options::webAssemblyBBQB3OptimizationLevel()
         : Options::webAssemblyOMGOptimizationLevel());
 
-    B3IRGenerator irGenerator(info, procedure, compilationContext.outgoingCalls, wasmToWasmExitStubs, osrEntryScratchBufferSize, mode, compilationMode, functionIndex, loopIndexForOSREntry, tierUp, throwWasmException);
+    B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, osrEntryScratchBufferSize, mode, compilationMode, functionIndex, loopIndexForOSREntry, tierUp, throwWasmException);
     FunctionParser<B3IRGenerator> parser(irGenerator, function.data.data(), function.data.size(), signature, info);
     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
 
@@ -1740,10 +1737,10 @@ Expected<void, String> parseAndCompile(CompilationContext& compilationContext, c
         B3::prepareForGeneration(procedure);
         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
-        compilationContext.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
+        result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
     }
 
-    return { };
+    return result;
 }
 
 // Custom wasm ops. These are the ones too messy to do in wasm.json.
index 348b547..ea55024 100644 (file)
@@ -49,12 +49,9 @@ struct CompilationContext {
     std::unique_ptr<B3::OpaqueByproducts> embedderEntrypointByproducts;
     std::unique_ptr<CCallHelpers> wasmEntrypointJIT;
     std::unique_ptr<B3::OpaqueByproducts> wasmEntrypointByproducts;
-    Vector<UnlinkedWasmToWasmCall> outgoingCalls;
-    Optional<UnlinkedMoveAndCall> embedderMoveAndCall;
-    RegisterAtOffsetList calleeSaveRegisters;
 };
 
-Expected<void, String> parseAndCompile(CompilationContext&, const FunctionData&, const Signature&, Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>>&, unsigned& osrEntryScratchBufferSize, const ModuleInformation&, MemoryMode, CompilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* = nullptr, ThrowWasmException = nullptr);
+Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext&, const FunctionData&, const Signature&, Vector<UnlinkedWasmToWasmCall>&, unsigned& osrEntryScratchBufferSize, const ModuleInformation&, MemoryMode, CompilationMode, uint32_t functionIndex, uint32_t loopIndexForOSREntry, TierUpCount* = nullptr, ThrowWasmException = nullptr);
 
 } } // namespace JSC::Wasm
 
index 7496203..4f9152d 100644 (file)
@@ -52,10 +52,9 @@ namespace WasmBBQPlanInternal {
 static constexpr bool verbose = false;
 }
 
-BBQPlan::BBQPlan(Context* context, const Vector<Ref<BBQCallee>>& callees, Ref<ModuleInformation> info, AsyncWork work, CompletionTask&& task, CreateEmbedderWrapper&& createEmbedderWrapper, ThrowWasmException throwWasmException)
+BBQPlan::BBQPlan(Context* context, Ref<ModuleInformation> info, AsyncWork work, CompletionTask&& task, CreateEmbedderWrapper&& createEmbedderWrapper, ThrowWasmException throwWasmException)
     : Base(context, WTFMove(info), WTFMove(task), WTFMove(createEmbedderWrapper), throwWasmException)
     , m_streamingParser(m_moduleInformation.get(), *this)
-    , m_callees(&callees)
     , m_state(State::Validated)
     , m_asyncWork(work)
 {
@@ -160,10 +159,16 @@ void BBQPlan::prepare()
 
     const auto& functions = m_moduleInformation->functions;
     if (!tryReserveCapacity(m_wasmToWasmExitStubs, m_moduleInformation->importFunctionSignatureIndices.size(), " WebAssembly to JavaScript stubs")
-        || !tryReserveCapacity(m_compilationContexts, functions.size(), " compilation contexts"))
+        || !tryReserveCapacity(m_unlinkedWasmToWasmCalls, functions.size(), " unlinked WebAssembly to WebAssembly calls")
+        || !tryReserveCapacity(m_wasmInternalFunctions, functions.size(), " WebAssembly functions")
+        || !tryReserveCapacity(m_compilationContexts, functions.size(), " compilation contexts")
+        || !tryReserveCapacity(m_tierUpCounts, functions.size(), " tier-up counts"))
         return;
 
+    m_unlinkedWasmToWasmCalls.resize(functions.size());
+    m_wasmInternalFunctions.resize(functions.size());
     m_compilationContexts.resize(functions.size());
+    m_tierUpCounts.resize(functions.size());
 
     for (unsigned importIndex = 0; importIndex < m_moduleInformation->imports.size(); ++importIndex) {
         Import* import = &m_moduleInformation->imports[importIndex];
@@ -225,7 +230,6 @@ public:
 
 void BBQPlan::compileFunctions(CompilationEffort effort)
 {
-    RELEASE_ASSERT(m_callees);
     ASSERT(m_state >= State::Prepared);
     dataLogLnIf(WasmBBQPlanInternal::verbose, "Starting compilation");
 
@@ -262,7 +266,13 @@ void BBQPlan::compileFunctions(CompilationEffort effort)
         ASSERT_UNUSED(functionIndexSpace, m_moduleInformation->signatureIndexFromFunctionIndexSpace(functionIndexSpace) == signatureIndex);
         ASSERT(validateFunction(function, signature, m_moduleInformation.get()));
 
-        Expected<void, String> parseAndCompileResult;
+        m_unlinkedWasmToWasmCalls[functionIndex] = Vector<UnlinkedWasmToWasmCall>();
+        if (Options::useBBQTierUpChecks())
+            m_tierUpCounts[functionIndex] = makeUnique<TierUpCount>();
+        else
+            m_tierUpCounts[functionIndex] = nullptr;
+        TierUpCount* tierUp = m_tierUpCounts[functionIndex].get();
+        Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileResult;
         unsigned osrEntryScratchBufferSize = 0;
 
         // FIXME: Some webpages use very large Wasm module, and it exhausts all executable memory in ARM64 devices since the size of executable memory region is only limited to 128MB.
@@ -272,11 +282,10 @@ void BBQPlan::compileFunctions(CompilationEffort effort)
         if (Options::webAssemblyBBQAirModeThreshold() && m_moduleInformation->codeSectionSize >= Options::webAssemblyBBQAirModeThreshold())
             forceUsingB3 = true;
 
-        TierUpCount* tierUp = m_callees->at(functionIndex)->tierUpCount();
         if (!forceUsingB3 && Options::wasmBBQUsesAir())
-            parseAndCompileResult = parseAndCompileAir(m_compilationContexts[functionIndex], function, signature, m_wasmToWasmExitStubs, m_moduleInformation.get(), m_mode, functionIndex, tierUp, m_throwWasmException);
+            parseAndCompileResult = parseAndCompileAir(m_compilationContexts[functionIndex], function, signature, m_unlinkedWasmToWasmCalls[functionIndex], m_moduleInformation.get(), m_mode, functionIndex, tierUp, m_throwWasmException);
         else
-            parseAndCompileResult = parseAndCompile(m_compilationContexts[functionIndex], function, signature, m_wasmToWasmExitStubs, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::BBQMode, functionIndex, UINT32_MAX, tierUp, m_throwWasmException);
+            parseAndCompileResult = parseAndCompile(m_compilationContexts[functionIndex], function, signature, m_unlinkedWasmToWasmCalls[functionIndex], osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::BBQMode, functionIndex, UINT32_MAX, tierUp, m_throwWasmException);
 
         if (UNLIKELY(!parseAndCompileResult)) {
             auto locker = holdLock(m_lock);
@@ -285,11 +294,11 @@ void BBQPlan::compileFunctions(CompilationEffort effort)
             return;
         }
 
+        m_wasmInternalFunctions[functionIndex] = WTFMove(*parseAndCompileResult);
+
         if (m_exportedFunctionIndices.contains(functionIndex) || m_moduleInformation->referencedFunctions().contains(functionIndex)) {
             auto locker = holdLock(m_lock);
-            RefPtr<EmbedderEntrypointCallee> embedderEntrypointCallee = Wasm::EmbedderEntrypointCallee::create();
-            m_createEmbedderWrapper(m_compilationContexts[functionIndex], signature, embedderEntrypointCallee.get(), m_moduleInformation.get(), m_mode);
-            auto result = m_embedderToWasmInternalFunctions.add(functionIndex, WTFMove(embedderEntrypointCallee));
+            auto result = m_embedderToWasmInternalFunctions.add(functionIndex, m_createEmbedderWrapper(m_compilationContexts[functionIndex], signature, &m_unlinkedWasmToWasmCalls[functionIndex], m_moduleInformation.get(), m_mode, functionIndex));
             ASSERT_UNUSED(result, result.isNewEntry);
         }
 
@@ -307,9 +316,7 @@ void BBQPlan::complete(const AbstractLocker& locker)
             CompilationContext& context = m_compilationContexts[functionIndex];
             SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
             const Signature& signature = SignatureInformation::get(signatureIndex);
-            const uint32_t importFunctionCount = m_moduleInformation->importFunctionCount();
-            const uint32_t functionIndexSpace = functionIndex + importFunctionCount;
-            BBQCallee& callee = m_callees->at(functionIndex).get();
+            const uint32_t functionIndexSpace = functionIndex + m_moduleInformation->importFunctionCount();
             ASSERT(functionIndexSpace < m_moduleInformation->functionIndexSpaceSize());
             {
                 LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, JITCompilationCanFail);
@@ -318,16 +325,9 @@ void BBQPlan::complete(const AbstractLocker& locker)
                     return;
                 }
 
-                callee.entrypoint().calleeSaveRegisters = WTFMove(context.calleeSaveRegisters);
-                callee.entrypoint().compilation = makeUnique<B3::Compilation>(
+                m_wasmInternalFunctions[functionIndex]->entrypoint.compilation = makeUnique<B3::Compilation>(
                     FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "WebAssembly BBQ function[%i] %s name %s", functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
                     WTFMove(context.wasmEntrypointByproducts));
-
-                for (auto& call : context.outgoingCalls) {
-                    ASSERT(call.targetFunctionIndexSpace >= importFunctionCount);
-                    const uint32_t targetFunctionIndex = call.targetFunctionIndexSpace - m_moduleInformation->importFunctionCount();
-                    m_callees->at(targetFunctionIndex)->addCaller(locker, linkBuffer, call.unlinkedMoveAndCall);
-                }
             }
 
             if (auto embedderToWasmInternalFunction = m_embedderToWasmInternalFunctions.get(functionIndex)) {
@@ -337,18 +337,23 @@ void BBQPlan::complete(const AbstractLocker& locker)
                     return;
                 }
 
-                embedderToWasmInternalFunction->entrypoint().compilation = makeUnique<B3::Compilation>(
+                embedderToWasmInternalFunction->entrypoint.compilation = makeUnique<B3::Compilation>(
                     FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "Embedder->WebAssembly entrypoint[%i] %s name %s", functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
                     WTFMove(context.embedderEntrypointByproducts));
-
-                if (context.embedderMoveAndCall)
-                    callee.addCaller(locker, linkBuffer, *context.embedderMoveAndCall);
             }
-
         }
 
-        for (auto& callee : *m_callees)
-            callee->repatchCallers(locker, callee.get());
+        for (auto& unlinked : m_unlinkedWasmToWasmCalls) {
+            for (auto& call : unlinked) {
+                MacroAssemblerCodePtr<WasmEntryPtrTag> executableAddress;
+                if (m_moduleInformation->isImportedFunctionFromFunctionIndexSpace(call.functionIndexSpace)) {
+                    // FIXME imports could have been linked in B3, instead of generating a patchpoint. This condition should be replaced by a RELEASE_ASSERT. https://bugs.webkit.org/show_bug.cgi?id=166462
+                    executableAddress = m_wasmToWasmExitStubs.at(call.functionIndexSpace).code();
+                } else
+                    executableAddress = m_wasmInternalFunctions.at(call.functionIndexSpace - m_moduleInformation->importFunctionCount())->entrypoint.compilation->code().retagged<WasmEntryPtrTag>();
+                MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(executableAddress));
+            }
+        }
     }
     
     if (!isComplete()) {
index e217462..883617f 100644 (file)
@@ -51,7 +51,7 @@ public:
     enum AsyncWork : uint8_t { FullCompile, Validation };
 
     // Note: CompletionTask should not hold a reference to the Plan otherwise there will be a reference cycle.
-    BBQPlan(Context*, const Vector<Ref<BBQCallee>>&, Ref<ModuleInformation>, AsyncWork, CompletionTask&&, CreateEmbedderWrapper&&, ThrowWasmException);
+    BBQPlan(Context*, Ref<ModuleInformation>, AsyncWork, CompletionTask&&, CreateEmbedderWrapper&&, ThrowWasmException);
     JS_EXPORT_PRIVATE BBQPlan(Context*, Vector<uint8_t>&&, AsyncWork, CompletionTask&&, CreateEmbedderWrapper&&, ThrowWasmException);
     BBQPlan(Context*, AsyncWork, CompletionTask&&);
 
@@ -98,6 +98,12 @@ public:
         return WTFMove(m_wasmToWasmExitStubs);
     }
 
+    Vector<Vector<UnlinkedWasmToWasmCall>> takeWasmToWasmCallsites()
+    {
+        RELEASE_ASSERT(!failed() && !hasWork());
+        return WTFMove(m_unlinkedWasmToWasmCalls);
+    }
+
     enum class State : uint8_t {
         Initial,
         Validated,
@@ -133,13 +139,14 @@ private:
     Vector<uint8_t> m_source;
     Bag<CallLinkInfo> m_callLinkInfos;
     Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>> m_wasmToWasmExitStubs;
+    Vector<std::unique_ptr<InternalFunction>> m_wasmInternalFunctions;
     HashSet<uint32_t, typename DefaultHash<uint32_t>::Hash, WTF::UnsignedWithZeroKeyHashTraits<uint32_t>> m_exportedFunctionIndices;
-    HashMap<uint32_t, RefPtr<EmbedderEntrypointCallee>, typename DefaultHash<uint32_t>::Hash, WTF::UnsignedWithZeroKeyHashTraits<uint32_t>> m_embedderToWasmInternalFunctions;
+    HashMap<uint32_t, std::unique_ptr<InternalFunction>, typename DefaultHash<uint32_t>::Hash, WTF::UnsignedWithZeroKeyHashTraits<uint32_t>> m_embedderToWasmInternalFunctions;
     Vector<CompilationContext> m_compilationContexts;
     Vector<std::unique_ptr<TierUpCount>> m_tierUpCounts;
 
+    Vector<Vector<UnlinkedWasmToWasmCall>> m_unlinkedWasmToWasmCalls;
     StreamingParser m_streamingParser;
-    const Vector<Ref<BBQCallee>>* m_callees { nullptr };
     State m_state;
 
     const AsyncWork m_asyncWork;
index 9bbc6f3..86601ac 100644 (file)
@@ -38,8 +38,21 @@ template<typename Functor>
 void BBQPlan::initializeCallees(const Functor& callback)
 {
     ASSERT(!failed());
-    for (unsigned internalFunctionIndex = 0; internalFunctionIndex < m_moduleInformation->functions.size(); ++internalFunctionIndex)
-        callback(internalFunctionIndex, m_embedderToWasmInternalFunctions.get(internalFunctionIndex));
+    for (unsigned internalFunctionIndex = 0; internalFunctionIndex < m_wasmInternalFunctions.size(); ++internalFunctionIndex) {
+
+        RefPtr<Wasm::Callee> embedderEntrypointCallee;
+        if (auto embedderToWasmFunction = m_embedderToWasmInternalFunctions.get(internalFunctionIndex)) {
+            embedderEntrypointCallee = Wasm::EmbedderEntrypointCallee::create(WTFMove(embedderToWasmFunction->entrypoint));
+            MacroAssembler::repatchPointer(embedderToWasmFunction->calleeMoveLocation, CalleeBits::boxWasm(embedderEntrypointCallee.get()));
+        }
+
+        InternalFunction* function = m_wasmInternalFunctions[internalFunctionIndex].get();
+        size_t functionIndexSpace = internalFunctionIndex + m_moduleInformation->importFunctionCount();
+        Ref<Wasm::Callee> wasmEntrypointCallee = Wasm::BBQCallee::create(WTFMove(function->entrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), WTFMove(m_tierUpCounts[internalFunctionIndex]));
+        MacroAssembler::repatchPointer(function->calleeMoveLocation, CalleeBits::boxWasm(wasmEntrypointCallee.ptr()));
+
+        callback(internalFunctionIndex, WTFMove(embedderEntrypointCallee), WTFMove(wasmEntrypointCallee));
+    }
 }
 
 } } // namespace JSC::Wasm
index eefb972..ce61076 100644 (file)
@@ -47,7 +47,6 @@ Expected<MacroAssemblerCodeRef<WasmEntryPtrTag>, BindingFailure> wasmToWasm(unsi
     JIT jit;
 
     GPRReg scratch = wasmCallingConventionAir().prologueScratch(0);
-    GPRReg callee = wasmCallingConventionAir().prologueScratch(1);
     GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
     ASSERT(baseMemory != scratch);
     ASSERT(pinnedRegs.sizeRegister != baseMemory);
@@ -58,20 +57,13 @@ Expected<MacroAssemblerCodeRef<WasmEntryPtrTag>, BindingFailure> wasmToWasm(unsi
     jit.loadWasmContextInstance(sizeRegAsScratch); // Old Instance*
     // Get the callee's Wasm::Instance and set it as WasmContext's instance. The caller will take care of restoring its own Instance.
     jit.loadPtr(JIT::Address(sizeRegAsScratch, Instance::offsetOfTargetInstance(importIndex)), baseMemory); // Instance*.
-    // While we're accessing that cacheline, also get the wasm entrypoint and callee so we can tail call to it below.
+    // While we're accessing that cacheline, also get the wasm entrypoint so we can tail call to it below.
     jit.loadPtr(JIT::Address(sizeRegAsScratch, Instance::offsetOfWasmEntrypointLoadLocation(importIndex)), scratch);
-    jit.loadPtr(JIT::Address(sizeRegAsScratch, Instance::offsetOfBoxedCalleeLoadLocation(importIndex)), callee);
     jit.storeWasmContextInstance(baseMemory);
 
     jit.loadPtr(JIT::Address(sizeRegAsScratch, Instance::offsetOfCachedStackLimit()), sizeRegAsScratch);
     jit.storePtr(sizeRegAsScratch, JIT::Address(baseMemory, Instance::offsetOfCachedStackLimit()));
 
-    // Set the callee slot in the call frame
-    jit.loadPtr(callee, callee);
-    // At this point, we have been called, so ReturnPC is on the stack, but we have not yet pushed the frame pointer,
-    // so we have to subtract the space for CallerFrame from the callee slot
-    jit.storePtr(callee, JIT::Address(JIT::stackPointerRegister, CallFrameSlot::callee * sizeof(Register) - sizeof(CPURegister)));
-
     // FIXME the following code assumes that all Wasm::Instance have the same pinned registers. https://bugs.webkit.org/show_bug.cgi?id=162952
     // Set up the callee's baseMemory register as well as the memory size registers.
     {
index 93314ab..e8ddf15 100644 (file)
 
 namespace JSC { namespace Wasm {
 
-Callee::Callee(Wasm::CompilationMode compilationMode)
+Callee::Callee(Wasm::CompilationMode compilationMode, Entrypoint&& entrypoint)
     : m_compilationMode(compilationMode)
+    , m_entrypoint(WTFMove(entrypoint))
 {
     CalleeRegistry::singleton().registerCallee(this);
 }
 
-Callee::Callee(Wasm::CompilationMode compilationMode, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name)
+Callee::Callee(Wasm::CompilationMode compilationMode, Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name)
     : m_compilationMode(compilationMode)
+    , m_entrypoint(WTFMove(entrypoint))
     , m_indexOrName(index, WTFMove(name))
 {
     CalleeRegistry::singleton().registerCallee(this);
@@ -50,48 +52,6 @@ Callee::~Callee()
     CalleeRegistry::singleton().unregisterCallee(this);
 }
 
-
-inline void repatchMove(const AbstractLocker&, CodeLocationDataLabelPtr<WasmEntryPtrTag> moveLocation, Callee& targetCallee)
-{
-    MacroAssembler::repatchPointer(moveLocation, CalleeBits::boxWasm(&targetCallee));
-}
-
-inline void repatchCall(const AbstractLocker&, CodeLocationNearCall<WasmEntryPtrTag> callLocation, Callee& targetCallee)
-{
-    MacroAssembler::repatchNearCall(callLocation, CodeLocationLabel<WasmEntryPtrTag>(targetCallee.code()));
-}
-
-void BBQCallee::addCaller(const AbstractLocker&, LinkBuffer& linkBuffer, UnlinkedMoveAndCall call)
-{
-    auto moveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(call.moveLocation);
-    auto callLocation = linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call.callLocation);
-    m_moveLocations.append(moveLocation);
-    m_callLocations.append(callLocation);
-}
-
-void BBQCallee::addAndLinkCaller(const AbstractLocker& locker, LinkBuffer& linkBuffer, UnlinkedMoveAndCall call)
-{
-    RELEASE_ASSERT(entrypoint().compilation);
-
-    auto moveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(call.moveLocation);
-    auto callLocation = linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call.callLocation);
-    m_moveLocations.append(moveLocation);
-    m_callLocations.append(callLocation);
-
-    Callee& targetCallee = m_replacement ? static_cast<Callee&>(*m_replacement) : *this;
-    repatchMove(locker, moveLocation, targetCallee);
-    repatchCall(locker, callLocation, targetCallee);
-}
-
-void BBQCallee::repatchCallers(const AbstractLocker& locker, Callee& targetCallee)
-{
-    RELEASE_ASSERT(targetCallee.entrypoint().compilation);
-    for (auto moveLocation : m_moveLocations)
-        repatchMove(locker, moveLocation, targetCallee);
-    for (auto callLocation : m_callLocations)
-        repatchCall(locker, callLocation, targetCallee);
-}
-
 } } // namespace JSC::Wasm
 
 #endif // ENABLE(WEBASSEMBLY)
index 7eeed52..f7ea4d0 100644 (file)
@@ -40,12 +40,11 @@ namespace JSC { namespace Wasm {
 class Callee : public ThreadSafeRefCounted<Callee> {
     WTF_MAKE_FAST_ALLOCATED;
 public:
-    Entrypoint& entrypoint() { return m_entrypoint; }
-    MacroAssemblerCodePtr<WasmEntryPtrTag> code() const { return m_entrypoint.compilation->code().retagged<WasmEntryPtrTag>(); }
-    IndexOrName indexOrName() const { return m_indexOrName; }
-    CompilationMode compilationMode() const { return m_compilationMode; }
+    MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint() const { return m_entrypoint.compilation->code().retagged<WasmEntryPtrTag>(); }
 
     RegisterAtOffsetList* calleeSaveRegisters() { return &m_entrypoint.calleeSaveRegisters; }
+    IndexOrName indexOrName() const { return m_indexOrName; }
+    CompilationMode compilationMode() const { return m_compilationMode; }
 
     std::tuple<void*, void*> range() const
     {
@@ -57,70 +56,78 @@ public:
     JS_EXPORT_PRIVATE virtual ~Callee();
 
 protected:
-    JS_EXPORT_PRIVATE Callee(CompilationMode);
-    JS_EXPORT_PRIVATE Callee(CompilationMode, size_t, std::pair<const Name*, RefPtr<NameSection>>&&);
+    JS_EXPORT_PRIVATE Callee(Wasm::CompilationMode, Wasm::Entrypoint&&);
+    JS_EXPORT_PRIVATE Callee(Wasm::CompilationMode, Wasm::Entrypoint&&, size_t, std::pair<const Name*, RefPtr<NameSection>>&&);
 
 private:
     CompilationMode m_compilationMode;
-    Entrypoint m_entrypoint;
+    Wasm::Entrypoint m_entrypoint;
     IndexOrName m_indexOrName;
 };
 
 class OMGCallee final : public Callee {
 public:
-    static Ref<OMGCallee> create(size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name)
+    static Ref<OMGCallee> create(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls)
     {
-        return adoptRef(*new OMGCallee(index, WTFMove(name)));
+        return adoptRef(*new OMGCallee(WTFMove(entrypoint), index, WTFMove(name), WTFMove(unlinkedCalls)));
     }
 
+    Vector<UnlinkedWasmToWasmCall>& wasmToWasmCallsites() { return m_wasmToWasmCallsites; }
+
 private:
-    OMGCallee(size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name)
-        : Callee(CompilationMode::OMGMode, index, WTFMove(name))
+    OMGCallee(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls)
+        : Callee(Wasm::CompilationMode::OMGMode, WTFMove(entrypoint), index, WTFMove(name))
+        , m_wasmToWasmCallsites(WTFMove(unlinkedCalls))
     {
     }
+
+    Vector<UnlinkedWasmToWasmCall> m_wasmToWasmCallsites;
 };
 
 class OMGForOSREntryCallee final : public Callee {
 public:
-    static Ref<OMGForOSREntryCallee> create(size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, unsigned osrEntryScratchBufferSize, uint32_t loopIndex)
+    static Ref<OMGForOSREntryCallee> create(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, unsigned osrEntryScratchBufferSize, uint32_t loopIndex, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls)
     {
-        return adoptRef(*new OMGForOSREntryCallee(index, WTFMove(name), osrEntryScratchBufferSize, loopIndex));
+        return adoptRef(*new OMGForOSREntryCallee(WTFMove(entrypoint), index, WTFMove(name), osrEntryScratchBufferSize, loopIndex, WTFMove(unlinkedCalls)));
     }
 
     unsigned osrEntryScratchBufferSize() const { return m_osrEntryScratchBufferSize; }
     uint32_t loopIndex() const { return m_loopIndex; }
+    Vector<UnlinkedWasmToWasmCall>& wasmToWasmCallsites() { return m_wasmToWasmCallsites; }
 
 private:
-    OMGForOSREntryCallee(size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, unsigned osrEntryScratchBufferSize, uint32_t loopIndex)
-        : Callee(CompilationMode::OMGForOSREntryMode, index, WTFMove(name))
+    OMGForOSREntryCallee(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, unsigned osrEntryScratchBufferSize, uint32_t loopIndex, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls)
+        : Callee(Wasm::CompilationMode::OMGForOSREntryMode, WTFMove(entrypoint), index, WTFMove(name))
+        , m_wasmToWasmCallsites(WTFMove(unlinkedCalls))
         , m_osrEntryScratchBufferSize(osrEntryScratchBufferSize)
         , m_loopIndex(loopIndex)
     {
     }
 
+    Vector<UnlinkedWasmToWasmCall> m_wasmToWasmCallsites;
     unsigned m_osrEntryScratchBufferSize;
     uint32_t m_loopIndex;
 };
 
 class EmbedderEntrypointCallee final : public Callee {
 public:
-    static Ref<EmbedderEntrypointCallee> create()
+    static Ref<EmbedderEntrypointCallee> create(Wasm::Entrypoint&& entrypoint)
     {
-        return adoptRef(*new EmbedderEntrypointCallee());
+        return adoptRef(*new EmbedderEntrypointCallee(WTFMove(entrypoint)));
     }
 
 private:
-    EmbedderEntrypointCallee()
-        : Callee(CompilationMode::EmbedderEntrypointMode)
+    EmbedderEntrypointCallee(Wasm::Entrypoint&& entrypoint)
+        : Callee(Wasm::CompilationMode::EmbedderEntrypointMode, WTFMove(entrypoint))
     {
     }
 };
 
 class BBQCallee final : public Callee {
 public:
-    static Ref<BBQCallee> create(size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, std::unique_ptr<TierUpCount>&& tierUpCount)
+    static Ref<BBQCallee> create(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, std::unique_ptr<TierUpCount>&& tierUpCount)
     {
-        return adoptRef(*new BBQCallee(index, WTFMove(name), WTFMove(tierUpCount)));
+        return adoptRef(*new BBQCallee(WTFMove(entrypoint), index, WTFMove(name), WTFMove(tierUpCount)));
     }
 
     OMGForOSREntryCallee* osrEntryCallee() { return m_osrEntryCallee.get(); }
@@ -133,27 +140,20 @@ public:
     void setDidStartCompilingOSREntryCallee(bool value) { m_didStartCompilingOSREntryCallee = value; }
 
     OMGCallee* replacement() { return m_replacement.get(); }
-    void setReplacement(const AbstractLocker&, Ref<OMGCallee>&& replacement)
+    void setReplacement(Ref<OMGCallee>&& replacement)
     {
         m_replacement = WTFMove(replacement);
     }
 
     TierUpCount* tierUpCount() { return m_tierUpCount.get(); }
 
-    void addCaller(const AbstractLocker&, LinkBuffer&, UnlinkedMoveAndCall);
-    void addAndLinkCaller(const AbstractLocker&, LinkBuffer&, UnlinkedMoveAndCall);
-
-    void repatchCallers(const AbstractLocker&, Callee&);
-
 private:
-    BBQCallee(size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, std::unique_ptr<TierUpCount>&& tierUpCount)
-        : Callee(CompilationMode::BBQMode, index, WTFMove(name))
+    BBQCallee(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, std::unique_ptr<TierUpCount>&& tierUpCount)
+        : Callee(Wasm::CompilationMode::BBQMode, WTFMove(entrypoint), index, WTFMove(name))
         , m_tierUpCount(WTFMove(tierUpCount))
     {
     }
 
-    Vector<CodeLocationDataLabelPtr<WasmEntryPtrTag>> m_moveLocations;
-    Vector<CodeLocationNearCall<WasmEntryPtrTag>> m_callLocations;
     RefPtr<OMGForOSREntryCallee> m_osrEntryCallee;
     RefPtr<OMGCallee> m_replacement;
     std::unique_ptr<TierUpCount> m_tierUpCount;
index 6fa4369..b442b61 100644 (file)
@@ -88,16 +88,34 @@ private:
 
 public:
     static unsigned headerSizeInBytes() { return headerSize; }
-    void setupFrameInPrologue(B3::Procedure& proc, B3::Origin origin, B3::BasicBlock* block) const
+    void setupFrameInPrologue(CodeLocationDataLabelPtr<WasmEntryPtrTag>* calleeMoveLocation, B3::Procedure& proc, B3::Origin origin, B3::BasicBlock* block) const
     {
+        static_assert(CallFrameSlot::callee * sizeof(Register) < headerSize, "We rely on this here for now.");
         static_assert(CallFrameSlot::codeBlock * sizeof(Register) < headerSize, "We rely on this here for now.");
 
+        B3::PatchpointValue* getCalleePatchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Int64, origin);
+        getCalleePatchpoint->resultConstraints = { B3::ValueRep::SomeRegister };
+        getCalleePatchpoint->effects = B3::Effects::none();
+        getCalleePatchpoint->setGenerator(
+            [=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
+                GPRReg result = params[0].gpr();
+                MacroAssembler::DataLabelPtr moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), result);
+                jit.addLinkTask([calleeMoveLocation, moveLocation] (LinkBuffer& linkBuffer) {
+                    *calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
+                });
+            });
+
+        B3::Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, origin);
+        B3::Value* offsetOfCallee = block->appendNew<B3::Const64Value>(proc, origin, CallFrameSlot::callee * sizeof(Register));
+        block->appendNew<B3::MemoryValue>(proc, B3::Store, origin,
+            getCalleePatchpoint,
+            block->appendNew<B3::Value>(proc, B3::Add, origin, framePointer, offsetOfCallee));
+
         // FIXME: We shouldn't have to store zero into the CodeBlock* spot in the call frame,
         // but there are places that interpret non-null CodeBlock slot to mean a valid CodeBlock.
         // When doing unwinding, we'll need to verify that the entire runtime is OK with a non-null
         // CodeBlock not implying that the CodeBlock is valid.
         // https://bugs.webkit.org/show_bug.cgi?id=165321
-        B3::Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, origin);
         B3::Value* offsetOfCodeBlock = block->appendNew<B3::Const64Value>(proc, origin, CallFrameSlot::codeBlock * sizeof(Register));
         block->appendNew<B3::MemoryValue>(proc, B3::Store, origin,
             block->appendNew<B3::Const64Value>(proc, origin, 0),
index 3a87f61..d4a9cd6 100644 (file)
@@ -29,6 +29,7 @@
 #if ENABLE(WEBASSEMBLY)
 
 #include "WasmBBQPlanInlines.h"
+#include "WasmCallee.h"
 #include "WasmFormat.h"
 #include "WasmWorklist.h"
 
@@ -46,15 +47,7 @@ CodeBlock::CodeBlock(Context* context, MemoryMode mode, ModuleInformation& modul
 {
     RefPtr<CodeBlock> protectedThis = this;
 
-    m_callees.resize(m_calleeCount);
-    m_boxedCallees.resize(m_calleeCount);
-    for (uint32_t i = 0; i < m_calleeCount; i++) {
-        uint32_t functionIndexSpace = moduleInformation.importFunctionCount() + i;
-        m_callees[i] = BBQCallee::create(functionIndexSpace, moduleInformation.nameSection->get(functionIndexSpace), makeUnique<TierUpCount>());
-        m_boxedCallees[i] = CalleeBits::boxWasm(m_callees[i].ptr());
-    }
-
-    m_plan = adoptRef(*new BBQPlan(context, m_callees, makeRef(moduleInformation), BBQPlan::FullCompile, createSharedTask<Plan::CallbackType>([this, protectedThis = WTFMove(protectedThis)] (Plan&) {
+    m_plan = adoptRef(*new BBQPlan(context, makeRef(moduleInformation), BBQPlan::FullCompile, createSharedTask<Plan::CallbackType>([this, protectedThis = WTFMove(protectedThis)] (Plan&) {
         auto locker = holdLock(m_lock);
         if (m_plan->failed()) {
             m_errorMessage = m_plan->errorMessage();
@@ -63,18 +56,21 @@ CodeBlock::CodeBlock(Context* context, MemoryMode mode, ModuleInformation& modul
         }
 
         // FIXME: we should eventually collect the BBQ code.
+        m_callees.resize(m_calleeCount);
         m_optimizedCallees.resize(m_calleeCount);
         m_wasmIndirectCallEntryPoints.resize(m_calleeCount);
 
-        m_plan->initializeCallees([&] (unsigned calleeIndex, RefPtr<EmbedderEntrypointCallee>&& embedderEntrypointCallee) {
+        m_plan->initializeCallees([&] (unsigned calleeIndex, RefPtr<Wasm::Callee>&& embedderEntrypointCallee, Ref<Wasm::Callee>&& wasmEntrypointCallee) {
             if (embedderEntrypointCallee) {
                 auto result = m_embedderCallees.set(calleeIndex, WTFMove(embedderEntrypointCallee));
                 ASSERT_UNUSED(result, result.isNewEntry);
             }
-            m_wasmIndirectCallEntryPoints[calleeIndex] = m_callees[calleeIndex]->code();
+            m_callees[calleeIndex] = WTFMove(wasmEntrypointCallee);
+            m_wasmIndirectCallEntryPoints[calleeIndex] = m_callees[calleeIndex]->entrypoint();
         });
 
         m_wasmToWasmExitStubs = m_plan->takeWasmToWasmExitStubs();
+        m_wasmToWasmCallsites = m_plan->takeWasmToWasmCallsites();
 
         setCompilationFinished();
     }), WTFMove(createEmbedderWrapper), throwWasmException));
index 10ef24c..dea2ffa 100644 (file)
@@ -28,7 +28,6 @@
 #if ENABLE(WEBASSEMBLY)
 
 #include "MacroAssemblerCodeRef.h"
-#include "WasmCallee.h"
 #include "WasmEmbedder.h"
 #include "WasmTierUpCount.h"
 #include <wtf/Lock.h>
@@ -41,8 +40,9 @@ namespace JSC {
 
 namespace Wasm {
 
-class BBQPlan;
+class Callee;
 struct Context;
+class BBQPlan;
 class OMGPlan;
 struct ModuleInformation;
 struct UnlinkedWasmToWasmCall;
@@ -76,7 +76,7 @@ public:
 
     // These two callee getters are only valid once the callees have been populated.
 
-    EmbedderEntrypointCallee& embedderEntrypointCalleeFromFunctionIndexSpace(unsigned functionIndexSpace)
+    Callee& embedderEntrypointCalleeFromFunctionIndexSpace(unsigned functionIndexSpace)
     {
         ASSERT(runnable());
         RELEASE_ASSERT(functionIndexSpace >= functionImportCount());
@@ -86,27 +86,29 @@ public:
         RELEASE_ASSERT(callee);
         return *callee;
     }
-
-    BBQCallee& wasmBBQCalleeFromFunctionIndexSpace(unsigned functionIndexSpace)
+    Callee& wasmEntrypointCalleeFromFunctionIndexSpace(unsigned functionIndexSpace)
     {
         ASSERT(runnable());
         RELEASE_ASSERT(functionIndexSpace >= functionImportCount());
         unsigned calleeIndex = functionIndexSpace - functionImportCount();
-        return m_callees[calleeIndex].get();
+        if (m_optimizedCallees[calleeIndex])
+            return *m_optimizedCallees[calleeIndex].get();
+        return *m_callees[calleeIndex].get();
     }
 
-    MacroAssemblerCodePtr<WasmEntryPtrTag>* entrypointLoadLocationFromFunctionIndexSpace(unsigned functionIndexSpace)
+    Callee& wasmBBQCalleeFromFunctionIndexSpace(unsigned functionIndexSpace)
     {
+        ASSERT(runnable());
         RELEASE_ASSERT(functionIndexSpace >= functionImportCount());
         unsigned calleeIndex = functionIndexSpace - functionImportCount();
-        return &m_wasmIndirectCallEntryPoints[calleeIndex];
+        return *m_callees[calleeIndex].get();
     }
 
-    void** boxedCalleeLoadLocationFromFunctionIndexSpace(unsigned functionIndexSpace)
+    MacroAssemblerCodePtr<WasmEntryPtrTag>* entrypointLoadLocationFromFunctionIndexSpace(unsigned functionIndexSpace)
     {
         RELEASE_ASSERT(functionIndexSpace >= functionImportCount());
         unsigned calleeIndex = functionIndexSpace - functionImportCount();
-        return &m_boxedCallees[calleeIndex];
+        return &m_wasmIndirectCallEntryPoints[calleeIndex];
     }
 
     bool isSafeToRun(MemoryMode);
@@ -122,11 +124,11 @@ private:
     void setCompilationFinished();
     unsigned m_calleeCount;
     MemoryMode m_mode;
-    Vector<Ref<BBQCallee>> m_callees;
-    Vector<RefPtr<OMGCallee>> m_optimizedCallees;
-    Vector<void*> m_boxedCallees;
-    HashMap<uint32_t, RefPtr<EmbedderEntrypointCallee>, typename DefaultHash<uint32_t>::Hash, WTF::UnsignedWithZeroKeyHashTraits<uint32_t>> m_embedderCallees;
+    Vector<RefPtr<Callee>> m_callees;
+    Vector<RefPtr<Callee>> m_optimizedCallees;
+    HashMap<uint32_t, RefPtr<Callee>, typename DefaultHash<uint32_t>::Hash, WTF::UnsignedWithZeroKeyHashTraits<uint32_t>> m_embedderCallees;
     Vector<MacroAssemblerCodePtr<WasmEntryPtrTag>> m_wasmIndirectCallEntryPoints;
+    Vector<Vector<UnlinkedWasmToWasmCall>> m_wasmToWasmCallsites;
     Vector<MacroAssemblerCodeRef<WasmEntryPtrTag>> m_wasmToWasmExitStubs;
     RefPtr<BBQPlan> m_plan;
     std::atomic<bool> m_compilationFinished { false };
index c78b443..a2379fd 100644 (file)
@@ -41,7 +41,6 @@ class ExecState;
 namespace Wasm {
 
 struct CompilationContext;
-class EmbedderEntrypointCallee;
 class Instance;
 struct InternalFunction;
 struct ModuleInformation;
@@ -49,7 +48,7 @@ class Signature;
 struct UnlinkedWasmToWasmCall;
 
 // Create wrapper code to call from embedder -> WebAssembly.
-using CreateEmbedderWrapper = WTF::Function<void(CompilationContext&, const Signature&, EmbedderEntrypointCallee*, const ModuleInformation&, MemoryMode)>;
+using CreateEmbedderWrapper = WTF::Function<std::unique_ptr<InternalFunction>(CompilationContext&, const Signature&, Vector<UnlinkedWasmToWasmCall>*, const ModuleInformation&, MemoryMode, uint32_t)>;
 
 // Called as soon as an exception is detected. The return value is the PC to continue at.
 using ThrowWasmException = void* (*)(ExecState*, Wasm::ExceptionType, Instance*);
index ea0e0b6..0669bb4 100644 (file)
@@ -287,15 +287,10 @@ inline bool isValidNameType(Int val)
     return false;
 }
 
-struct UnlinkedMoveAndCall {
-    MacroAssembler::DataLabelPtr moveLocation;
-    MacroAssembler::Call callLocation;
-};
-
 struct UnlinkedWasmToWasmCall {
     WTF_MAKE_STRUCT_FAST_ALLOCATED;
-    size_t targetFunctionIndexSpace;
-    UnlinkedMoveAndCall unlinkedMoveAndCall;
+    CodeLocationNearCall<WasmEntryPtrTag> callLocation;
+    size_t functionIndexSpace;
 };
 
 struct Entrypoint {
@@ -318,12 +313,10 @@ struct WasmToWasmImportableFunction {
     using LoadLocation = MacroAssemblerCodePtr<WasmEntryPtrTag>*;
     static ptrdiff_t offsetOfSignatureIndex() { return OBJECT_OFFSETOF(WasmToWasmImportableFunction, signatureIndex); }
     static ptrdiff_t offsetOfEntrypointLoadLocation() { return OBJECT_OFFSETOF(WasmToWasmImportableFunction, entrypointLoadLocation); }
-    static ptrdiff_t offsetOfBoxedCalleeLoadLocation() { return OBJECT_OFFSETOF(WasmToWasmImportableFunction, boxedCalleeLoadLocation); }
 
     // FIXME: Pack signature index and code pointer into one 64-bit value. See <https://bugs.webkit.org/show_bug.cgi?id=165511>.
     SignatureIndex signatureIndex { Signature::invalidIndex };
     LoadLocation entrypointLoadLocation;
-    void** boxedCalleeLoadLocation;
 };
 using FunctionIndexSpace = Vector<WasmToWasmImportableFunction>;
 
index 7e2124f..f4f3508 100644 (file)
@@ -130,7 +130,6 @@ public:
         WasmToWasmImportableFunction::LoadLocation wasmEntrypointLoadLocation { nullptr };
         MacroAssemblerCodePtr<WasmEntryPtrTag> wasmToEmbedderStub;
         void* importFunction { nullptr }; // In a JS embedding, this is a WriteBarrier<JSObject>.
-        void** boxedCalleeLoadLocation { nullptr };
     };
     unsigned numImportFunctions() const { return m_numImportFunctions; }
     ImportFunctionInfo* importFunctionInfo(size_t importFunctionNum)
@@ -142,7 +141,6 @@ public:
     static size_t offsetOfWasmEntrypointLoadLocation(size_t importFunctionNum) { return offsetOfTail() + importFunctionNum * sizeof(ImportFunctionInfo) + OBJECT_OFFSETOF(ImportFunctionInfo, wasmEntrypointLoadLocation); }
     static size_t offsetOfWasmToEmbedderStub(size_t importFunctionNum) { return offsetOfTail() + importFunctionNum * sizeof(ImportFunctionInfo) + OBJECT_OFFSETOF(ImportFunctionInfo, wasmToEmbedderStub); }
     static size_t offsetOfImportFunction(size_t importFunctionNum) { return offsetOfTail() + importFunctionNum * sizeof(ImportFunctionInfo) + OBJECT_OFFSETOF(ImportFunctionInfo, importFunction); }
-    static size_t offsetOfBoxedCalleeLoadLocation(size_t importFunctionNum) { return offsetOfTail() + importFunctionNum * sizeof(ImportFunctionInfo) + OBJECT_OFFSETOF(ImportFunctionInfo, boxedCalleeLoadLocation); }
     template<typename T> T* importFunction(unsigned importFunctionNum) { return reinterpret_cast<T*>(&importFunctionInfo(importFunctionNum)->importFunction); }
 
     static_assert(sizeof(ImportFunctionInfo) == WTF::roundUpToMultipleOf<sizeof(uint64_t)>(sizeof(ImportFunctionInfo)), "We rely on this for the alignment to be correct");
index 3eb4c1a..db0d68c 100644 (file)
@@ -54,15 +54,15 @@ namespace WasmOMGForOSREntryPlanInternal {
 static constexpr bool verbose = false;
 }
 
-OMGForOSREntryPlan::OMGForOSREntryPlan(Context* context, Ref<Module>&& module, Ref<BBQCallee>&& callee, uint32_t functionIndex, uint32_t loopIndex, CodeBlock& codeBlock, CompletionTask&& task)
+OMGForOSREntryPlan::OMGForOSREntryPlan(Context* context, Ref<Module>&& module, Ref<BBQCallee>&& callee, uint32_t functionIndex, uint32_t loopIndex, MemoryMode mode, CompletionTask&& task)
     : Base(context, makeRef(const_cast<ModuleInformation&>(module->moduleInformation())), WTFMove(task))
     , m_module(WTFMove(module))
-    , m_codeBlock(codeBlock)
+    , m_codeBlock(*m_module->codeBlockFor(mode))
     , m_callee(WTFMove(callee))
     , m_functionIndex(functionIndex)
     , m_loopIndex(loopIndex)
 {
-    setMode(codeBlock.mode());
+    setMode(mode);
     ASSERT(m_codeBlock->runnable());
     ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(m_mode));
     dataLogLnIf(WasmOMGForOSREntryPlanInternal::verbose, "Starting OMGForOSREntry plan for ", functionIndex, " of module: ", RawPointer(&m_module.get()));
@@ -81,33 +81,45 @@ void OMGForOSREntryPlan::work(CompilationEffort)
     const Signature& signature = SignatureInformation::get(signatureIndex);
     ASSERT(validateFunction(function, signature, m_moduleInformation.get()));
 
+    Vector<UnlinkedWasmToWasmCall> unlinkedCalls;
     CompilationContext context;
     unsigned osrEntryScratchBufferSize = 0;
-    auto parseAndCompileResult = parseAndCompile(context, function, signature, m_codeBlock->m_wasmToWasmExitStubs, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::OMGForOSREntryMode, m_functionIndex, m_loopIndex);
+    auto parseAndCompileResult = parseAndCompile(context, function, signature, unlinkedCalls, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::OMGForOSREntryMode, m_functionIndex, m_loopIndex);
 
     if (UNLIKELY(!parseAndCompileResult)) {
         fail(holdLock(m_lock), makeString(parseAndCompileResult.error(), "when trying to tier up ", String::number(m_functionIndex)));
         return;
     }
 
+    Entrypoint omgEntrypoint;
     LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, JITCompilationCanFail);
     if (UNLIKELY(linkBuffer.didFailToAllocate())) {
         Base::fail(holdLock(m_lock), makeString("Out of executable memory while tiering up function at index ", String::number(m_functionIndex)));
         return;
     }
 
-    Ref<OMGForOSREntryCallee> callee = OMGForOSREntryCallee::create(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), osrEntryScratchBufferSize, m_loopIndex);
-    callee->entrypoint().calleeSaveRegisters = WTFMove(context.calleeSaveRegisters);
-    callee->entrypoint().compilation = makeUnique<B3::Compilation>(
+    omgEntrypoint.compilation = makeUnique<B3::Compilation>(
         FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "WebAssembly OMGForOSREntry function[%i] %s name %s", m_functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
         WTFMove(context.wasmEntrypointByproducts));
 
+    omgEntrypoint.calleeSaveRegisters = WTFMove(parseAndCompileResult.value()->entrypoint.calleeSaveRegisters);
+
+    MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
     ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(mode()));
+    Ref<OMGForOSREntryCallee> callee = OMGForOSREntryCallee::create(WTFMove(omgEntrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), osrEntryScratchBufferSize, m_loopIndex, WTFMove(unlinkedCalls));
     {
+        MacroAssembler::repatchPointer(parseAndCompileResult.value()->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
+        entrypoint = callee->entrypoint();
+
         auto locker = holdLock(m_codeBlock->m_lock);
-        for (auto& call : context.outgoingCalls) {
-            BBQCallee& targetCallee = m_codeBlock->wasmBBQCalleeFromFunctionIndexSpace(call.targetFunctionIndexSpace);
-            targetCallee.addAndLinkCaller(locker, linkBuffer, call.unlinkedMoveAndCall);
+        for (auto& call : callee->wasmToWasmCallsites()) {
+            MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
+            if (call.functionIndexSpace < m_module->moduleInformation().importFunctionCount())
+                entrypoint = m_codeBlock->m_wasmToWasmExitStubs[call.functionIndexSpace].code();
+            else
+                entrypoint = m_codeBlock->wasmEntrypointCalleeFromFunctionIndexSpace(call.functionIndexSpace).entrypoint().retagged<WasmEntryPtrTag>();
+
+            MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
         }
     }
     resetInstructionCacheOnAllThreads();
@@ -120,6 +132,19 @@ void OMGForOSREntryPlan::work(CompilationEffort)
             m_callee->tierUpCount()->osrEntryTriggers()[m_loopIndex] = TierUpCount::TriggerReason::CompilationDone;
             m_callee->tierUpCount()->m_compilationStatusForOMGForOSREntry = TierUpCount::CompilationStatus::Compiled;
         }
+        WTF::storeStoreFence();
+        // It is possible that a new OMG callee is added while we release m_codeBlock->lock.
+        // Until we add OMGForOSREntry callee to BBQCallee's m_osrEntryCallee, this new OMG function linking does not happen for this OMGForOSREntry callee.
+        // We re-link this OMGForOSREntry callee again not to miss that chance.
+        for (auto& call : callee->wasmToWasmCallsites()) {
+            MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
+            if (call.functionIndexSpace < m_module->moduleInformation().importFunctionCount())
+                entrypoint = m_codeBlock->m_wasmToWasmExitStubs[call.functionIndexSpace].code();
+            else
+                entrypoint = m_codeBlock->wasmEntrypointCalleeFromFunctionIndexSpace(call.functionIndexSpace).entrypoint().retagged<WasmEntryPtrTag>();
+
+            MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
+        }
     }
     dataLogLnIf(WasmOMGForOSREntryPlanInternal::verbose, "Finished OMGForOSREntry ", m_functionIndex, " with tier up count at: ", m_callee->tierUpCount()->count());
     complete(holdLock(m_lock));
index d56ea42..0151da6 100644 (file)
@@ -48,7 +48,7 @@ public:
     bool multiThreaded() const override { return false; }
 
     // Note: CompletionTask should not hold a reference to the Plan otherwise there will be a reference cycle.
-    OMGForOSREntryPlan(Context*, Ref<Module>&&, Ref<BBQCallee>&&, uint32_t functionIndex, uint32_t loopIndex, CodeBlock&, CompletionTask&&);
+    OMGForOSREntryPlan(Context*, Ref<Module>&&, Ref<BBQCallee>&&, uint32_t functionIndex, uint32_t loopIndex, MemoryMode, CompletionTask&&);
 
 private:
     // For some reason friendship doesn't extend to parent classes...
index 748e549..983a36a 100644 (file)
@@ -54,13 +54,13 @@ namespace WasmOMGPlanInternal {
 static constexpr bool verbose = false;
 }
 
-OMGPlan::OMGPlan(Context* context, Ref<Module>&& module, uint32_t functionIndex, CodeBlock& codeBlock, CompletionTask&& task)
+OMGPlan::OMGPlan(Context* context, Ref<Module>&& module, uint32_t functionIndex, MemoryMode mode, CompletionTask&& task)
     : Base(context, makeRef(const_cast<ModuleInformation&>(module->moduleInformation())), WTFMove(task))
     , m_module(WTFMove(module))
-    , m_codeBlock(codeBlock)
+    , m_codeBlock(*m_module->codeBlockFor(mode))
     , m_functionIndex(functionIndex)
 {
-    setMode(codeBlock.mode());
+    setMode(mode);
     ASSERT(m_codeBlock->runnable());
     ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(m_mode));
     dataLogLnIf(WasmOMGPlanInternal::verbose, "Starting OMG plan for ", functionIndex, " of module: ", RawPointer(&m_module.get()));
@@ -79,48 +79,57 @@ void OMGPlan::work(CompilationEffort)
     const Signature& signature = SignatureInformation::get(signatureIndex);
     ASSERT(validateFunction(function, signature, m_moduleInformation.get()));
 
+    Vector<UnlinkedWasmToWasmCall> unlinkedCalls;
     unsigned osrEntryScratchBufferSize;
     CompilationContext context;
-    auto parseAndCompileResult = parseAndCompile(context, function, signature, m_codeBlock->m_wasmToWasmExitStubs, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::OMGMode, m_functionIndex, UINT32_MAX);
+    auto parseAndCompileResult = parseAndCompile(context, function, signature, unlinkedCalls, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::OMGMode, m_functionIndex, UINT32_MAX);
 
     if (UNLIKELY(!parseAndCompileResult)) {
         fail(holdLock(m_lock), makeString(parseAndCompileResult.error(), "when trying to tier up ", String::number(m_functionIndex)));
         return;
     }
 
+    Entrypoint omgEntrypoint;
     LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, JITCompilationCanFail);
     if (UNLIKELY(linkBuffer.didFailToAllocate())) {
         Base::fail(holdLock(m_lock), makeString("Out of executable memory while tiering up function at index ", String::number(m_functionIndex)));
         return;
     }
 
-    Ref<OMGCallee> omgCallee = OMGCallee::create(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace));
-    omgCallee->entrypoint().compilation = makeUnique<B3::Compilation>(
+    omgEntrypoint.compilation = makeUnique<B3::Compilation>(
         FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "WebAssembly OMG function[%i] %s name %s", m_functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
         WTFMove(context.wasmEntrypointByproducts));
 
-    omgCallee->entrypoint().calleeSaveRegisters = WTFMove(context.calleeSaveRegisters);
+    omgEntrypoint.calleeSaveRegisters = WTFMove(parseAndCompileResult.value()->entrypoint.calleeSaveRegisters);
 
-    BBQCallee& bbqCallee = m_codeBlock->m_callees[m_functionIndex].get();
+    MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
     {
         ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(mode()));
+        Ref<OMGCallee> callee = OMGCallee::create(WTFMove(omgEntrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), WTFMove(unlinkedCalls));
+        MacroAssembler::repatchPointer(parseAndCompileResult.value()->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
         ASSERT(!m_codeBlock->m_optimizedCallees[m_functionIndex]);
+        entrypoint = callee->entrypoint();
 
         // We want to make sure we publish our callee at the same time as we link our callsites. This enables us to ensure we
         // always call the fastest code. Any function linked after us will see our new code and the new callsites, which they
         // will update. It's also ok if they publish their code before we reset the instruction caches because after we release
         // the lock our code is ready to be published too.
         LockHolder holder(m_codeBlock->m_lock);
-        m_codeBlock->m_optimizedCallees[m_functionIndex] = omgCallee.copyRef();
+        m_codeBlock->m_optimizedCallees[m_functionIndex] = callee.copyRef();
         {
+            BBQCallee& bbqCallee = *static_cast<BBQCallee*>(m_codeBlock->m_callees[m_functionIndex].get());
             auto locker = holdLock(bbqCallee.tierUpCount()->getLock());
-            bbqCallee.setReplacement(holder, omgCallee.copyRef());
+            bbqCallee.setReplacement(callee.copyRef());
             bbqCallee.tierUpCount()->m_compilationStatusForOMG = TierUpCount::CompilationStatus::Compiled;
         }
-
-        for (auto& call : context.outgoingCalls) {
-            BBQCallee& targetCallee = m_codeBlock->wasmBBQCalleeFromFunctionIndexSpace(call.targetFunctionIndexSpace);
-            targetCallee.addAndLinkCaller(holder, linkBuffer, call.unlinkedMoveAndCall);
+        for (auto& call : callee->wasmToWasmCallsites()) {
+            MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
+            if (call.functionIndexSpace < m_module->moduleInformation().importFunctionCount())
+                entrypoint = m_codeBlock->m_wasmToWasmExitStubs[call.functionIndexSpace].code();
+            else
+                entrypoint = m_codeBlock->wasmEntrypointCalleeFromFunctionIndexSpace(call.functionIndexSpace).entrypoint().retagged<WasmEntryPtrTag>();
+
+            MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
         }
     }
 
@@ -130,14 +139,30 @@ void OMGPlan::work(CompilationEffort)
     resetInstructionCacheOnAllThreads();
     WTF::storeStoreFence(); // This probably isn't necessary but it's good to be paranoid.
 
-    m_codeBlock->m_wasmIndirectCallEntryPoints[m_functionIndex] = omgCallee->code();
-    m_codeBlock->m_boxedCallees[m_functionIndex] = CalleeBits::boxWasm(omgCallee.ptr());
+    m_codeBlock->m_wasmIndirectCallEntryPoints[m_functionIndex] = entrypoint;
     {
         LockHolder holder(m_codeBlock->m_lock);
-        bbqCallee.repatchCallers(holder, omgCallee.get());
+
+        auto repatchCalls = [&] (const Vector<UnlinkedWasmToWasmCall>& callsites) {
+            for (auto& call : callsites) {
+                dataLogLnIf(WasmOMGPlanInternal::verbose, "Considering repatching call at: ", RawPointer(call.callLocation.dataLocation()), " that targets ", call.functionIndexSpace);
+                if (call.functionIndexSpace == functionIndexSpace) {
+                    dataLogLnIf(WasmOMGPlanInternal::verbose, "Repatching call at: ", RawPointer(call.callLocation.dataLocation()), " to ", RawPointer(entrypoint.executableAddress()));
+                    MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
+                }
+            }
+        };
+
+        for (unsigned i = 0; i < m_codeBlock->m_wasmToWasmCallsites.size(); ++i) {
+            repatchCalls(m_codeBlock->m_wasmToWasmCallsites[i]);
+            if (OMGCallee* replacementCallee = static_cast<BBQCallee*>(m_codeBlock->m_callees[i].get())->replacement())
+                repatchCalls(replacementCallee->wasmToWasmCallsites());
+            if (OMGForOSREntryCallee* osrEntryCallee = static_cast<BBQCallee*>(m_codeBlock->m_callees[i].get())->osrEntryCallee())
+                repatchCalls(osrEntryCallee->wasmToWasmCallsites());
+        }
     }
 
-    dataLogLnIf(WasmOMGPlanInternal::verbose, "Finished OMG ", m_functionIndex, " with tier up count at: ", m_codeBlock->m_callees[m_functionIndex]->tierUpCount()->count());
+    dataLogLnIf(WasmOMGPlanInternal::verbose, "Finished OMG ", m_functionIndex, " with tier up count at: ", static_cast<BBQCallee*>(m_codeBlock->m_callees[m_functionIndex].get())->tierUpCount()->count());
     complete(holdLock(m_lock));
 }
 
index e689d56..cb79b43 100644 (file)
@@ -46,7 +46,7 @@ public:
     bool multiThreaded() const override { return false; }
 
     // Note: CompletionTask should not hold a reference to the Plan otherwise there will be a reference cycle.
-    OMGPlan(Context*, Ref<Module>&&, uint32_t functionIndex, CodeBlock&, CompletionTask&&);
+    OMGPlan(Context*, Ref<Module>&&, uint32_t functionIndex, MemoryMode, CompletionTask&&);
 
 private:
     // For some reason friendship doesn't extend to parent classes...
index 79c640b..1ade8a8 100644 (file)
@@ -82,7 +82,7 @@ static void triggerOMGReplacementCompile(TierUpCount& tierUp, OMGCallee* replace
     if (compile) {
         dataLogLnIf(Options::verboseOSR(), "triggerOMGReplacement for ", functionIndex);
         // We need to compile the code.
-        Ref<Plan> plan = adoptRef(*new OMGPlan(instance->context(), Ref<Wasm::Module>(instance->module()), functionIndex, codeBlock, Plan::dontFinalize()));
+        Ref<Plan> plan = adoptRef(*new OMGPlan(instance->context(), Ref<Wasm::Module>(instance->module()), functionIndex, codeBlock.mode(), Plan::dontFinalize()));
         ensureWorklist().enqueue(plan.copyRef());
         if (UNLIKELY(!Options::useConcurrentJIT()))
             plan->waitForCompletion();
@@ -197,14 +197,9 @@ static void doOSREntry(Instance* instance, Probe::Context& context, BBQCallee& c
 #else
 #error Unsupported architecture.
 #endif
-
-    // 4. Update the callee slot in the call frame to the OSR entry callee.
-    Register* stackPointer = bitwise_cast<Register*>(context.sp());
-    *(stackPointer + CallFrameSlot::callee - 1) = bitwise_cast<Register>(CalleeBits::boxWasm(&osrEntryCallee));
-
-    // 5. Configure argument registers to jump to OSR entry from the caller of this runtime function.
+    // 4. Configure argument registers to jump to OSR entry from the caller of this runtime function.
     context.gpr(GPRInfo::argumentGPR0) = bitwise_cast<UCPURegister>(buffer);
-    context.gpr(GPRInfo::argumentGPR1) = bitwise_cast<UCPURegister>(osrEntryCallee.code().executableAddress<>());
+    context.gpr(GPRInfo::argumentGPR1) = bitwise_cast<UCPURegister>(osrEntryCallee.entrypoint().executableAddress<>());
 }
 
 void JIT_OPERATION triggerOSREntryNow(Probe::Context& context)
@@ -370,7 +365,7 @@ void JIT_OPERATION triggerOSREntryNow(Probe::Context& context)
 
     if (startOSREntryCompilation) {
         dataLogLnIf(Options::verboseOSR(), "triggerOMGOSR for ", functionIndex);
-        Ref<Plan> plan = adoptRef(*new OMGForOSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::BBQCallee>(callee), functionIndex, loopIndex, codeBlock, Plan::dontFinalize()));
+        Ref<Plan> plan = adoptRef(*new OMGForOSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::BBQCallee>(callee), functionIndex, loopIndex, codeBlock.mode(), Plan::dontFinalize()));
         ensureWorklist().enqueue(plan.copyRef());
         if (UNLIKELY(!Options::useConcurrentJIT()))
             plan->waitForCompletion();
index f13f645..7352202 100644 (file)
 
 namespace JSC { namespace Wasm {
 
-void createJSToWasmWrapper(CompilationContext& compilationContext, const Signature& signature, EmbedderEntrypointCallee* self, const ModuleInformation& info, MemoryMode mode)
+std::unique_ptr<InternalFunction> createJSToWasmWrapper(CompilationContext& compilationContext, const Signature& signature, Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, unsigned functionIndex)
 {
     CCallHelpers& jit = *compilationContext.embedderEntrypointJIT;
 
+    auto result = makeUnique<InternalFunction>();
     jit.emitFunctionPrologue();
 
     // FIXME Stop using 0 as codeBlocks. https://bugs.webkit.org/show_bug.cgi?id=165321
     jit.store64(CCallHelpers::TrustedImm64(0), CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::codeBlock * static_cast<int>(sizeof(Register))));
-    jit.storePtr(CCallHelpers::TrustedImmPtr(CalleeBits::boxWasm(self)), CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::callee * static_cast<int>(sizeof(Register))));
+    MacroAssembler::DataLabelPtr calleeMoveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), GPRInfo::nonPreservedNonReturnGPR);
+    jit.storePtr(GPRInfo::nonPreservedNonReturnGPR, CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::callee * static_cast<int>(sizeof(Register))));
+    CodeLocationDataLabelPtr<WasmEntryPtrTag>* linkedCalleeMove = &result->calleeMoveLocation;
+    jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
+        *linkedCalleeMove = linkBuffer.locationOf<WasmEntryPtrTag>(calleeMoveLocation);
+    });
 
     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
     RegisterSet toSave = pinnedRegs.toSave(mode);
@@ -63,7 +69,7 @@ void createJSToWasmWrapper(CompilationContext& compilationContext, const Signatu
 #endif
 
     RegisterAtOffsetList registersToSpill(toSave, RegisterAtOffsetList::OffsetBaseType::FramePointerBased);
-    self->entrypoint().calleeSaveRegisters = registersToSpill;
+    result->entrypoint.calleeSaveRegisters = registersToSpill;
 
     unsigned totalFrameSize = registersToSpill.size() * sizeof(void*);
     totalFrameSize += WasmCallingConvention::headerSizeInBytes();
@@ -119,10 +125,9 @@ void createJSToWasmWrapper(CompilationContext& compilationContext, const Signatu
         }
 
         emitThrowWasmToJSException(jit, GPRInfo::argumentGPR2, argumentsIncludeI64 ? ExceptionType::I64ArgumentType : ExceptionType::I64ReturnType);
-        return;
+        return result;
     }
 
-    compilationContext.embedderMoveAndCall = UnlinkedMoveAndCall { };
     GPRReg wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
 
     {
@@ -193,10 +198,6 @@ void createJSToWasmWrapper(CompilationContext& compilationContext, const Signatu
 
             jsOffset += sizeof(EncodedJSValue);
         }
-
-        GPRReg calleeGPR = wasmCallingConventionAir().prologueScratch(0);
-        compilationContext.embedderMoveAndCall->moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
-        jit.storePtr(calleeGPR, calleeFrame.withOffset(CallFrameSlot::callee * sizeof(Register)));
     }
 
     if (!!info.memory) {
@@ -220,7 +221,12 @@ void createJSToWasmWrapper(CompilationContext& compilationContext, const Signatu
         jit.cageConditionally(Gigacage::Primitive, baseMemory, scratchOrSize, scratchOrSize);
     }
 
-    compilationContext.embedderMoveAndCall->callLocation = jit.threadSafePatchableNearCall();
+    CCallHelpers::Call call = jit.threadSafePatchableNearCall();
+    unsigned functionIndexSpace = functionIndex + info.importFunctionCount();
+    ASSERT(functionIndexSpace < info.functionIndexSpaceSize());
+    jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndexSpace] (LinkBuffer& linkBuffer) {
+        unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndexSpace });
+    });
 
     for (const RegisterAtOffset& regAtOffset : registersToSpill) {
         GPRReg reg = regAtOffset.reg().gpr();
@@ -260,6 +266,8 @@ void createJSToWasmWrapper(CompilationContext& compilationContext, const Signatu
 
     jit.emitFunctionEpilogue();
     jit.ret();
+
+    return result;
 }
 
 } } // namespace JSC::Wasm
index 93aa24d..b4b98d1 100644 (file)
@@ -41,7 +41,7 @@ namespace JSC {
 
 namespace Wasm {
 
-void createJSToWasmWrapper(CompilationContext&, const Signature&, EmbedderEntrypointCallee*, const ModuleInformation&, MemoryMode);
+std::unique_ptr<InternalFunction> createJSToWasmWrapper(CompilationContext&, const Signature&, Vector<UnlinkedWasmToWasmCall>*, const ModuleInformation&, MemoryMode, uint32_t functionIndex);
 
 } } // namespace JSC::Wasm
 
index fd3ae18..057aa70 100644 (file)
@@ -465,8 +465,6 @@ MacroAssemblerCodePtr<JSEntryPtrTag> WebAssemblyFunction::jsCallEntrypointSlow()
         // FIXME: Currently we just do an indirect jump. But we should teach the Module
         // how to repatch us:
         // https://bugs.webkit.org/show_bug.cgi?id=196570
-        jit.loadPtr(boxedCalleeLoadLocation(), scratchGPR);
-        jit.storePtr(scratchGPR, CCallHelpers::Address(CCallHelpers::stackPointerRegister, CallFrameSlot::callee * sizeof(Register) - sizeof(CallerFrameAndPC)));
         jit.loadPtr(entrypointLoadLocation(), scratchGPR);
         jit.call(scratchGPR, WasmEntryPtrTag);
     }
@@ -538,10 +536,10 @@ MacroAssemblerCodePtr<JSEntryPtrTag> WebAssemblyFunction::jsCallEntrypointSlow()
     return m_jsCallEntrypoint.code();
 }
 
-WebAssemblyFunction* WebAssemblyFunction::create(VM& vm, JSGlobalObject* globalObject, Structure* structure, unsigned length, const String& name, JSWebAssemblyInstance* instance, Wasm::Callee& jsEntrypoint, Wasm::WasmToWasmImportableFunction::LoadLocation wasmToWasmEntrypointLoadLocation, void** boxedCalleeLoadLocation, Wasm::SignatureIndex signatureIndex)
+WebAssemblyFunction* WebAssemblyFunction::create(VM& vm, JSGlobalObject* globalObject, Structure* structure, unsigned length, const String& name, JSWebAssemblyInstance* instance, Wasm::Callee& jsEntrypoint, Wasm::WasmToWasmImportableFunction::LoadLocation wasmToWasmEntrypointLoadLocation, Wasm::SignatureIndex signatureIndex)
 {
     NativeExecutable* executable = vm.getHostFunction(callWebAssemblyFunction, NoIntrinsic, callHostFunctionAsConstructor, nullptr, name);
-    WebAssemblyFunction* function = new (NotNull, allocateCell<WebAssemblyFunction>(vm.heap)) WebAssemblyFunction(vm, globalObject, structure, jsEntrypoint, wasmToWasmEntrypointLoadLocation, boxedCalleeLoadLocation, signatureIndex);
+    WebAssemblyFunction* function = new (NotNull, allocateCell<WebAssemblyFunction>(vm.heap)) WebAssemblyFunction(vm, globalObject, structure, jsEntrypoint, wasmToWasmEntrypointLoadLocation, signatureIndex);
     function->finishCreation(vm, executable, length, name, instance);
     ASSERT_WITH_MESSAGE(!function->isLargeAllocation(), "WebAssemblyFunction should be allocated not in large allocation since it is JSCallee.");
     return function;
@@ -553,10 +551,10 @@ Structure* WebAssemblyFunction::createStructure(VM& vm, JSGlobalObject* globalOb
     return Structure::create(vm, globalObject, prototype, TypeInfo(JSFunctionType, StructureFlags), info());
 }
 
-WebAssemblyFunction::WebAssemblyFunction(VM& vm, JSGlobalObject* globalObject, Structure* structure, Wasm::Callee& jsEntrypoint, Wasm::WasmToWasmImportableFunction::LoadLocation wasmToWasmEntrypointLoadLocation, void** boxedCalleeLoadLocation, Wasm::SignatureIndex signatureIndex)
+WebAssemblyFunction::WebAssemblyFunction(VM& vm, JSGlobalObject* globalObject, Structure* structure, Wasm::Callee& jsEntrypoint, Wasm::WasmToWasmImportableFunction::LoadLocation wasmToWasmEntrypointLoadLocation, Wasm::SignatureIndex signatureIndex)
     : Base { vm, globalObject, structure }
-    , m_jsEntrypoint { jsEntrypoint.code() }
-    , m_importableFunction { signatureIndex, wasmToWasmEntrypointLoadLocation, boxedCalleeLoadLocation }
+    , m_jsEntrypoint { jsEntrypoint.entrypoint() }
+    , m_importableFunction { signatureIndex, wasmToWasmEntrypointLoadLocation }
 { }
 
 void WebAssemblyFunction::visitChildren(JSCell* cell, SlotVisitor& visitor)
index f5adccc..395a5bd 100644 (file)
@@ -62,12 +62,11 @@ public:
 
     DECLARE_EXPORT_INFO;
 
-    JS_EXPORT_PRIVATE static WebAssemblyFunction* create(VM&, JSGlobalObject*, Structure*, unsigned, const String&, JSWebAssemblyInstance*, Wasm::Callee& jsEntrypoint, WasmToWasmImportableFunction::LoadLocation wasmEntrypointLoadLocation, void** boxedCalleeLoadLocation, Wasm::SignatureIndex);
+    JS_EXPORT_PRIVATE static WebAssemblyFunction* create(VM&, JSGlobalObject*, Structure*, unsigned, const String&, JSWebAssemblyInstance*, Wasm::Callee& jsEntrypoint, WasmToWasmImportableFunction::LoadLocation, Wasm::SignatureIndex);
     static Structure* createStructure(VM&, JSGlobalObject*, JSValue);
 
     Wasm::SignatureIndex signatureIndex() const { return m_importableFunction.signatureIndex; }
     WasmToWasmImportableFunction::LoadLocation entrypointLoadLocation() const { return m_importableFunction.entrypointLoadLocation; }
-    void** boxedCalleeLoadLocation() const { return m_importableFunction.boxedCalleeLoadLocation; }
     WasmToWasmImportableFunction importableFunction() const { return m_importableFunction; }
 
     MacroAssemblerCodePtr<WasmEntryPtrTag> jsEntrypoint(ArityCheckMode arity)
@@ -79,7 +78,6 @@ public:
     }
 
     static ptrdiff_t offsetOfEntrypointLoadLocation() { return OBJECT_OFFSETOF(WebAssemblyFunction, m_importableFunction) + WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation(); }
-    static ptrdiff_t offsetOfBoxedCalleeLoadLocation() { return OBJECT_OFFSETOF(WebAssemblyFunction, m_importableFunction) + WasmToWasmImportableFunction::offsetOfBoxedCalleeLoadLocation(); }
 
     MacroAssemblerCodePtr<JSEntryPtrTag> jsCallEntrypoint()
     {
@@ -93,7 +91,7 @@ public:
 
 private:
     static void visitChildren(JSCell*, SlotVisitor&);
-    WebAssemblyFunction(VM&, JSGlobalObject*, Structure*, Wasm::Callee& jsEntrypoint, WasmToWasmImportableFunction::LoadLocation entrypointLoadLocation, void** boxedCalleeLoadLocation, Wasm::SignatureIndex);
+    WebAssemblyFunction(VM&, JSGlobalObject*, Structure*, Wasm::Callee& jsEntrypoint, WasmToWasmImportableFunction::LoadLocation entrypointLoadLocation, Wasm::SignatureIndex);
 
     MacroAssemblerCodePtr<JSEntryPtrTag> jsCallEntrypointSlow();
     ptrdiff_t previousInstanceOffset() const;
index 7af1a04..92ddcf7 100644 (file)
@@ -193,7 +193,6 @@ void WebAssemblyModuleRecord::link(ExecState* exec, JSValue, JSObject* importObj
 
             Wasm::Instance* calleeInstance = nullptr;
             WasmToWasmImportableFunction::LoadLocation entrypointLoadLocation = nullptr;
-            void** boxedCalleeLoadLocation = nullptr;
             JSObject* function = jsCast<JSObject*>(value);
 
             // ii. If v is an Exported Function Exotic Object:
@@ -206,7 +205,6 @@ void WebAssemblyModuleRecord::link(ExecState* exec, JSValue, JSObject* importObj
                     importedSignatureIndex = wasmFunction->signatureIndex();
                     calleeInstance = &wasmFunction->instance()->instance();
                     entrypointLoadLocation = wasmFunction->entrypointLoadLocation();
-                    boxedCalleeLoadLocation = wasmFunction->boxedCalleeLoadLocation();
                 } else {
                     importedSignatureIndex = wasmWrapperFunction->signatureIndex();
                     // b. Let closure be v.[[Closure]].
@@ -225,7 +223,6 @@ void WebAssemblyModuleRecord::link(ExecState* exec, JSValue, JSObject* importObj
             auto* info = m_instance->instance().importFunctionInfo(import.kindIndex);
             info->targetInstance = calleeInstance;
             info->wasmEntrypointLoadLocation = entrypointLoadLocation;
-            info->boxedCalleeLoadLocation = boxedCalleeLoadLocation;
             m_instance->instance().importFunction<WriteBarrier<JSObject>>(import.kindIndex)->set(vm, m_instance.get(), function);
             break;
         }
@@ -350,10 +347,9 @@ void WebAssemblyModuleRecord::link(ExecState* exec, JSValue, JSObject* importObj
             //     c. Return func.
             Wasm::Callee& embedderEntrypointCallee = codeBlock->embedderEntrypointCalleeFromFunctionIndexSpace(index);
             Wasm::WasmToWasmImportableFunction::LoadLocation entrypointLoadLocation = codeBlock->entrypointLoadLocationFromFunctionIndexSpace(index);
-            void** boxedCalleeLoadLocation = codeBlock->boxedCalleeLoadLocationFromFunctionIndexSpace(index);
             Wasm::SignatureIndex signatureIndex = module->signatureIndexFromFunctionIndexSpace(index);
             const Wasm::Signature& signature = Wasm::SignatureInformation::get(signatureIndex);
-            WebAssemblyFunction* function = WebAssemblyFunction::create(vm, globalObject, globalObject->webAssemblyFunctionStructure(), signature.argumentCount(), field, m_instance.get(), embedderEntrypointCallee, entrypointLoadLocation, boxedCalleeLoadLocation, signatureIndex);
+            WebAssemblyFunction* function = WebAssemblyFunction::create(vm, globalObject, globalObject->webAssemblyFunctionStructure(), signature.argumentCount(), field, m_instance.get(), embedderEntrypointCallee, entrypointLoadLocation, signatureIndex);
             wrapper = function;
         }
 
@@ -464,8 +460,7 @@ void WebAssemblyModuleRecord::link(ExecState* exec, JSValue, JSObject* importObj
         } else {
             Wasm::Callee& embedderEntrypointCallee = codeBlock->embedderEntrypointCalleeFromFunctionIndexSpace(startFunctionIndexSpace);
             Wasm::WasmToWasmImportableFunction::LoadLocation entrypointLoadLocation = codeBlock->entrypointLoadLocationFromFunctionIndexSpace(startFunctionIndexSpace);
-            void** boxedCalleeLoadLocation = codeBlock->boxedCalleeLoadLocationFromFunctionIndexSpace(startFunctionIndexSpace);
-            WebAssemblyFunction* function = WebAssemblyFunction::create(vm, globalObject, globalObject->webAssemblyFunctionStructure(), signature.argumentCount(), "start", m_instance.get(), embedderEntrypointCallee, entrypointLoadLocation, boxedCalleeLoadLocation, signatureIndex);
+            WebAssemblyFunction* function = WebAssemblyFunction::create(vm, globalObject, globalObject->webAssemblyFunctionStructure(), signature.argumentCount(), "start", m_instance.get(), embedderEntrypointCallee, entrypointLoadLocation, signatureIndex);
             m_startFunction.set(vm, this, function);
         }
     }
@@ -580,14 +575,13 @@ JSValue WebAssemblyModuleRecord::evaluate(ExecState* exec)
 
             Wasm::Callee& embedderEntrypointCallee = codeBlock->embedderEntrypointCalleeFromFunctionIndexSpace(functionIndex);
             Wasm::WasmToWasmImportableFunction::LoadLocation entrypointLoadLocation = codeBlock->entrypointLoadLocationFromFunctionIndexSpace(functionIndex);
-            void** boxedCalleeLoadLocation = codeBlock->boxedCalleeLoadLocationFromFunctionIndexSpace(functionIndex);
             const Wasm::Signature& signature = Wasm::SignatureInformation::get(signatureIndex);
             // FIXME: Say we export local function "foo" at function index 0.
             // What if we also set it to the table an Element w/ index 0.
             // Does (new Instance(...)).exports.foo === table.get(0)?
             // https://bugs.webkit.org/show_bug.cgi?id=165825
             WebAssemblyFunction* function = WebAssemblyFunction::create(
-                vm, globalObject, globalObject->webAssemblyFunctionStructure(), signature.argumentCount(), String(), m_instance.get(), embedderEntrypointCallee, entrypointLoadLocation, boxedCalleeLoadLocation, signatureIndex);
+                vm, globalObject, globalObject->webAssemblyFunctionStructure(), signature.argumentCount(), String(), m_instance.get(), embedderEntrypointCallee, entrypointLoadLocation, signatureIndex);
 
             m_instance->table(tableIndex)->set(elementIndex, function);
             ++elementIndex;
index 62862b4..cde4f3c 100644 (file)
@@ -60,9 +60,7 @@ WebAssemblyWrapperFunction* WebAssemblyWrapperFunction::create(VM& vm, JSGlobalO
     ASSERT_WITH_MESSAGE(!function->inherits<WebAssemblyWrapperFunction>(vm), "We should never double wrap a wrapper function.");
     String name = "";
     NativeExecutable* executable = vm.getHostFunction(callWebAssemblyWrapperFunction, NoIntrinsic, callHostFunctionAsConstructor, nullptr, name);
-    auto importFunctionInfo = instance->instance().importFunctionInfo(importIndex);
-    static void* nullBoxedCallee = nullptr;
-    WebAssemblyWrapperFunction* result = new (NotNull, allocateCell<WebAssemblyWrapperFunction>(vm.heap)) WebAssemblyWrapperFunction(vm, globalObject, structure, Wasm::WasmToWasmImportableFunction { signatureIndex, &importFunctionInfo->wasmToEmbedderStub, &nullBoxedCallee });
+    WebAssemblyWrapperFunction* result = new (NotNull, allocateCell<WebAssemblyWrapperFunction>(vm.heap)) WebAssemblyWrapperFunction(vm, globalObject, structure, Wasm::WasmToWasmImportableFunction { signatureIndex, &instance->instance().importFunctionInfo(importIndex)->wasmToEmbedderStub } );
     const Wasm::Signature& signature = Wasm::SignatureInformation::get(signatureIndex);
     result->finishCreation(vm, executable, signature.argumentCount(), name, function, instance);
     return result;