We need to set topCallFrame when calling Wasm::Memory::grow from the JIT
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "AllowMacroScratchRegisterUsageIf.h"
32 #include "B3BasicBlockInlines.h"
33 #include "B3CCallValue.h"
34 #include "B3Compile.h"
35 #include "B3ConstPtrValue.h"
36 #include "B3FixSSA.h"
37 #include "B3Generate.h"
38 #include "B3InsertionSet.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3SwitchValue.h"
42 #include "B3UpsilonValue.h"
43 #include "B3Validate.h"
44 #include "B3ValueInlines.h"
45 #include "B3ValueKey.h"
46 #include "B3Variable.h"
47 #include "B3VariableValue.h"
48 #include "B3WasmAddressValue.h"
49 #include "B3WasmBoundsCheckValue.h"
50 #include "JSCInlines.h"
51 #include "ScratchRegisterAllocator.h"
52 #include "VirtualRegister.h"
53 #include "WasmCallingConvention.h"
54 #include "WasmContext.h"
55 #include "WasmExceptionType.h"
56 #include "WasmFunctionParser.h"
57 #include "WasmInstance.h"
58 #include "WasmMemory.h"
59 #include "WasmOMGPlan.h"
60 #include "WasmOpcodeOrigin.h"
61 #include "WasmThunks.h"
62 #include <limits>
63 #include <wtf/Optional.h>
64 #include <wtf/StdLibExtras.h>
65
66 void dumpProcedure(void* ptr)
67 {
68     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
69     proc->dump(WTF::dataFile());
70 }
71
72 namespace JSC { namespace Wasm {
73
74 using namespace B3;
75
76 namespace {
77 namespace WasmB3IRGeneratorInternal {
78 static const bool verbose = false;
79 }
80 }
81
82 class B3IRGenerator {
83 public:
84     struct ControlData {
85         ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
86             : blockType(type)
87             , continuation(continuation)
88             , special(special)
89         {
90             if (signature != Void)
91                 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
92         }
93
94         ControlData()
95         {
96         }
97
98         void dump(PrintStream& out) const
99         {
100             switch (type()) {
101             case BlockType::If:
102                 out.print("If:       ");
103                 break;
104             case BlockType::Block:
105                 out.print("Block:    ");
106                 break;
107             case BlockType::Loop:
108                 out.print("Loop:     ");
109                 break;
110             case BlockType::TopLevel:
111                 out.print("TopLevel: ");
112                 break;
113             }
114             out.print("Continuation: ", *continuation, ", Special: ");
115             if (special)
116                 out.print(*special);
117             else
118                 out.print("None");
119         }
120
121         BlockType type() const { return blockType; }
122
123         bool hasNonVoidSignature() const { return result.size(); }
124
125         BasicBlock* targetBlockForBranch()
126         {
127             if (type() == BlockType::Loop)
128                 return special;
129             return continuation;
130         }
131
132         void convertIfToBlock()
133         {
134             ASSERT(type() == BlockType::If);
135             blockType = BlockType::Block;
136             special = nullptr;
137         }
138
139         using ResultList = Vector<Value*, 1>; // Value must be a Phi
140
141         ResultList resultForBranch() const
142         {
143             if (type() == BlockType::Loop)
144                 return ResultList();
145             return result;
146         }
147
148     private:
149         friend class B3IRGenerator;
150         BlockType blockType;
151         BasicBlock* continuation;
152         BasicBlock* special;
153         ResultList result;
154     };
155
156     typedef Value* ExpressionType;
157     typedef ControlData ControlType;
158     typedef Vector<ExpressionType, 1> ExpressionList;
159     typedef ControlData::ResultList ResultList;
160     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
161
162     static constexpr ExpressionType emptyExpression = nullptr;
163
164     typedef String ErrorType;
165     typedef Unexpected<ErrorType> UnexpectedResult;
166     typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
167     typedef Expected<void, ErrorType> PartialResult;
168     template <typename ...Args>
169     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
170     {
171         using namespace FailureHelper; // See ADL comment in WasmParser.h.
172         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
173     }
174 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
175         if (UNLIKELY(condition))                  \
176             return fail(__VA_ARGS__);             \
177     } while (0)
178
179     B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, CompilationMode, unsigned functionIndex, TierUpCount*, ThrowWasmException);
180
181     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
182     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
183     ExpressionType addConstant(Type, uint64_t);
184
185     // Locals
186     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
187     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
188
189     // Globals
190     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
191     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
192
193     // Memory
194     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
195     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
196     PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
197     PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
198
199     // Basic operators
200     template<OpType>
201     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
202     template<OpType>
203     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
204     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
205
206     // Control flow
207     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
208     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
209     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
210     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
211     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
212     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
213
214     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
215     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
216     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
217     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
218     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
219
220     // Calls
221     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
222     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
223     PartialResult WARN_UNUSED_RETURN addUnreachable();
224
225     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
226     void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
227
228     Value* constant(B3::Type, uint64_t bits, std::optional<Origin> = std::nullopt);
229     void insertConstants();
230
231 private:
232     void emitExceptionCheck(CCallHelpers&, ExceptionType);
233
234     void emitTierUpCheck(uint32_t decrementCount, Origin);
235
236     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
237     B3::Kind memoryKind(B3::Opcode memoryOp);
238     ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
239     void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
240
241     void unify(const ExpressionType phi, const ExpressionType source);
242     void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
243
244     void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
245
246     int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
247
248     void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*);
249     void restoreWebAssemblyGlobalState(const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
250
251     Origin origin();
252
253     FunctionParser<B3IRGenerator>* m_parser { nullptr };
254     const ModuleInformation& m_info;
255     const MemoryMode m_mode { MemoryMode::BoundsChecking };
256     const CompilationMode m_compilationMode { CompilationMode::BBQMode };
257     const unsigned m_functionIndex { UINT_MAX };
258     const TierUpCount* m_tierUp { nullptr };
259
260     Procedure& m_proc;
261     BasicBlock* m_currentBlock { nullptr };
262     Vector<Variable*> m_locals;
263     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
264     HashMap<ValueKey, Value*> m_constantPool;
265     InsertionSet m_constantInsertionValues;
266     GPRReg m_memoryBaseGPR { InvalidGPRReg };
267     GPRReg m_memorySizeGPR { InvalidGPRReg };
268     GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
269     bool m_makesCalls { false };
270
271     Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used.
272     bool m_usesInstanceValue { false };
273     Value* instanceValue()
274     {
275         m_usesInstanceValue = true;
276         return m_instanceValue;
277     }
278
279     uint32_t m_maxNumJSCallArguments { 0 };
280 };
281
282 // Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
283 int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
284 {
285     if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
286         ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
287         return 0;
288     }
289     return offset;
290 }
291
292 void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg)
293 {
294     if (Context::useFastTLS()) {
295         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
296         if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
297             patchpoint->clobber(RegisterSet::macroScratchRegisters());
298         patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
299         patchpoint->setGenerator(
300             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
301                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
302                 jit.storeWasmContextInstance(params[0].gpr());
303             });
304         return;
305     }
306
307     // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
308     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
309     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
310     Effects effects = Effects::none();
311     effects.writesPinned = true;
312     effects.reads = B3::HeapRange::top();
313     patchpoint->effects = effects;
314     patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
315     patchpoint->append(instanceValue(), ValueRep::SomeRegister);
316     GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
317     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
318         jit.move(param[0].gpr(), wasmContextInstanceGPR);
319     });
320 }
321
322 B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
323     : m_info(info)
324     , m_mode(mode)
325     , m_compilationMode(compilationMode)
326     , m_functionIndex(functionIndex)
327     , m_tierUp(tierUp)
328     , m_proc(procedure)
329     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
330     , m_constantInsertionValues(m_proc)
331 {
332     m_currentBlock = m_proc.addBlock();
333
334     // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
335     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
336
337     m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
338     m_proc.pinRegister(m_memoryBaseGPR);
339
340     m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
341     if (!Context::useFastTLS())
342         m_proc.pinRegister(m_wasmContextInstanceGPR);
343
344     if (mode != MemoryMode::Signaling) {
345         ASSERT(!pinnedRegs.sizeRegisters[0].sizeOffset);
346         m_memorySizeGPR = pinnedRegs.sizeRegisters[0].sizeRegister;
347         for (const PinnedSizeRegisterInfo& regInfo : pinnedRegs.sizeRegisters)
348             m_proc.pinRegister(regInfo.sizeRegister);
349     }
350
351     if (throwWasmException)
352         Thunks::singleton().setThrowWasmException(throwWasmException);
353
354     if (info.memory) {
355         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
356             AllowMacroScratchRegisterUsage allowScratch(jit);
357             switch (m_mode) {
358             case MemoryMode::BoundsChecking:
359                 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
360                 break;
361             case MemoryMode::Signaling:
362                 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
363                 break;
364             }
365             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
366         });
367
368         switch (m_mode) {
369         case MemoryMode::BoundsChecking:
370             break;
371         case MemoryMode::Signaling:
372             // Most memory accesses in signaling mode don't do an explicit
373             // exception check because they can rely on fault handling to detect
374             // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
375             // thunk to exist so that it can jump to that thunk.
376             if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
377                 CRASH();
378             break;
379         }
380     }
381
382     wasmCallingConvention().setupFrameInPrologue(&compilation->calleeMoveLocation, m_proc, Origin(), m_currentBlock);
383
384     {
385         B3::Value* framePointer = m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin());
386         B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, pointerType(), Origin());
387         m_instanceValue = stackOverflowCheck;
388         stackOverflowCheck->appendSomeRegister(framePointer);
389         stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters());
390         if (!Context::useFastTLS()) {
391             // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
392             // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
393             stackOverflowCheck->effects.writesPinned = false;
394             stackOverflowCheck->effects.readsPinned = true;
395             stackOverflowCheck->resultConstraint = ValueRep::reg(m_wasmContextInstanceGPR);
396         }
397         stackOverflowCheck->numGPScratchRegisters = 2;
398         stackOverflowCheck->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
399             const Checked<int32_t> wasmFrameSize = params.proc().frameSize();
400             const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
401             const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
402                 // This allows us to elide stack checks for functions that are terminal nodes in the call
403                 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
404                 // having any such terminal node have its parent caller include some extra size in its
405                 // own check for it. The goal here is twofold:
406                 // 1. Emit less code.
407                 // 2. Try to speed things up by skipping stack checks.
408                 minimumParentCheckSize,
409                 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
410                 // spill all arguments to the stack, we ensure that a stack check here covers the
411                 // stack that such a stub would use.
412                 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
413             ));
414             const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
415             bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
416             bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
417
418             GPRReg contextInstance = Context::useFastTLS() ? params[0].gpr() : m_wasmContextInstanceGPR;
419
420             // This allows leaf functions to not do stack checks if their frame size is within
421             // certain limits since their caller would have already done the check.
422             if (needsOverflowCheck) {
423                 AllowMacroScratchRegisterUsage allowScratch(jit);
424                 GPRReg fp = params[1].gpr();
425                 GPRReg scratch1 = params.gpScratch(0);
426                 GPRReg scratch2 = params.gpScratch(1);
427
428                 if (Context::useFastTLS())
429                     jit.loadWasmContextInstance(contextInstance);
430
431                 jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2);
432                 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1);
433                 MacroAssembler::JumpList overflow;
434                 if (UNLIKELY(needUnderflowCheck))
435                     overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp));
436                 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2));
437                 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
438                     linkBuffer.link(overflow, CodeLocationLabel(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
439                 });
440             } else if (m_usesInstanceValue && Context::useFastTLS()) {
441                 // No overflow check is needed, but the instance values still needs to be correct.
442                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister());
443                 jit.loadWasmContextInstance(contextInstance);
444             } else {
445                 // We said we'd return a pointer. We don't actually need to because it isn't used, but the patchpoint conservatively said it had effects (potential stack check) which prevent it from getting removed.
446             }
447         });
448     }
449
450     emitTierUpCheck(TierUpCount::functionEntryDecrement(), Origin());
451 }
452
453 void B3IRGenerator::restoreWebAssemblyGlobalState(const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
454 {
455     restoreWasmContextInstance(proc, block, instance);
456
457     if (!!memory) {
458         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
459         RegisterSet clobbers;
460         clobbers.set(pinnedRegs->baseMemoryPointer);
461         for (auto info : pinnedRegs->sizeRegisters)
462             clobbers.set(info.sizeRegister);
463
464         B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
465         Effects effects = Effects::none();
466         effects.writesPinned = true;
467         effects.reads = B3::HeapRange::top();
468         patchpoint->effects = effects;
469         patchpoint->clobber(clobbers);
470
471         patchpoint->append(instance, ValueRep::SomeRegister);
472
473         patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
474             GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
475             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfMemory()), baseMemory);
476             const auto& sizeRegs = pinnedRegs->sizeRegisters;
477             ASSERT(sizeRegs.size() >= 1);
478             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
479             jit.loadPtr(CCallHelpers::Address(baseMemory, Memory::offsetOfSize()), sizeRegs[0].sizeRegister);
480             jit.loadPtr(CCallHelpers::Address(baseMemory, Memory::offsetOfMemory()), baseMemory);
481             for (unsigned i = 1; i < sizeRegs.size(); ++i)
482                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
483         });
484     }
485 }
486
487 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
488 {
489     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
490     auto jumpToExceptionStub = jit.jump();
491
492     jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
493         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
494     });
495 }
496
497 Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, std::optional<Origin> maybeOrigin)
498 {
499     auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
500         Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits);
501         m_constantInsertionValues.insertValue(0, result);
502         return result;
503     });
504     return result.iterator->value;
505 }
506
507 void B3IRGenerator::insertConstants()
508 {
509     m_constantInsertionValues.execute(m_proc.at(0));
510 }
511
512 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
513 {
514     Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
515     totalBytesChecked += m_locals.size();
516     uint32_t totalBytes;
517     WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
518
519     for (uint32_t i = 0; i < count; ++i) {
520         Variable* local = m_proc.addVariable(toB3Type(type));
521         m_locals.uncheckedAppend(local);
522         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, constant(toB3Type(type), 0, Origin()));
523     }
524     return { };
525 }
526
527 auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
528 {
529     ASSERT(!m_locals.size());
530     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
531
532     m_locals.grow(signature.argumentCount());
533     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
534         [=] (ExpressionType argument, unsigned i) {
535             Variable* argumentVariable = m_proc.addVariable(argument->type());
536             m_locals[i] = argumentVariable;
537             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
538         });
539     return { };
540 }
541
542 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
543 {
544     ASSERT(m_locals[index]);
545     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
546     return { };
547 }
548
549 auto B3IRGenerator::addUnreachable() -> PartialResult
550 {
551     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
552     unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
553         this->emitExceptionCheck(jit, ExceptionType::Unreachable);
554     });
555     unreachable->effects.terminal = true;
556     return { };
557 }
558
559 auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
560 {
561     int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
562         instance->storeTopCallFrame(callFrame);
563
564         if (delta < 0)
565             return -1;
566
567         auto grown = instance->memory()->grow(PageCount(delta));
568         if (!grown) {
569             switch (grown.error()) {
570             case Memory::GrowFailReason::InvalidDelta:
571             case Memory::GrowFailReason::InvalidGrowSize:
572             case Memory::GrowFailReason::WouldExceedMaximum:
573             case Memory::GrowFailReason::OutOfMemory:
574                 return -1;
575             }
576             RELEASE_ASSERT_NOT_REACHED();
577         }
578
579         return grown.value().pageCount();
580     };
581
582     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
583         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(growMemory)),
584         m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, origin()), instanceValue(), delta);
585
586     restoreWebAssemblyGlobalState(m_info.memory, instanceValue(), m_proc, m_currentBlock);
587
588     return { };
589 }
590
591 auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
592 {
593     Value* memoryObject = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfMemory()));
594
595     static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
596     Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), memoryObject, safeCast<int32_t>(Memory::offsetOfSize()));
597     
598     constexpr uint32_t shiftValue = 16;
599     static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
600     Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
601         size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
602
603     result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
604
605     return { };
606 }
607
608 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
609 {
610     ASSERT(m_locals[index]);
611     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
612     return { };
613 }
614
615 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
616 {
617     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
618     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
619     return { };
620 }
621
622 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
623 {
624     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
625     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
626     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
627     return { };
628 }
629
630 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
631 {
632     ASSERT(m_memoryBaseGPR);
633
634     switch (m_mode) {
635     case MemoryMode::BoundsChecking:
636         // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
637         ASSERT(m_memorySizeGPR);
638         ASSERT(sizeOfOperation + offset > offset);
639         m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), m_memorySizeGPR, pointer, sizeOfOperation + offset - 1);
640         break;
641
642     case MemoryMode::Signaling:
643         // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
644         // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
645         // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
646         // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
647         // register + large offset access. We don't think this will be generated frequently.
648         //
649         // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
650         // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
651         // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
652         // any access equal to or greater than 4GiB will trap, no need to add the redzone.
653         if (offset >= Memory::fastMappedRedzoneBytes()) {
654             size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
655             m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
656         }
657         break;
658     }
659     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
660     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
661 }
662
663 inline uint32_t sizeOfLoadOp(LoadOpType op)
664 {
665     switch (op) {
666     case LoadOpType::I32Load8S:
667     case LoadOpType::I32Load8U:
668     case LoadOpType::I64Load8S:
669     case LoadOpType::I64Load8U:
670         return 1;
671     case LoadOpType::I32Load16S:
672     case LoadOpType::I64Load16S:
673     case LoadOpType::I32Load16U:
674     case LoadOpType::I64Load16U:
675         return 2;
676     case LoadOpType::I32Load:
677     case LoadOpType::I64Load32S:
678     case LoadOpType::I64Load32U:
679     case LoadOpType::F32Load:
680         return 4;
681     case LoadOpType::I64Load:
682     case LoadOpType::F64Load:
683         return 8;
684     }
685     RELEASE_ASSERT_NOT_REACHED();
686 }
687
688 inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
689 {
690     if (m_mode == MemoryMode::Signaling)
691         return trapping(memoryOp);
692     return memoryOp;
693 }
694
695 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
696 {
697     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
698
699     switch (op) {
700     case LoadOpType::I32Load8S: {
701         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
702     }
703
704     case LoadOpType::I64Load8S: {
705         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
706         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
707     }
708
709     case LoadOpType::I32Load8U: {
710         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
711     }
712
713     case LoadOpType::I64Load8U: {
714         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
715         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
716     }
717
718     case LoadOpType::I32Load16S: {
719         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
720     }
721     case LoadOpType::I64Load16S: {
722         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
723         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
724     }
725
726     case LoadOpType::I32Load: {
727         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
728     }
729
730     case LoadOpType::I64Load32U: {
731         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
732         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
733     }
734
735     case LoadOpType::I64Load32S: {
736         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
737         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
738     }
739
740     case LoadOpType::I64Load: {
741         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
742     }
743
744     case LoadOpType::F32Load: {
745         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
746     }
747
748     case LoadOpType::F64Load: {
749         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
750     }
751
752     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
753     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
754     case LoadOpType::I32Load16U: {
755         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
756         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
757             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
758     }
759     case LoadOpType::I64Load16U: {
760         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
761         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
762             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
763
764         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), partialResult);
765     }
766     }
767     RELEASE_ASSERT_NOT_REACHED();
768 }
769
770 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
771 {
772     ASSERT(pointer->type() == Int32);
773
774     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
775         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
776         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
777         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
778         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
779             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
780         });
781
782         switch (op) {
783         case LoadOpType::I32Load8S:
784         case LoadOpType::I32Load16S:
785         case LoadOpType::I32Load:
786         case LoadOpType::I32Load16U:
787         case LoadOpType::I32Load8U:
788             result = constant(Int32, 0);
789             break;
790         case LoadOpType::I64Load8S:
791         case LoadOpType::I64Load8U:
792         case LoadOpType::I64Load16S:
793         case LoadOpType::I64Load32U:
794         case LoadOpType::I64Load32S:
795         case LoadOpType::I64Load:
796         case LoadOpType::I64Load16U:
797             result = constant(Int64, 0);
798             break;
799         case LoadOpType::F32Load:
800             result = constant(Float, 0);
801             break;
802         case LoadOpType::F64Load:
803             result = constant(Double, 0);
804             break;
805         }
806
807     } else
808         result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
809
810     return { };
811 }
812
813 inline uint32_t sizeOfStoreOp(StoreOpType op)
814 {
815     switch (op) {
816     case StoreOpType::I32Store8:
817     case StoreOpType::I64Store8:
818         return 1;
819     case StoreOpType::I32Store16:
820     case StoreOpType::I64Store16:
821         return 2;
822     case StoreOpType::I32Store:
823     case StoreOpType::I64Store32:
824     case StoreOpType::F32Store:
825         return 4;
826     case StoreOpType::I64Store:
827     case StoreOpType::F64Store:
828         return 8;
829     }
830     RELEASE_ASSERT_NOT_REACHED();
831 }
832
833
834 inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
835 {
836     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
837
838     switch (op) {
839     case StoreOpType::I64Store8:
840         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
841         FALLTHROUGH;
842
843     case StoreOpType::I32Store8:
844         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
845         return;
846
847     case StoreOpType::I64Store16:
848         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
849         FALLTHROUGH;
850
851     case StoreOpType::I32Store16:
852         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
853         return;
854
855     case StoreOpType::I64Store32:
856         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
857         FALLTHROUGH;
858
859     case StoreOpType::I64Store:
860     case StoreOpType::I32Store:
861     case StoreOpType::F32Store:
862     case StoreOpType::F64Store:
863         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
864         return;
865     }
866     RELEASE_ASSERT_NOT_REACHED();
867 }
868
869 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
870 {
871     ASSERT(pointer->type() == Int32);
872
873     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
874         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
875         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
876         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
877         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
878             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
879         });
880     } else
881         emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
882
883     return { };
884 }
885
886 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
887 {
888     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
889     return { };
890 }
891
892 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
893 {
894     return constant(toB3Type(type), value);
895 }
896
897 void B3IRGenerator::emitTierUpCheck(uint32_t decrementCount, Origin origin)
898 {
899     if (!m_tierUp)
900         return;
901
902     ASSERT(m_tierUp);
903     Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(m_tierUp), origin);
904     Value* oldCountDown = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, countDownLocation);
905     Value* newCountDown = m_currentBlock->appendNew<Value>(m_proc, Sub, origin, oldCountDown, constant(Int32, decrementCount, origin));
906     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, newCountDown, countDownLocation);
907
908     PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, origin);
909     Effects effects = Effects::none();
910     // FIXME: we should have a more precise heap range for the tier up count.
911     effects.reads = B3::HeapRange::top();
912     effects.writes = B3::HeapRange::top();
913     patch->effects = effects;
914
915     patch->append(newCountDown, ValueRep::SomeRegister);
916     patch->append(oldCountDown, ValueRep::SomeRegister);
917     patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
918         MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
919         MacroAssembler::Label tierUpResume = jit.label();
920
921         params.addLatePath([=] (CCallHelpers& jit) {
922             tierUp.link(&jit);
923
924             const unsigned extraPaddingBytes = 0;
925             RegisterSet registersToSpill = { };
926             registersToSpill.add(GPRInfo::argumentGPR1);
927             unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
928
929             jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
930             MacroAssembler::Call call = jit.nearCall();
931
932             ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
933             jit.jump(tierUpResume);
934
935             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
936                 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall(call), CodeLocationLabel(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
937
938             });
939         });
940     });
941 }
942
943 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
944 {
945     BasicBlock* body = m_proc.addBlock();
946     BasicBlock* continuation = m_proc.addBlock();
947
948     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
949
950     m_currentBlock = body;
951     emitTierUpCheck(TierUpCount::loopDecrement(), origin());
952
953     return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
954 }
955
956 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
957 {
958     return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
959 }
960
961 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
962 {
963     return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
964 }
965
966 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
967 {
968     // FIXME: This needs to do some kind of stack passing.
969
970     BasicBlock* taken = m_proc.addBlock();
971     BasicBlock* notTaken = m_proc.addBlock();
972     BasicBlock* continuation = m_proc.addBlock();
973
974     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
975     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
976     taken->addPredecessor(m_currentBlock);
977     notTaken->addPredecessor(m_currentBlock);
978
979     m_currentBlock = taken;
980     result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
981     return { };
982 }
983
984 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
985 {
986     unifyValuesWithBlock(currentStack, data.result);
987     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
988     return addElseToUnreachable(data);
989 }
990
991 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
992 {
993     ASSERT(data.type() == BlockType::If);
994     m_currentBlock = data.special;
995     data.convertIfToBlock();
996     return { };
997 }
998
999 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
1000 {
1001     ASSERT(returnValues.size() <= 1);
1002     if (returnValues.size())
1003         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
1004     else
1005         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
1006     return { };
1007 }
1008
1009 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1010 {
1011     unifyValuesWithBlock(returnValues, data.resultForBranch());
1012
1013     BasicBlock* target = data.targetBlockForBranch();
1014     if (condition) {
1015         BasicBlock* continuation = m_proc.addBlock();
1016         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1017         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
1018         target->addPredecessor(m_currentBlock);
1019         continuation->addPredecessor(m_currentBlock);
1020         m_currentBlock = continuation;
1021     } else {
1022         m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
1023         target->addPredecessor(m_currentBlock);
1024     }
1025
1026     return { };
1027 }
1028
1029 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1030 {
1031     for (size_t i = 0; i < targets.size(); ++i)
1032         unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
1033     unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1034
1035     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
1036     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
1037     for (size_t i = 0; i < targets.size(); ++i)
1038         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
1039
1040     return { };
1041 }
1042
1043 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1044 {
1045     ControlData& data = entry.controlData;
1046
1047     unifyValuesWithBlock(expressionStack, data.result);
1048     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1049     data.continuation->addPredecessor(m_currentBlock);
1050
1051     return addEndToUnreachable(entry);
1052 }
1053
1054
1055 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1056 {
1057     ControlData& data = entry.controlData;
1058     m_currentBlock = data.continuation;
1059
1060     if (data.type() == BlockType::If) {
1061         data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
1062         m_currentBlock->addPredecessor(data.special);
1063     }
1064
1065     for (Value* result : data.result) {
1066         m_currentBlock->append(result);
1067         entry.enclosedExpressionStack.append(result);
1068     }
1069
1070     // TopLevel does not have any code after this so we need to make sure we emit a return here.
1071     if (data.type() == BlockType::TopLevel)
1072         return addReturn(entry.controlData, entry.enclosedExpressionStack);
1073
1074     return { };
1075 }
1076
1077 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1078 {
1079     ASSERT(signature.argumentCount() == args.size());
1080
1081     m_makesCalls = true;
1082
1083     Type returnType = signature.returnType();
1084     Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1085
1086     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1087         m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1088
1089         // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
1090         Value* targetInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfTargetInstance(functionIndex)));
1091         // The target instance is 0 unless the call is wasm->wasm.
1092         Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0));
1093
1094         BasicBlock* isWasmBlock = m_proc.addBlock();
1095         BasicBlock* isEmbedderBlock = m_proc.addBlock();
1096         BasicBlock* continuation = m_proc.addBlock();
1097         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
1098
1099         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
1100             [=] (PatchpointValue* patchpoint) {
1101                 patchpoint->effects.writesPinned = true;
1102                 patchpoint->effects.readsPinned = true;
1103                 // We need to clobber all potential pinned registers since we might be leaving the instance.
1104                 // We pessimistically assume we could be calling to something that is bounds checking.
1105                 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1106                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1107                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1108                     AllowMacroScratchRegisterUsage allowScratch(jit);
1109                     CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1110                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1111                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall(call), functionIndex });
1112                     });
1113                 });
1114             });
1115         UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
1116         isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1117
1118         // FIXME: Let's remove this indirection by creating a PIC friendly IC
1119         // for calls out to the embedder. This shouldn't be that hard to do. We could probably
1120         // implement the IC to be over Context*.
1121         // https://bugs.webkit.org/show_bug.cgi?id=170375
1122         Value* jumpDestination = isEmbedderBlock->appendNew<MemoryValue>(m_proc,
1123             Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfWasmToEmbedderStubExecutableAddress(functionIndex)));
1124         Value* embedderCallResult = wasmCallingConvention().setupCall(m_proc, isEmbedderBlock, origin(), args, toB3Type(returnType),
1125             [&] (PatchpointValue* patchpoint) {
1126                 patchpoint->effects.writesPinned = true;
1127                 patchpoint->effects.readsPinned = true;
1128                 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
1129                 // We need to clobber all potential pinned registers since we might be leaving the instance.
1130                 // We pessimistically assume we could be calling to something that is bounds checking.
1131                 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1132                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1133                 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1134                     AllowMacroScratchRegisterUsage allowScratch(jit);
1135                     jit.call(params[returnType == Void ? 0 : 1].gpr());
1136                 });
1137             });
1138         UpsilonValue* embedderCallResultUpsilon = returnType == Void ? nullptr : isEmbedderBlock->appendNew<UpsilonValue>(m_proc, origin(), embedderCallResult);
1139         isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1140
1141         m_currentBlock = continuation;
1142
1143         if (returnType == Void)
1144             result = nullptr;
1145         else {
1146             result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1147             wasmCallResultUpsilon->setPhi(result);
1148             embedderCallResultUpsilon->setPhi(result);
1149         }
1150
1151         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1152         restoreWebAssemblyGlobalState(m_info.memory, instanceValue(), m_proc, continuation);
1153     } else {
1154         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1155             [=] (PatchpointValue* patchpoint) {
1156                 patchpoint->effects.writesPinned = true;
1157                 patchpoint->effects.readsPinned = true;
1158
1159                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1160                     AllowMacroScratchRegisterUsage allowScratch(jit);
1161                     CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1162                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1163                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall(call), functionIndex });
1164                     });
1165                 });
1166             });
1167     }
1168
1169     return { };
1170 }
1171
1172 auto B3IRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1173 {
1174     ExpressionType calleeIndex = args.takeLast();
1175     ASSERT(signature.argumentCount() == args.size());
1176
1177     m_makesCalls = true;
1178     // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1179     // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1180     // can be to the embedder for our stack check calculation.
1181     m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1182
1183     ExpressionType callableFunctionBuffer;
1184     ExpressionType instancesBuffer;
1185     ExpressionType callableFunctionBufferSize;
1186     {
1187         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1188             instanceValue(), safeCast<int32_t>(Instance::offsetOfTable()));
1189         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1190             table, safeCast<int32_t>(Table::offsetOfFunctions()));
1191         instancesBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1192             table, safeCast<int32_t>(Table::offsetOfInstances()));
1193         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1194             table, safeCast<int32_t>(Table::offsetOfSize()));
1195     }
1196
1197     // Check the index we are looking for is valid.
1198     {
1199         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1200             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferSize));
1201
1202         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1203             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1204         });
1205     }
1206
1207     // Compute the offset in the table index space we are looking for.
1208     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1209         m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex),
1210         constant(pointerType(), sizeof(CallableFunction)));
1211     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1212
1213     // Check that the CallableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1214     // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1215     static_assert(sizeof(CallableFunction::signatureIndex) == sizeof(uint32_t), "Load codegen assumes i32");
1216     ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), callableFunction, safeCast<int32_t>(OBJECT_OFFSETOF(CallableFunction, signatureIndex)));
1217     {
1218         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1219             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1220                 calleeSignatureIndex,
1221                 m_currentBlock->appendNew<Const32Value>(m_proc, origin(), Signature::invalidIndex)));
1222
1223         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1224             this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1225         });
1226     }
1227
1228     // Check the signature matches the value we expect.
1229     {
1230         ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const32Value>(m_proc, origin(), SignatureInformation::get(signature));
1231         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1232             m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1233
1234         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1235             this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1236         });
1237     }
1238
1239     // Do a context switch if needed.
1240     {
1241         Value* offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1242             m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex),
1243             constant(pointerType(), sizeof(Instance*)));
1244         Value* newContextInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1245             m_currentBlock->appendNew<Value>(m_proc, Add, origin(), instancesBuffer, offset));
1246
1247         BasicBlock* continuation = m_proc.addBlock();
1248         BasicBlock* doContextSwitch = m_proc.addBlock();
1249
1250         Value* isSameContextInstance = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1251             newContextInstance, instanceValue());
1252         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1253             isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch));
1254
1255         PatchpointValue* patchpoint = doContextSwitch->appendNew<PatchpointValue>(m_proc, B3::Void, origin());
1256         patchpoint->effects.writesPinned = true;
1257         // We pessimistically assume we're calling something with BoundsChecking memory.
1258         // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1259         patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1260         patchpoint->clobber(RegisterSet::macroScratchRegisters());
1261         patchpoint->append(newContextInstance, ValueRep::SomeRegister);
1262         patchpoint->append(instanceValue(), ValueRep::SomeRegister);
1263         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1264             AllowMacroScratchRegisterUsage allowScratch(jit);
1265             GPRReg newContextInstance = params[0].gpr();
1266             GPRReg oldContextInstance = params[1].gpr();
1267             const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1268             const auto& sizeRegs = pinnedRegs.sizeRegisters;
1269             GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1270             ASSERT(newContextInstance != baseMemory);
1271             jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1272             jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1273             jit.storeWasmContextInstance(newContextInstance);
1274             jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfMemory()), baseMemory); // Memory*.
1275             ASSERT(sizeRegs.size() == 1);
1276             ASSERT(sizeRegs[0].sizeRegister != baseMemory);
1277             ASSERT(sizeRegs[0].sizeRegister != newContextInstance);
1278             ASSERT(!sizeRegs[0].sizeOffset);
1279             jit.loadPtr(CCallHelpers::Address(baseMemory, Memory::offsetOfSize()), sizeRegs[0].sizeRegister); // Memory size.
1280             jit.loadPtr(CCallHelpers::Address(baseMemory, Memory::offsetOfMemory()), baseMemory); // Memory::void*.
1281         });
1282         doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation);
1283
1284         m_currentBlock = continuation;
1285     }
1286
1287     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1288         m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
1289             safeCast<int32_t>(OBJECT_OFFSETOF(CallableFunction, code))));
1290
1291     Type returnType = signature.returnType();
1292     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1293         [=] (PatchpointValue* patchpoint) {
1294             patchpoint->effects.writesPinned = true;
1295             patchpoint->effects.readsPinned = true;
1296             // We need to clobber all potential pinned registers since we might be leaving the instance.
1297             // We pessimistically assume we're always calling something that is bounds checking so
1298             // because the wasm->wasm thunk unconditionally overrides the size registers.
1299             // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1300             // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1301             patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1302
1303             patchpoint->append(calleeCode, ValueRep::SomeRegister);
1304             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1305                 AllowMacroScratchRegisterUsage allowScratch(jit);
1306                 jit.call(params[returnType == Void ? 0 : 1].gpr());
1307             });
1308         });
1309
1310     // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1311     restoreWebAssemblyGlobalState(m_info.memory, instanceValue(), m_proc, m_currentBlock);
1312
1313     return { };
1314 }
1315
1316 void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1317 {
1318     m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1319 }
1320
1321 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1322 {
1323     ASSERT(result.size() <= resultStack.size());
1324
1325     for (size_t i = 0; i < result.size(); ++i)
1326         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1327 }
1328
1329 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1330 {
1331     dataLog(comma, "ExpressionStack:");
1332     for (const auto& expression : expressionStack)
1333         dataLog(comma, *expression);
1334 }
1335
1336 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1337 {
1338     dataLogLn("Constants:");
1339     for (const auto& constant : m_constantPool)
1340         dataLogLn(deepDump(m_proc, constant.value));
1341
1342     dataLogLn("Processing Graph:");
1343     dataLog(m_proc);
1344     dataLogLn("With current block:", *m_currentBlock);
1345     dataLogLn("Control stack:");
1346     ASSERT(controlStack.size());
1347     for (size_t i = controlStack.size(); i--;) {
1348         dataLog("  ", controlStack[i].controlData, ": ");
1349         CommaPrinter comma(", ", "");
1350         dumpExpressionStack(comma, *expressionStack);
1351         expressionStack = &controlStack[i].enclosedExpressionStack;
1352         dataLogLn();
1353     }
1354     dataLogLn();
1355 }
1356
1357 auto B3IRGenerator::origin() -> Origin
1358 {
1359     OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset());
1360     ASSERT(isValidOpType(static_cast<uint8_t>(origin.opcode())));
1361     return bitwise_cast<Origin>(origin);
1362 }
1363
1364 Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1365 {
1366     auto result = std::make_unique<InternalFunction>();
1367
1368     compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
1369     compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1370
1371     Procedure procedure;
1372
1373     procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1374         if (origin.data())
1375             out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1376     });
1377     
1378     // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1379     // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1380     // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1381     // optLevel=1.
1382     procedure.setNeedsUsedRegisters(false);
1383     
1384     procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
1385         ? Options::webAssemblyBBQOptimizationLevel()
1386         : Options::webAssemblyOMGOptimizationLevel());
1387
1388     B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, compilationMode, functionIndex, tierUp, throwWasmException);
1389     FunctionParser<B3IRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1390     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1391
1392     irGenerator.insertConstants();
1393
1394     procedure.resetReachability();
1395     if (!ASSERT_DISABLED)
1396         validate(procedure, "After parsing:\n");
1397
1398     dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure);
1399     fixSSA(procedure);
1400     dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure);
1401     
1402     {
1403         B3::prepareForGeneration(procedure);
1404         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1405         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1406         result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1407     }
1408
1409     return WTFMove(result);
1410 }
1411
1412 // Custom wasm ops. These are the ones too messy to do in wasm.json.
1413
1414 void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1415 {
1416     ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1417     const B3::Type type = left->type();
1418
1419     {
1420         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1421             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1422
1423         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1424             this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1425         });
1426     }
1427
1428     if (operation == Div) {
1429         int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1430
1431         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1432             m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1433                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1434                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1435
1436         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1437             this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1438         });
1439     }
1440 }
1441
1442 template<>
1443 auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1444 {
1445     const B3::Opcode op = Div;
1446     emitChecksForModOrDiv(op, left, right);
1447     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1448     return { };
1449 }
1450
1451 template<>
1452 auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1453 {
1454     const B3::Opcode op = Mod;
1455     emitChecksForModOrDiv(op, left, right);
1456     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1457     return { };
1458 }
1459
1460 template<>
1461 auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1462 {
1463     const B3::Opcode op = UDiv;
1464     emitChecksForModOrDiv(op, left, right);
1465     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1466     return { };
1467 }
1468
1469 template<>
1470 auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1471 {
1472     const B3::Opcode op = UMod;
1473     emitChecksForModOrDiv(op, left, right);
1474     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1475     return { };
1476 }
1477
1478 template<>
1479 auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1480 {
1481     const B3::Opcode op = Div;
1482     emitChecksForModOrDiv(op, left, right);
1483     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1484     return { };
1485 }
1486
1487 template<>
1488 auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1489 {
1490     const B3::Opcode op = Mod;
1491     emitChecksForModOrDiv(op, left, right);
1492     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1493     return { };
1494 }
1495
1496 template<>
1497 auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1498 {
1499     const B3::Opcode op = UDiv;
1500     emitChecksForModOrDiv(op, left, right);
1501     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1502     return { };
1503 }
1504
1505 template<>
1506 auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1507 {
1508     const B3::Opcode op = UMod;
1509     emitChecksForModOrDiv(op, left, right);
1510     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1511     return { };
1512 }
1513
1514 template<>
1515 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1516 {
1517     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1518     patchpoint->append(arg, ValueRep::SomeRegister);
1519     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1520         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1521     });
1522     patchpoint->effects = Effects::none();
1523     result = patchpoint;
1524     return { };
1525 }
1526
1527 template<>
1528 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1529 {
1530     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1531     patchpoint->append(arg, ValueRep::SomeRegister);
1532     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1533         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1534     });
1535     patchpoint->effects = Effects::none();
1536     result = patchpoint;
1537     return { };
1538 }
1539
1540 template<>
1541 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1542 {
1543     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1544     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1545     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1546     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1547     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1548     return { };
1549 }
1550
1551 template<>
1552 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1553 {
1554     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1555     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1556     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1557     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1558     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1559     return { };
1560 }
1561
1562 template<>
1563 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1564 {
1565     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1566     if (isX86())
1567         patchpoint->numGPScratchRegisters = 1;
1568     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1569     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1570         AllowMacroScratchRegisterUsage allowScratch(jit);
1571 #if CPU(X86_64)
1572         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1573 #else
1574         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1575 #endif
1576     });
1577     patchpoint->effects = Effects::none();
1578     result = patchpoint;
1579     return { };
1580 }
1581
1582 template<>
1583 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1584 {
1585     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1586     if (isX86())
1587         patchpoint->numGPScratchRegisters = 1;
1588     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1589     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1590         AllowMacroScratchRegisterUsage allowScratch(jit);
1591 #if CPU(X86_64)
1592         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1593 #else
1594         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1595 #endif
1596     });
1597     patchpoint->effects = Effects::none();
1598     result = patchpoint;
1599     return { };
1600 }
1601
1602 template<>
1603 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1604 {
1605     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1606     patchpoint->append(arg, ValueRep::SomeRegister);
1607     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1608         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1609     });
1610     patchpoint->effects = Effects::none();
1611     result = patchpoint;
1612     return { };
1613 }
1614
1615 template<>
1616 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1617 {
1618     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1619     patchpoint->append(arg, ValueRep::SomeRegister);
1620     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1621         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1622     });
1623     patchpoint->effects = Effects::none();
1624     result = patchpoint;
1625     return { };
1626 }
1627
1628 template<>
1629 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1630 {
1631     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1632     patchpoint->append(arg, ValueRep::SomeRegister);
1633     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1634         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1635     });
1636     patchpoint->effects = Effects::none();
1637     result = patchpoint;
1638     return { };
1639 }
1640
1641 template<>
1642 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1643 {
1644     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1645     patchpoint->append(arg, ValueRep::SomeRegister);
1646     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1647         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1648     });
1649     patchpoint->effects = Effects::none();
1650     result = patchpoint;
1651     return { };
1652 }
1653
1654 template<>
1655 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1656 {
1657     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1658     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1659     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1660         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1661         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1662     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1663     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1664     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1665         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1666     });
1667     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1668     patchpoint->append(arg, ValueRep::SomeRegister);
1669     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1670         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1671     });
1672     patchpoint->effects = Effects::none();
1673     result = patchpoint;
1674     return { };
1675 }
1676
1677 template<>
1678 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1679 {
1680     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1681     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1682     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1683         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1684         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1685     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1686     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1687     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1688         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1689     });
1690     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1691     patchpoint->append(arg, ValueRep::SomeRegister);
1692     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1693         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1694     });
1695     patchpoint->effects = Effects::none();
1696     result = patchpoint;
1697     return { };
1698 }
1699
1700
1701 template<>
1702 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1703 {
1704     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1705     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1706     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1707         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1708         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1709     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1710     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1711     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1712         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1713     });
1714     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1715     patchpoint->append(arg, ValueRep::SomeRegister);
1716     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1717         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1718     });
1719     patchpoint->effects = Effects::none();
1720     result = patchpoint;
1721     return { };
1722 }
1723
1724 template<>
1725 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1726 {
1727     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1728     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1729     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1730         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1731         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1732     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1733     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1734     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1735         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1736     });
1737     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1738     patchpoint->append(arg, ValueRep::SomeRegister);
1739     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1740         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1741     });
1742     patchpoint->effects = Effects::none();
1743     result = patchpoint;
1744     return { };
1745 }
1746
1747 template<>
1748 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1749 {
1750     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1751     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1752     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1753         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1754         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1755     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1756     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1757     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1758         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1759     });
1760     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1761     patchpoint->append(arg, ValueRep::SomeRegister);
1762     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1763         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1764     });
1765     patchpoint->effects = Effects::none();
1766     result = patchpoint;
1767     return { };
1768 }
1769
1770 template<>
1771 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1772 {
1773     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1774     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1775     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1776         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1777         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1778     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1779     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1780     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1781         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1782     });
1783
1784     Value* signBitConstant;
1785     if (isX86()) {
1786         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1787         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1788         // so we can pool them if needed.
1789         signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1790     }
1791     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1792     patchpoint->append(arg, ValueRep::SomeRegister);
1793     if (isX86()) {
1794         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1795         patchpoint->numFPScratchRegisters = 1;
1796     }
1797     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1798         AllowMacroScratchRegisterUsage allowScratch(jit);
1799         FPRReg scratch = InvalidFPRReg;
1800         FPRReg constant = InvalidFPRReg;
1801         if (isX86()) {
1802             scratch = params.fpScratch(0);
1803             constant = params[2].fpr();
1804         }
1805         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1806     });
1807     patchpoint->effects = Effects::none();
1808     result = patchpoint;
1809     return { };
1810 }
1811
1812 template<>
1813 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1814 {
1815     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1816     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1817     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1818         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1819         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1820     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1821     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1822     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1823         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1824     });
1825     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1826     patchpoint->append(arg, ValueRep::SomeRegister);
1827     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1828         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1829     });
1830     patchpoint->effects = Effects::none();
1831     result = patchpoint;
1832     return { };
1833 }
1834
1835 template<>
1836 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1837 {
1838     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1839     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1840     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1841         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1842         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1843     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1844     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1845     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1846         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1847     });
1848
1849     Value* signBitConstant;
1850     if (isX86()) {
1851         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1852         // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1853         // so we can pool them if needed.
1854         signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1855     }
1856     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1857     patchpoint->append(arg, ValueRep::SomeRegister);
1858     if (isX86()) {
1859         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1860         patchpoint->numFPScratchRegisters = 1;
1861     }
1862     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1863         AllowMacroScratchRegisterUsage allowScratch(jit);
1864         FPRReg scratch = InvalidFPRReg;
1865         FPRReg constant = InvalidFPRReg;
1866         if (isX86()) {
1867             scratch = params.fpScratch(0);
1868             constant = params[2].fpr();
1869         }
1870         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1871     });
1872     patchpoint->effects = Effects::none();
1873     result = patchpoint;
1874     return { };
1875 }
1876
1877 } } // namespace JSC::Wasm
1878
1879 #include "WasmB3IRGeneratorInlines.h"
1880
1881 #endif // ENABLE(WEBASSEMBLY)