Teach Call ICs how to call Wasm
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "AllowMacroScratchRegisterUsageIf.h"
32 #include "B3BasicBlockInlines.h"
33 #include "B3CCallValue.h"
34 #include "B3Compile.h"
35 #include "B3ConstPtrValue.h"
36 #include "B3FixSSA.h"
37 #include "B3Generate.h"
38 #include "B3InsertionSet.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3SwitchValue.h"
42 #include "B3UpsilonValue.h"
43 #include "B3Validate.h"
44 #include "B3ValueInlines.h"
45 #include "B3ValueKey.h"
46 #include "B3Variable.h"
47 #include "B3VariableValue.h"
48 #include "B3WasmAddressValue.h"
49 #include "B3WasmBoundsCheckValue.h"
50 #include "JSCInlines.h"
51 #include "ScratchRegisterAllocator.h"
52 #include "VirtualRegister.h"
53 #include "WasmCallingConvention.h"
54 #include "WasmContextInlines.h"
55 #include "WasmExceptionType.h"
56 #include "WasmFunctionParser.h"
57 #include "WasmInstance.h"
58 #include "WasmMemory.h"
59 #include "WasmOMGPlan.h"
60 #include "WasmOpcodeOrigin.h"
61 #include "WasmSignatureInlines.h"
62 #include "WasmThunks.h"
63 #include <limits>
64 #include <wtf/Optional.h>
65 #include <wtf/StdLibExtras.h>
66
67 void dumpProcedure(void* ptr)
68 {
69     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
70     proc->dump(WTF::dataFile());
71 }
72
73 namespace JSC { namespace Wasm {
74
75 using namespace B3;
76
77 namespace {
78 namespace WasmB3IRGeneratorInternal {
79 static const bool verbose = false;
80 }
81 }
82
83 class B3IRGenerator {
84 public:
85     struct ControlData {
86         ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
87             : blockType(type)
88             , continuation(continuation)
89             , special(special)
90         {
91             if (signature != Void)
92                 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
93         }
94
95         ControlData()
96         {
97         }
98
99         void dump(PrintStream& out) const
100         {
101             switch (type()) {
102             case BlockType::If:
103                 out.print("If:       ");
104                 break;
105             case BlockType::Block:
106                 out.print("Block:    ");
107                 break;
108             case BlockType::Loop:
109                 out.print("Loop:     ");
110                 break;
111             case BlockType::TopLevel:
112                 out.print("TopLevel: ");
113                 break;
114             }
115             out.print("Continuation: ", *continuation, ", Special: ");
116             if (special)
117                 out.print(*special);
118             else
119                 out.print("None");
120         }
121
122         BlockType type() const { return blockType; }
123
124         bool hasNonVoidSignature() const { return result.size(); }
125
126         BasicBlock* targetBlockForBranch()
127         {
128             if (type() == BlockType::Loop)
129                 return special;
130             return continuation;
131         }
132
133         void convertIfToBlock()
134         {
135             ASSERT(type() == BlockType::If);
136             blockType = BlockType::Block;
137             special = nullptr;
138         }
139
140         using ResultList = Vector<Value*, 1>; // Value must be a Phi
141
142         ResultList resultForBranch() const
143         {
144             if (type() == BlockType::Loop)
145                 return ResultList();
146             return result;
147         }
148
149     private:
150         friend class B3IRGenerator;
151         BlockType blockType;
152         BasicBlock* continuation;
153         BasicBlock* special;
154         ResultList result;
155     };
156
157     typedef Value* ExpressionType;
158     typedef ControlData ControlType;
159     typedef Vector<ExpressionType, 1> ExpressionList;
160     typedef ControlData::ResultList ResultList;
161     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
162
163     static constexpr ExpressionType emptyExpression() { return nullptr; }
164
165     typedef String ErrorType;
166     typedef Unexpected<ErrorType> UnexpectedResult;
167     typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
168     typedef Expected<void, ErrorType> PartialResult;
169     template <typename ...Args>
170     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
171     {
172         using namespace FailureHelper; // See ADL comment in WasmParser.h.
173         return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
174     }
175 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
176         if (UNLIKELY(condition))                  \
177             return fail(__VA_ARGS__);             \
178     } while (0)
179
180     B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, CompilationMode, unsigned functionIndex, TierUpCount*, ThrowWasmException);
181
182     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
183     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
184     ExpressionType addConstant(Type, uint64_t);
185
186     // Locals
187     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
188     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
189
190     // Globals
191     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
192     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
193
194     // Memory
195     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
196     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
197     PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
198     PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
199
200     // Basic operators
201     template<OpType>
202     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
203     template<OpType>
204     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
205     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
206
207     // Control flow
208     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
209     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
210     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
211     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
212     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
213     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
214
215     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
216     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
217     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
218     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
219     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
220
221     // Calls
222     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
223     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
224     PartialResult WARN_UNUSED_RETURN addUnreachable();
225
226     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
227     void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
228
229     Value* constant(B3::Type, uint64_t bits, Optional<Origin> = WTF::nullopt);
230     void insertConstants();
231
232     ALWAYS_INLINE void didKill(ExpressionType) { }
233
234 private:
235     void emitExceptionCheck(CCallHelpers&, ExceptionType);
236
237     void emitTierUpCheck(uint32_t decrementCount, Origin);
238
239     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
240     B3::Kind memoryKind(B3::Opcode memoryOp);
241     ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
242     void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
243
244     void unify(const ExpressionType phi, const ExpressionType source);
245     void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
246
247     void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
248
249     int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
250
251     void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*);
252     enum class RestoreCachedStackLimit { No, Yes };
253     void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
254
255     Origin origin();
256
257     FunctionParser<B3IRGenerator>* m_parser { nullptr };
258     const ModuleInformation& m_info;
259     const MemoryMode m_mode { MemoryMode::BoundsChecking };
260     const CompilationMode m_compilationMode { CompilationMode::BBQMode };
261     const unsigned m_functionIndex { UINT_MAX };
262     const TierUpCount* m_tierUp { nullptr };
263
264     Procedure& m_proc;
265     BasicBlock* m_currentBlock { nullptr };
266     Vector<Variable*> m_locals;
267     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
268     HashMap<ValueKey, Value*> m_constantPool;
269     InsertionSet m_constantInsertionValues;
270     GPRReg m_memoryBaseGPR { InvalidGPRReg };
271     GPRReg m_memorySizeGPR { InvalidGPRReg };
272     GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
273     bool m_makesCalls { false };
274
275     Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used.
276     bool m_usesInstanceValue { false };
277     Value* instanceValue()
278     {
279         m_usesInstanceValue = true;
280         return m_instanceValue;
281     }
282
283     uint32_t m_maxNumJSCallArguments { 0 };
284 };
285
286 // Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
287 int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
288 {
289     if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
290         ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
291         return 0;
292     }
293     return offset;
294 }
295
296 void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg)
297 {
298     if (Context::useFastTLS()) {
299         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
300         if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
301             patchpoint->clobber(RegisterSet::macroScratchRegisters());
302         patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
303         patchpoint->setGenerator(
304             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
305                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
306                 jit.storeWasmContextInstance(params[0].gpr());
307             });
308         return;
309     }
310
311     // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
312     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
313     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
314     Effects effects = Effects::none();
315     effects.writesPinned = true;
316     effects.reads = B3::HeapRange::top();
317     patchpoint->effects = effects;
318     patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
319     patchpoint->append(arg, ValueRep::SomeRegister);
320     GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
321     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
322         jit.move(param[0].gpr(), wasmContextInstanceGPR);
323     });
324 }
325
326 B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
327     : m_info(info)
328     , m_mode(mode)
329     , m_compilationMode(compilationMode)
330     , m_functionIndex(functionIndex)
331     , m_tierUp(tierUp)
332     , m_proc(procedure)
333     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
334     , m_constantInsertionValues(m_proc)
335 {
336     m_currentBlock = m_proc.addBlock();
337
338     // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
339     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
340
341     m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
342     m_proc.pinRegister(m_memoryBaseGPR);
343
344     m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
345     if (!Context::useFastTLS())
346         m_proc.pinRegister(m_wasmContextInstanceGPR);
347
348     if (mode != MemoryMode::Signaling) {
349         m_memorySizeGPR = pinnedRegs.sizeRegister;
350         m_proc.pinRegister(m_memorySizeGPR);
351     }
352
353     if (throwWasmException)
354         Thunks::singleton().setThrowWasmException(throwWasmException);
355
356     if (info.memory) {
357         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
358             AllowMacroScratchRegisterUsage allowScratch(jit);
359             switch (m_mode) {
360             case MemoryMode::BoundsChecking:
361                 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
362                 break;
363             case MemoryMode::Signaling:
364                 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
365                 break;
366             }
367             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
368         });
369
370         switch (m_mode) {
371         case MemoryMode::BoundsChecking:
372             break;
373         case MemoryMode::Signaling:
374             // Most memory accesses in signaling mode don't do an explicit
375             // exception check because they can rely on fault handling to detect
376             // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
377             // thunk to exist so that it can jump to that thunk.
378             if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
379                 CRASH();
380             break;
381         }
382     }
383
384     wasmCallingConvention().setupFrameInPrologue(&compilation->calleeMoveLocation, m_proc, Origin(), m_currentBlock);
385
386     {
387         B3::Value* framePointer = m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin());
388         B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, pointerType(), Origin());
389         m_instanceValue = stackOverflowCheck;
390         stackOverflowCheck->appendSomeRegister(framePointer);
391         stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters());
392         if (!Context::useFastTLS()) {
393             // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
394             // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
395             stackOverflowCheck->effects.writesPinned = false;
396             stackOverflowCheck->effects.readsPinned = true;
397             stackOverflowCheck->resultConstraint = ValueRep::reg(m_wasmContextInstanceGPR);
398         }
399         stackOverflowCheck->numGPScratchRegisters = 2;
400         stackOverflowCheck->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
401             const Checked<int32_t> wasmFrameSize = params.proc().frameSize();
402             const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
403             const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
404                 // This allows us to elide stack checks for functions that are terminal nodes in the call
405                 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
406                 // having any such terminal node have its parent caller include some extra size in its
407                 // own check for it. The goal here is twofold:
408                 // 1. Emit less code.
409                 // 2. Try to speed things up by skipping stack checks.
410                 minimumParentCheckSize,
411                 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
412                 // spill all arguments to the stack, we ensure that a stack check here covers the
413                 // stack that such a stub would use.
414                 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
415             ));
416             const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
417             bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
418             bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
419
420             GPRReg contextInstance = Context::useFastTLS() ? params[0].gpr() : m_wasmContextInstanceGPR;
421
422             // This allows leaf functions to not do stack checks if their frame size is within
423             // certain limits since their caller would have already done the check.
424             if (needsOverflowCheck) {
425                 AllowMacroScratchRegisterUsage allowScratch(jit);
426                 GPRReg fp = params[1].gpr();
427                 GPRReg scratch1 = params.gpScratch(0);
428                 GPRReg scratch2 = params.gpScratch(1);
429
430                 if (Context::useFastTLS())
431                     jit.loadWasmContextInstance(contextInstance);
432
433                 jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2);
434                 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1);
435                 MacroAssembler::JumpList overflow;
436                 if (UNLIKELY(needUnderflowCheck))
437                     overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp));
438                 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2));
439                 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
440                     linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
441                 });
442             } else if (m_usesInstanceValue && Context::useFastTLS()) {
443                 // No overflow check is needed, but the instance values still needs to be correct.
444                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister());
445                 jit.loadWasmContextInstance(contextInstance);
446             } else {
447                 // We said we'd return a pointer. We don't actually need to because it isn't used, but the patchpoint conservatively said it had effects (potential stack check) which prevent it from getting removed.
448             }
449         });
450     }
451
452     emitTierUpCheck(TierUpCount::functionEntryDecrement(), Origin());
453 }
454
455 void B3IRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
456 {
457     restoreWasmContextInstance(proc, block, instance);
458
459     if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
460         // The Instance caches the stack limit, but also knows where its canonical location is.
461         Value* pointerToActualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfPointerToActualStackLimit()));
462         Value* actualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), pointerToActualStackLimit);
463         block->appendNew<MemoryValue>(m_proc, Store, origin(), actualStackLimit, instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedStackLimit()));
464     }
465
466     if (!!memory) {
467         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
468         RegisterSet clobbers;
469         clobbers.set(pinnedRegs->baseMemoryPointer);
470         clobbers.set(pinnedRegs->sizeRegister);
471
472         B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
473         Effects effects = Effects::none();
474         effects.writesPinned = true;
475         effects.reads = B3::HeapRange::top();
476         patchpoint->effects = effects;
477         patchpoint->clobber(clobbers);
478
479         patchpoint->append(instance, ValueRep::SomeRegister);
480
481         patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
482             GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
483             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
484             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
485         });
486     }
487 }
488
489 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
490 {
491     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
492     auto jumpToExceptionStub = jit.jump();
493
494     jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
495         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
496     });
497 }
498
499 Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, Optional<Origin> maybeOrigin)
500 {
501     auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
502         Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits);
503         m_constantInsertionValues.insertValue(0, result);
504         return result;
505     });
506     return result.iterator->value;
507 }
508
509 void B3IRGenerator::insertConstants()
510 {
511     m_constantInsertionValues.execute(m_proc.at(0));
512 }
513
514 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
515 {
516     Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
517     totalBytesChecked += m_locals.size();
518     uint32_t totalBytes;
519     WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
520
521     for (uint32_t i = 0; i < count; ++i) {
522         Variable* local = m_proc.addVariable(toB3Type(type));
523         m_locals.uncheckedAppend(local);
524         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, constant(toB3Type(type), 0, Origin()));
525     }
526     return { };
527 }
528
529 auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
530 {
531     ASSERT(!m_locals.size());
532     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
533
534     m_locals.grow(signature.argumentCount());
535     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
536         [=] (ExpressionType argument, unsigned i) {
537             Variable* argumentVariable = m_proc.addVariable(argument->type());
538             m_locals[i] = argumentVariable;
539             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
540         });
541     return { };
542 }
543
544 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
545 {
546     ASSERT(m_locals[index]);
547     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
548     return { };
549 }
550
551 auto B3IRGenerator::addUnreachable() -> PartialResult
552 {
553     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
554     unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
555         this->emitExceptionCheck(jit, ExceptionType::Unreachable);
556     });
557     unreachable->effects.terminal = true;
558     return { };
559 }
560
561 auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
562 {
563     int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
564         instance->storeTopCallFrame(callFrame);
565
566         if (delta < 0)
567             return -1;
568
569         auto grown = instance->memory()->grow(PageCount(delta));
570         if (!grown) {
571             switch (grown.error()) {
572             case Memory::GrowFailReason::InvalidDelta:
573             case Memory::GrowFailReason::InvalidGrowSize:
574             case Memory::GrowFailReason::WouldExceedMaximum:
575             case Memory::GrowFailReason::OutOfMemory:
576                 return -1;
577             }
578             RELEASE_ASSERT_NOT_REACHED();
579         }
580
581         return grown.value().pageCount();
582     };
583
584     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
585         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(growMemory, B3CCallPtrTag)),
586         m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, origin()), instanceValue(), delta);
587
588     restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_proc, m_currentBlock);
589
590     return { };
591 }
592
593 auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
594 {
595     static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
596     Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedMemorySize()));
597
598     constexpr uint32_t shiftValue = 16;
599     static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
600     Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
601         size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
602
603     result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
604
605     return { };
606 }
607
608 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
609 {
610     ASSERT(m_locals[index]);
611     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
612     return { };
613 }
614
615 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
616 {
617     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
618     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
619     return { };
620 }
621
622 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
623 {
624     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
625     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
626     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
627     return { };
628 }
629
630 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
631 {
632     ASSERT(m_memoryBaseGPR);
633
634     switch (m_mode) {
635     case MemoryMode::BoundsChecking: {
636         // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
637         ASSERT(m_memorySizeGPR);
638         ASSERT(sizeOfOperation + offset > offset);
639         m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), m_memorySizeGPR, pointer, sizeOfOperation + offset - 1);
640         break;
641     }
642
643     case MemoryMode::Signaling: {
644         // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
645         // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
646         // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
647         // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
648         // register + large offset access. We don't think this will be generated frequently.
649         //
650         // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
651         // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
652         // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
653         // any access equal to or greater than 4GiB will trap, no need to add the redzone.
654         if (offset >= Memory::fastMappedRedzoneBytes()) {
655             size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
656             m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
657         }
658         break;
659     }
660     }
661
662     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
663     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
664 }
665
666 inline uint32_t sizeOfLoadOp(LoadOpType op)
667 {
668     switch (op) {
669     case LoadOpType::I32Load8S:
670     case LoadOpType::I32Load8U:
671     case LoadOpType::I64Load8S:
672     case LoadOpType::I64Load8U:
673         return 1;
674     case LoadOpType::I32Load16S:
675     case LoadOpType::I64Load16S:
676     case LoadOpType::I32Load16U:
677     case LoadOpType::I64Load16U:
678         return 2;
679     case LoadOpType::I32Load:
680     case LoadOpType::I64Load32S:
681     case LoadOpType::I64Load32U:
682     case LoadOpType::F32Load:
683         return 4;
684     case LoadOpType::I64Load:
685     case LoadOpType::F64Load:
686         return 8;
687     }
688     RELEASE_ASSERT_NOT_REACHED();
689 }
690
691 inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
692 {
693     if (m_mode == MemoryMode::Signaling)
694         return trapping(memoryOp);
695     return memoryOp;
696 }
697
698 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
699 {
700     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
701
702     switch (op) {
703     case LoadOpType::I32Load8S: {
704         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
705     }
706
707     case LoadOpType::I64Load8S: {
708         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
709         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
710     }
711
712     case LoadOpType::I32Load8U: {
713         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
714     }
715
716     case LoadOpType::I64Load8U: {
717         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
718         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
719     }
720
721     case LoadOpType::I32Load16S: {
722         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
723     }
724
725     case LoadOpType::I64Load16S: {
726         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
727         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
728     }
729
730     case LoadOpType::I32Load16U: {
731         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
732     }
733
734     case LoadOpType::I64Load16U: {
735         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
736         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
737     }
738
739     case LoadOpType::I32Load: {
740         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
741     }
742
743     case LoadOpType::I64Load32U: {
744         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
745         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
746     }
747
748     case LoadOpType::I64Load32S: {
749         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
750         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
751     }
752
753     case LoadOpType::I64Load: {
754         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
755     }
756
757     case LoadOpType::F32Load: {
758         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
759     }
760
761     case LoadOpType::F64Load: {
762         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
763     }
764     }
765     RELEASE_ASSERT_NOT_REACHED();
766 }
767
768 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
769 {
770     ASSERT(pointer->type() == Int32);
771
772     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
773         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
774         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
775         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
776         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
777             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
778         });
779
780         switch (op) {
781         case LoadOpType::I32Load8S:
782         case LoadOpType::I32Load16S:
783         case LoadOpType::I32Load:
784         case LoadOpType::I32Load16U:
785         case LoadOpType::I32Load8U:
786             result = constant(Int32, 0);
787             break;
788         case LoadOpType::I64Load8S:
789         case LoadOpType::I64Load8U:
790         case LoadOpType::I64Load16S:
791         case LoadOpType::I64Load32U:
792         case LoadOpType::I64Load32S:
793         case LoadOpType::I64Load:
794         case LoadOpType::I64Load16U:
795             result = constant(Int64, 0);
796             break;
797         case LoadOpType::F32Load:
798             result = constant(Float, 0);
799             break;
800         case LoadOpType::F64Load:
801             result = constant(Double, 0);
802             break;
803         }
804
805     } else
806         result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
807
808     return { };
809 }
810
811 inline uint32_t sizeOfStoreOp(StoreOpType op)
812 {
813     switch (op) {
814     case StoreOpType::I32Store8:
815     case StoreOpType::I64Store8:
816         return 1;
817     case StoreOpType::I32Store16:
818     case StoreOpType::I64Store16:
819         return 2;
820     case StoreOpType::I32Store:
821     case StoreOpType::I64Store32:
822     case StoreOpType::F32Store:
823         return 4;
824     case StoreOpType::I64Store:
825     case StoreOpType::F64Store:
826         return 8;
827     }
828     RELEASE_ASSERT_NOT_REACHED();
829 }
830
831
832 inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
833 {
834     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
835
836     switch (op) {
837     case StoreOpType::I64Store8:
838         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
839         FALLTHROUGH;
840
841     case StoreOpType::I32Store8:
842         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
843         return;
844
845     case StoreOpType::I64Store16:
846         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
847         FALLTHROUGH;
848
849     case StoreOpType::I32Store16:
850         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
851         return;
852
853     case StoreOpType::I64Store32:
854         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
855         FALLTHROUGH;
856
857     case StoreOpType::I64Store:
858     case StoreOpType::I32Store:
859     case StoreOpType::F32Store:
860     case StoreOpType::F64Store:
861         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
862         return;
863     }
864     RELEASE_ASSERT_NOT_REACHED();
865 }
866
867 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
868 {
869     ASSERT(pointer->type() == Int32);
870
871     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
872         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
873         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
874         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
875         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
876             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
877         });
878     } else
879         emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
880
881     return { };
882 }
883
884 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
885 {
886     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
887     return { };
888 }
889
890 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
891 {
892     return constant(toB3Type(type), value);
893 }
894
895 void B3IRGenerator::emitTierUpCheck(uint32_t decrementCount, Origin origin)
896 {
897     if (!m_tierUp)
898         return;
899
900     ASSERT(m_tierUp);
901     Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(m_tierUp), origin);
902     Value* oldCountDown = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, countDownLocation);
903     Value* newCountDown = m_currentBlock->appendNew<Value>(m_proc, Sub, origin, oldCountDown, constant(Int32, decrementCount, origin));
904     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, newCountDown, countDownLocation);
905
906     PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, origin);
907     Effects effects = Effects::none();
908     // FIXME: we should have a more precise heap range for the tier up count.
909     effects.reads = B3::HeapRange::top();
910     effects.writes = B3::HeapRange::top();
911     patch->effects = effects;
912
913     patch->append(newCountDown, ValueRep::SomeRegister);
914     patch->append(oldCountDown, ValueRep::SomeRegister);
915     patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
916         MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
917         MacroAssembler::Label tierUpResume = jit.label();
918
919         params.addLatePath([=] (CCallHelpers& jit) {
920             tierUp.link(&jit);
921
922             const unsigned extraPaddingBytes = 0;
923             RegisterSet registersToSpill = { };
924             registersToSpill.add(GPRInfo::argumentGPR1);
925             unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
926
927             jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
928             MacroAssembler::Call call = jit.nearCall();
929
930             ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
931             jit.jump(tierUpResume);
932
933             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
934                 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
935
936             });
937         });
938     });
939 }
940
941 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
942 {
943     BasicBlock* body = m_proc.addBlock();
944     BasicBlock* continuation = m_proc.addBlock();
945
946     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
947
948     m_currentBlock = body;
949     emitTierUpCheck(TierUpCount::loopDecrement(), origin());
950
951     return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
952 }
953
954 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
955 {
956     return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
957 }
958
959 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
960 {
961     return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
962 }
963
964 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
965 {
966     // FIXME: This needs to do some kind of stack passing.
967
968     BasicBlock* taken = m_proc.addBlock();
969     BasicBlock* notTaken = m_proc.addBlock();
970     BasicBlock* continuation = m_proc.addBlock();
971
972     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
973     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
974     taken->addPredecessor(m_currentBlock);
975     notTaken->addPredecessor(m_currentBlock);
976
977     m_currentBlock = taken;
978     result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
979     return { };
980 }
981
982 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
983 {
984     unifyValuesWithBlock(currentStack, data.result);
985     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
986     return addElseToUnreachable(data);
987 }
988
989 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
990 {
991     ASSERT(data.type() == BlockType::If);
992     m_currentBlock = data.special;
993     data.convertIfToBlock();
994     return { };
995 }
996
997 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
998 {
999     ASSERT(returnValues.size() <= 1);
1000     if (returnValues.size())
1001         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
1002     else
1003         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
1004     return { };
1005 }
1006
1007 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1008 {
1009     unifyValuesWithBlock(returnValues, data.resultForBranch());
1010
1011     BasicBlock* target = data.targetBlockForBranch();
1012     if (condition) {
1013         BasicBlock* continuation = m_proc.addBlock();
1014         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1015         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
1016         target->addPredecessor(m_currentBlock);
1017         continuation->addPredecessor(m_currentBlock);
1018         m_currentBlock = continuation;
1019     } else {
1020         m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
1021         target->addPredecessor(m_currentBlock);
1022     }
1023
1024     return { };
1025 }
1026
1027 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1028 {
1029     for (size_t i = 0; i < targets.size(); ++i)
1030         unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
1031     unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1032
1033     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
1034     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
1035     for (size_t i = 0; i < targets.size(); ++i)
1036         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
1037
1038     return { };
1039 }
1040
1041 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1042 {
1043     ControlData& data = entry.controlData;
1044
1045     unifyValuesWithBlock(expressionStack, data.result);
1046     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1047     data.continuation->addPredecessor(m_currentBlock);
1048
1049     return addEndToUnreachable(entry);
1050 }
1051
1052
1053 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1054 {
1055     ControlData& data = entry.controlData;
1056     m_currentBlock = data.continuation;
1057
1058     if (data.type() == BlockType::If) {
1059         data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
1060         m_currentBlock->addPredecessor(data.special);
1061     }
1062
1063     for (Value* result : data.result) {
1064         m_currentBlock->append(result);
1065         entry.enclosedExpressionStack.append(result);
1066     }
1067
1068     // TopLevel does not have any code after this so we need to make sure we emit a return here.
1069     if (data.type() == BlockType::TopLevel)
1070         return addReturn(entry.controlData, entry.enclosedExpressionStack);
1071
1072     return { };
1073 }
1074
1075 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1076 {
1077     ASSERT(signature.argumentCount() == args.size());
1078
1079     m_makesCalls = true;
1080
1081     Type returnType = signature.returnType();
1082     Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1083
1084     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1085         m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1086
1087         // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
1088         Value* targetInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfTargetInstance(functionIndex)));
1089         // The target instance is 0 unless the call is wasm->wasm.
1090         Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0));
1091
1092         BasicBlock* isWasmBlock = m_proc.addBlock();
1093         BasicBlock* isEmbedderBlock = m_proc.addBlock();
1094         BasicBlock* continuation = m_proc.addBlock();
1095         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
1096
1097         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
1098             [=] (PatchpointValue* patchpoint) {
1099                 patchpoint->effects.writesPinned = true;
1100                 patchpoint->effects.readsPinned = true;
1101                 // We need to clobber all potential pinned registers since we might be leaving the instance.
1102                 // We pessimistically assume we could be calling to something that is bounds checking.
1103                 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1104                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1105                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1106                     AllowMacroScratchRegisterUsage allowScratch(jit);
1107                     CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1108                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1109                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1110                     });
1111                 });
1112             });
1113         UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
1114         isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1115
1116         // FIXME: Let's remove this indirection by creating a PIC friendly IC
1117         // for calls out to the embedder. This shouldn't be that hard to do. We could probably
1118         // implement the IC to be over Context*.
1119         // https://bugs.webkit.org/show_bug.cgi?id=170375
1120         Value* jumpDestination = isEmbedderBlock->appendNew<MemoryValue>(m_proc,
1121             Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfWasmToEmbedderStub(functionIndex)));
1122
1123         Value* embedderCallResult = wasmCallingConvention().setupCall(m_proc, isEmbedderBlock, origin(), args, toB3Type(returnType),
1124             [=] (PatchpointValue* patchpoint) {
1125                 patchpoint->effects.writesPinned = true;
1126                 patchpoint->effects.readsPinned = true;
1127                 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
1128                 // We need to clobber all potential pinned registers since we might be leaving the instance.
1129                 // We pessimistically assume we could be calling to something that is bounds checking.
1130                 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1131                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1132                 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1133                     AllowMacroScratchRegisterUsage allowScratch(jit);
1134                     jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1135                 });
1136             });
1137         UpsilonValue* embedderCallResultUpsilon = returnType == Void ? nullptr : isEmbedderBlock->appendNew<UpsilonValue>(m_proc, origin(), embedderCallResult);
1138         isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1139
1140         m_currentBlock = continuation;
1141
1142         if (returnType == Void)
1143             result = nullptr;
1144         else {
1145             result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1146             wasmCallResultUpsilon->setPhi(result);
1147             embedderCallResultUpsilon->setPhi(result);
1148         }
1149
1150         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1151         restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation);
1152     } else {
1153         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1154             [=] (PatchpointValue* patchpoint) {
1155                 patchpoint->effects.writesPinned = true;
1156                 patchpoint->effects.readsPinned = true;
1157
1158                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1159                     AllowMacroScratchRegisterUsage allowScratch(jit);
1160                     CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1161                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1162                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1163                     });
1164                 });
1165             });
1166     }
1167
1168     return { };
1169 }
1170
1171 auto B3IRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1172 {
1173     ExpressionType calleeIndex = args.takeLast();
1174     ASSERT(signature.argumentCount() == args.size());
1175
1176     m_makesCalls = true;
1177     // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1178     // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1179     // can be to the embedder for our stack check calculation.
1180     m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1181
1182     ExpressionType callableFunctionBuffer;
1183     ExpressionType instancesBuffer;
1184     ExpressionType callableFunctionBufferLength;
1185     ExpressionType mask;
1186     {
1187         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1188             instanceValue(), safeCast<int32_t>(Instance::offsetOfTable()));
1189         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1190             table, safeCast<int32_t>(Table::offsetOfFunctions()));
1191         instancesBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1192             table, safeCast<int32_t>(Table::offsetOfInstances()));
1193         callableFunctionBufferLength = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1194             table, safeCast<int32_t>(Table::offsetOfLength()));
1195         mask = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(),
1196             m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1197                 table, safeCast<int32_t>(Table::offsetOfMask())));
1198     }
1199
1200     // Check the index we are looking for is valid.
1201     {
1202         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1203             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferLength));
1204
1205         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1206             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1207         });
1208     }
1209
1210     calleeIndex = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex);
1211
1212     if (Options::enableSpectreMitigations())
1213         calleeIndex = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), mask, calleeIndex);
1214
1215     ExpressionType callableFunction;
1216     {
1217         // Compute the offset in the table index space we are looking for.
1218         ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1219             calleeIndex, constant(pointerType(), sizeof(WasmToWasmImportableFunction)));
1220         callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1221
1222         // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1223         // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1224         static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1225         ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), callableFunction, safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfSignatureIndex()));
1226         {
1227             CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1228                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1229                     calleeSignatureIndex,
1230                     m_currentBlock->appendNew<Const64Value>(m_proc, origin(), Signature::invalidIndex)));
1231
1232             check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1233                 this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1234             });
1235         }
1236
1237         // Check the signature matches the value we expect.
1238         {
1239             ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const64Value>(m_proc, origin(), SignatureInformation::get(signature));
1240             CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1241                 m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1242
1243             check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1244                 this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1245             });
1246         }
1247     }
1248
1249     // Do a context switch if needed.
1250     {
1251         Value* offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1252             calleeIndex, constant(pointerType(), sizeof(Instance*)));
1253         Value* newContextInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1254             m_currentBlock->appendNew<Value>(m_proc, Add, origin(), instancesBuffer, offset));
1255
1256         BasicBlock* continuation = m_proc.addBlock();
1257         BasicBlock* doContextSwitch = m_proc.addBlock();
1258
1259         Value* isSameContextInstance = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1260             newContextInstance, instanceValue());
1261         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1262             isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch));
1263
1264         PatchpointValue* patchpoint = doContextSwitch->appendNew<PatchpointValue>(m_proc, B3::Void, origin());
1265         patchpoint->effects.writesPinned = true;
1266         // We pessimistically assume we're calling something with BoundsChecking memory.
1267         // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1268         patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1269         patchpoint->clobber(RegisterSet::macroScratchRegisters());
1270         patchpoint->append(newContextInstance, ValueRep::SomeRegister);
1271         patchpoint->append(instanceValue(), ValueRep::SomeRegister);
1272         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1273             AllowMacroScratchRegisterUsage allowScratch(jit);
1274             GPRReg newContextInstance = params[0].gpr();
1275             GPRReg oldContextInstance = params[1].gpr();
1276             const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1277             GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1278             ASSERT(newContextInstance != baseMemory);
1279             jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1280             jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1281             jit.storeWasmContextInstance(newContextInstance);
1282             ASSERT(pinnedRegs.sizeRegister != baseMemory);
1283             // FIXME: We should support more than one memory size register
1284             //   see: https://bugs.webkit.org/show_bug.cgi?id=162952
1285             ASSERT(pinnedRegs.sizeRegister != newContextInstance);
1286             jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
1287             jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1288         });
1289         doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation);
1290
1291         m_currentBlock = continuation;
1292     }
1293
1294     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1295         m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
1296             safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation())));
1297
1298     Type returnType = signature.returnType();
1299     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1300         [=] (PatchpointValue* patchpoint) {
1301             patchpoint->effects.writesPinned = true;
1302             patchpoint->effects.readsPinned = true;
1303             // We need to clobber all potential pinned registers since we might be leaving the instance.
1304             // We pessimistically assume we're always calling something that is bounds checking so
1305             // because the wasm->wasm thunk unconditionally overrides the size registers.
1306             // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1307             // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1308             patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1309
1310             patchpoint->append(calleeCode, ValueRep::SomeRegister);
1311             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1312                 AllowMacroScratchRegisterUsage allowScratch(jit);
1313                 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1314             });
1315         });
1316
1317     // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1318     restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock);
1319
1320     return { };
1321 }
1322
1323 void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1324 {
1325     m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1326 }
1327
1328 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1329 {
1330     ASSERT(result.size() <= resultStack.size());
1331
1332     for (size_t i = 0; i < result.size(); ++i)
1333         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1334 }
1335
1336 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1337 {
1338     dataLog(comma, "ExpressionStack:");
1339     for (const auto& expression : expressionStack)
1340         dataLog(comma, *expression);
1341 }
1342
1343 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1344 {
1345     dataLogLn("Constants:");
1346     for (const auto& constant : m_constantPool)
1347         dataLogLn(deepDump(m_proc, constant.value));
1348
1349     dataLogLn("Processing Graph:");
1350     dataLog(m_proc);
1351     dataLogLn("With current block:", *m_currentBlock);
1352     dataLogLn("Control stack:");
1353     ASSERT(controlStack.size());
1354     for (size_t i = controlStack.size(); i--;) {
1355         dataLog("  ", controlStack[i].controlData, ": ");
1356         CommaPrinter comma(", ", "");
1357         dumpExpressionStack(comma, *expressionStack);
1358         expressionStack = &controlStack[i].enclosedExpressionStack;
1359         dataLogLn();
1360     }
1361     dataLogLn();
1362 }
1363
1364 auto B3IRGenerator::origin() -> Origin
1365 {
1366     OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset());
1367     ASSERT(isValidOpType(static_cast<uint8_t>(origin.opcode())));
1368     return bitwise_cast<Origin>(origin);
1369 }
1370
1371 Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1372 {
1373     auto result = std::make_unique<InternalFunction>();
1374
1375     compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
1376     compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1377
1378     Procedure procedure;
1379
1380     procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1381         if (origin.data())
1382             out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1383     });
1384     
1385     // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1386     // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1387     // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1388     // optLevel=1.
1389     procedure.setNeedsUsedRegisters(false);
1390     
1391     procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
1392         ? Options::webAssemblyBBQOptimizationLevel()
1393         : Options::webAssemblyOMGOptimizationLevel());
1394
1395     B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, compilationMode, functionIndex, tierUp, throwWasmException);
1396     FunctionParser<B3IRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1397     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1398
1399     irGenerator.insertConstants();
1400
1401     procedure.resetReachability();
1402     if (!ASSERT_DISABLED)
1403         validate(procedure, "After parsing:\n");
1404
1405     dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure);
1406     fixSSA(procedure);
1407     dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure);
1408     
1409     {
1410         B3::prepareForGeneration(procedure);
1411         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1412         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1413         result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1414     }
1415
1416     return result;
1417 }
1418
1419 // Custom wasm ops. These are the ones too messy to do in wasm.json.
1420
1421 void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1422 {
1423     ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1424     const B3::Type type = left->type();
1425
1426     {
1427         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1428             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1429
1430         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1431             this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1432         });
1433     }
1434
1435     if (operation == Div) {
1436         int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1437
1438         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1439             m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1440                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1441                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1442
1443         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1444             this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1445         });
1446     }
1447 }
1448
1449 template<>
1450 auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1451 {
1452     const B3::Opcode op = Div;
1453     emitChecksForModOrDiv(op, left, right);
1454     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1455     return { };
1456 }
1457
1458 template<>
1459 auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1460 {
1461     const B3::Opcode op = Mod;
1462     emitChecksForModOrDiv(op, left, right);
1463     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1464     return { };
1465 }
1466
1467 template<>
1468 auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1469 {
1470     const B3::Opcode op = UDiv;
1471     emitChecksForModOrDiv(op, left, right);
1472     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1473     return { };
1474 }
1475
1476 template<>
1477 auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1478 {
1479     const B3::Opcode op = UMod;
1480     emitChecksForModOrDiv(op, left, right);
1481     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1482     return { };
1483 }
1484
1485 template<>
1486 auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1487 {
1488     const B3::Opcode op = Div;
1489     emitChecksForModOrDiv(op, left, right);
1490     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1491     return { };
1492 }
1493
1494 template<>
1495 auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1496 {
1497     const B3::Opcode op = Mod;
1498     emitChecksForModOrDiv(op, left, right);
1499     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1500     return { };
1501 }
1502
1503 template<>
1504 auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1505 {
1506     const B3::Opcode op = UDiv;
1507     emitChecksForModOrDiv(op, left, right);
1508     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1509     return { };
1510 }
1511
1512 template<>
1513 auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1514 {
1515     const B3::Opcode op = UMod;
1516     emitChecksForModOrDiv(op, left, right);
1517     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1518     return { };
1519 }
1520
1521 template<>
1522 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1523 {
1524     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1525     patchpoint->append(arg, ValueRep::SomeRegister);
1526     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1527         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1528     });
1529     patchpoint->effects = Effects::none();
1530     result = patchpoint;
1531     return { };
1532 }
1533
1534 template<>
1535 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1536 {
1537     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1538     patchpoint->append(arg, ValueRep::SomeRegister);
1539     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1540         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1541     });
1542     patchpoint->effects = Effects::none();
1543     result = patchpoint;
1544     return { };
1545 }
1546
1547 template<>
1548 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1549 {
1550 #if CPU(X86_64)
1551     if (MacroAssembler::supportsCountPopulation()) {
1552         PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1553         patchpoint->append(arg, ValueRep::SomeRegister);
1554         patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1555             jit.countPopulation32(params[1].gpr(), params[0].gpr());
1556         });
1557         patchpoint->effects = Effects::none();
1558         result = patchpoint;
1559         return { };
1560     }
1561 #endif
1562
1563     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1564     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1565     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1566     return { };
1567 }
1568
1569 template<>
1570 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1571 {
1572 #if CPU(X86_64)
1573     if (MacroAssembler::supportsCountPopulation()) {
1574         PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1575         patchpoint->append(arg, ValueRep::SomeRegister);
1576         patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1577             jit.countPopulation64(params[1].gpr(), params[0].gpr());
1578         });
1579         patchpoint->effects = Effects::none();
1580         result = patchpoint;
1581         return { };
1582     }
1583 #endif
1584
1585     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1586     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1587     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1588     return { };
1589 }
1590
1591 template<>
1592 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1593 {
1594     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1595     if (isX86())
1596         patchpoint->numGPScratchRegisters = 1;
1597     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1598     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1599     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1600         AllowMacroScratchRegisterUsage allowScratch(jit);
1601 #if CPU(X86_64)
1602         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1603 #else
1604         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1605 #endif
1606     });
1607     patchpoint->effects = Effects::none();
1608     result = patchpoint;
1609     return { };
1610 }
1611
1612 template<>
1613 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1614 {
1615     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1616     if (isX86())
1617         patchpoint->numGPScratchRegisters = 1;
1618     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1619     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1620     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1621         AllowMacroScratchRegisterUsage allowScratch(jit);
1622 #if CPU(X86_64)
1623         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1624 #else
1625         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1626 #endif
1627     });
1628     patchpoint->effects = Effects::none();
1629     result = patchpoint;
1630     return { };
1631 }
1632
1633 template<>
1634 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1635 {
1636     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1637     patchpoint->append(arg, ValueRep::SomeRegister);
1638     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1639         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1640     });
1641     patchpoint->effects = Effects::none();
1642     result = patchpoint;
1643     return { };
1644 }
1645
1646 template<>
1647 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1648 {
1649     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1650     patchpoint->append(arg, ValueRep::SomeRegister);
1651     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1652         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1653     });
1654     patchpoint->effects = Effects::none();
1655     result = patchpoint;
1656     return { };
1657 }
1658
1659 template<>
1660 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1661 {
1662     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1663     patchpoint->append(arg, ValueRep::SomeRegister);
1664     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1665         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1666     });
1667     patchpoint->effects = Effects::none();
1668     result = patchpoint;
1669     return { };
1670 }
1671
1672 template<>
1673 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1674 {
1675     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1676     patchpoint->append(arg, ValueRep::SomeRegister);
1677     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1678         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1679     });
1680     patchpoint->effects = Effects::none();
1681     result = patchpoint;
1682     return { };
1683 }
1684
1685 template<>
1686 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1687 {
1688     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1689     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1690     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1691         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1692         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1693     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1694     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1695     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1696         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1697     });
1698     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1699     patchpoint->append(arg, ValueRep::SomeRegister);
1700     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1701         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1702     });
1703     patchpoint->effects = Effects::none();
1704     result = patchpoint;
1705     return { };
1706 }
1707
1708 template<>
1709 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1710 {
1711     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1712     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1713     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1714         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1715         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1716     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1717     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1718     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1719         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1720     });
1721     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1722     patchpoint->append(arg, ValueRep::SomeRegister);
1723     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1724         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1725     });
1726     patchpoint->effects = Effects::none();
1727     result = patchpoint;
1728     return { };
1729 }
1730
1731
1732 template<>
1733 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1734 {
1735     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1736     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1737     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1738         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1739         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1740     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1741     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1742     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1743         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1744     });
1745     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1746     patchpoint->append(arg, ValueRep::SomeRegister);
1747     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1748         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1749     });
1750     patchpoint->effects = Effects::none();
1751     result = patchpoint;
1752     return { };
1753 }
1754
1755 template<>
1756 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1757 {
1758     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1759     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1760     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1761         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1762         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1763     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1764     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1765     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1766         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1767     });
1768     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1769     patchpoint->append(arg, ValueRep::SomeRegister);
1770     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1771         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1772     });
1773     patchpoint->effects = Effects::none();
1774     result = patchpoint;
1775     return { };
1776 }
1777
1778 template<>
1779 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1780 {
1781     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1782     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1783     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1784         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1785         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1786     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1787     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1788     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1789         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1790     });
1791     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1792     patchpoint->append(arg, ValueRep::SomeRegister);
1793     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1794         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1795     });
1796     patchpoint->effects = Effects::none();
1797     result = patchpoint;
1798     return { };
1799 }
1800
1801 template<>
1802 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1803 {
1804     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1805     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1806     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1807         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1808         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1809     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1810     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1811     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1812         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1813     });
1814
1815     Value* signBitConstant;
1816     if (isX86()) {
1817         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1818         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1819         // so we can pool them if needed.
1820         signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1821     }
1822     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1823     patchpoint->append(arg, ValueRep::SomeRegister);
1824     if (isX86()) {
1825         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1826         patchpoint->numFPScratchRegisters = 1;
1827     }
1828     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1829     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1830         AllowMacroScratchRegisterUsage allowScratch(jit);
1831         FPRReg scratch = InvalidFPRReg;
1832         FPRReg constant = InvalidFPRReg;
1833         if (isX86()) {
1834             scratch = params.fpScratch(0);
1835             constant = params[2].fpr();
1836         }
1837         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1838     });
1839     patchpoint->effects = Effects::none();
1840     result = patchpoint;
1841     return { };
1842 }
1843
1844 template<>
1845 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1846 {
1847     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1848     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1849     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1850         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1851         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1852     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1853     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1854     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1855         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1856     });
1857     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1858     patchpoint->append(arg, ValueRep::SomeRegister);
1859     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1860         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1861     });
1862     patchpoint->effects = Effects::none();
1863     result = patchpoint;
1864     return { };
1865 }
1866
1867 template<>
1868 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1869 {
1870     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1871     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1872     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1873         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1874         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1875     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1876     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1877     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1878         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1879     });
1880
1881     Value* signBitConstant;
1882     if (isX86()) {
1883         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1884         // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1885         // so we can pool them if needed.
1886         signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1887     }
1888     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1889     patchpoint->append(arg, ValueRep::SomeRegister);
1890     if (isX86()) {
1891         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1892         patchpoint->numFPScratchRegisters = 1;
1893     }
1894     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1895     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1896         AllowMacroScratchRegisterUsage allowScratch(jit);
1897         FPRReg scratch = InvalidFPRReg;
1898         FPRReg constant = InvalidFPRReg;
1899         if (isX86()) {
1900             scratch = params.fpScratch(0);
1901             constant = params[2].fpr();
1902         }
1903         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1904     });
1905     patchpoint->effects = Effects::none();
1906     result = patchpoint;
1907     return { };
1908 }
1909
1910 } } // namespace JSC::Wasm
1911
1912 #include "WasmB3IRGeneratorInlines.h"
1913
1914 #endif // ENABLE(WEBASSEMBLY)