7a24319b7c7b1748d8937640b6a3fd88c7c4ce70
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "AllowMacroScratchRegisterUsageIf.h"
32 #include "B3BasicBlockInlines.h"
33 #include "B3CCallValue.h"
34 #include "B3Compile.h"
35 #include "B3ConstPtrValue.h"
36 #include "B3FixSSA.h"
37 #include "B3Generate.h"
38 #include "B3InsertionSet.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3SwitchValue.h"
42 #include "B3UpsilonValue.h"
43 #include "B3Validate.h"
44 #include "B3ValueInlines.h"
45 #include "B3ValueKey.h"
46 #include "B3Variable.h"
47 #include "B3VariableValue.h"
48 #include "B3WasmAddressValue.h"
49 #include "B3WasmBoundsCheckValue.h"
50 #include "DisallowMacroScratchRegisterUsage.h"
51 #include "JSCInlines.h"
52 #include "JSWebAssemblyInstance.h"
53 #include "ScratchRegisterAllocator.h"
54 #include "VirtualRegister.h"
55 #include "WasmCallingConvention.h"
56 #include "WasmContextInlines.h"
57 #include "WasmExceptionType.h"
58 #include "WasmFunctionParser.h"
59 #include "WasmInstance.h"
60 #include "WasmMemory.h"
61 #include "WasmOMGPlan.h"
62 #include "WasmOpcodeOrigin.h"
63 #include "WasmSignatureInlines.h"
64 #include "WasmThunks.h"
65 #include <limits>
66 #include <wtf/Optional.h>
67 #include <wtf/StdLibExtras.h>
68
69 void dumpProcedure(void* ptr)
70 {
71     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
72     proc->dump(WTF::dataFile());
73 }
74
75 namespace JSC { namespace Wasm {
76
77 using namespace B3;
78
79 namespace {
80 namespace WasmB3IRGeneratorInternal {
81 static const bool verbose = false;
82 }
83 }
84
85 class B3IRGenerator {
86 public:
87     struct ControlData {
88         ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
89             : blockType(type)
90             , continuation(continuation)
91             , special(special)
92         {
93             if (signature != Void)
94                 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
95         }
96
97         ControlData()
98         {
99         }
100
101         void dump(PrintStream& out) const
102         {
103             switch (type()) {
104             case BlockType::If:
105                 out.print("If:       ");
106                 break;
107             case BlockType::Block:
108                 out.print("Block:    ");
109                 break;
110             case BlockType::Loop:
111                 out.print("Loop:     ");
112                 break;
113             case BlockType::TopLevel:
114                 out.print("TopLevel: ");
115                 break;
116             }
117             out.print("Continuation: ", *continuation, ", Special: ");
118             if (special)
119                 out.print(*special);
120             else
121                 out.print("None");
122         }
123
124         BlockType type() const { return blockType; }
125
126         bool hasNonVoidSignature() const { return result.size(); }
127
128         BasicBlock* targetBlockForBranch()
129         {
130             if (type() == BlockType::Loop)
131                 return special;
132             return continuation;
133         }
134
135         void convertIfToBlock()
136         {
137             ASSERT(type() == BlockType::If);
138             blockType = BlockType::Block;
139             special = nullptr;
140         }
141
142         using ResultList = Vector<Value*, 1>; // Value must be a Phi
143
144         ResultList resultForBranch() const
145         {
146             if (type() == BlockType::Loop)
147                 return ResultList();
148             return result;
149         }
150
151     private:
152         friend class B3IRGenerator;
153         BlockType blockType;
154         BasicBlock* continuation;
155         BasicBlock* special;
156         ResultList result;
157     };
158
159     typedef Value* ExpressionType;
160     typedef ControlData ControlType;
161     typedef Vector<ExpressionType, 1> ExpressionList;
162     typedef ControlData::ResultList ResultList;
163     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
164
165     static constexpr ExpressionType emptyExpression() { return nullptr; }
166
167     typedef String ErrorType;
168     typedef Unexpected<ErrorType> UnexpectedResult;
169     typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
170     typedef Expected<void, ErrorType> PartialResult;
171     template <typename ...Args>
172     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
173     {
174         using namespace FailureHelper; // See ADL comment in WasmParser.h.
175         return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
176     }
177 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
178         if (UNLIKELY(condition))                  \
179             return fail(__VA_ARGS__);             \
180     } while (0)
181
182     B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, CompilationMode, unsigned functionIndex, TierUpCount*, ThrowWasmException);
183
184     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
185     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
186     ExpressionType addConstant(Type, uint64_t);
187
188     // References
189     PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType& value, ExpressionType& result);
190     PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result);
191
192     // Tables
193     PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType& index, ExpressionType& result);
194     PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType& index, ExpressionType& value);
195     PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result);
196     PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType& fill, ExpressionType& delta, ExpressionType& result);
197     PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType& offset, ExpressionType& fill, ExpressionType& count);
198     // Locals
199     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
200     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
201
202     // Globals
203     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
204     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
205
206     // Memory
207     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
208     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
209     PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
210     PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
211
212     // Basic operators
213     template<OpType>
214     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
215     template<OpType>
216     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
217     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
218
219     // Control flow
220     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
221     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
222     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
223     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
224     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
225     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
226
227     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
228     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
229     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
230     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
231     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
232
233     // Calls
234     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
235     PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
236     PartialResult WARN_UNUSED_RETURN addUnreachable();
237
238     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
239     void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
240
241     Value* constant(B3::Type, uint64_t bits, Optional<Origin> = WTF::nullopt);
242     void insertConstants();
243
244     ALWAYS_INLINE void didKill(ExpressionType) { }
245
246 private:
247     void emitExceptionCheck(CCallHelpers&, ExceptionType);
248
249     void emitTierUpCheck(uint32_t decrementCount, Origin);
250
251     void emitWriteBarrierForJSWrapper();
252     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
253     B3::Kind memoryKind(B3::Opcode memoryOp);
254     ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
255     void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
256
257     void unify(const ExpressionType phi, const ExpressionType source);
258     void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
259
260     void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
261
262     int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
263
264     void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*);
265     enum class RestoreCachedStackLimit { No, Yes };
266     void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
267
268     Origin origin();
269
270     FunctionParser<B3IRGenerator>* m_parser { nullptr };
271     const ModuleInformation& m_info;
272     const MemoryMode m_mode { MemoryMode::BoundsChecking };
273     const CompilationMode m_compilationMode { CompilationMode::BBQMode };
274     const unsigned m_functionIndex { UINT_MAX };
275     const TierUpCount* m_tierUp { nullptr };
276
277     Procedure& m_proc;
278     BasicBlock* m_currentBlock { nullptr };
279     Vector<Variable*> m_locals;
280     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
281     HashMap<ValueKey, Value*> m_constantPool;
282     InsertionSet m_constantInsertionValues;
283     GPRReg m_memoryBaseGPR { InvalidGPRReg };
284     GPRReg m_memorySizeGPR { InvalidGPRReg };
285     GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
286     bool m_makesCalls { false };
287
288     Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used.
289     bool m_usesInstanceValue { false };
290     Value* instanceValue()
291     {
292         m_usesInstanceValue = true;
293         return m_instanceValue;
294     }
295
296     uint32_t m_maxNumJSCallArguments { 0 };
297     unsigned m_numImportFunctions;
298 };
299
300 // Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
301 int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
302 {
303     if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
304         ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
305         return 0;
306     }
307     return offset;
308 }
309
310 void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg)
311 {
312     if (Context::useFastTLS()) {
313         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
314         if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
315             patchpoint->clobber(RegisterSet::macroScratchRegisters());
316         patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
317         patchpoint->setGenerator(
318             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
319                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
320                 jit.storeWasmContextInstance(params[0].gpr());
321             });
322         return;
323     }
324
325     // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
326     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
327     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
328     Effects effects = Effects::none();
329     effects.writesPinned = true;
330     effects.reads = B3::HeapRange::top();
331     patchpoint->effects = effects;
332     patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
333     patchpoint->append(arg, ValueRep::SomeRegister);
334     GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
335     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
336         jit.move(param[0].gpr(), wasmContextInstanceGPR);
337     });
338 }
339
340 B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
341     : m_info(info)
342     , m_mode(mode)
343     , m_compilationMode(compilationMode)
344     , m_functionIndex(functionIndex)
345     , m_tierUp(tierUp)
346     , m_proc(procedure)
347     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
348     , m_constantInsertionValues(m_proc)
349     , m_numImportFunctions(info.importFunctionCount())
350 {
351     m_currentBlock = m_proc.addBlock();
352
353     // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
354     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
355
356     m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
357     m_proc.pinRegister(m_memoryBaseGPR);
358
359     m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
360     if (!Context::useFastTLS())
361         m_proc.pinRegister(m_wasmContextInstanceGPR);
362
363     if (mode != MemoryMode::Signaling) {
364         m_memorySizeGPR = pinnedRegs.sizeRegister;
365         m_proc.pinRegister(m_memorySizeGPR);
366     }
367
368     if (throwWasmException)
369         Thunks::singleton().setThrowWasmException(throwWasmException);
370
371     if (info.memory) {
372         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
373             AllowMacroScratchRegisterUsage allowScratch(jit);
374             switch (m_mode) {
375             case MemoryMode::BoundsChecking:
376                 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
377                 break;
378             case MemoryMode::Signaling:
379                 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
380                 break;
381             }
382             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
383         });
384
385         switch (m_mode) {
386         case MemoryMode::BoundsChecking:
387             break;
388         case MemoryMode::Signaling:
389             // Most memory accesses in signaling mode don't do an explicit
390             // exception check because they can rely on fault handling to detect
391             // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
392             // thunk to exist so that it can jump to that thunk.
393             if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
394                 CRASH();
395             break;
396         }
397     }
398
399     wasmCallingConvention().setupFrameInPrologue(&compilation->calleeMoveLocation, m_proc, Origin(), m_currentBlock);
400
401     {
402         B3::Value* framePointer = m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin());
403         B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, pointerType(), Origin());
404         m_instanceValue = stackOverflowCheck;
405         stackOverflowCheck->appendSomeRegister(framePointer);
406         stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters());
407         if (!Context::useFastTLS()) {
408             // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
409             // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
410             stackOverflowCheck->effects.writesPinned = false;
411             stackOverflowCheck->effects.readsPinned = true;
412             stackOverflowCheck->resultConstraints = { ValueRep::reg(m_wasmContextInstanceGPR) };
413         }
414         stackOverflowCheck->numGPScratchRegisters = 2;
415         stackOverflowCheck->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
416             const Checked<int32_t> wasmFrameSize = params.proc().frameSize();
417             const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
418             const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
419                 // This allows us to elide stack checks for functions that are terminal nodes in the call
420                 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
421                 // having any such terminal node have its parent caller include some extra size in its
422                 // own check for it. The goal here is twofold:
423                 // 1. Emit less code.
424                 // 2. Try to speed things up by skipping stack checks.
425                 minimumParentCheckSize,
426                 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
427                 // spill all arguments to the stack, we ensure that a stack check here covers the
428                 // stack that such a stub would use.
429                 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
430             ));
431             const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
432             bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
433             bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
434
435             GPRReg contextInstance = Context::useFastTLS() ? params[0].gpr() : m_wasmContextInstanceGPR;
436
437             // This allows leaf functions to not do stack checks if their frame size is within
438             // certain limits since their caller would have already done the check.
439             if (needsOverflowCheck) {
440                 AllowMacroScratchRegisterUsage allowScratch(jit);
441                 GPRReg fp = params[1].gpr();
442                 GPRReg scratch1 = params.gpScratch(0);
443                 GPRReg scratch2 = params.gpScratch(1);
444
445                 if (Context::useFastTLS())
446                     jit.loadWasmContextInstance(contextInstance);
447
448                 jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2);
449                 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1);
450                 MacroAssembler::JumpList overflow;
451                 if (UNLIKELY(needUnderflowCheck))
452                     overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp));
453                 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2));
454                 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
455                     linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
456                 });
457             } else if (m_usesInstanceValue && Context::useFastTLS()) {
458                 // No overflow check is needed, but the instance values still needs to be correct.
459                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister());
460                 jit.loadWasmContextInstance(contextInstance);
461             } else {
462                 // We said we'd return a pointer. We don't actually need to because it isn't used, but the patchpoint conservatively said it had effects (potential stack check) which prevent it from getting removed.
463             }
464         });
465     }
466
467     emitTierUpCheck(TierUpCount::functionEntryDecrement(), Origin());
468 }
469
470 void B3IRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
471 {
472     restoreWasmContextInstance(proc, block, instance);
473
474     if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
475         // The Instance caches the stack limit, but also knows where its canonical location is.
476         Value* pointerToActualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfPointerToActualStackLimit()));
477         Value* actualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), pointerToActualStackLimit);
478         block->appendNew<MemoryValue>(m_proc, Store, origin(), actualStackLimit, instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedStackLimit()));
479     }
480
481     if (!!memory) {
482         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
483         RegisterSet clobbers;
484         clobbers.set(pinnedRegs->baseMemoryPointer);
485         clobbers.set(pinnedRegs->sizeRegister);
486         if (!isARM64())
487             clobbers.set(RegisterSet::macroScratchRegisters());
488
489         B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
490         Effects effects = Effects::none();
491         effects.writesPinned = true;
492         effects.reads = B3::HeapRange::top();
493         patchpoint->effects = effects;
494         patchpoint->clobber(clobbers);
495         patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
496
497         patchpoint->append(instance, ValueRep::SomeRegister);
498         patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
499             AllowMacroScratchRegisterUsage allowScratch(jit);
500             GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
501             GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs->sizeRegister;
502
503             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
504             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
505
506             jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs->sizeRegister, scratchOrSize);
507         });
508     }
509 }
510
511 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
512 {
513     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
514     auto jumpToExceptionStub = jit.jump();
515
516     jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
517         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
518     });
519 }
520
521 Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, Optional<Origin> maybeOrigin)
522 {
523     auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
524         Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits);
525         m_constantInsertionValues.insertValue(0, result);
526         return result;
527     });
528     return result.iterator->value;
529 }
530
531 void B3IRGenerator::insertConstants()
532 {
533     m_constantInsertionValues.execute(m_proc.at(0));
534 }
535
536 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
537 {
538     Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
539     totalBytesChecked += m_locals.size();
540     uint32_t totalBytes;
541     WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
542
543     for (uint32_t i = 0; i < count; ++i) {
544         Variable* local = m_proc.addVariable(toB3Type(type));
545         m_locals.uncheckedAppend(local);
546         auto val = isSubtype(type, Anyref) ? JSValue::encode(jsNull()) : 0;
547         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, constant(toB3Type(type), val, Origin()));
548     }
549     return { };
550 }
551
552 auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
553 {
554     ASSERT(!m_locals.size());
555     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
556
557     m_locals.grow(signature.argumentCount());
558     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
559         [=] (ExpressionType argument, unsigned i) {
560             Variable* argumentVariable = m_proc.addVariable(argument->type());
561             m_locals[i] = argumentVariable;
562             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
563         });
564     return { };
565 }
566
567 auto B3IRGenerator::addRefIsNull(ExpressionType& value, ExpressionType& result) -> PartialResult
568 {
569     result = m_currentBlock->appendNew<Value>(m_proc, B3::Equal, origin(), value, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), JSValue::encode(jsNull())));
570     return { };
571 }
572
573 auto B3IRGenerator::addTableGet(unsigned tableIndex, ExpressionType& index, ExpressionType& result) -> PartialResult
574 {
575     // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
576     result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(Anyref), origin(),
577         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&getWasmTableElement, B3CCallPtrTag)),
578         instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), index);
579
580     {
581         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
582             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0)));
583
584         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
585             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
586         });
587     }
588
589     return { };
590 }
591
592 auto B3IRGenerator::addTableSet(unsigned tableIndex, ExpressionType& index, ExpressionType& value) -> PartialResult
593 {
594     // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
595     auto shouldThrow = m_currentBlock->appendNew<CCallValue>(m_proc, B3::Int32, origin(),
596         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&setWasmTableElement, B3CCallPtrTag)),
597         instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), index, value);
598
599     {
600         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
601             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), shouldThrow, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
602
603         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
604             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
605         });
606     }
607
608     return { };
609 }
610
611 auto B3IRGenerator::addRefFunc(uint32_t index, ExpressionType& result) -> PartialResult
612 {
613     // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
614
615     result = m_currentBlock->appendNew<CCallValue>(m_proc, B3::Int64, origin(),
616         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&doWasmRefFunc, B3CCallPtrTag)),
617         instanceValue(), addConstant(Type::I32, index));
618
619     return { };
620 }
621
622 auto B3IRGenerator::addTableSize(unsigned tableIndex, ExpressionType& result) -> PartialResult
623 {
624     // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
625     uint32_t (*doSize)(Instance*, unsigned) = [] (Instance* instance, unsigned tableIndex) -> uint32_t {
626         return instance->table(tableIndex)->length();
627     };
628
629     result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(I32), origin(),
630         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(doSize, B3CCallPtrTag)),
631         instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex));
632
633     return { };
634 }
635
636 auto B3IRGenerator::addTableGrow(unsigned tableIndex, ExpressionType& fill, ExpressionType& delta, ExpressionType& result) -> PartialResult
637 {
638     result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(I32), origin(),
639         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&doWasmTableGrow, B3CCallPtrTag)),
640         instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), fill, delta);
641
642     return { };
643 }
644
645 auto B3IRGenerator::addTableFill(unsigned tableIndex, ExpressionType& offset, ExpressionType& fill, ExpressionType& count) -> PartialResult
646 {
647     auto result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(I32), origin(),
648         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&doWasmTableFill, B3CCallPtrTag)),
649         instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), offset, fill, count);
650
651     {
652         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
653             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
654
655         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
656             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
657         });
658     }
659
660     return { };
661 }
662
663 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
664 {
665     ASSERT(m_locals[index]);
666     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
667     return { };
668 }
669
670 auto B3IRGenerator::addUnreachable() -> PartialResult
671 {
672     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
673     unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
674         this->emitExceptionCheck(jit, ExceptionType::Unreachable);
675     });
676     unreachable->effects.terminal = true;
677     return { };
678 }
679
680 auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
681 {
682     int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
683         instance->storeTopCallFrame(callFrame);
684
685         if (delta < 0)
686             return -1;
687
688         auto grown = instance->memory()->grow(PageCount(delta));
689         if (!grown) {
690             switch (grown.error()) {
691             case Memory::GrowFailReason::InvalidDelta:
692             case Memory::GrowFailReason::InvalidGrowSize:
693             case Memory::GrowFailReason::WouldExceedMaximum:
694             case Memory::GrowFailReason::OutOfMemory:
695                 return -1;
696             }
697             RELEASE_ASSERT_NOT_REACHED();
698         }
699
700         return grown.value().pageCount();
701     };
702
703     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
704         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(growMemory, B3CCallPtrTag)),
705         m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, origin()), instanceValue(), delta);
706
707     restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_proc, m_currentBlock);
708
709     return { };
710 }
711
712 auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
713 {
714     static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
715     Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedMemorySize()));
716
717     constexpr uint32_t shiftValue = 16;
718     static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
719     Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
720         size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
721
722     result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
723
724     return { };
725 }
726
727 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
728 {
729     ASSERT(m_locals[index]);
730     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
731     return { };
732 }
733
734 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
735 {
736     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
737     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
738     return { };
739 }
740
741 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
742 {
743     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
744     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
745     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
746
747     if (isSubtype(m_info.globals[index].type, Anyref))
748         emitWriteBarrierForJSWrapper();
749
750     return { };
751 }
752
753 inline void B3IRGenerator::emitWriteBarrierForJSWrapper()
754 {
755     Value* cell = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfOwner()));
756     Value* cellState = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
757     Value* vm = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), cell, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfVM()));
758     Value* threshold = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapBarrierThreshold()));
759
760     BasicBlock* fenceCheckPath = m_proc.addBlock();
761     BasicBlock* fencePath = m_proc.addBlock();
762     BasicBlock* doSlowPath = m_proc.addBlock();
763     BasicBlock* continuation = m_proc.addBlock();
764
765     m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
766         m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellState, threshold),
767         FrequentedBlock(continuation), FrequentedBlock(fenceCheckPath, FrequencyClass::Rare));
768     fenceCheckPath->addPredecessor(m_currentBlock);
769     continuation->addPredecessor(m_currentBlock);
770     m_currentBlock = fenceCheckPath;
771
772     Value* shouldFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapMutatorShouldBeFenced()));
773     m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
774         shouldFence,
775         FrequentedBlock(fencePath), FrequentedBlock(doSlowPath));
776     fencePath->addPredecessor(m_currentBlock);
777     doSlowPath->addPredecessor(m_currentBlock);
778     m_currentBlock = fencePath;
779
780     B3::PatchpointValue* doFence = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
781     doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
782         jit.memoryFence();
783     });
784
785     Value* cellStateLoadAfterFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
786     m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
787         m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellStateLoadAfterFence, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), blackThreshold)),
788         FrequentedBlock(continuation), FrequentedBlock(doSlowPath, FrequencyClass::Rare));
789     doSlowPath->addPredecessor(m_currentBlock);
790     continuation->addPredecessor(m_currentBlock);
791     m_currentBlock = doSlowPath;
792
793     void (*writeBarrier)(JSWebAssemblyInstance*, VM*) = [] (JSWebAssemblyInstance* cell, VM* vm) -> void {
794         vm->heap.writeBarrierSlowPath(cell);
795     };
796
797     Value* writeBarrierAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(writeBarrier, B3CCallPtrTag));
798     m_currentBlock->appendNew<CCallValue>(m_proc, B3::Void, origin(), writeBarrierAddress, cell, vm);
799     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
800
801     continuation->addPredecessor(m_currentBlock);
802     m_currentBlock = continuation;
803 }
804
805 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
806 {
807     ASSERT(m_memoryBaseGPR);
808
809     switch (m_mode) {
810     case MemoryMode::BoundsChecking: {
811         // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
812         ASSERT(m_memorySizeGPR);
813         ASSERT(sizeOfOperation + offset > offset);
814         m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), m_memorySizeGPR, pointer, sizeOfOperation + offset - 1);
815         break;
816     }
817
818     case MemoryMode::Signaling: {
819         // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
820         // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
821         // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
822         // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
823         // register + large offset access. We don't think this will be generated frequently.
824         //
825         // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
826         // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
827         // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
828         // any access equal to or greater than 4GiB will trap, no need to add the redzone.
829         if (offset >= Memory::fastMappedRedzoneBytes()) {
830             size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
831             m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
832         }
833         break;
834     }
835     }
836
837     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
838     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
839 }
840
841 inline uint32_t sizeOfLoadOp(LoadOpType op)
842 {
843     switch (op) {
844     case LoadOpType::I32Load8S:
845     case LoadOpType::I32Load8U:
846     case LoadOpType::I64Load8S:
847     case LoadOpType::I64Load8U:
848         return 1;
849     case LoadOpType::I32Load16S:
850     case LoadOpType::I64Load16S:
851     case LoadOpType::I32Load16U:
852     case LoadOpType::I64Load16U:
853         return 2;
854     case LoadOpType::I32Load:
855     case LoadOpType::I64Load32S:
856     case LoadOpType::I64Load32U:
857     case LoadOpType::F32Load:
858         return 4;
859     case LoadOpType::I64Load:
860     case LoadOpType::F64Load:
861         return 8;
862     }
863     RELEASE_ASSERT_NOT_REACHED();
864 }
865
866 inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
867 {
868     if (m_mode == MemoryMode::Signaling)
869         return trapping(memoryOp);
870     return memoryOp;
871 }
872
873 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
874 {
875     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
876
877     switch (op) {
878     case LoadOpType::I32Load8S: {
879         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
880     }
881
882     case LoadOpType::I64Load8S: {
883         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
884         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
885     }
886
887     case LoadOpType::I32Load8U: {
888         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
889     }
890
891     case LoadOpType::I64Load8U: {
892         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
893         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
894     }
895
896     case LoadOpType::I32Load16S: {
897         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
898     }
899
900     case LoadOpType::I64Load16S: {
901         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
902         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
903     }
904
905     case LoadOpType::I32Load16U: {
906         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
907     }
908
909     case LoadOpType::I64Load16U: {
910         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
911         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
912     }
913
914     case LoadOpType::I32Load: {
915         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
916     }
917
918     case LoadOpType::I64Load32U: {
919         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
920         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
921     }
922
923     case LoadOpType::I64Load32S: {
924         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
925         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
926     }
927
928     case LoadOpType::I64Load: {
929         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
930     }
931
932     case LoadOpType::F32Load: {
933         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
934     }
935
936     case LoadOpType::F64Load: {
937         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
938     }
939     }
940     RELEASE_ASSERT_NOT_REACHED();
941 }
942
943 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
944 {
945     ASSERT(pointer->type() == Int32);
946
947     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
948         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
949         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
950         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
951         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
952             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
953         });
954
955         switch (op) {
956         case LoadOpType::I32Load8S:
957         case LoadOpType::I32Load16S:
958         case LoadOpType::I32Load:
959         case LoadOpType::I32Load16U:
960         case LoadOpType::I32Load8U:
961             result = constant(Int32, 0);
962             break;
963         case LoadOpType::I64Load8S:
964         case LoadOpType::I64Load8U:
965         case LoadOpType::I64Load16S:
966         case LoadOpType::I64Load32U:
967         case LoadOpType::I64Load32S:
968         case LoadOpType::I64Load:
969         case LoadOpType::I64Load16U:
970             result = constant(Int64, 0);
971             break;
972         case LoadOpType::F32Load:
973             result = constant(Float, 0);
974             break;
975         case LoadOpType::F64Load:
976             result = constant(Double, 0);
977             break;
978         }
979
980     } else
981         result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
982
983     return { };
984 }
985
986 inline uint32_t sizeOfStoreOp(StoreOpType op)
987 {
988     switch (op) {
989     case StoreOpType::I32Store8:
990     case StoreOpType::I64Store8:
991         return 1;
992     case StoreOpType::I32Store16:
993     case StoreOpType::I64Store16:
994         return 2;
995     case StoreOpType::I32Store:
996     case StoreOpType::I64Store32:
997     case StoreOpType::F32Store:
998         return 4;
999     case StoreOpType::I64Store:
1000     case StoreOpType::F64Store:
1001         return 8;
1002     }
1003     RELEASE_ASSERT_NOT_REACHED();
1004 }
1005
1006
1007 inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1008 {
1009     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1010
1011     switch (op) {
1012     case StoreOpType::I64Store8:
1013         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
1014         FALLTHROUGH;
1015
1016     case StoreOpType::I32Store8:
1017         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
1018         return;
1019
1020     case StoreOpType::I64Store16:
1021         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
1022         FALLTHROUGH;
1023
1024     case StoreOpType::I32Store16:
1025         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
1026         return;
1027
1028     case StoreOpType::I64Store32:
1029         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
1030         FALLTHROUGH;
1031
1032     case StoreOpType::I64Store:
1033     case StoreOpType::I32Store:
1034     case StoreOpType::F32Store:
1035     case StoreOpType::F64Store:
1036         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
1037         return;
1038     }
1039     RELEASE_ASSERT_NOT_REACHED();
1040 }
1041
1042 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
1043 {
1044     ASSERT(pointer->type() == Int32);
1045
1046     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
1047         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1048         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1049         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1050         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1051             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1052         });
1053     } else
1054         emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
1055
1056     return { };
1057 }
1058
1059 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
1060 {
1061     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
1062     return { };
1063 }
1064
1065 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
1066 {
1067     return constant(toB3Type(type), value);
1068 }
1069
1070 void B3IRGenerator::emitTierUpCheck(uint32_t decrementCount, Origin origin)
1071 {
1072     if (!m_tierUp)
1073         return;
1074
1075     ASSERT(m_tierUp);
1076     Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(m_tierUp), origin);
1077     Value* oldCountDown = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, countDownLocation);
1078     Value* newCountDown = m_currentBlock->appendNew<Value>(m_proc, Sub, origin, oldCountDown, constant(Int32, decrementCount, origin));
1079     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, newCountDown, countDownLocation);
1080
1081     PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, origin);
1082     Effects effects = Effects::none();
1083     // FIXME: we should have a more precise heap range for the tier up count.
1084     effects.reads = B3::HeapRange::top();
1085     effects.writes = B3::HeapRange::top();
1086     patch->effects = effects;
1087
1088     patch->append(newCountDown, ValueRep::SomeRegister);
1089     patch->append(oldCountDown, ValueRep::SomeRegister);
1090     patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1091         MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
1092         MacroAssembler::Label tierUpResume = jit.label();
1093
1094         params.addLatePath([=] (CCallHelpers& jit) {
1095             tierUp.link(&jit);
1096
1097             const unsigned extraPaddingBytes = 0;
1098             RegisterSet registersToSpill = { };
1099             registersToSpill.add(GPRInfo::argumentGPR1);
1100             unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
1101
1102             jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
1103             MacroAssembler::Call call = jit.nearCall();
1104
1105             ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
1106             jit.jump(tierUpResume);
1107
1108             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1109                 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
1110
1111             });
1112         });
1113     });
1114 }
1115
1116 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
1117 {
1118     BasicBlock* body = m_proc.addBlock();
1119     BasicBlock* continuation = m_proc.addBlock();
1120
1121     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
1122
1123     m_currentBlock = body;
1124     emitTierUpCheck(TierUpCount::loopDecrement(), origin());
1125
1126     return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
1127 }
1128
1129 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
1130 {
1131     return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
1132 }
1133
1134 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
1135 {
1136     return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
1137 }
1138
1139 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
1140 {
1141     // FIXME: This needs to do some kind of stack passing.
1142
1143     BasicBlock* taken = m_proc.addBlock();
1144     BasicBlock* notTaken = m_proc.addBlock();
1145     BasicBlock* continuation = m_proc.addBlock();
1146
1147     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1148     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
1149     taken->addPredecessor(m_currentBlock);
1150     notTaken->addPredecessor(m_currentBlock);
1151
1152     m_currentBlock = taken;
1153     result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
1154     return { };
1155 }
1156
1157 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
1158 {
1159     unifyValuesWithBlock(currentStack, data.result);
1160     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1161     return addElseToUnreachable(data);
1162 }
1163
1164 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
1165 {
1166     ASSERT(data.type() == BlockType::If);
1167     m_currentBlock = data.special;
1168     data.convertIfToBlock();
1169     return { };
1170 }
1171
1172 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
1173 {
1174     ASSERT(returnValues.size() <= 1);
1175     if (returnValues.size())
1176         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
1177     else
1178         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
1179     return { };
1180 }
1181
1182 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1183 {
1184     unifyValuesWithBlock(returnValues, data.resultForBranch());
1185
1186     BasicBlock* target = data.targetBlockForBranch();
1187     if (condition) {
1188         BasicBlock* continuation = m_proc.addBlock();
1189         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1190         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
1191         target->addPredecessor(m_currentBlock);
1192         continuation->addPredecessor(m_currentBlock);
1193         m_currentBlock = continuation;
1194     } else {
1195         m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
1196         target->addPredecessor(m_currentBlock);
1197     }
1198
1199     return { };
1200 }
1201
1202 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1203 {
1204     for (size_t i = 0; i < targets.size(); ++i)
1205         unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
1206     unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1207
1208     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
1209     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
1210     for (size_t i = 0; i < targets.size(); ++i)
1211         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
1212
1213     return { };
1214 }
1215
1216 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1217 {
1218     ControlData& data = entry.controlData;
1219
1220     unifyValuesWithBlock(expressionStack, data.result);
1221     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1222     data.continuation->addPredecessor(m_currentBlock);
1223
1224     return addEndToUnreachable(entry);
1225 }
1226
1227
1228 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1229 {
1230     ControlData& data = entry.controlData;
1231     m_currentBlock = data.continuation;
1232
1233     if (data.type() == BlockType::If) {
1234         data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
1235         m_currentBlock->addPredecessor(data.special);
1236     }
1237
1238     for (Value* result : data.result) {
1239         m_currentBlock->append(result);
1240         entry.enclosedExpressionStack.append(result);
1241     }
1242
1243     // TopLevel does not have any code after this so we need to make sure we emit a return here.
1244     if (data.type() == BlockType::TopLevel)
1245         return addReturn(entry.controlData, entry.enclosedExpressionStack);
1246
1247     return { };
1248 }
1249
1250 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1251 {
1252     ASSERT(signature.argumentCount() == args.size());
1253
1254     m_makesCalls = true;
1255
1256     Type returnType = signature.returnType();
1257     Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1258
1259     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1260         m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1261
1262         // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
1263         Value* targetInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfTargetInstance(functionIndex)));
1264         // The target instance is 0 unless the call is wasm->wasm.
1265         Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0));
1266
1267         BasicBlock* isWasmBlock = m_proc.addBlock();
1268         BasicBlock* isEmbedderBlock = m_proc.addBlock();
1269         BasicBlock* continuation = m_proc.addBlock();
1270         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
1271
1272         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
1273             [=] (PatchpointValue* patchpoint) {
1274                 patchpoint->effects.writesPinned = true;
1275                 patchpoint->effects.readsPinned = true;
1276                 // We need to clobber all potential pinned registers since we might be leaving the instance.
1277                 // We pessimistically assume we could be calling to something that is bounds checking.
1278                 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1279                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1280                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1281                     AllowMacroScratchRegisterUsage allowScratch(jit);
1282                     CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1283                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1284                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1285                     });
1286                 });
1287             });
1288         UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
1289         isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1290
1291         // FIXME: Let's remove this indirection by creating a PIC friendly IC
1292         // for calls out to the embedder. This shouldn't be that hard to do. We could probably
1293         // implement the IC to be over Context*.
1294         // https://bugs.webkit.org/show_bug.cgi?id=170375
1295         Value* jumpDestination = isEmbedderBlock->appendNew<MemoryValue>(m_proc,
1296             Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfWasmToEmbedderStub(functionIndex)));
1297
1298         Value* embedderCallResult = wasmCallingConvention().setupCall(m_proc, isEmbedderBlock, origin(), args, toB3Type(returnType),
1299             [=] (PatchpointValue* patchpoint) {
1300                 patchpoint->effects.writesPinned = true;
1301                 patchpoint->effects.readsPinned = true;
1302                 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
1303                 // We need to clobber all potential pinned registers since we might be leaving the instance.
1304                 // We pessimistically assume we could be calling to something that is bounds checking.
1305                 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1306                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1307                 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1308                     AllowMacroScratchRegisterUsage allowScratch(jit);
1309                     jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1310                 });
1311             });
1312         UpsilonValue* embedderCallResultUpsilon = returnType == Void ? nullptr : isEmbedderBlock->appendNew<UpsilonValue>(m_proc, origin(), embedderCallResult);
1313         isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1314
1315         m_currentBlock = continuation;
1316
1317         if (returnType == Void)
1318             result = nullptr;
1319         else {
1320             result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1321             wasmCallResultUpsilon->setPhi(result);
1322             embedderCallResultUpsilon->setPhi(result);
1323         }
1324
1325         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1326         restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation);
1327     } else {
1328         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1329             [=] (PatchpointValue* patchpoint) {
1330                 patchpoint->effects.writesPinned = true;
1331                 patchpoint->effects.readsPinned = true;
1332
1333                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1334                     AllowMacroScratchRegisterUsage allowScratch(jit);
1335                     CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1336                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1337                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1338                     });
1339                 });
1340             });
1341     }
1342
1343     return { };
1344 }
1345
1346 auto B3IRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1347 {
1348     ExpressionType calleeIndex = args.takeLast();
1349     ASSERT(signature.argumentCount() == args.size());
1350
1351     m_makesCalls = true;
1352     // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1353     // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1354     // can be to the embedder for our stack check calculation.
1355     m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1356
1357     ExpressionType callableFunctionBuffer;
1358     ExpressionType instancesBuffer;
1359     ExpressionType callableFunctionBufferLength;
1360     ExpressionType mask;
1361     {
1362         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1363             instanceValue(), safeCast<int32_t>(Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex)));
1364         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1365             table, safeCast<int32_t>(FuncRefTable::offsetOfFunctions()));
1366         instancesBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1367             table, safeCast<int32_t>(FuncRefTable::offsetOfInstances()));
1368         callableFunctionBufferLength = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1369             table, safeCast<int32_t>(Table::offsetOfLength()));
1370         mask = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(),
1371             m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1372                 table, safeCast<int32_t>(Table::offsetOfMask())));
1373     }
1374
1375     // Check the index we are looking for is valid.
1376     {
1377         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1378             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferLength));
1379
1380         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1381             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1382         });
1383     }
1384
1385     calleeIndex = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex);
1386
1387     if (Options::enableSpectreMitigations())
1388         calleeIndex = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), mask, calleeIndex);
1389
1390     ExpressionType callableFunction;
1391     {
1392         // Compute the offset in the table index space we are looking for.
1393         ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1394             calleeIndex, constant(pointerType(), sizeof(WasmToWasmImportableFunction)));
1395         callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1396
1397         // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1398         // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1399         static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1400         ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), callableFunction, safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfSignatureIndex()));
1401         {
1402             CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1403                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1404                     calleeSignatureIndex,
1405                     m_currentBlock->appendNew<Const64Value>(m_proc, origin(), Signature::invalidIndex)));
1406
1407             check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1408                 this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1409             });
1410         }
1411
1412         // Check the signature matches the value we expect.
1413         {
1414             ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const64Value>(m_proc, origin(), SignatureInformation::get(signature));
1415             CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1416                 m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1417
1418             check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1419                 this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1420             });
1421         }
1422     }
1423
1424     // Do a context switch if needed.
1425     {
1426         Value* offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1427             calleeIndex, constant(pointerType(), sizeof(Instance*)));
1428         Value* newContextInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1429             m_currentBlock->appendNew<Value>(m_proc, Add, origin(), instancesBuffer, offset));
1430
1431         BasicBlock* continuation = m_proc.addBlock();
1432         BasicBlock* doContextSwitch = m_proc.addBlock();
1433
1434         Value* isSameContextInstance = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1435             newContextInstance, instanceValue());
1436         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1437             isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch));
1438
1439         PatchpointValue* patchpoint = doContextSwitch->appendNew<PatchpointValue>(m_proc, B3::Void, origin());
1440         patchpoint->effects.writesPinned = true;
1441         // We pessimistically assume we're calling something with BoundsChecking memory.
1442         // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1443         patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1444         patchpoint->clobber(RegisterSet::macroScratchRegisters());
1445         patchpoint->append(newContextInstance, ValueRep::SomeRegister);
1446         patchpoint->append(instanceValue(), ValueRep::SomeRegister);
1447         patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
1448
1449         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1450             AllowMacroScratchRegisterUsage allowScratch(jit);
1451             GPRReg newContextInstance = params[0].gpr();
1452             GPRReg oldContextInstance = params[1].gpr();
1453             const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1454             GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1455             ASSERT(newContextInstance != baseMemory);
1456             jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1457             jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1458             jit.storeWasmContextInstance(newContextInstance);
1459             ASSERT(pinnedRegs.sizeRegister != baseMemory);
1460             // FIXME: We should support more than one memory size register
1461             //   see: https://bugs.webkit.org/show_bug.cgi?id=162952
1462             ASSERT(pinnedRegs.sizeRegister != newContextInstance);
1463             GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs.sizeRegister;
1464
1465             jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
1466             jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1467
1468             jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs.sizeRegister, scratchOrSize);
1469         });
1470         doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation);
1471
1472         m_currentBlock = continuation;
1473     }
1474
1475     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1476         m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
1477             safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation())));
1478
1479     Type returnType = signature.returnType();
1480     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1481         [=] (PatchpointValue* patchpoint) {
1482             patchpoint->effects.writesPinned = true;
1483             patchpoint->effects.readsPinned = true;
1484             // We need to clobber all potential pinned registers since we might be leaving the instance.
1485             // We pessimistically assume we're always calling something that is bounds checking so
1486             // because the wasm->wasm thunk unconditionally overrides the size registers.
1487             // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1488             // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1489             patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1490
1491             patchpoint->append(calleeCode, ValueRep::SomeRegister);
1492             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1493                 AllowMacroScratchRegisterUsage allowScratch(jit);
1494                 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1495             });
1496         });
1497
1498     // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1499     restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock);
1500
1501     return { };
1502 }
1503
1504 void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1505 {
1506     m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1507 }
1508
1509 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1510 {
1511     ASSERT(result.size() <= resultStack.size());
1512
1513     for (size_t i = 0; i < result.size(); ++i)
1514         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1515 }
1516
1517 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1518 {
1519     dataLog(comma, "ExpressionStack:");
1520     for (const auto& expression : expressionStack)
1521         dataLog(comma, *expression);
1522 }
1523
1524 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1525 {
1526     dataLogLn("Constants:");
1527     for (const auto& constant : m_constantPool)
1528         dataLogLn(deepDump(m_proc, constant.value));
1529
1530     dataLogLn("Processing Graph:");
1531     dataLog(m_proc);
1532     dataLogLn("With current block:", *m_currentBlock);
1533     dataLogLn("Control stack:");
1534     ASSERT(controlStack.size());
1535     for (size_t i = controlStack.size(); i--;) {
1536         dataLog("  ", controlStack[i].controlData, ": ");
1537         CommaPrinter comma(", ", "");
1538         dumpExpressionStack(comma, *expressionStack);
1539         expressionStack = &controlStack[i].enclosedExpressionStack;
1540         dataLogLn();
1541     }
1542     dataLogLn();
1543 }
1544
1545 auto B3IRGenerator::origin() -> Origin
1546 {
1547     OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset());
1548     ASSERT(isValidOpType(static_cast<uint8_t>(origin.opcode())));
1549     return bitwise_cast<Origin>(origin);
1550 }
1551
1552 Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1553 {
1554     auto result = makeUnique<InternalFunction>();
1555
1556     compilationContext.embedderEntrypointJIT = makeUnique<CCallHelpers>();
1557     compilationContext.wasmEntrypointJIT = makeUnique<CCallHelpers>();
1558
1559     Procedure procedure;
1560
1561     procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1562         if (origin.data())
1563             out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1564     });
1565     
1566     // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1567     // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1568     // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1569     // optLevel=1.
1570     procedure.setNeedsUsedRegisters(false);
1571     
1572     procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
1573         ? Options::webAssemblyBBQB3OptimizationLevel()
1574         : Options::webAssemblyOMGOptimizationLevel());
1575
1576     B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, compilationMode, functionIndex, tierUp, throwWasmException);
1577     FunctionParser<B3IRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1578     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1579
1580     irGenerator.insertConstants();
1581
1582     procedure.resetReachability();
1583     if (!ASSERT_DISABLED)
1584         validate(procedure, "After parsing:\n");
1585
1586     dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure);
1587     fixSSA(procedure);
1588     dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure);
1589     
1590     {
1591         B3::prepareForGeneration(procedure);
1592         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1593         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1594         result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1595     }
1596
1597     return result;
1598 }
1599
1600 // Custom wasm ops. These are the ones too messy to do in wasm.json.
1601
1602 void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1603 {
1604     ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1605     const B3::Type type = left->type();
1606
1607     {
1608         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1609             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1610
1611         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1612             this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1613         });
1614     }
1615
1616     if (operation == Div) {
1617         int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1618
1619         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1620             m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1621                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1622                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1623
1624         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1625             this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1626         });
1627     }
1628 }
1629
1630 template<>
1631 auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1632 {
1633     const B3::Opcode op = Div;
1634     emitChecksForModOrDiv(op, left, right);
1635     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1636     return { };
1637 }
1638
1639 template<>
1640 auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1641 {
1642     const B3::Opcode op = Mod;
1643     emitChecksForModOrDiv(op, left, right);
1644     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1645     return { };
1646 }
1647
1648 template<>
1649 auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1650 {
1651     const B3::Opcode op = UDiv;
1652     emitChecksForModOrDiv(op, left, right);
1653     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1654     return { };
1655 }
1656
1657 template<>
1658 auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1659 {
1660     const B3::Opcode op = UMod;
1661     emitChecksForModOrDiv(op, left, right);
1662     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1663     return { };
1664 }
1665
1666 template<>
1667 auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1668 {
1669     const B3::Opcode op = Div;
1670     emitChecksForModOrDiv(op, left, right);
1671     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1672     return { };
1673 }
1674
1675 template<>
1676 auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1677 {
1678     const B3::Opcode op = Mod;
1679     emitChecksForModOrDiv(op, left, right);
1680     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1681     return { };
1682 }
1683
1684 template<>
1685 auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1686 {
1687     const B3::Opcode op = UDiv;
1688     emitChecksForModOrDiv(op, left, right);
1689     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1690     return { };
1691 }
1692
1693 template<>
1694 auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1695 {
1696     const B3::Opcode op = UMod;
1697     emitChecksForModOrDiv(op, left, right);
1698     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1699     return { };
1700 }
1701
1702 template<>
1703 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1704 {
1705     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1706     patchpoint->append(arg, ValueRep::SomeRegister);
1707     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1708         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1709     });
1710     patchpoint->effects = Effects::none();
1711     result = patchpoint;
1712     return { };
1713 }
1714
1715 template<>
1716 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1717 {
1718     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1719     patchpoint->append(arg, ValueRep::SomeRegister);
1720     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1721         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1722     });
1723     patchpoint->effects = Effects::none();
1724     result = patchpoint;
1725     return { };
1726 }
1727
1728 template<>
1729 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1730 {
1731 #if CPU(X86_64)
1732     if (MacroAssembler::supportsCountPopulation()) {
1733         PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1734         patchpoint->append(arg, ValueRep::SomeRegister);
1735         patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1736             jit.countPopulation32(params[1].gpr(), params[0].gpr());
1737         });
1738         patchpoint->effects = Effects::none();
1739         result = patchpoint;
1740         return { };
1741     }
1742 #endif
1743
1744     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1745     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1746     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1747     return { };
1748 }
1749
1750 template<>
1751 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1752 {
1753 #if CPU(X86_64)
1754     if (MacroAssembler::supportsCountPopulation()) {
1755         PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1756         patchpoint->append(arg, ValueRep::SomeRegister);
1757         patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1758             jit.countPopulation64(params[1].gpr(), params[0].gpr());
1759         });
1760         patchpoint->effects = Effects::none();
1761         result = patchpoint;
1762         return { };
1763     }
1764 #endif
1765
1766     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1767     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1768     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1769     return { };
1770 }
1771
1772 template<>
1773 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1774 {
1775     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1776     if (isX86())
1777         patchpoint->numGPScratchRegisters = 1;
1778     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1779     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1780     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1781         AllowMacroScratchRegisterUsage allowScratch(jit);
1782 #if CPU(X86_64)
1783         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1784 #else
1785         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1786 #endif
1787     });
1788     patchpoint->effects = Effects::none();
1789     result = patchpoint;
1790     return { };
1791 }
1792
1793 template<>
1794 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1795 {
1796     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1797     if (isX86())
1798         patchpoint->numGPScratchRegisters = 1;
1799     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1800     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1801     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1802         AllowMacroScratchRegisterUsage allowScratch(jit);
1803 #if CPU(X86_64)
1804         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1805 #else
1806         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1807 #endif
1808     });
1809     patchpoint->effects = Effects::none();
1810     result = patchpoint;
1811     return { };
1812 }
1813
1814 template<>
1815 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1816 {
1817     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1818     patchpoint->append(arg, ValueRep::SomeRegister);
1819     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1820         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1821     });
1822     patchpoint->effects = Effects::none();
1823     result = patchpoint;
1824     return { };
1825 }
1826
1827 template<>
1828 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1829 {
1830     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1831     patchpoint->append(arg, ValueRep::SomeRegister);
1832     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1833         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1834     });
1835     patchpoint->effects = Effects::none();
1836     result = patchpoint;
1837     return { };
1838 }
1839
1840 template<>
1841 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1842 {
1843     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1844     patchpoint->append(arg, ValueRep::SomeRegister);
1845     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1846         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1847     });
1848     patchpoint->effects = Effects::none();
1849     result = patchpoint;
1850     return { };
1851 }
1852
1853 template<>
1854 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1855 {
1856     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1857     patchpoint->append(arg, ValueRep::SomeRegister);
1858     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1859         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1860     });
1861     patchpoint->effects = Effects::none();
1862     result = patchpoint;
1863     return { };
1864 }
1865
1866 template<>
1867 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1868 {
1869     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1870     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1871     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1872         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1873         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1874     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1875     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1876     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1877         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1878     });
1879     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1880     patchpoint->append(arg, ValueRep::SomeRegister);
1881     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1882         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1883     });
1884     patchpoint->effects = Effects::none();
1885     result = patchpoint;
1886     return { };
1887 }
1888
1889 template<>
1890 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1891 {
1892     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1893     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1894     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1895         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1896         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1897     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1898     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1899     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1900         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1901     });
1902     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1903     patchpoint->append(arg, ValueRep::SomeRegister);
1904     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1905         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1906     });
1907     patchpoint->effects = Effects::none();
1908     result = patchpoint;
1909     return { };
1910 }
1911
1912
1913 template<>
1914 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1915 {
1916     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1917     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1918     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1919         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1920         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1921     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1922     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1923     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1924         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1925     });
1926     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1927     patchpoint->append(arg, ValueRep::SomeRegister);
1928     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1929         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1930     });
1931     patchpoint->effects = Effects::none();
1932     result = patchpoint;
1933     return { };
1934 }
1935
1936 template<>
1937 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1938 {
1939     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1940     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1941     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1942         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1943         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1944     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1945     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1946     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1947         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1948     });
1949     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1950     patchpoint->append(arg, ValueRep::SomeRegister);
1951     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1952         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1953     });
1954     patchpoint->effects = Effects::none();
1955     result = patchpoint;
1956     return { };
1957 }
1958
1959 template<>
1960 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1961 {
1962     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1963     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1964     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1965         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1966         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1967     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1968     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1969     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1970         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1971     });
1972     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1973     patchpoint->append(arg, ValueRep::SomeRegister);
1974     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1975         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1976     });
1977     patchpoint->effects = Effects::none();
1978     result = patchpoint;
1979     return { };
1980 }
1981
1982 template<>
1983 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1984 {
1985     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1986     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1987     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1988         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1989         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1990     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1991     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1992     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1993         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1994     });
1995
1996     Value* signBitConstant;
1997     if (isX86()) {
1998         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1999         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
2000         // so we can pool them if needed.
2001         signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2002     }
2003     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2004     patchpoint->append(arg, ValueRep::SomeRegister);
2005     if (isX86()) {
2006         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
2007         patchpoint->numFPScratchRegisters = 1;
2008     }
2009     patchpoint->clobber(RegisterSet::macroScratchRegisters());
2010     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2011         AllowMacroScratchRegisterUsage allowScratch(jit);
2012         FPRReg scratch = InvalidFPRReg;
2013         FPRReg constant = InvalidFPRReg;
2014         if (isX86()) {
2015             scratch = params.fpScratch(0);
2016             constant = params[2].fpr();
2017         }
2018         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2019     });
2020     patchpoint->effects = Effects::none();
2021     result = patchpoint;
2022     return { };
2023 }
2024
2025 template<>
2026 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2027 {
2028     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
2029     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
2030     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2031         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2032         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
2033     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2034     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2035     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2036         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2037     });
2038     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2039     patchpoint->append(arg, ValueRep::SomeRegister);
2040     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2041         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
2042     });
2043     patchpoint->effects = Effects::none();
2044     result = patchpoint;
2045     return { };
2046 }
2047
2048 template<>
2049 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2050 {
2051     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
2052     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2053     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2054         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2055         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
2056     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2057     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2058     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2059         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2060     });
2061
2062     Value* signBitConstant;
2063     if (isX86()) {
2064         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
2065         // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
2066         // so we can pool them if needed.
2067         signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2068     }
2069     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2070     patchpoint->append(arg, ValueRep::SomeRegister);
2071     if (isX86()) {
2072         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
2073         patchpoint->numFPScratchRegisters = 1;
2074     }
2075     patchpoint->clobber(RegisterSet::macroScratchRegisters());
2076     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2077         AllowMacroScratchRegisterUsage allowScratch(jit);
2078         FPRReg scratch = InvalidFPRReg;
2079         FPRReg constant = InvalidFPRReg;
2080         if (isX86()) {
2081             scratch = params.fpScratch(0);
2082             constant = params[2].fpr();
2083         }
2084         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2085     });
2086     patchpoint->effects = Effects::none();
2087     result = patchpoint;
2088     return { };
2089 }
2090
2091 } } // namespace JSC::Wasm
2092
2093 #include "WasmB3IRGeneratorInlines.h"
2094
2095 #endif // ENABLE(WEBASSEMBLY)