Remove unused lamda captures
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "AllowMacroScratchRegisterUsageIf.h"
32 #include "B3BasicBlockInlines.h"
33 #include "B3CCallValue.h"
34 #include "B3Compile.h"
35 #include "B3ConstPtrValue.h"
36 #include "B3FixSSA.h"
37 #include "B3Generate.h"
38 #include "B3InsertionSet.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3SwitchValue.h"
42 #include "B3UpsilonValue.h"
43 #include "B3Validate.h"
44 #include "B3ValueInlines.h"
45 #include "B3ValueKey.h"
46 #include "B3Variable.h"
47 #include "B3VariableValue.h"
48 #include "B3WasmAddressValue.h"
49 #include "B3WasmBoundsCheckValue.h"
50 #include "JSCInlines.h"
51 #include "JSWebAssemblyInstance.h"
52 #include "JSWebAssemblyModule.h"
53 #include "JSWebAssemblyRuntimeError.h"
54 #include "VirtualRegister.h"
55 #include "WasmCallingConvention.h"
56 #include "WasmContext.h"
57 #include "WasmExceptionType.h"
58 #include "WasmFunctionParser.h"
59 #include "WasmMemory.h"
60 #include "WasmOpcodeOrigin.h"
61 #include "WasmThunks.h"
62 #include <limits>
63 #include <wtf/Optional.h>
64 #include <wtf/StdLibExtras.h>
65
66 void dumpProcedure(void* ptr)
67 {
68     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
69     proc->dump(WTF::dataFile());
70 }
71
72 namespace JSC { namespace Wasm {
73
74 using namespace B3;
75
76 namespace {
77 const bool verbose = false;
78 }
79
80 class B3IRGenerator {
81 public:
82     struct ControlData {
83         ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
84             : blockType(type)
85             , continuation(continuation)
86             , special(special)
87         {
88             if (signature != Void)
89                 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
90         }
91
92         ControlData()
93         {
94         }
95
96         void dump(PrintStream& out) const
97         {
98             switch (type()) {
99             case BlockType::If:
100                 out.print("If:       ");
101                 break;
102             case BlockType::Block:
103                 out.print("Block:    ");
104                 break;
105             case BlockType::Loop:
106                 out.print("Loop:     ");
107                 break;
108             case BlockType::TopLevel:
109                 out.print("TopLevel: ");
110                 break;
111             }
112             out.print("Continuation: ", *continuation, ", Special: ");
113             if (special)
114                 out.print(*special);
115             else
116                 out.print("None");
117         }
118
119         BlockType type() const { return blockType; }
120
121         bool hasNonVoidSignature() const { return result.size(); }
122
123         BasicBlock* targetBlockForBranch()
124         {
125             if (type() == BlockType::Loop)
126                 return special;
127             return continuation;
128         }
129
130         void convertIfToBlock()
131         {
132             ASSERT(type() == BlockType::If);
133             blockType = BlockType::Block;
134             special = nullptr;
135         }
136
137         using ResultList = Vector<Value*, 1>; // Value must be a Phi
138
139         ResultList resultForBranch() const
140         {
141             if (type() == BlockType::Loop)
142                 return ResultList();
143             return result;
144         }
145
146     private:
147         friend class B3IRGenerator;
148         BlockType blockType;
149         BasicBlock* continuation;
150         BasicBlock* special;
151         ResultList result;
152     };
153
154     typedef Value* ExpressionType;
155     typedef ControlData ControlType;
156     typedef Vector<ExpressionType, 1> ExpressionList;
157     typedef ControlData::ResultList ResultList;
158     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
159
160     static constexpr ExpressionType emptyExpression = nullptr;
161
162     typedef String ErrorType;
163     typedef UnexpectedType<ErrorType> UnexpectedResult;
164     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
165     typedef Expected<void, ErrorType> PartialResult;
166     template <typename ...Args>
167     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
168     {
169         using namespace FailureHelper; // See ADL comment in WasmParser.h.
170         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
171     }
172 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
173         if (UNLIKELY(condition))                  \
174             return fail(__VA_ARGS__);             \
175     } while (0)
176
177     B3IRGenerator(const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode);
178
179     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
180     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
181     ExpressionType addConstant(Type, uint64_t);
182
183     // Locals
184     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
185     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
186
187     // Globals
188     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
189     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
190
191     // Memory
192     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
193     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
194     PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
195     PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
196
197     // Basic operators
198     template<OpType>
199     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
200     template<OpType>
201     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
202     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
203
204     // Control flow
205     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
206     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
207     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
208     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
209     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
210     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
211
212     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
213     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
214     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
215     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
216     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
217
218     // Calls
219     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
220     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
221     PartialResult WARN_UNUSED_RETURN addUnreachable();
222
223     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
224     void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
225
226     Value* constant(B3::Type, uint64_t bits);
227     void insertConstants();
228
229 private:
230     void emitExceptionCheck(CCallHelpers&, ExceptionType);
231
232     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
233     B3::Kind memoryKind(B3::Opcode memoryOp);
234     ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
235     void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
236
237     void unify(const ExpressionType phi, const ExpressionType source);
238     void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
239
240     void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
241
242     int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
243
244     Value* materializeWasmContext(Procedure&, BasicBlock*);
245     void restoreWasmContext(Procedure&, BasicBlock*, Value*);
246     void restoreWebAssemblyGlobalState(const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
247
248     Origin origin();
249
250     FunctionParser<B3IRGenerator>* m_parser;
251     const ModuleInformation& m_info;
252     MemoryMode m_mode;
253     Procedure& m_proc;
254     BasicBlock* m_currentBlock;
255     Vector<Variable*> m_locals;
256     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
257     HashMap<ValueKey, Value*> m_constantPool;
258     InsertionSet m_constantInsertionValues;
259     GPRReg m_memoryBaseGPR;
260     GPRReg m_memorySizeGPR { InvalidGPRReg };
261     GPRReg m_wasmContextGPR;
262     Value* m_instanceValue; // FIXME: make this lazy https://bugs.webkit.org/show_bug.cgi?id=169792
263 };
264
265 // Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
266 int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
267 {
268     if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
269         ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
270         return 0;
271     }
272     return offset;
273 }
274
275 Value* B3IRGenerator::materializeWasmContext(Procedure& proc, BasicBlock* block)
276 {
277     if (useFastTLSForContext()) {
278         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, pointerType(), Origin());
279         if (CCallHelpers::loadWasmContextNeedsMacroScratchRegister())
280             patchpoint->clobber(RegisterSet::macroScratchRegisters());
281         patchpoint->setGenerator(
282             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
283                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextNeedsMacroScratchRegister());
284                 jit.loadWasmContext(params[0].gpr());
285             });
286         return patchpoint;
287     }
288
289     // FIXME: Because WasmToWasm call clobbers wasmContext register and does not restore it, we need to restore it in the caller side.
290     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
291     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, pointerType(), Origin());
292     patchpoint->effects.writesPinned = false;
293     patchpoint->effects.readsPinned = true;
294     patchpoint->resultConstraint = ValueRep::reg(m_wasmContextGPR);
295     patchpoint->setGenerator([] (CCallHelpers&, const StackmapGenerationParams&) { });
296     return patchpoint;
297 }
298
299 void B3IRGenerator::restoreWasmContext(Procedure& proc, BasicBlock* block, Value* arg)
300 {
301     if (useFastTLSForContext()) {
302         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
303         if (CCallHelpers::storeWasmContextNeedsMacroScratchRegister())
304             patchpoint->clobber(RegisterSet::macroScratchRegisters());
305         patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
306         patchpoint->setGenerator(
307             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
308                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextNeedsMacroScratchRegister());
309                 jit.storeWasmContext(params[0].gpr());
310             });
311         return;
312     }
313
314     // FIXME: Because WasmToWasm call clobbers wasmContext register and does not restore it, we need to restore it in the caller side.
315     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
316     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
317     Effects effects = Effects::none();
318     effects.writesPinned = true;
319     effects.reads = B3::HeapRange::top();
320     patchpoint->effects = effects;
321     patchpoint->clobberLate(RegisterSet(m_wasmContextGPR));
322     patchpoint->append(m_instanceValue, ValueRep::SomeRegister);
323     GPRReg wasmContextGPR = m_wasmContextGPR;
324     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
325         jit.move(param[0].gpr(), wasmContextGPR);
326     });
327 }
328
329 B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode)
330     : m_info(info)
331     , m_mode(mode)
332     , m_proc(procedure)
333     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
334     , m_constantInsertionValues(m_proc)
335 {
336     m_currentBlock = m_proc.addBlock();
337
338     // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
339     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
340
341     m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
342     m_proc.pinRegister(m_memoryBaseGPR);
343
344     m_wasmContextGPR = pinnedRegs.wasmContextPointer;
345     if (!useFastTLSForContext())
346         m_proc.pinRegister(m_wasmContextGPR);
347
348     if (mode != MemoryMode::Signaling) {
349         ASSERT(!pinnedRegs.sizeRegisters[0].sizeOffset);
350         m_memorySizeGPR = pinnedRegs.sizeRegisters[0].sizeRegister;
351         for (const PinnedSizeRegisterInfo& regInfo : pinnedRegs.sizeRegisters)
352             m_proc.pinRegister(regInfo.sizeRegister);
353     }
354
355     if (info.memory) {
356         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
357             AllowMacroScratchRegisterUsage allowScratch(jit);
358             switch (m_mode) {
359             case MemoryMode::BoundsChecking:
360                 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
361                 break;
362             case MemoryMode::Signaling:
363                 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
364                 break;
365             case MemoryMode::NumberOfMemoryModes:
366                 ASSERT_NOT_REACHED();
367             }
368             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
369         });
370     }
371
372     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
373
374     m_instanceValue = materializeWasmContext(m_proc, m_currentBlock);
375 }
376
377 void B3IRGenerator::restoreWebAssemblyGlobalState(const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
378 {
379     restoreWasmContext(proc, block, instance);
380
381     if (!!memory) {
382         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
383         RegisterSet clobbers;
384         clobbers.set(pinnedRegs->baseMemoryPointer);
385         for (auto info : pinnedRegs->sizeRegisters)
386             clobbers.set(info.sizeRegister);
387
388         B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
389         Effects effects = Effects::none();
390         effects.writesPinned = true;
391         effects.reads = B3::HeapRange::top();
392         patchpoint->effects = effects;
393         patchpoint->clobber(clobbers);
394
395         patchpoint->append(instance, ValueRep::SomeRegister);
396
397         patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
398             GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
399             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
400             const auto& sizeRegs = pinnedRegs->sizeRegisters;
401             ASSERT(sizeRegs.size() >= 1);
402             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
403             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfSize()), sizeRegs[0].sizeRegister);
404             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfMemory()), baseMemory);
405             for (unsigned i = 1; i < sizeRegs.size(); ++i)
406                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
407         });
408     }
409 }
410
411 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
412 {
413     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
414     auto jumpToExceptionStub = jit.jump();
415
416     jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
417         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
418     });
419 }
420
421 Value* B3IRGenerator::constant(B3::Type type, uint64_t bits)
422 {
423     auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
424         Value* result = m_proc.addConstant(origin(), type, bits);
425         m_constantInsertionValues.insertValue(0, result);
426         return result;
427     });
428     return result.iterator->value;
429 }
430
431 void B3IRGenerator::insertConstants()
432 {
433     m_constantInsertionValues.execute(m_proc.at(0));
434 }
435
436 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
437 {
438     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
439
440     for (uint32_t i = 0; i < count; ++i) {
441         Variable* local = m_proc.addVariable(toB3Type(type));
442         m_locals.uncheckedAppend(local);
443         m_currentBlock->appendNew<VariableValue>(m_proc, Set, origin(), local, addConstant(type, 0));
444     }
445     return { };
446 }
447
448 auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
449 {
450     ASSERT(!m_locals.size());
451     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
452
453     m_locals.grow(signature.argumentCount());
454     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
455         [=] (ExpressionType argument, unsigned i) {
456             Variable* argumentVariable = m_proc.addVariable(argument->type());
457             m_locals[i] = argumentVariable;
458             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
459         });
460     return { };
461 }
462
463 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
464 {
465     ASSERT(m_locals[index]);
466     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
467     return { };
468 }
469
470 auto B3IRGenerator::addUnreachable() -> PartialResult
471 {
472     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
473     unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
474         this->emitExceptionCheck(jit, ExceptionType::Unreachable);
475     });
476     unreachable->effects.terminal = true;
477     return { };
478 }
479
480 auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
481 {
482     int32_t (*growMemory) (Context*, int32_t) = [] (Context* wasmContext, int32_t delta) -> int32_t {
483         VM& vm = *wasmContext->vm();
484         auto scope = DECLARE_THROW_SCOPE(vm);
485
486         JSWebAssemblyMemory* wasmMemory = wasmContext->memory();
487
488         if (delta < 0)
489             return -1;
490
491         bool shouldThrowExceptionsOnFailure = false;
492         // grow() does not require ExecState* if it doesn't throw exceptions.
493         ExecState* exec = nullptr; 
494         PageCount result = wasmMemory->grow(vm, exec, static_cast<uint32_t>(delta), shouldThrowExceptionsOnFailure);
495         RELEASE_ASSERT(!scope.exception());
496         if (!result)
497             return -1;
498
499         return result.pageCount();
500     };
501
502     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
503         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(growMemory)),
504         m_instanceValue, delta);
505
506     restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, m_currentBlock);
507
508     return { };
509 }
510
511 auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
512 {
513     Value* memoryObject = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfMemory()));
514
515     static_assert(sizeof(decltype(static_cast<JSWebAssemblyInstance*>(nullptr)->memory()->memory().size())) == sizeof(uint64_t), "codegen relies on this size");
516     Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), memoryObject, safeCast<int32_t>(JSWebAssemblyMemory::offsetOfSize()));
517     
518     constexpr uint32_t shiftValue = 16;
519     static_assert(PageCount::pageSize == 1 << shiftValue, "This must hold for the code below to be correct.");
520     Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
521         size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
522
523     result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
524
525     return { };
526 }
527
528 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
529 {
530     ASSERT(m_locals[index]);
531     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
532     return { };
533 }
534
535 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
536 {
537     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfGlobals()));
538     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
539     return { };
540 }
541
542 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
543 {
544     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
545     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfGlobals()));
546     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
547     return { };
548 }
549
550 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
551 {
552     ASSERT(m_memoryBaseGPR);
553
554     switch (m_mode) {
555     case MemoryMode::BoundsChecking:
556         // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
557         ASSERT(m_memorySizeGPR);
558         ASSERT(sizeOfOperation + offset > offset);
559         m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, m_memorySizeGPR);
560         break;
561
562     case MemoryMode::Signaling:
563         // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
564         // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
565         // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
566         // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
567         // register + large offset access. We don't think this will be generated frequently.
568         //
569         // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
570         // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
571         // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
572         // any access equal to or greater than 4GiB will trap, no need to add the redzone.
573         if (offset >= Memory::fastMappedRedzoneBytes()) {
574             size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
575             m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
576         }
577         break;
578
579     case MemoryMode::NumberOfMemoryModes:
580         RELEASE_ASSERT_NOT_REACHED();
581     }
582     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
583     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
584 }
585
586 inline uint32_t sizeOfLoadOp(LoadOpType op)
587 {
588     switch (op) {
589     case LoadOpType::I32Load8S:
590     case LoadOpType::I32Load8U:
591     case LoadOpType::I64Load8S:
592     case LoadOpType::I64Load8U:
593         return 1;
594     case LoadOpType::I32Load16S:
595     case LoadOpType::I64Load16S:
596     case LoadOpType::I32Load16U:
597     case LoadOpType::I64Load16U:
598         return 2;
599     case LoadOpType::I32Load:
600     case LoadOpType::I64Load32S:
601     case LoadOpType::I64Load32U:
602     case LoadOpType::F32Load:
603         return 4;
604     case LoadOpType::I64Load:
605     case LoadOpType::F64Load:
606         return 8;
607     }
608     RELEASE_ASSERT_NOT_REACHED();
609 }
610
611 inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
612 {
613     if (m_mode == MemoryMode::Signaling)
614         return trapping(memoryOp);
615     return memoryOp;
616 }
617
618 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
619 {
620     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
621
622     switch (op) {
623     case LoadOpType::I32Load8S: {
624         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
625     }
626
627     case LoadOpType::I64Load8S: {
628         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
629         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
630     }
631
632     case LoadOpType::I32Load8U: {
633         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
634     }
635
636     case LoadOpType::I64Load8U: {
637         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
638         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
639     }
640
641     case LoadOpType::I32Load16S: {
642         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
643     }
644     case LoadOpType::I64Load16S: {
645         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
646         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
647     }
648
649     case LoadOpType::I32Load: {
650         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
651     }
652
653     case LoadOpType::I64Load32U: {
654         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
655         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
656     }
657
658     case LoadOpType::I64Load32S: {
659         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
660         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
661     }
662
663     case LoadOpType::I64Load: {
664         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
665     }
666
667     case LoadOpType::F32Load: {
668         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
669     }
670
671     case LoadOpType::F64Load: {
672         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
673     }
674
675     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
676     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
677     case LoadOpType::I32Load16U: {
678         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
679         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
680             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
681     }
682     case LoadOpType::I64Load16U: {
683         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
684         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
685             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
686
687         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), partialResult);
688     }
689     }
690     RELEASE_ASSERT_NOT_REACHED();
691 }
692
693 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
694 {
695     ASSERT(pointer->type() == Int32);
696
697     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
698         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
699         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
700         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
701         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
702             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
703         });
704
705         switch (op) {
706         case LoadOpType::I32Load8S:
707         case LoadOpType::I32Load16S:
708         case LoadOpType::I32Load:
709         case LoadOpType::I32Load16U:
710         case LoadOpType::I32Load8U:
711             result = constant(Int32, 0);
712             break;
713         case LoadOpType::I64Load8S:
714         case LoadOpType::I64Load8U:
715         case LoadOpType::I64Load16S:
716         case LoadOpType::I64Load32U:
717         case LoadOpType::I64Load32S:
718         case LoadOpType::I64Load:
719         case LoadOpType::I64Load16U:
720             result = constant(Int64, 0);
721             break;
722         case LoadOpType::F32Load:
723             result = constant(Float, 0);
724             break;
725         case LoadOpType::F64Load:
726             result = constant(Double, 0);
727             break;
728         }
729
730     } else
731         result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
732
733     return { };
734 }
735
736 inline uint32_t sizeOfStoreOp(StoreOpType op)
737 {
738     switch (op) {
739     case StoreOpType::I32Store8:
740     case StoreOpType::I64Store8:
741         return 1;
742     case StoreOpType::I32Store16:
743     case StoreOpType::I64Store16:
744         return 2;
745     case StoreOpType::I32Store:
746     case StoreOpType::I64Store32:
747     case StoreOpType::F32Store:
748         return 4;
749     case StoreOpType::I64Store:
750     case StoreOpType::F64Store:
751         return 8;
752     }
753     RELEASE_ASSERT_NOT_REACHED();
754 }
755
756
757 inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
758 {
759     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
760
761     switch (op) {
762     case StoreOpType::I64Store8:
763         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
764         FALLTHROUGH;
765
766     case StoreOpType::I32Store8:
767         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
768         return;
769
770     case StoreOpType::I64Store16:
771         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
772         FALLTHROUGH;
773
774     case StoreOpType::I32Store16:
775         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
776         return;
777
778     case StoreOpType::I64Store32:
779         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
780         FALLTHROUGH;
781
782     case StoreOpType::I64Store:
783     case StoreOpType::I32Store:
784     case StoreOpType::F32Store:
785     case StoreOpType::F64Store:
786         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
787         return;
788     }
789     RELEASE_ASSERT_NOT_REACHED();
790 }
791
792 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
793 {
794     ASSERT(pointer->type() == Int32);
795
796     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
797         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
798         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
799         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
800         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
801             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
802         });
803     } else
804         emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
805
806     return { };
807 }
808
809 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
810 {
811     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
812     return { };
813 }
814
815 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
816 {
817     return constant(toB3Type(type), value);
818 }
819
820 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
821 {
822     return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
823 }
824
825 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
826 {
827     return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
828 }
829
830 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
831 {
832     BasicBlock* body = m_proc.addBlock();
833     BasicBlock* continuation = m_proc.addBlock();
834     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
835     body->addPredecessor(m_currentBlock);
836     m_currentBlock = body;
837     return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
838 }
839
840 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
841 {
842     // FIXME: This needs to do some kind of stack passing.
843
844     BasicBlock* taken = m_proc.addBlock();
845     BasicBlock* notTaken = m_proc.addBlock();
846     BasicBlock* continuation = m_proc.addBlock();
847
848     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
849     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
850     taken->addPredecessor(m_currentBlock);
851     notTaken->addPredecessor(m_currentBlock);
852
853     m_currentBlock = taken;
854     result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
855     return { };
856 }
857
858 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
859 {
860     unifyValuesWithBlock(currentStack, data.result);
861     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
862     return addElseToUnreachable(data);
863 }
864
865 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
866 {
867     ASSERT(data.type() == BlockType::If);
868     m_currentBlock = data.special;
869     data.convertIfToBlock();
870     return { };
871 }
872
873 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
874 {
875     ASSERT(returnValues.size() <= 1);
876     if (returnValues.size())
877         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
878     else
879         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
880     return { };
881 }
882
883 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
884 {
885     unifyValuesWithBlock(returnValues, data.resultForBranch());
886
887     BasicBlock* target = data.targetBlockForBranch();
888     if (condition) {
889         BasicBlock* continuation = m_proc.addBlock();
890         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
891         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
892         target->addPredecessor(m_currentBlock);
893         continuation->addPredecessor(m_currentBlock);
894         m_currentBlock = continuation;
895     } else {
896         m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
897         target->addPredecessor(m_currentBlock);
898     }
899
900     return { };
901 }
902
903 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
904 {
905     for (size_t i = 0; i < targets.size(); ++i)
906         unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
907     unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
908
909     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
910     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
911     for (size_t i = 0; i < targets.size(); ++i)
912         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
913
914     return { };
915 }
916
917 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
918 {
919     ControlData& data = entry.controlData;
920
921     unifyValuesWithBlock(expressionStack, data.result);
922     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
923     data.continuation->addPredecessor(m_currentBlock);
924
925     return addEndToUnreachable(entry);
926 }
927
928
929 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
930 {
931     ControlData& data = entry.controlData;
932     m_currentBlock = data.continuation;
933
934     if (data.type() == BlockType::If) {
935         data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
936         m_currentBlock->addPredecessor(data.special);
937     }
938
939     for (Value* result : data.result) {
940         m_currentBlock->append(result);
941         entry.enclosedExpressionStack.append(result);
942     }
943
944     // TopLevel does not have any code after this so we need to make sure we emit a return here.
945     if (data.type() == BlockType::TopLevel)
946         return addReturn(entry.controlData, entry.enclosedExpressionStack);
947
948     return { };
949 }
950
951 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
952 {
953     ASSERT(signature.argumentCount() == args.size());
954
955     Type returnType = signature.returnType();
956     Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
957
958     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
959         // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
960         Value* functionImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfImportFunction(functionIndex)));
961         Value* jsTypeOfImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin(), functionImport, safeCast<int32_t>(JSCell::typeInfoTypeOffset()));
962         Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), jsTypeOfImport, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), WebAssemblyFunctionType));
963
964         BasicBlock* isWasmBlock = m_proc.addBlock();
965         BasicBlock* isJSBlock = m_proc.addBlock();
966         BasicBlock* continuation = m_proc.addBlock();
967         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isJSBlock));
968
969         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
970             [=] (PatchpointValue* patchpoint) {
971                 patchpoint->effects.writesPinned = true;
972                 patchpoint->effects.readsPinned = true;
973                 // We need to clobber all potential pinned registers since we might be leaving the instance.
974                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
975                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
976                     AllowMacroScratchRegisterUsage allowScratch(jit);
977                     CCallHelpers::Call call = jit.call();
978                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
979                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex });
980                     });
981                 });
982             });
983         UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
984         isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
985
986         // FIXME: Lets remove this indirection by creating a PIC friendly IC
987         // for calls out to JS. This shouldn't be that hard to do. We could probably
988         // implement the IC to be over Wasm::Context*.
989         // https://bugs.webkit.org/show_bug.cgi?id=170375
990         Value* codeBlock = isJSBlock->appendNew<MemoryValue>(m_proc,
991             Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfCodeBlock()));
992         Value* jumpDestination = isJSBlock->appendNew<MemoryValue>(m_proc,
993             Load, pointerType(), origin(), codeBlock, safeCast<int32_t>(JSWebAssemblyCodeBlock::offsetOfImportWasmToJSStub(functionIndex)));
994         Value* jsCallResult = wasmCallingConvention().setupCall(m_proc, isJSBlock, origin(), args, toB3Type(returnType),
995             [&] (PatchpointValue* patchpoint) {
996                 patchpoint->effects.writesPinned = true;
997                 patchpoint->effects.readsPinned = true;
998                 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
999                 // We need to clobber all potential pinned registers since we might be leaving the instance.
1000                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
1001                 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1002                     AllowMacroScratchRegisterUsage allowScratch(jit);
1003                     jit.call(params[returnType == Void ? 0 : 1].gpr());
1004                 });
1005             });
1006         UpsilonValue* jsCallResultUpsilon = returnType == Void ? nullptr : isJSBlock->appendNew<UpsilonValue>(m_proc, origin(), jsCallResult);
1007         isJSBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1008
1009         m_currentBlock = continuation;
1010
1011         if (returnType == Void)
1012             result = nullptr;
1013         else {
1014             result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1015             wasmCallResultUpsilon->setPhi(result);
1016             jsCallResultUpsilon->setPhi(result);
1017         }
1018
1019         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1020         restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, continuation);
1021     } else {
1022         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1023             [=] (PatchpointValue* patchpoint) {
1024                 patchpoint->effects.writesPinned = true;
1025                 patchpoint->effects.readsPinned = true;
1026
1027                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1028                     AllowMacroScratchRegisterUsage allowScratch(jit);
1029                     CCallHelpers::Call call = jit.call();
1030                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1031                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex });
1032                     });
1033                 });
1034             });
1035     }
1036
1037     return { };
1038 }
1039
1040 auto B3IRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1041 {
1042     ExpressionType calleeIndex = args.takeLast();
1043     ASSERT(signature.argumentCount() == args.size());
1044
1045     ExpressionType callableFunctionBuffer;
1046     ExpressionType callableFunctionBufferSize;
1047     {
1048         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1049             m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfTable()));
1050         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1051             table, safeCast<int32_t>(JSWebAssemblyTable::offsetOfFunctions()));
1052         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1053             table, safeCast<int32_t>(JSWebAssemblyTable::offsetOfSize()));
1054     }
1055
1056     // Check the index we are looking for is valid.
1057     {
1058         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1059             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferSize));
1060
1061         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1062             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1063         });
1064     }
1065
1066     // Compute the offset in the table index space we are looking for.
1067     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1068         m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex),
1069         constant(pointerType(), sizeof(CallableFunction)));
1070     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1071
1072     // Check that the CallableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1073     static_assert(sizeof(CallableFunction::signatureIndex) == sizeof(uint32_t), "Load codegen assumes i32");
1074     ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), callableFunction, safeCast<int32_t>(OBJECT_OFFSETOF(CallableFunction, signatureIndex)));
1075     {
1076         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1077             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1078                 calleeSignatureIndex,
1079                 m_currentBlock->appendNew<Const32Value>(m_proc, origin(), Signature::invalidIndex)));
1080
1081         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1082             this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1083         });
1084     }
1085
1086     // Check the signature matches the value we expect.
1087     {
1088         ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const32Value>(m_proc, origin(), SignatureInformation::get(signature));
1089         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1090             m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1091
1092         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1093             this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1094         });
1095     }
1096
1097     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction, safeCast<int32_t>(OBJECT_OFFSETOF(CallableFunction, code)));
1098
1099     Type returnType = signature.returnType();
1100     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1101         [=] (PatchpointValue* patchpoint) {
1102             patchpoint->effects.writesPinned = true;
1103             patchpoint->effects.readsPinned = true;
1104             // We need to clobber all potential pinned registers since we might be leaving the instance.
1105             patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
1106
1107             patchpoint->append(calleeCode, ValueRep::SomeRegister);
1108             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1109                 AllowMacroScratchRegisterUsage allowScratch(jit);
1110                 jit.call(params[returnType == Void ? 0 : 1].gpr());
1111             });
1112         });
1113
1114     // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1115     restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, m_currentBlock);
1116
1117     return { };
1118 }
1119
1120 void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1121 {
1122     m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1123 }
1124
1125 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1126 {
1127     ASSERT(result.size() <= resultStack.size());
1128
1129     for (size_t i = 0; i < result.size(); ++i)
1130         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1131 }
1132
1133 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1134 {
1135     dataLog(comma, "ExpressionStack:");
1136     for (const auto& expression : expressionStack)
1137         dataLog(comma, *expression);
1138 }
1139
1140 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1141 {
1142     dataLogLn("Constants:");
1143     for (const auto& constant : m_constantPool)
1144         dataLogLn(deepDump(m_proc, constant.value));
1145
1146     dataLogLn("Processing Graph:");
1147     dataLog(m_proc);
1148     dataLogLn("With current block:", *m_currentBlock);
1149     dataLogLn("Control stack:");
1150     ASSERT(controlStack.size());
1151     for (size_t i = controlStack.size(); i--;) {
1152         dataLog("  ", controlStack[i].controlData, ": ");
1153         CommaPrinter comma(", ", "");
1154         dumpExpressionStack(comma, *expressionStack);
1155         expressionStack = &controlStack[i].enclosedExpressionStack;
1156         dataLogLn();
1157     }
1158     dataLogLn();
1159 }
1160
1161 static void createJSToWasmWrapper(CompilationContext& compilationContext, WasmInternalFunction& function, const Signature& signature, const ModuleInformation& info, MemoryMode mode)
1162 {
1163     CCallHelpers& jit = *compilationContext.jsEntrypointJIT;
1164
1165     jit.emitFunctionPrologue();
1166
1167     // FIXME Stop using 0 as codeBlocks. https://bugs.webkit.org/show_bug.cgi?id=165321
1168     jit.store64(CCallHelpers::TrustedImm64(0), CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::codeBlock * static_cast<int>(sizeof(Register))));
1169     MacroAssembler::DataLabelPtr calleeMoveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), GPRInfo::nonPreservedNonReturnGPR);
1170     jit.storePtr(GPRInfo::nonPreservedNonReturnGPR, CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::callee * static_cast<int>(sizeof(Register))));
1171     CodeLocationDataLabelPtr* linkedCalleeMove = &function.jsToWasmCalleeMoveLocation;
1172     jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1173         *linkedCalleeMove = linkBuffer.locationOf(calleeMoveLocation);
1174     });
1175
1176     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1177     RegisterSet toSave = pinnedRegs.toSave(mode);
1178
1179 #if !ASSERT_DISABLED
1180     unsigned toSaveSize = toSave.numberOfSetGPRs();
1181     // They should all be callee saves.
1182     toSave.filter(RegisterSet::calleeSaveRegisters());
1183     ASSERT(toSave.numberOfSetGPRs() == toSaveSize);
1184 #endif
1185
1186     RegisterAtOffsetList registersToSpill(toSave, RegisterAtOffsetList::OffsetBaseType::FramePointerBased);
1187     function.jsToWasmEntrypoint.calleeSaveRegisters = registersToSpill;
1188
1189     unsigned totalFrameSize = registersToSpill.size() * sizeof(void*);
1190     totalFrameSize += WasmCallingConvention::headerSizeInBytes();
1191     totalFrameSize -= sizeof(CallerFrameAndPC);
1192     unsigned numGPRs = 0;
1193     unsigned numFPRs = 0;
1194     for (unsigned i = 0; i < signature.argumentCount(); i++) {
1195         switch (signature.argument(i)) {
1196         case Wasm::I64:
1197         case Wasm::I32:
1198             if (numGPRs >= wasmCallingConvention().m_gprArgs.size())
1199                 totalFrameSize += sizeof(void*);
1200             ++numGPRs;
1201             break;
1202         case Wasm::F32:
1203         case Wasm::F64:
1204             if (numFPRs >= wasmCallingConvention().m_fprArgs.size())
1205                 totalFrameSize += sizeof(void*);
1206             ++numFPRs;
1207             break;
1208         default:
1209             RELEASE_ASSERT_NOT_REACHED();
1210         }
1211     }
1212
1213     totalFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), totalFrameSize);
1214     jit.subPtr(MacroAssembler::TrustedImm32(totalFrameSize), MacroAssembler::stackPointerRegister);
1215
1216     // We save all these registers regardless of having a memory or not.
1217     // The reason is that we use one of these as a scratch. That said,
1218     // almost all real wasm programs use memory, so it's not really
1219     // worth optimizing for the case that they don't.
1220     for (const RegisterAtOffset& regAtOffset : registersToSpill) {
1221         GPRReg reg = regAtOffset.reg().gpr();
1222         ptrdiff_t offset = regAtOffset.offset();
1223         jit.storePtr(reg, CCallHelpers::Address(GPRInfo::callFrameRegister, offset));
1224     }
1225
1226     GPRReg wasmContextGPR = pinnedRegs.wasmContextPointer;
1227
1228     {
1229         CCallHelpers::Address calleeFrame = CCallHelpers::Address(MacroAssembler::stackPointerRegister, -static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC)));
1230         numGPRs = 0;
1231         numFPRs = 0;
1232         // We're going to set the pinned registers after this. So
1233         // we can use this as a scratch for now since we saved it above.
1234         GPRReg scratchReg = pinnedRegs.baseMemoryPointer;
1235
1236         ptrdiff_t jsOffset = CallFrameSlot::thisArgument * sizeof(EncodedJSValue);
1237
1238         // vmEntryToWasm passes Wasm::Context* as the first JS argument when we're
1239         // not using fast TLS to hold the Wasm::Context*.
1240         if (!useFastTLSForContext()) {
1241             jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmContextGPR);
1242             jsOffset += sizeof(EncodedJSValue);
1243         }
1244
1245         ptrdiff_t wasmOffset = CallFrame::headerSizeInRegisters * sizeof(void*);
1246         for (unsigned i = 0; i < signature.argumentCount(); i++) {
1247             switch (signature.argument(i)) {
1248             case Wasm::I32:
1249             case Wasm::I64:
1250                 if (numGPRs >= wasmCallingConvention().m_gprArgs.size()) {
1251                     if (signature.argument(i) == Wasm::I32) {
1252                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1253                         jit.store32(scratchReg, calleeFrame.withOffset(wasmOffset));
1254                     } else {
1255                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1256                         jit.store64(scratchReg, calleeFrame.withOffset(wasmOffset));
1257                     }
1258                     wasmOffset += sizeof(void*);
1259                 } else {
1260                     if (signature.argument(i) == Wasm::I32)
1261                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_gprArgs[numGPRs].gpr());
1262                     else
1263                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_gprArgs[numGPRs].gpr());
1264                 }
1265                 ++numGPRs;
1266                 break;
1267             case Wasm::F32:
1268             case Wasm::F64:
1269                 if (numFPRs >= wasmCallingConvention().m_fprArgs.size()) {
1270                     if (signature.argument(i) == Wasm::F32) {
1271                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1272                         jit.store32(scratchReg, calleeFrame.withOffset(wasmOffset));
1273                     } else {
1274                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1275                         jit.store64(scratchReg, calleeFrame.withOffset(wasmOffset));
1276                     }
1277                     wasmOffset += sizeof(void*);
1278                 } else {
1279                     if (signature.argument(i) == Wasm::F32)
1280                         jit.loadFloat(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_fprArgs[numFPRs].fpr());
1281                     else
1282                         jit.loadDouble(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_fprArgs[numFPRs].fpr());
1283                 }
1284                 ++numFPRs;
1285                 break;
1286             default:
1287                 RELEASE_ASSERT_NOT_REACHED();
1288             }
1289
1290             jsOffset += sizeof(EncodedJSValue);
1291         }
1292     }
1293
1294     if (!!info.memory) {
1295         GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1296
1297         if (!useFastTLSForContext())
1298             jit.loadPtr(CCallHelpers::Address(wasmContextGPR, JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
1299         else {
1300             jit.loadWasmContext(baseMemory);
1301             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
1302         }
1303
1304         if (mode != MemoryMode::Signaling) {
1305             const auto& sizeRegs = pinnedRegs.sizeRegisters;
1306             ASSERT(sizeRegs.size() >= 1);
1307             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
1308             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfSize()), sizeRegs[0].sizeRegister);
1309             for (unsigned i = 1; i < sizeRegs.size(); ++i)
1310                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
1311         }
1312
1313         jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfMemory()), baseMemory);
1314     }
1315
1316     compilationContext.jsEntrypointToWasmEntrypointCall = jit.call();
1317
1318     for (const RegisterAtOffset& regAtOffset : registersToSpill) {
1319         GPRReg reg = regAtOffset.reg().gpr();
1320         ASSERT(reg != GPRInfo::returnValueGPR);
1321         ptrdiff_t offset = regAtOffset.offset();
1322         jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, offset), reg);
1323     }
1324
1325     switch (signature.returnType()) {
1326     case Wasm::F32:
1327         jit.moveFloatTo32(FPRInfo::returnValueFPR, GPRInfo::returnValueGPR);
1328         break;
1329     case Wasm::F64:
1330         jit.moveDoubleTo64(FPRInfo::returnValueFPR, GPRInfo::returnValueGPR);
1331         break;
1332     default:
1333         break;
1334     }
1335
1336     jit.emitFunctionEpilogue();
1337     jit.ret();
1338 }
1339
1340 auto B3IRGenerator::origin() -> Origin
1341 {
1342     return bitwise_cast<Origin>(OpcodeOrigin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset()));
1343 }
1344
1345 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, unsigned optLevel)
1346 {
1347     auto result = std::make_unique<WasmInternalFunction>();
1348
1349     compilationContext.jsEntrypointJIT = std::make_unique<CCallHelpers>();
1350     compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1351
1352     Procedure procedure;
1353
1354     procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1355         if (origin.data())
1356             out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1357     });
1358     
1359     // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1360     // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1361     // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1362     // optLevel=1.
1363     procedure.setNeedsUsedRegisters(false);
1364     
1365     procedure.setOptLevel(optLevel);
1366
1367     B3IRGenerator context(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode);
1368     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, info);
1369     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1370
1371     context.insertConstants();
1372
1373     procedure.resetReachability();
1374     if (!ASSERT_DISABLED)
1375         validate(procedure, "After parsing:\n");
1376
1377     dataLogIf(verbose, "Pre SSA: ", procedure);
1378     fixSSA(procedure);
1379     dataLogIf(verbose, "Post SSA: ", procedure);
1380     
1381     {
1382         B3::prepareForGeneration(procedure);
1383         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1384         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1385         result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1386     }
1387
1388     createJSToWasmWrapper(compilationContext, *result, signature, info, mode);
1389     return WTFMove(result);
1390 }
1391
1392 // Custom wasm ops. These are the ones too messy to do in wasm.json.
1393
1394 void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1395 {
1396     ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1397     const B3::Type type = left->type();
1398
1399     {
1400         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1401             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1402
1403         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1404             this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1405         });
1406     }
1407
1408     if (operation == Div) {
1409         int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1410
1411         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1412             m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1413                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1414                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1415
1416         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1417             this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1418         });
1419     }
1420 }
1421
1422 template<>
1423 auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1424 {
1425     const B3::Opcode op = Div;
1426     emitChecksForModOrDiv(op, left, right);
1427     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1428     return { };
1429 }
1430
1431 template<>
1432 auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1433 {
1434     const B3::Opcode op = Mod;
1435     emitChecksForModOrDiv(op, left, right);
1436     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1437     return { };
1438 }
1439
1440 template<>
1441 auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1442 {
1443     const B3::Opcode op = UDiv;
1444     emitChecksForModOrDiv(op, left, right);
1445     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1446     return { };
1447 }
1448
1449 template<>
1450 auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1451 {
1452     const B3::Opcode op = UMod;
1453     emitChecksForModOrDiv(op, left, right);
1454     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1455     return { };
1456 }
1457
1458 template<>
1459 auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1460 {
1461     const B3::Opcode op = Div;
1462     emitChecksForModOrDiv(op, left, right);
1463     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1464     return { };
1465 }
1466
1467 template<>
1468 auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1469 {
1470     const B3::Opcode op = Mod;
1471     emitChecksForModOrDiv(op, left, right);
1472     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1473     return { };
1474 }
1475
1476 template<>
1477 auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1478 {
1479     const B3::Opcode op = UDiv;
1480     emitChecksForModOrDiv(op, left, right);
1481     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1482     return { };
1483 }
1484
1485 template<>
1486 auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1487 {
1488     const B3::Opcode op = UMod;
1489     emitChecksForModOrDiv(op, left, right);
1490     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1491     return { };
1492 }
1493
1494 template<>
1495 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1496 {
1497     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1498     patchpoint->append(arg, ValueRep::SomeRegister);
1499     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1500         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1501     });
1502     patchpoint->effects = Effects::none();
1503     result = patchpoint;
1504     return { };
1505 }
1506
1507 template<>
1508 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1509 {
1510     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1511     patchpoint->append(arg, ValueRep::SomeRegister);
1512     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1513         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1514     });
1515     patchpoint->effects = Effects::none();
1516     result = patchpoint;
1517     return { };
1518 }
1519
1520 template<>
1521 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1522 {
1523     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1524     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1525     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1526     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1527     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1528     return { };
1529 }
1530
1531 template<>
1532 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1533 {
1534     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1535     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1536     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1537     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1538     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1539     return { };
1540 }
1541
1542 template<>
1543 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1544 {
1545     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1546     if (isX86())
1547         patchpoint->numGPScratchRegisters = 1;
1548     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1549     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1550         AllowMacroScratchRegisterUsage allowScratch(jit);
1551 #if CPU(X86_64)
1552         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1553 #else
1554         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1555 #endif
1556     });
1557     patchpoint->effects = Effects::none();
1558     result = patchpoint;
1559     return { };
1560 }
1561
1562 template<>
1563 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1564 {
1565     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1566     if (isX86())
1567         patchpoint->numGPScratchRegisters = 1;
1568     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1569     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1570         AllowMacroScratchRegisterUsage allowScratch(jit);
1571 #if CPU(X86_64)
1572         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1573 #else
1574         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1575 #endif
1576     });
1577     patchpoint->effects = Effects::none();
1578     result = patchpoint;
1579     return { };
1580 }
1581
1582 template<>
1583 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1584 {
1585     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1586     patchpoint->append(arg, ValueRep::SomeRegister);
1587     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1588         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1589     });
1590     patchpoint->effects = Effects::none();
1591     result = patchpoint;
1592     return { };
1593 }
1594
1595 template<>
1596 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1597 {
1598     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1599     patchpoint->append(arg, ValueRep::SomeRegister);
1600     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1601         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1602     });
1603     patchpoint->effects = Effects::none();
1604     result = patchpoint;
1605     return { };
1606 }
1607
1608 template<>
1609 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1610 {
1611     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1612     patchpoint->append(arg, ValueRep::SomeRegister);
1613     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1614         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1615     });
1616     patchpoint->effects = Effects::none();
1617     result = patchpoint;
1618     return { };
1619 }
1620
1621 template<>
1622 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1623 {
1624     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1625     patchpoint->append(arg, ValueRep::SomeRegister);
1626     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1627         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1628     });
1629     patchpoint->effects = Effects::none();
1630     result = patchpoint;
1631     return { };
1632 }
1633
1634 template<>
1635 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1636 {
1637     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1638     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1639     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1640         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1641         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1642     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1643     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1644     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1645         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1646     });
1647     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1648     patchpoint->append(arg, ValueRep::SomeRegister);
1649     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1650         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1651     });
1652     patchpoint->effects = Effects::none();
1653     result = patchpoint;
1654     return { };
1655 }
1656
1657 template<>
1658 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1659 {
1660     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1661     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1662     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1663         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1664         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1665     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1666     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1667     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1668         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1669     });
1670     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1671     patchpoint->append(arg, ValueRep::SomeRegister);
1672     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1673         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1674     });
1675     patchpoint->effects = Effects::none();
1676     result = patchpoint;
1677     return { };
1678 }
1679
1680
1681 template<>
1682 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1683 {
1684     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1685     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1686     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1687         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1688         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1689     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1690     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1691     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1692         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1693     });
1694     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1695     patchpoint->append(arg, ValueRep::SomeRegister);
1696     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1697         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1698     });
1699     patchpoint->effects = Effects::none();
1700     result = patchpoint;
1701     return { };
1702 }
1703
1704 template<>
1705 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1706 {
1707     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1708     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1709     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1710         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1711         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1712     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1713     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1714     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1715         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1716     });
1717     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1718     patchpoint->append(arg, ValueRep::SomeRegister);
1719     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1720         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1721     });
1722     patchpoint->effects = Effects::none();
1723     result = patchpoint;
1724     return { };
1725 }
1726
1727 template<>
1728 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1729 {
1730     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1731     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1732     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1733         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1734         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1735     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1736     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1737     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1738         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1739     });
1740     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1741     patchpoint->append(arg, ValueRep::SomeRegister);
1742     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1743         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1744     });
1745     patchpoint->effects = Effects::none();
1746     result = patchpoint;
1747     return { };
1748 }
1749
1750 template<>
1751 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1752 {
1753     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1754     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1755     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1756         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1757         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1758     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1759     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1760     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1761         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1762     });
1763
1764     Value* signBitConstant;
1765     if (isX86()) {
1766         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1767         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1768         // so we can pool them if needed.
1769         signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1770     }
1771     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1772     patchpoint->append(arg, ValueRep::SomeRegister);
1773     if (isX86()) {
1774         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1775         patchpoint->numFPScratchRegisters = 1;
1776     }
1777     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1778         AllowMacroScratchRegisterUsage allowScratch(jit);
1779         FPRReg scratch = InvalidFPRReg;
1780         FPRReg constant = InvalidFPRReg;
1781         if (isX86()) {
1782             scratch = params.fpScratch(0);
1783             constant = params[2].fpr();
1784         }
1785         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1786     });
1787     patchpoint->effects = Effects::none();
1788     result = patchpoint;
1789     return { };
1790 }
1791
1792 template<>
1793 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1794 {
1795     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1796     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1797     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1798         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1799         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1800     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1801     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1802     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1803         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1804     });
1805     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1806     patchpoint->append(arg, ValueRep::SomeRegister);
1807     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1808         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1809     });
1810     patchpoint->effects = Effects::none();
1811     result = patchpoint;
1812     return { };
1813 }
1814
1815 template<>
1816 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1817 {
1818     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1819     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1820     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1821         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1822         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1823     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1824     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1825     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1826         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1827     });
1828
1829     Value* signBitConstant;
1830     if (isX86()) {
1831         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1832         // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1833         // so we can pool them if needed.
1834         signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1835     }
1836     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1837     patchpoint->append(arg, ValueRep::SomeRegister);
1838     if (isX86()) {
1839         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1840         patchpoint->numFPScratchRegisters = 1;
1841     }
1842     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1843         AllowMacroScratchRegisterUsage allowScratch(jit);
1844         FPRReg scratch = InvalidFPRReg;
1845         FPRReg constant = InvalidFPRReg;
1846         if (isX86()) {
1847             scratch = params.fpScratch(0);
1848             constant = params[2].fpr();
1849         }
1850         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1851     });
1852     patchpoint->effects = Effects::none();
1853     result = patchpoint;
1854     return { };
1855 }
1856
1857 } } // namespace JSC::Wasm
1858
1859 #include "WasmB3IRGeneratorInlines.h"
1860
1861 #endif // ENABLE(WEBASSEMBLY)