B3: don't allow unsigned offsets in Value
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "AllowMacroScratchRegisterUsageIf.h"
32 #include "B3BasicBlockInlines.h"
33 #include "B3CCallValue.h"
34 #include "B3Compile.h"
35 #include "B3ConstPtrValue.h"
36 #include "B3FixSSA.h"
37 #include "B3Generate.h"
38 #include "B3InsertionSet.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3SwitchValue.h"
42 #include "B3UpsilonValue.h"
43 #include "B3Validate.h"
44 #include "B3ValueInlines.h"
45 #include "B3ValueKey.h"
46 #include "B3Variable.h"
47 #include "B3VariableValue.h"
48 #include "B3WasmAddressValue.h"
49 #include "B3WasmBoundsCheckValue.h"
50 #include "JSCInlines.h"
51 #include "JSWebAssemblyInstance.h"
52 #include "JSWebAssemblyModule.h"
53 #include "JSWebAssemblyRuntimeError.h"
54 #include "VirtualRegister.h"
55 #include "WasmCallingConvention.h"
56 #include "WasmContext.h"
57 #include "WasmExceptionType.h"
58 #include "WasmFunctionParser.h"
59 #include "WasmMemory.h"
60 #include "WasmOpcodeOrigin.h"
61 #include "WasmThunks.h"
62 #include <wtf/Optional.h>
63 #include <wtf/StdLibExtras.h>
64
65 void dumpProcedure(void* ptr)
66 {
67     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
68     proc->dump(WTF::dataFile());
69 }
70
71 namespace JSC { namespace Wasm {
72
73 using namespace B3;
74
75 namespace {
76 const bool verbose = false;
77 }
78
79 class B3IRGenerator {
80 public:
81     struct ControlData {
82         ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
83             : blockType(type)
84             , continuation(continuation)
85             , special(special)
86         {
87             if (signature != Void)
88                 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
89         }
90
91         ControlData()
92         {
93         }
94
95         void dump(PrintStream& out) const
96         {
97             switch (type()) {
98             case BlockType::If:
99                 out.print("If:       ");
100                 break;
101             case BlockType::Block:
102                 out.print("Block:    ");
103                 break;
104             case BlockType::Loop:
105                 out.print("Loop:     ");
106                 break;
107             case BlockType::TopLevel:
108                 out.print("TopLevel: ");
109                 break;
110             }
111             out.print("Continuation: ", *continuation, ", Special: ");
112             if (special)
113                 out.print(*special);
114             else
115                 out.print("None");
116         }
117
118         BlockType type() const { return blockType; }
119
120         bool hasNonVoidSignature() const { return result.size(); }
121
122         BasicBlock* targetBlockForBranch()
123         {
124             if (type() == BlockType::Loop)
125                 return special;
126             return continuation;
127         }
128
129         void convertIfToBlock()
130         {
131             ASSERT(type() == BlockType::If);
132             blockType = BlockType::Block;
133             special = nullptr;
134         }
135
136         using ResultList = Vector<Value*, 1>; // Value must be a Phi
137
138         ResultList resultForBranch() const
139         {
140             if (type() == BlockType::Loop)
141                 return ResultList();
142             return result;
143         }
144
145     private:
146         friend class B3IRGenerator;
147         BlockType blockType;
148         BasicBlock* continuation;
149         BasicBlock* special;
150         ResultList result;
151     };
152
153     typedef Value* ExpressionType;
154     typedef ControlData ControlType;
155     typedef Vector<ExpressionType, 1> ExpressionList;
156     typedef ControlData::ResultList ResultList;
157     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
158
159     static constexpr ExpressionType emptyExpression = nullptr;
160
161     typedef String ErrorType;
162     typedef UnexpectedType<ErrorType> UnexpectedResult;
163     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
164     typedef Expected<void, ErrorType> PartialResult;
165     template <typename ...Args>
166     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
167     {
168         using namespace FailureHelper; // See ADL comment in WasmParser.h.
169         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
170     }
171 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
172         if (UNLIKELY(condition))                  \
173             return fail(__VA_ARGS__);             \
174     } while (0)
175
176     B3IRGenerator(const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode);
177
178     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
179     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
180     ExpressionType addConstant(Type, uint64_t);
181
182     // Locals
183     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
184     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
185
186     // Globals
187     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
188     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
189
190     // Memory
191     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
192     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
193     PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
194     PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
195
196     // Basic operators
197     template<OpType>
198     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
199     template<OpType>
200     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
201     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
202
203     // Control flow
204     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
205     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
206     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
207     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
208     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
209     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
210
211     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
212     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
213     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
214     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
215     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
216
217     // Calls
218     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
219     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
220     PartialResult WARN_UNUSED_RETURN addUnreachable();
221
222     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
223     void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
224
225     Value* constant(B3::Type, uint64_t bits);
226     void insertConstants();
227
228 private:
229     void emitExceptionCheck(CCallHelpers&, ExceptionType);
230
231     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
232     B3::Kind memoryKind(B3::Opcode memoryOp);
233     ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
234     void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
235
236     void unify(const ExpressionType phi, const ExpressionType source);
237     void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
238
239     void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
240
241     int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
242
243     Value* materializeWasmContext(Procedure&, BasicBlock*);
244     void restoreWasmContext(Procedure&, BasicBlock*, Value*);
245     void restoreWebAssemblyGlobalState(const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
246
247     Origin origin();
248
249     FunctionParser<B3IRGenerator>* m_parser;
250     const ModuleInformation& m_info;
251     MemoryMode m_mode;
252     Procedure& m_proc;
253     BasicBlock* m_currentBlock;
254     Vector<Variable*> m_locals;
255     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
256     HashMap<ValueKey, Value*> m_constantPool;
257     InsertionSet m_constantInsertionValues;
258     GPRReg m_memoryBaseGPR;
259     GPRReg m_memorySizeGPR { InvalidGPRReg };
260     GPRReg m_wasmContextGPR;
261     Value* m_instanceValue; // FIXME: make this lazy https://bugs.webkit.org/show_bug.cgi?id=169792
262 };
263
264 // Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
265 int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
266 {
267     if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
268         ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
269         return 0;
270     }
271     return offset;
272 }
273
274 Value* B3IRGenerator::materializeWasmContext(Procedure& proc, BasicBlock* block)
275 {
276     if (useFastTLSForContext()) {
277         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, pointerType(), Origin());
278         if (CCallHelpers::loadWasmContextNeedsMacroScratchRegister())
279             patchpoint->clobber(RegisterSet::macroScratchRegisters());
280         patchpoint->setGenerator(
281             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
282                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextNeedsMacroScratchRegister());
283                 jit.loadWasmContext(params[0].gpr());
284             });
285         return patchpoint;
286     }
287
288     // FIXME: Because WasmToWasm call clobbers wasmContext register and does not restore it, we need to restore it in the caller side.
289     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
290     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, pointerType(), Origin());
291     patchpoint->effects.writesPinned = false;
292     patchpoint->effects.readsPinned = true;
293     patchpoint->resultConstraint = ValueRep::reg(m_wasmContextGPR);
294     patchpoint->setGenerator([] (CCallHelpers&, const StackmapGenerationParams&) { });
295     return patchpoint;
296 }
297
298 void B3IRGenerator::restoreWasmContext(Procedure& proc, BasicBlock* block, Value* arg)
299 {
300     if (useFastTLSForContext()) {
301         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
302         if (CCallHelpers::storeWasmContextNeedsMacroScratchRegister())
303             patchpoint->clobber(RegisterSet::macroScratchRegisters());
304         patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
305         patchpoint->setGenerator(
306             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
307                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextNeedsMacroScratchRegister());
308                 jit.storeWasmContext(params[0].gpr());
309             });
310         return;
311     }
312
313     // FIXME: Because WasmToWasm call clobbers wasmContext register and does not restore it, we need to restore it in the caller side.
314     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
315     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
316     Effects effects = Effects::none();
317     effects.writesPinned = true;
318     effects.reads = B3::HeapRange::top();
319     patchpoint->effects = effects;
320     patchpoint->clobberLate(RegisterSet(m_wasmContextGPR));
321     patchpoint->append(m_instanceValue, ValueRep::SomeRegister);
322     GPRReg wasmContextGPR = m_wasmContextGPR;
323     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
324         jit.move(param[0].gpr(), wasmContextGPR);
325     });
326 }
327
328 B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode)
329     : m_info(info)
330     , m_mode(mode)
331     , m_proc(procedure)
332     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
333     , m_constantInsertionValues(m_proc)
334 {
335     m_currentBlock = m_proc.addBlock();
336
337     // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
338     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
339
340     m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
341     m_proc.pinRegister(m_memoryBaseGPR);
342
343     m_wasmContextGPR = pinnedRegs.wasmContextPointer;
344     if (!useFastTLSForContext())
345         m_proc.pinRegister(m_wasmContextGPR);
346
347     if (mode != MemoryMode::Signaling) {
348         ASSERT(!pinnedRegs.sizeRegisters[0].sizeOffset);
349         m_memorySizeGPR = pinnedRegs.sizeRegisters[0].sizeRegister;
350         for (const PinnedSizeRegisterInfo& regInfo : pinnedRegs.sizeRegisters)
351             m_proc.pinRegister(regInfo.sizeRegister);
352     }
353
354     if (info.memory) {
355         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
356             AllowMacroScratchRegisterUsage allowScratch(jit);
357             switch (m_mode) {
358             case MemoryMode::BoundsChecking:
359                 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
360                 break;
361             case MemoryMode::Signaling:
362                 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
363                 break;
364             case MemoryMode::NumberOfMemoryModes:
365                 ASSERT_NOT_REACHED();
366             }
367             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
368         });
369     }
370
371     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
372
373     m_instanceValue = materializeWasmContext(m_proc, m_currentBlock);
374 }
375
376 void B3IRGenerator::restoreWebAssemblyGlobalState(const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
377 {
378     restoreWasmContext(proc, block, instance);
379
380     if (!!memory) {
381         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
382         RegisterSet clobbers;
383         clobbers.set(pinnedRegs->baseMemoryPointer);
384         for (auto info : pinnedRegs->sizeRegisters)
385             clobbers.set(info.sizeRegister);
386
387         B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
388         Effects effects = Effects::none();
389         effects.writesPinned = true;
390         effects.reads = B3::HeapRange::top();
391         patchpoint->effects = effects;
392         patchpoint->clobber(clobbers);
393
394         patchpoint->append(instance, ValueRep::SomeRegister);
395
396         patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
397             GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
398             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
399             const auto& sizeRegs = pinnedRegs->sizeRegisters;
400             ASSERT(sizeRegs.size() >= 1);
401             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
402             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfSize()), sizeRegs[0].sizeRegister);
403             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfMemory()), baseMemory);
404             for (unsigned i = 1; i < sizeRegs.size(); ++i)
405                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
406         });
407     }
408 }
409
410 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
411 {
412     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
413     auto jumpToExceptionStub = jit.jump();
414
415     jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
416         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
417     });
418 }
419
420 Value* B3IRGenerator::constant(B3::Type type, uint64_t bits)
421 {
422     auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
423         Value* result = m_proc.addConstant(origin(), type, bits);
424         m_constantInsertionValues.insertValue(0, result);
425         return result;
426     });
427     return result.iterator->value;
428 }
429
430 void B3IRGenerator::insertConstants()
431 {
432     m_constantInsertionValues.execute(m_proc.at(0));
433 }
434
435 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
436 {
437     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
438
439     for (uint32_t i = 0; i < count; ++i) {
440         Variable* local = m_proc.addVariable(toB3Type(type));
441         m_locals.uncheckedAppend(local);
442         m_currentBlock->appendNew<VariableValue>(m_proc, Set, origin(), local, addConstant(type, 0));
443     }
444     return { };
445 }
446
447 auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
448 {
449     ASSERT(!m_locals.size());
450     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
451
452     m_locals.grow(signature.argumentCount());
453     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
454         [=] (ExpressionType argument, unsigned i) {
455             Variable* argumentVariable = m_proc.addVariable(argument->type());
456             m_locals[i] = argumentVariable;
457             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
458         });
459     return { };
460 }
461
462 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
463 {
464     ASSERT(m_locals[index]);
465     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
466     return { };
467 }
468
469 auto B3IRGenerator::addUnreachable() -> PartialResult
470 {
471     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
472     unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
473         this->emitExceptionCheck(jit, ExceptionType::Unreachable);
474     });
475     unreachable->effects.terminal = true;
476     return { };
477 }
478
479 auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
480 {
481     int32_t (*growMemory) (Context*, int32_t) = [] (Context* wasmContext, int32_t delta) -> int32_t {
482         VM& vm = *wasmContext->vm();
483         auto scope = DECLARE_THROW_SCOPE(vm);
484
485         JSWebAssemblyMemory* wasmMemory = wasmContext->memory();
486
487         if (delta < 0)
488             return -1;
489
490         bool shouldThrowExceptionsOnFailure = false;
491         // grow() does not require ExecState* if it doesn't throw exceptions.
492         ExecState* exec = nullptr; 
493         PageCount result = wasmMemory->grow(vm, exec, static_cast<uint32_t>(delta), shouldThrowExceptionsOnFailure);
494         RELEASE_ASSERT(!scope.exception());
495         if (!result)
496             return -1;
497
498         return result.pageCount();
499     };
500
501     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
502         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(growMemory)),
503         m_instanceValue, delta);
504
505     restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, m_currentBlock);
506
507     return { };
508 }
509
510 auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
511 {
512     Value* memoryObject = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfMemory()));
513
514     static_assert(sizeof(decltype(static_cast<JSWebAssemblyInstance*>(nullptr)->memory()->memory().size())) == sizeof(uint64_t), "codegen relies on this size");
515     Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), memoryObject, safeCast<int32_t>(JSWebAssemblyMemory::offsetOfSize()));
516     
517     constexpr uint32_t shiftValue = 16;
518     static_assert(PageCount::pageSize == 1 << shiftValue, "This must hold for the code below to be correct.");
519     Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
520         size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
521
522     result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
523
524     return { };
525 }
526
527 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
528 {
529     ASSERT(m_locals[index]);
530     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
531     return { };
532 }
533
534 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
535 {
536     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfGlobals()));
537     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
538     return { };
539 }
540
541 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
542 {
543     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
544     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfGlobals()));
545     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
546     return { };
547 }
548
549 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
550 {
551     ASSERT(m_memoryBaseGPR);
552     switch (m_mode) {
553     case MemoryMode::BoundsChecking:
554         // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
555         ASSERT(m_memorySizeGPR);
556         ASSERT(sizeOfOperation + offset > offset);
557         m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1, m_info.memory.maximum());
558         break;
559     case MemoryMode::Signaling:
560         // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current], and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register memory accesses are 32-bit. However WebAssembly register+immediate accesses perform the addition in 64-bit which can push an access above the 32-bit limit. The redzone will catch most small immediates, and we'll explicitly bounds check any register + large immediate access.
561         if (offset >= Memory::fastMappedRedzoneBytes())
562             m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, InvalidGPRReg, sizeOfOperation + offset - 1, m_info.memory.maximum());
563         break;
564     case MemoryMode::NumberOfMemoryModes:
565         RELEASE_ASSERT_NOT_REACHED();
566     }
567     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
568     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
569 }
570
571 inline uint32_t sizeOfLoadOp(LoadOpType op)
572 {
573     switch (op) {
574     case LoadOpType::I32Load8S:
575     case LoadOpType::I32Load8U:
576     case LoadOpType::I64Load8S:
577     case LoadOpType::I64Load8U:
578         return 1;
579     case LoadOpType::I32Load16S:
580     case LoadOpType::I64Load16S:
581     case LoadOpType::I32Load16U:
582     case LoadOpType::I64Load16U:
583         return 2;
584     case LoadOpType::I32Load:
585     case LoadOpType::I64Load32S:
586     case LoadOpType::I64Load32U:
587     case LoadOpType::F32Load:
588         return 4;
589     case LoadOpType::I64Load:
590     case LoadOpType::F64Load:
591         return 8;
592     }
593     RELEASE_ASSERT_NOT_REACHED();
594 }
595
596 inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
597 {
598     if (m_mode == MemoryMode::Signaling)
599         return trapping(memoryOp);
600     return memoryOp;
601 }
602
603 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
604 {
605     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
606
607     switch (op) {
608     case LoadOpType::I32Load8S: {
609         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
610     }
611
612     case LoadOpType::I64Load8S: {
613         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
614         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
615     }
616
617     case LoadOpType::I32Load8U: {
618         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
619     }
620
621     case LoadOpType::I64Load8U: {
622         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
623         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
624     }
625
626     case LoadOpType::I32Load16S: {
627         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
628     }
629     case LoadOpType::I64Load16S: {
630         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
631         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
632     }
633
634     case LoadOpType::I32Load: {
635         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
636     }
637
638     case LoadOpType::I64Load32U: {
639         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
640         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
641     }
642
643     case LoadOpType::I64Load32S: {
644         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
645         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
646     }
647
648     case LoadOpType::I64Load: {
649         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
650     }
651
652     case LoadOpType::F32Load: {
653         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
654     }
655
656     case LoadOpType::F64Load: {
657         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
658     }
659
660     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
661     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
662     case LoadOpType::I32Load16U: {
663         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
664         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
665             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
666     }
667     case LoadOpType::I64Load16U: {
668         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
669         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
670             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
671
672         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), partialResult);
673     }
674     }
675     RELEASE_ASSERT_NOT_REACHED();
676 }
677
678 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
679 {
680     ASSERT(pointer->type() == Int32);
681
682     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
683         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
684         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
685         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
686         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
687             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
688         });
689
690         switch (op) {
691         case LoadOpType::I32Load8S:
692         case LoadOpType::I32Load16S:
693         case LoadOpType::I32Load:
694         case LoadOpType::I32Load16U:
695         case LoadOpType::I32Load8U:
696             result = constant(Int32, 0);
697             break;
698         case LoadOpType::I64Load8S:
699         case LoadOpType::I64Load8U:
700         case LoadOpType::I64Load16S:
701         case LoadOpType::I64Load32U:
702         case LoadOpType::I64Load32S:
703         case LoadOpType::I64Load:
704         case LoadOpType::I64Load16U:
705             result = constant(Int64, 0);
706             break;
707         case LoadOpType::F32Load:
708             result = constant(Float, 0);
709             break;
710         case LoadOpType::F64Load:
711             result = constant(Double, 0);
712             break;
713         }
714
715     } else
716         result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
717
718     return { };
719 }
720
721 inline uint32_t sizeOfStoreOp(StoreOpType op)
722 {
723     switch (op) {
724     case StoreOpType::I32Store8:
725     case StoreOpType::I64Store8:
726         return 1;
727     case StoreOpType::I32Store16:
728     case StoreOpType::I64Store16:
729         return 2;
730     case StoreOpType::I32Store:
731     case StoreOpType::I64Store32:
732     case StoreOpType::F32Store:
733         return 4;
734     case StoreOpType::I64Store:
735     case StoreOpType::F64Store:
736         return 8;
737     }
738     RELEASE_ASSERT_NOT_REACHED();
739 }
740
741
742 inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
743 {
744     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
745
746     switch (op) {
747     case StoreOpType::I64Store8:
748         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
749         FALLTHROUGH;
750
751     case StoreOpType::I32Store8:
752         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
753         return;
754
755     case StoreOpType::I64Store16:
756         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
757         FALLTHROUGH;
758
759     case StoreOpType::I32Store16:
760         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
761         return;
762
763     case StoreOpType::I64Store32:
764         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
765         FALLTHROUGH;
766
767     case StoreOpType::I64Store:
768     case StoreOpType::I32Store:
769     case StoreOpType::F32Store:
770     case StoreOpType::F64Store:
771         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
772         return;
773     }
774     RELEASE_ASSERT_NOT_REACHED();
775 }
776
777 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
778 {
779     ASSERT(pointer->type() == Int32);
780
781     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
782         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
783         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
784         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
785         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
786             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
787         });
788     } else
789         emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
790
791     return { };
792 }
793
794 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
795 {
796     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
797     return { };
798 }
799
800 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
801 {
802     return constant(toB3Type(type), value);
803 }
804
805 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
806 {
807     return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
808 }
809
810 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
811 {
812     return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
813 }
814
815 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
816 {
817     BasicBlock* body = m_proc.addBlock();
818     BasicBlock* continuation = m_proc.addBlock();
819     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
820     body->addPredecessor(m_currentBlock);
821     m_currentBlock = body;
822     return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
823 }
824
825 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
826 {
827     // FIXME: This needs to do some kind of stack passing.
828
829     BasicBlock* taken = m_proc.addBlock();
830     BasicBlock* notTaken = m_proc.addBlock();
831     BasicBlock* continuation = m_proc.addBlock();
832
833     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
834     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
835     taken->addPredecessor(m_currentBlock);
836     notTaken->addPredecessor(m_currentBlock);
837
838     m_currentBlock = taken;
839     result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
840     return { };
841 }
842
843 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
844 {
845     unifyValuesWithBlock(currentStack, data.result);
846     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
847     return addElseToUnreachable(data);
848 }
849
850 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
851 {
852     ASSERT(data.type() == BlockType::If);
853     m_currentBlock = data.special;
854     data.convertIfToBlock();
855     return { };
856 }
857
858 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
859 {
860     ASSERT(returnValues.size() <= 1);
861     if (returnValues.size())
862         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
863     else
864         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
865     return { };
866 }
867
868 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
869 {
870     unifyValuesWithBlock(returnValues, data.resultForBranch());
871
872     BasicBlock* target = data.targetBlockForBranch();
873     if (condition) {
874         BasicBlock* continuation = m_proc.addBlock();
875         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
876         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
877         target->addPredecessor(m_currentBlock);
878         continuation->addPredecessor(m_currentBlock);
879         m_currentBlock = continuation;
880     } else {
881         m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
882         target->addPredecessor(m_currentBlock);
883     }
884
885     return { };
886 }
887
888 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
889 {
890     for (size_t i = 0; i < targets.size(); ++i)
891         unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
892     unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
893
894     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
895     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
896     for (size_t i = 0; i < targets.size(); ++i)
897         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
898
899     return { };
900 }
901
902 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
903 {
904     ControlData& data = entry.controlData;
905
906     unifyValuesWithBlock(expressionStack, data.result);
907     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
908     data.continuation->addPredecessor(m_currentBlock);
909
910     return addEndToUnreachable(entry);
911 }
912
913
914 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
915 {
916     ControlData& data = entry.controlData;
917     m_currentBlock = data.continuation;
918
919     if (data.type() == BlockType::If) {
920         data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
921         m_currentBlock->addPredecessor(data.special);
922     }
923
924     for (Value* result : data.result) {
925         m_currentBlock->append(result);
926         entry.enclosedExpressionStack.append(result);
927     }
928
929     // TopLevel does not have any code after this so we need to make sure we emit a return here.
930     if (data.type() == BlockType::TopLevel)
931         return addReturn(entry.controlData, entry.enclosedExpressionStack);
932
933     return { };
934 }
935
936 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
937 {
938     ASSERT(signature.argumentCount() == args.size());
939
940     Type returnType = signature.returnType();
941     Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
942
943     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
944         // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
945         Value* functionImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfImportFunction(functionIndex)));
946         Value* jsTypeOfImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin(), functionImport, safeCast<int32_t>(JSCell::typeInfoTypeOffset()));
947         Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), jsTypeOfImport, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), WebAssemblyFunctionType));
948
949         BasicBlock* isWasmBlock = m_proc.addBlock();
950         BasicBlock* isJSBlock = m_proc.addBlock();
951         BasicBlock* continuation = m_proc.addBlock();
952         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isJSBlock));
953
954         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
955             [=] (PatchpointValue* patchpoint) {
956                 patchpoint->effects.writesPinned = true;
957                 patchpoint->effects.readsPinned = true;
958                 // We need to clobber all potential pinned registers since we might be leaving the instance.
959                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
960                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
961                     AllowMacroScratchRegisterUsage allowScratch(jit);
962                     CCallHelpers::Call call = jit.call();
963                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
964                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex });
965                     });
966                 });
967             });
968         UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
969         isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
970
971         // FIXME: Lets remove this indirection by creating a PIC friendly IC
972         // for calls out to JS. This shouldn't be that hard to do. We could probably
973         // implement the IC to be over Wasm::Context*.
974         // https://bugs.webkit.org/show_bug.cgi?id=170375
975         Value* codeBlock = isJSBlock->appendNew<MemoryValue>(m_proc,
976             Load, pointerType(), origin(), m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfCodeBlock()));
977         Value* jumpDestination = isJSBlock->appendNew<MemoryValue>(m_proc,
978             Load, pointerType(), origin(), codeBlock, safeCast<int32_t>(JSWebAssemblyCodeBlock::offsetOfImportWasmToJSStub(functionIndex)));
979         Value* jsCallResult = wasmCallingConvention().setupCall(m_proc, isJSBlock, origin(), args, toB3Type(returnType),
980             [&] (PatchpointValue* patchpoint) {
981                 patchpoint->effects.writesPinned = true;
982                 patchpoint->effects.readsPinned = true;
983                 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
984                 // We need to clobber all potential pinned registers since we might be leaving the instance.
985                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
986                 patchpoint->setGenerator([functionIndex, returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
987                     AllowMacroScratchRegisterUsage allowScratch(jit);
988                     jit.call(params[returnType == Void ? 0 : 1].gpr());
989                 });
990             });
991         UpsilonValue* jsCallResultUpsilon = returnType == Void ? nullptr : isJSBlock->appendNew<UpsilonValue>(m_proc, origin(), jsCallResult);
992         isJSBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
993
994         m_currentBlock = continuation;
995
996         if (returnType == Void)
997             result = nullptr;
998         else {
999             result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1000             wasmCallResultUpsilon->setPhi(result);
1001             jsCallResultUpsilon->setPhi(result);
1002         }
1003
1004         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1005         restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, continuation);
1006     } else {
1007         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1008             [=] (PatchpointValue* patchpoint) {
1009                 patchpoint->effects.writesPinned = true;
1010                 patchpoint->effects.readsPinned = true;
1011
1012                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1013                     AllowMacroScratchRegisterUsage allowScratch(jit);
1014                     CCallHelpers::Call call = jit.call();
1015                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1016                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex });
1017                     });
1018                 });
1019             });
1020     }
1021
1022     return { };
1023 }
1024
1025 auto B3IRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1026 {
1027     ExpressionType calleeIndex = args.takeLast();
1028     ASSERT(signature.argumentCount() == args.size());
1029
1030     ExpressionType callableFunctionBuffer;
1031     ExpressionType callableFunctionBufferSize;
1032     {
1033         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1034             m_instanceValue, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfTable()));
1035         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1036             table, safeCast<int32_t>(JSWebAssemblyTable::offsetOfFunctions()));
1037         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1038             table, safeCast<int32_t>(JSWebAssemblyTable::offsetOfSize()));
1039     }
1040
1041     // Check the index we are looking for is valid.
1042     {
1043         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1044             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferSize));
1045
1046         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1047             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1048         });
1049     }
1050
1051     // Compute the offset in the table index space we are looking for.
1052     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1053         m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex),
1054         constant(pointerType(), sizeof(CallableFunction)));
1055     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1056
1057     // Check that the CallableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1058     static_assert(sizeof(CallableFunction::signatureIndex) == sizeof(uint32_t), "Load codegen assumes i32");
1059     ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), callableFunction, safeCast<int32_t>(OBJECT_OFFSETOF(CallableFunction, signatureIndex)));
1060     {
1061         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1062             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1063                 calleeSignatureIndex,
1064                 m_currentBlock->appendNew<Const32Value>(m_proc, origin(), Signature::invalidIndex)));
1065
1066         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1067             this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1068         });
1069     }
1070
1071     // Check the signature matches the value we expect.
1072     {
1073         ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const32Value>(m_proc, origin(), SignatureInformation::get(signature));
1074         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1075             m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1076
1077         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1078             this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1079         });
1080     }
1081
1082     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction, safeCast<int32_t>(OBJECT_OFFSETOF(CallableFunction, code)));
1083
1084     Type returnType = signature.returnType();
1085     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1086         [=] (PatchpointValue* patchpoint) {
1087             patchpoint->effects.writesPinned = true;
1088             patchpoint->effects.readsPinned = true;
1089             // We need to clobber all potential pinned registers since we might be leaving the instance.
1090             patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
1091
1092             patchpoint->append(calleeCode, ValueRep::SomeRegister);
1093             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1094                 AllowMacroScratchRegisterUsage allowScratch(jit);
1095                 jit.call(params[returnType == Void ? 0 : 1].gpr());
1096             });
1097         });
1098
1099     // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1100     restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, m_currentBlock);
1101
1102     return { };
1103 }
1104
1105 void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1106 {
1107     m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1108 }
1109
1110 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1111 {
1112     ASSERT(result.size() <= resultStack.size());
1113
1114     for (size_t i = 0; i < result.size(); ++i)
1115         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1116 }
1117
1118 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1119 {
1120     dataLog(comma, "ExpressionStack:");
1121     for (const auto& expression : expressionStack)
1122         dataLog(comma, *expression);
1123 }
1124
1125 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1126 {
1127     dataLogLn("Constants:");
1128     for (const auto& constant : m_constantPool)
1129         dataLogLn(deepDump(m_proc, constant.value));
1130
1131     dataLogLn("Processing Graph:");
1132     dataLog(m_proc);
1133     dataLogLn("With current block:", *m_currentBlock);
1134     dataLogLn("Control stack:");
1135     ASSERT(controlStack.size());
1136     for (size_t i = controlStack.size(); i--;) {
1137         dataLog("  ", controlStack[i].controlData, ": ");
1138         CommaPrinter comma(", ", "");
1139         dumpExpressionStack(comma, *expressionStack);
1140         expressionStack = &controlStack[i].enclosedExpressionStack;
1141         dataLogLn();
1142     }
1143     dataLogLn();
1144 }
1145
1146 static void createJSToWasmWrapper(CompilationContext& compilationContext, WasmInternalFunction& function, const Signature& signature, const ModuleInformation& info, MemoryMode mode)
1147 {
1148     CCallHelpers& jit = *compilationContext.jsEntrypointJIT;
1149
1150     jit.emitFunctionPrologue();
1151
1152     // FIXME Stop using 0 as codeBlocks. https://bugs.webkit.org/show_bug.cgi?id=165321
1153     jit.store64(CCallHelpers::TrustedImm64(0), CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::codeBlock * static_cast<int>(sizeof(Register))));
1154     MacroAssembler::DataLabelPtr calleeMoveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), GPRInfo::nonPreservedNonReturnGPR);
1155     jit.storePtr(GPRInfo::nonPreservedNonReturnGPR, CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::callee * static_cast<int>(sizeof(Register))));
1156     CodeLocationDataLabelPtr* linkedCalleeMove = &function.jsToWasmCalleeMoveLocation;
1157     jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1158         *linkedCalleeMove = linkBuffer.locationOf(calleeMoveLocation);
1159     });
1160
1161     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1162     RegisterSet toSave = pinnedRegs.toSave(mode);
1163
1164 #if !ASSERT_DISABLED
1165     unsigned toSaveSize = toSave.numberOfSetGPRs();
1166     // They should all be callee saves.
1167     toSave.filter(RegisterSet::calleeSaveRegisters());
1168     ASSERT(toSave.numberOfSetGPRs() == toSaveSize);
1169 #endif
1170
1171     RegisterAtOffsetList registersToSpill(toSave, RegisterAtOffsetList::OffsetBaseType::FramePointerBased);
1172     function.jsToWasmEntrypoint.calleeSaveRegisters = registersToSpill;
1173
1174     unsigned totalFrameSize = registersToSpill.size() * sizeof(void*);
1175     totalFrameSize += WasmCallingConvention::headerSizeInBytes();
1176     totalFrameSize -= sizeof(CallerFrameAndPC);
1177     unsigned numGPRs = 0;
1178     unsigned numFPRs = 0;
1179     for (unsigned i = 0; i < signature.argumentCount(); i++) {
1180         switch (signature.argument(i)) {
1181         case Wasm::I64:
1182         case Wasm::I32:
1183             if (numGPRs >= wasmCallingConvention().m_gprArgs.size())
1184                 totalFrameSize += sizeof(void*);
1185             ++numGPRs;
1186             break;
1187         case Wasm::F32:
1188         case Wasm::F64:
1189             if (numFPRs >= wasmCallingConvention().m_fprArgs.size())
1190                 totalFrameSize += sizeof(void*);
1191             ++numFPRs;
1192             break;
1193         default:
1194             RELEASE_ASSERT_NOT_REACHED();
1195         }
1196     }
1197
1198     totalFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), totalFrameSize);
1199     jit.subPtr(MacroAssembler::TrustedImm32(totalFrameSize), MacroAssembler::stackPointerRegister);
1200
1201     // We save all these registers regardless of having a memory or not.
1202     // The reason is that we use one of these as a scratch. That said,
1203     // almost all real wasm programs use memory, so it's not really
1204     // worth optimizing for the case that they don't.
1205     for (const RegisterAtOffset& regAtOffset : registersToSpill) {
1206         GPRReg reg = regAtOffset.reg().gpr();
1207         ptrdiff_t offset = regAtOffset.offset();
1208         jit.storePtr(reg, CCallHelpers::Address(GPRInfo::callFrameRegister, offset));
1209     }
1210
1211     GPRReg wasmContextGPR = pinnedRegs.wasmContextPointer;
1212
1213     {
1214         CCallHelpers::Address calleeFrame = CCallHelpers::Address(MacroAssembler::stackPointerRegister, -static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC)));
1215         numGPRs = 0;
1216         numFPRs = 0;
1217         // We're going to set the pinned registers after this. So
1218         // we can use this as a scratch for now since we saved it above.
1219         GPRReg scratchReg = pinnedRegs.baseMemoryPointer;
1220
1221         ptrdiff_t jsOffset = CallFrameSlot::thisArgument * sizeof(EncodedJSValue);
1222
1223         // vmEntryToWasm passes Wasm::Context* as the first JS argument when we're
1224         // not using fast TLS to hold the Wasm::Context*.
1225         if (!useFastTLSForContext()) {
1226             jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmContextGPR);
1227             jsOffset += sizeof(EncodedJSValue);
1228         }
1229
1230         ptrdiff_t wasmOffset = CallFrame::headerSizeInRegisters * sizeof(void*);
1231         for (unsigned i = 0; i < signature.argumentCount(); i++) {
1232             switch (signature.argument(i)) {
1233             case Wasm::I32:
1234             case Wasm::I64:
1235                 if (numGPRs >= wasmCallingConvention().m_gprArgs.size()) {
1236                     if (signature.argument(i) == Wasm::I32) {
1237                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1238                         jit.store32(scratchReg, calleeFrame.withOffset(wasmOffset));
1239                     } else {
1240                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1241                         jit.store64(scratchReg, calleeFrame.withOffset(wasmOffset));
1242                     }
1243                     wasmOffset += sizeof(void*);
1244                 } else {
1245                     if (signature.argument(i) == Wasm::I32)
1246                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_gprArgs[numGPRs].gpr());
1247                     else
1248                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_gprArgs[numGPRs].gpr());
1249                 }
1250                 ++numGPRs;
1251                 break;
1252             case Wasm::F32:
1253             case Wasm::F64:
1254                 if (numFPRs >= wasmCallingConvention().m_fprArgs.size()) {
1255                     if (signature.argument(i) == Wasm::F32) {
1256                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1257                         jit.store32(scratchReg, calleeFrame.withOffset(wasmOffset));
1258                     } else {
1259                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1260                         jit.store64(scratchReg, calleeFrame.withOffset(wasmOffset));
1261                     }
1262                     wasmOffset += sizeof(void*);
1263                 } else {
1264                     if (signature.argument(i) == Wasm::F32)
1265                         jit.loadFloat(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_fprArgs[numFPRs].fpr());
1266                     else
1267                         jit.loadDouble(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_fprArgs[numFPRs].fpr());
1268                 }
1269                 ++numFPRs;
1270                 break;
1271             default:
1272                 RELEASE_ASSERT_NOT_REACHED();
1273             }
1274
1275             jsOffset += sizeof(EncodedJSValue);
1276         }
1277     }
1278
1279     if (!!info.memory) {
1280         GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1281
1282         if (!useFastTLSForContext())
1283             jit.loadPtr(CCallHelpers::Address(wasmContextGPR, JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
1284         else {
1285             jit.loadWasmContext(baseMemory);
1286             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
1287         }
1288
1289         if (mode != MemoryMode::Signaling) {
1290             const auto& sizeRegs = pinnedRegs.sizeRegisters;
1291             ASSERT(sizeRegs.size() >= 1);
1292             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
1293             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfSize()), sizeRegs[0].sizeRegister);
1294             for (unsigned i = 1; i < sizeRegs.size(); ++i)
1295                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
1296         }
1297
1298         jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfMemory()), baseMemory);
1299     }
1300
1301     compilationContext.jsEntrypointToWasmEntrypointCall = jit.call();
1302
1303     if (!!info.memory) {
1304         // Resetting the register prevents the GC from mistakenly thinking that the context is still live.
1305         GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1306         jit.move(CCallHelpers::TrustedImm32(0), baseMemory);
1307     }
1308
1309     for (const RegisterAtOffset& regAtOffset : registersToSpill) {
1310         GPRReg reg = regAtOffset.reg().gpr();
1311         ASSERT(reg != GPRInfo::returnValueGPR);
1312         ptrdiff_t offset = regAtOffset.offset();
1313         jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, offset), reg);
1314     }
1315
1316     switch (signature.returnType()) {
1317     case Wasm::F32:
1318         jit.moveFloatTo32(FPRInfo::returnValueFPR, GPRInfo::returnValueGPR);
1319         break;
1320     case Wasm::F64:
1321         jit.moveDoubleTo64(FPRInfo::returnValueFPR, GPRInfo::returnValueGPR);
1322         break;
1323     default:
1324         break;
1325     }
1326
1327     jit.emitFunctionEpilogue();
1328     jit.ret();
1329 }
1330
1331 auto B3IRGenerator::origin() -> Origin
1332 {
1333     return bitwise_cast<Origin>(OpcodeOrigin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset()));
1334 }
1335
1336 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, unsigned optLevel)
1337 {
1338     auto result = std::make_unique<WasmInternalFunction>();
1339
1340     compilationContext.jsEntrypointJIT = std::make_unique<CCallHelpers>();
1341     compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1342
1343     Procedure procedure;
1344
1345     procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1346         if (origin.data())
1347             out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1348     });
1349     
1350     // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1351     // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1352     // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1353     // optLevel=1.
1354     procedure.setNeedsUsedRegisters(false);
1355     
1356     procedure.setOptLevel(optLevel);
1357
1358     B3IRGenerator context(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode);
1359     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, info);
1360     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1361
1362     context.insertConstants();
1363
1364     procedure.resetReachability();
1365     if (!ASSERT_DISABLED)
1366         validate(procedure, "After parsing:\n");
1367
1368     dataLogIf(verbose, "Pre SSA: ", procedure);
1369     fixSSA(procedure);
1370     dataLogIf(verbose, "Post SSA: ", procedure);
1371     
1372     {
1373         B3::prepareForGeneration(procedure);
1374         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1375         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1376         result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1377     }
1378
1379     createJSToWasmWrapper(compilationContext, *result, signature, info, mode);
1380     return WTFMove(result);
1381 }
1382
1383 // Custom wasm ops. These are the ones too messy to do in wasm.json.
1384
1385 void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1386 {
1387     ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1388     const B3::Type type = left->type();
1389
1390     {
1391         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1392             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1393
1394         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1395             this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1396         });
1397     }
1398
1399     if (operation == Div) {
1400         int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1401
1402         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1403             m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1404                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1405                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1406
1407         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1408             this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1409         });
1410     }
1411 }
1412
1413 template<>
1414 auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1415 {
1416     const B3::Opcode op = Div;
1417     emitChecksForModOrDiv(op, left, right);
1418     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1419     return { };
1420 }
1421
1422 template<>
1423 auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1424 {
1425     const B3::Opcode op = Mod;
1426     emitChecksForModOrDiv(op, left, right);
1427     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1428     return { };
1429 }
1430
1431 template<>
1432 auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1433 {
1434     const B3::Opcode op = UDiv;
1435     emitChecksForModOrDiv(op, left, right);
1436     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1437     return { };
1438 }
1439
1440 template<>
1441 auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1442 {
1443     const B3::Opcode op = UMod;
1444     emitChecksForModOrDiv(op, left, right);
1445     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1446     return { };
1447 }
1448
1449 template<>
1450 auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1451 {
1452     const B3::Opcode op = Div;
1453     emitChecksForModOrDiv(op, left, right);
1454     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1455     return { };
1456 }
1457
1458 template<>
1459 auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1460 {
1461     const B3::Opcode op = Mod;
1462     emitChecksForModOrDiv(op, left, right);
1463     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1464     return { };
1465 }
1466
1467 template<>
1468 auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1469 {
1470     const B3::Opcode op = UDiv;
1471     emitChecksForModOrDiv(op, left, right);
1472     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1473     return { };
1474 }
1475
1476 template<>
1477 auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1478 {
1479     const B3::Opcode op = UMod;
1480     emitChecksForModOrDiv(op, left, right);
1481     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1482     return { };
1483 }
1484
1485 template<>
1486 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1487 {
1488     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1489     patchpoint->append(arg, ValueRep::SomeRegister);
1490     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1491         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1492     });
1493     patchpoint->effects = Effects::none();
1494     result = patchpoint;
1495     return { };
1496 }
1497
1498 template<>
1499 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1500 {
1501     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1502     patchpoint->append(arg, ValueRep::SomeRegister);
1503     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1504         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1505     });
1506     patchpoint->effects = Effects::none();
1507     result = patchpoint;
1508     return { };
1509 }
1510
1511 template<>
1512 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1513 {
1514     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1515     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1516     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1517     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1518     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1519     return { };
1520 }
1521
1522 template<>
1523 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1524 {
1525     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1526     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1527     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1528     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1529     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1530     return { };
1531 }
1532
1533 template<>
1534 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1535 {
1536     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1537     if (isX86())
1538         patchpoint->numGPScratchRegisters = 1;
1539     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1540     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1541         AllowMacroScratchRegisterUsage allowScratch(jit);
1542 #if CPU(X86_64)
1543         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1544 #else
1545         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1546 #endif
1547     });
1548     patchpoint->effects = Effects::none();
1549     result = patchpoint;
1550     return { };
1551 }
1552
1553 template<>
1554 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1555 {
1556     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1557     if (isX86())
1558         patchpoint->numGPScratchRegisters = 1;
1559     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1560     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1561         AllowMacroScratchRegisterUsage allowScratch(jit);
1562 #if CPU(X86_64)
1563         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1564 #else
1565         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1566 #endif
1567     });
1568     patchpoint->effects = Effects::none();
1569     result = patchpoint;
1570     return { };
1571 }
1572
1573 template<>
1574 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1575 {
1576     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1577     patchpoint->append(arg, ValueRep::SomeRegister);
1578     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1579         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1580     });
1581     patchpoint->effects = Effects::none();
1582     result = patchpoint;
1583     return { };
1584 }
1585
1586 template<>
1587 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1588 {
1589     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1590     patchpoint->append(arg, ValueRep::SomeRegister);
1591     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1592         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1593     });
1594     patchpoint->effects = Effects::none();
1595     result = patchpoint;
1596     return { };
1597 }
1598
1599 template<>
1600 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1601 {
1602     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1603     patchpoint->append(arg, ValueRep::SomeRegister);
1604     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1605         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1606     });
1607     patchpoint->effects = Effects::none();
1608     result = patchpoint;
1609     return { };
1610 }
1611
1612 template<>
1613 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1614 {
1615     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1616     patchpoint->append(arg, ValueRep::SomeRegister);
1617     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1618         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1619     });
1620     patchpoint->effects = Effects::none();
1621     result = patchpoint;
1622     return { };
1623 }
1624
1625 template<>
1626 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1627 {
1628     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1629     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1630     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1631         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1632         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1633     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1634     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1635     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1636         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1637     });
1638     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1639     patchpoint->append(arg, ValueRep::SomeRegister);
1640     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1641         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1642     });
1643     patchpoint->effects = Effects::none();
1644     result = patchpoint;
1645     return { };
1646 }
1647
1648 template<>
1649 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1650 {
1651     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1652     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1653     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1654         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1655         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1656     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1657     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1658     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1659         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1660     });
1661     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1662     patchpoint->append(arg, ValueRep::SomeRegister);
1663     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1664         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1665     });
1666     patchpoint->effects = Effects::none();
1667     result = patchpoint;
1668     return { };
1669 }
1670
1671
1672 template<>
1673 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1674 {
1675     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1676     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1677     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1678         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1679         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1680     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1681     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1682     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1683         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1684     });
1685     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1686     patchpoint->append(arg, ValueRep::SomeRegister);
1687     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1688         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1689     });
1690     patchpoint->effects = Effects::none();
1691     result = patchpoint;
1692     return { };
1693 }
1694
1695 template<>
1696 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1697 {
1698     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1699     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1700     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1701         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1702         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1703     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1704     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1705     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1706         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1707     });
1708     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1709     patchpoint->append(arg, ValueRep::SomeRegister);
1710     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1711         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1712     });
1713     patchpoint->effects = Effects::none();
1714     result = patchpoint;
1715     return { };
1716 }
1717
1718 template<>
1719 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1720 {
1721     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1722     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1723     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1724         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1725         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1726     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1727     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1728     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1729         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1730     });
1731     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1732     patchpoint->append(arg, ValueRep::SomeRegister);
1733     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1734         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1735     });
1736     patchpoint->effects = Effects::none();
1737     result = patchpoint;
1738     return { };
1739 }
1740
1741 template<>
1742 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1743 {
1744     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1745     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1746     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1747         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1748         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1749     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1750     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1751     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1752         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1753     });
1754
1755     Value* signBitConstant;
1756     if (isX86()) {
1757         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1758         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1759         // so we can pool them if needed.
1760         signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1761     }
1762     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1763     patchpoint->append(arg, ValueRep::SomeRegister);
1764     if (isX86()) {
1765         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1766         patchpoint->numFPScratchRegisters = 1;
1767     }
1768     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1769         AllowMacroScratchRegisterUsage allowScratch(jit);
1770         FPRReg scratch = InvalidFPRReg;
1771         FPRReg constant = InvalidFPRReg;
1772         if (isX86()) {
1773             scratch = params.fpScratch(0);
1774             constant = params[2].fpr();
1775         }
1776         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1777     });
1778     patchpoint->effects = Effects::none();
1779     result = patchpoint;
1780     return { };
1781 }
1782
1783 template<>
1784 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1785 {
1786     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1787     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1788     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1789         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1790         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1791     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1792     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1793     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1794         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1795     });
1796     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1797     patchpoint->append(arg, ValueRep::SomeRegister);
1798     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1799         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1800     });
1801     patchpoint->effects = Effects::none();
1802     result = patchpoint;
1803     return { };
1804 }
1805
1806 template<>
1807 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1808 {
1809     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1810     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1811     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1812         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1813         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1814     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1815     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1816     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1817         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1818     });
1819
1820     Value* signBitConstant;
1821     if (isX86()) {
1822         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1823         // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1824         // so we can pool them if needed.
1825         signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1826     }
1827     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1828     patchpoint->append(arg, ValueRep::SomeRegister);
1829     if (isX86()) {
1830         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1831         patchpoint->numFPScratchRegisters = 1;
1832     }
1833     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1834         AllowMacroScratchRegisterUsage allowScratch(jit);
1835         FPRReg scratch = InvalidFPRReg;
1836         FPRReg constant = InvalidFPRReg;
1837         if (isX86()) {
1838             scratch = params.fpScratch(0);
1839             constant = params[2].fpr();
1840         }
1841         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1842     });
1843     patchpoint->effects = Effects::none();
1844     result = patchpoint;
1845     return { };
1846 }
1847
1848 } } // namespace JSC::Wasm
1849
1850 #include "WasmB3IRGeneratorInlines.h"
1851
1852 #endif // ENABLE(WEBASSEMBLY)