REGRESSION (r210244): Release JSC Stress test failure: wasm.yaml/wasm/js-api/wasm...
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "B3BasicBlockInlines.h"
32 #include "B3CCallValue.h"
33 #include "B3Compile.h"
34 #include "B3ConstPtrValue.h"
35 #include "B3FixSSA.h"
36 #include "B3Generate.h"
37 #include "B3StackmapGenerationParams.h"
38 #include "B3SwitchValue.h"
39 #include "B3Validate.h"
40 #include "B3ValueInlines.h"
41 #include "B3Variable.h"
42 #include "B3VariableValue.h"
43 #include "B3WasmAddressValue.h"
44 #include "B3WasmBoundsCheckValue.h"
45 #include "JSCInlines.h"
46 #include "JSWebAssemblyInstance.h"
47 #include "JSWebAssemblyModule.h"
48 #include "JSWebAssemblyRuntimeError.h"
49 #include "VirtualRegister.h"
50 #include "WasmCallingConvention.h"
51 #include "WasmExceptionType.h"
52 #include "WasmFunctionParser.h"
53 #include "WasmMemory.h"
54 #include <wtf/Optional.h>
55
56 void dumpProcedure(void* ptr)
57 {
58     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
59     proc->dump(WTF::dataFile());
60 }
61
62 namespace JSC { namespace Wasm {
63
64 using namespace B3;
65
66 namespace {
67 const bool verbose = false;
68 }
69
70 class B3IRGenerator {
71 public:
72     struct ControlData {
73         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
74             : blockType(type)
75             , continuation(continuation)
76             , special(special)
77         {
78             if (signature != Void)
79                 result.append(proc.addVariable(toB3Type(signature)));
80         }
81
82         ControlData()
83         {
84         }
85
86         void dump(PrintStream& out) const
87         {
88             switch (type()) {
89             case BlockType::If:
90                 out.print("If:       ");
91                 break;
92             case BlockType::Block:
93                 out.print("Block:    ");
94                 break;
95             case BlockType::Loop:
96                 out.print("Loop:     ");
97                 break;
98             case BlockType::TopLevel:
99                 out.print("TopLevel: ");
100                 break;
101             }
102             out.print("Continuation: ", *continuation, ", Special: ");
103             if (special)
104                 out.print(*special);
105             else
106                 out.print("None");
107         }
108
109         BlockType type() const { return blockType; }
110
111         bool hasNonVoidSignature() const { return result.size(); }
112
113         BasicBlock* targetBlockForBranch()
114         {
115             if (type() == BlockType::Loop)
116                 return special;
117             return continuation;
118         }
119
120         void convertIfToBlock()
121         {
122             ASSERT(type() == BlockType::If);
123             blockType = BlockType::Block;
124             special = nullptr;
125         }
126
127     private:
128         friend class B3IRGenerator;
129         BlockType blockType;
130         BasicBlock* continuation;
131         BasicBlock* special;
132         Vector<Variable*, 1> result;
133     };
134
135     typedef Value* ExpressionType;
136     typedef ControlData ControlType;
137     typedef Vector<ExpressionType, 1> ExpressionList;
138     typedef Vector<Variable*, 1> ResultList;
139     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
140
141     static constexpr ExpressionType emptyExpression = nullptr;
142
143     typedef String ErrorType;
144     typedef UnexpectedType<ErrorType> UnexpectedResult;
145     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
146     typedef Expected<void, ErrorType> PartialResult;
147     template <typename ...Args>
148     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
149     {
150         using namespace FailureHelper; // See ADL comment in WasmParser.h.
151         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
152     }
153 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
154         if (UNLIKELY(condition))                  \
155             return fail(__VA_ARGS__);             \
156     } while (0)
157
158     B3IRGenerator(VM&, const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&);
159
160     PartialResult WARN_UNUSED_RETURN addArguments(const Signature*);
161     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
162     ExpressionType addConstant(Type, uint64_t);
163
164     // Locals
165     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
166     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
167
168     // Globals
169     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
170     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
171
172     // Memory
173     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
174     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
175     PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
176     PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
177
178     // Basic operators
179     template<OpType>
180     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
181     template<OpType>
182     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
183     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
184
185     // Control flow
186     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
187     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
188     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
189     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
190     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
191     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
192
193     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
194     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
195     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
196     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
197     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
198
199     // Calls
200     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
201     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature*, SignatureIndex, Vector<ExpressionType>& args, ExpressionType& result);
202     PartialResult WARN_UNUSED_RETURN addUnreachable();
203
204     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
205
206     void emitExceptionCheck(CCallHelpers&, ExceptionType);
207
208 private:
209     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
210     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
211     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
212
213     void unify(Variable* target, const ExpressionType source);
214     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
215     Value* zeroForType(Type);
216
217     void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
218
219     VM& m_vm;
220     const ModuleInformation& m_info;
221     Procedure& m_proc;
222     BasicBlock* m_currentBlock;
223     Vector<Variable*> m_locals;
224     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
225     GPRReg m_memoryBaseGPR;
226     GPRReg m_memorySizeGPR;
227     Value* m_zeroValues[numTypes];
228     Value* m_instanceValue;
229 };
230
231 B3IRGenerator::B3IRGenerator(VM& vm, const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls)
232     : m_vm(vm)
233     , m_info(info)
234     , m_proc(procedure)
235     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
236 {
237     m_currentBlock = m_proc.addBlock();
238
239     for (unsigned i = 0; i < numTypes; ++i) {
240         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
241         case B3::Int32:
242         case B3::Int64:
243         case B3::Float:
244         case B3::Double:
245             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
246             break;
247         case B3::Void:
248             m_zeroValues[i] = nullptr;
249             break;
250         }
251     }
252
253     // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
254     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
255     m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
256     m_proc.pinRegister(m_memoryBaseGPR);
257     ASSERT(!pinnedRegs.sizeRegisters[0].sizeOffset);
258     m_memorySizeGPR = pinnedRegs.sizeRegisters[0].sizeRegister;
259     for (const PinnedSizeRegisterInfo& regInfo : pinnedRegs.sizeRegisters)
260         m_proc.pinRegister(regInfo.sizeRegister);
261
262     if (info.hasMemory()) {
263         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
264             AllowMacroScratchRegisterUsage allowScratch(jit);
265             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
266             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
267         });
268     }
269
270     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
271
272     m_instanceValue = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
273         m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), &m_vm.topJSWebAssemblyInstance));
274 }
275
276 struct MemoryBaseAndSize {
277     Value* base;
278     Value* size;
279 };
280
281 static MemoryBaseAndSize getMemoryBaseAndSize(VM& vm, Value* instance, Procedure& proc, BasicBlock* block)
282 {
283     Value* memoryObject = block->appendNew<MemoryValue>(proc, Load, pointerType(), Origin(), instance, JSWebAssemblyInstance::offsetOfMemory());
284
285     static_assert(sizeof(decltype(vm.topJSWebAssemblyInstance->memory()->memory()->memory())) == sizeof(void*), "codegen relies on this size");
286     static_assert(sizeof(decltype(vm.topJSWebAssemblyInstance->memory()->memory()->size())) == sizeof(uint64_t), "codegen relies on this size");
287     MemoryBaseAndSize result;
288     result.base = block->appendNew<MemoryValue>(proc, Load, pointerType(), Origin(), memoryObject, JSWebAssemblyMemory::offsetOfMemory());
289     result.size = block->appendNew<MemoryValue>(proc, Load, Int64, Origin(), memoryObject, JSWebAssemblyMemory::offsetOfSize());
290
291     return result;
292 }
293
294 static void restoreWebAssemblyGlobalState(VM& vm, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
295 {
296     block->appendNew<MemoryValue>(proc, Store, Origin(), instance, block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topJSWebAssemblyInstance));
297
298     if (!!memory) {
299         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
300         RegisterSet clobbers;
301         clobbers.set(pinnedRegs->baseMemoryPointer);
302         for (auto info : pinnedRegs->sizeRegisters)
303             clobbers.set(info.sizeRegister);
304
305         B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, Origin());
306         patchpoint->effects = Effects::none();
307         patchpoint->effects.writesPinned = true;
308         patchpoint->clobber(clobbers);
309
310         patchpoint->append(instance, ValueRep::SomeRegister);
311
312         patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
313             GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
314             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
315             const auto& sizeRegs = pinnedRegs->sizeRegisters;
316             ASSERT(sizeRegs.size() >= 1);
317             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
318             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfSize()), sizeRegs[0].sizeRegister);
319             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfMemory()), baseMemory);
320             for (unsigned i = 1; i < sizeRegs.size(); ++i)
321                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
322         });
323     }
324 }
325
326 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
327 {
328     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
329     auto jumpToExceptionStub = jit.jump();
330
331     VM* vm = &m_vm;
332     jit.addLinkTask([vm, jumpToExceptionStub] (LinkBuffer& linkBuffer) {
333         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(vm->getCTIStub(throwExceptionFromWasmThunkGenerator).code()));
334     });
335 }
336
337 Value* B3IRGenerator::zeroForType(Type type)
338 {
339     ASSERT(type != Void);
340     Value* zeroValue = m_zeroValues[linearizeType(type)];
341     ASSERT(zeroValue);
342     return zeroValue;
343 }
344
345 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
346 {
347     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
348
349     for (uint32_t i = 0; i < count; ++i) {
350         Variable* local = m_proc.addVariable(toB3Type(type));
351         m_locals.uncheckedAppend(local);
352         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
353     }
354     return { };
355 }
356
357 auto B3IRGenerator::addArguments(const Signature* signature) -> PartialResult
358 {
359     ASSERT(!m_locals.size());
360     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature->argumentCount()), "can't allocate memory for ", signature->argumentCount(), " arguments");
361
362     m_locals.grow(signature->argumentCount());
363     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
364         [&] (ExpressionType argument, unsigned i) {
365             Variable* argumentVariable = m_proc.addVariable(argument->type());
366             m_locals[i] = argumentVariable;
367             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
368         });
369     return { };
370 }
371
372 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
373 {
374     ASSERT(m_locals[index]);
375     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
376     return { };
377 }
378
379 auto B3IRGenerator::addUnreachable() -> PartialResult
380 {
381     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
382     unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
383         this->emitExceptionCheck(jit, ExceptionType::Unreachable);
384     });
385     unreachable->effects.terminal = true;
386     return { };
387 }
388
389 auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
390 {
391     int32_t (*growMemory) (ExecState*, int32_t) = [] (ExecState* exec, int32_t delta) -> int32_t {
392         VM& vm = exec->vm();
393         auto scope = DECLARE_THROW_SCOPE(vm);
394
395         JSWebAssemblyInstance* instance = vm.topJSWebAssemblyInstance;
396         JSWebAssemblyMemory* wasmMemory = instance->memory();
397
398         if (delta < 0)
399             return -1;
400
401         bool shouldThrowExceptionsOnFailure = false;
402         PageCount result = wasmMemory->grow(exec, static_cast<uint32_t>(delta), shouldThrowExceptionsOnFailure);
403         RELEASE_ASSERT(!scope.exception());
404         if (!result)
405             return -1;
406
407         return result.pageCount();
408     };
409
410     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(),
411         m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(growMemory)),
412         m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin()), delta);
413
414     restoreWebAssemblyGlobalState(m_vm, m_info.memory, m_instanceValue, m_proc, m_currentBlock);
415
416     return { };
417 }
418
419 auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
420 {
421     auto memoryValue = getMemoryBaseAndSize(m_vm, m_instanceValue, m_proc, m_currentBlock);
422
423     constexpr uint32_t shiftValue = 16;
424     static_assert(PageCount::pageSize == 1 << shiftValue, "This must hold for the code below to be correct.");
425     Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, Origin(),
426         memoryValue.size, m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), shiftValue));
427
428     result = m_currentBlock->appendNew<Value>(m_proc, Trunc, Origin(), numPages);
429
430     return { };
431 }
432
433 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
434 {
435     ASSERT(m_locals[index]);
436     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
437     return { };
438 }
439
440 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
441 {
442     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
443     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), Origin(), globalsArray, index * sizeof(Register));
444     return { };
445 }
446
447 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
448 {
449     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
450     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
451     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, Origin(), value, globalsArray, index * sizeof(Register));
452     return { };
453 }
454
455 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
456 {
457     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
458     ASSERT(sizeOfOperation + offset > offset);
459     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
460     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
461     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
462 }
463
464 inline uint32_t sizeOfLoadOp(LoadOpType op)
465 {
466     switch (op) {
467     case LoadOpType::I32Load8S:
468     case LoadOpType::I32Load8U:
469     case LoadOpType::I64Load8S:
470     case LoadOpType::I64Load8U:
471         return 1;
472     case LoadOpType::I32Load16S:
473     case LoadOpType::I64Load16S:
474     case LoadOpType::I32Load16U:
475     case LoadOpType::I64Load16U:
476         return 2;
477     case LoadOpType::I32Load:
478     case LoadOpType::I64Load32S:
479     case LoadOpType::I64Load32U:
480     case LoadOpType::F32Load:
481         return 4;
482     case LoadOpType::I64Load:
483     case LoadOpType::F64Load:
484         return 8;
485     }
486     RELEASE_ASSERT_NOT_REACHED();
487 }
488
489 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
490 {
491     switch (op) {
492     case LoadOpType::I32Load8S: {
493         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
494     }
495
496     case LoadOpType::I64Load8S: {
497         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
498         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
499     }
500
501     case LoadOpType::I32Load8U: {
502         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
503     }
504
505     case LoadOpType::I64Load8U: {
506         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
507         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
508     }
509
510     case LoadOpType::I32Load16S: {
511         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
512     }
513     case LoadOpType::I64Load16S: {
514         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
515         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
516     }
517
518     case LoadOpType::I32Load: {
519         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer, offset);
520     }
521
522     case LoadOpType::I64Load32U: {
523         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer, offset);
524         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
525     }
526
527     case LoadOpType::I64Load32S: {
528         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer, offset);
529         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
530     }
531
532     case LoadOpType::I64Load: {
533         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer, offset);
534     }
535
536     case LoadOpType::F32Load: {
537         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer, offset);
538     }
539
540     case LoadOpType::F64Load: {
541         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer, offset);
542     }
543
544     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
545     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
546     case LoadOpType::I32Load16U: {
547         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
548         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
549                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
550     }
551     case LoadOpType::I64Load16U: {
552         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
553         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
554                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
555
556         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), partialResult);
557     }
558     }
559     RELEASE_ASSERT_NOT_REACHED();
560 }
561
562 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
563 {
564     ASSERT(pointer->type() == Int32);
565
566     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
567         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
568         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
569         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
570         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
571             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
572         });
573
574         switch (op) {
575         case LoadOpType::I32Load8S:
576         case LoadOpType::I32Load16S:
577         case LoadOpType::I32Load:
578         case LoadOpType::I32Load16U:
579         case LoadOpType::I32Load8U:
580             result = zeroForType(I32);
581             break;
582         case LoadOpType::I64Load8S:
583         case LoadOpType::I64Load8U:
584         case LoadOpType::I64Load16S:
585         case LoadOpType::I64Load32U:
586         case LoadOpType::I64Load32S:
587         case LoadOpType::I64Load:
588         case LoadOpType::I64Load16U:
589             result = zeroForType(I64);
590             break;
591         case LoadOpType::F32Load:
592             result = zeroForType(F32);
593             break;
594         case LoadOpType::F64Load:
595             result = zeroForType(F64);
596             break;
597         }
598
599     } else
600         result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
601
602     return { };
603 }
604
605 inline uint32_t sizeOfStoreOp(StoreOpType op)
606 {
607     switch (op) {
608     case StoreOpType::I32Store8:
609     case StoreOpType::I64Store8:
610         return 1;
611     case StoreOpType::I32Store16:
612     case StoreOpType::I64Store16:
613         return 2;
614     case StoreOpType::I32Store:
615     case StoreOpType::I64Store32:
616     case StoreOpType::F32Store:
617         return 4;
618     case StoreOpType::I64Store:
619     case StoreOpType::F64Store:
620         return 8;
621     }
622     RELEASE_ASSERT_NOT_REACHED();
623 }
624
625
626 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
627 {
628     switch (op) {
629     case StoreOpType::I64Store8:
630         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
631         FALLTHROUGH;
632
633     case StoreOpType::I32Store8:
634         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
635         return;
636
637     case StoreOpType::I64Store16:
638         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
639         FALLTHROUGH;
640
641     case StoreOpType::I32Store16:
642         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
643         return;
644
645     case StoreOpType::I64Store32:
646         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
647         FALLTHROUGH;
648
649     case StoreOpType::I64Store:
650     case StoreOpType::I32Store:
651     case StoreOpType::F32Store:
652     case StoreOpType::F64Store:
653         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
654         return;
655     }
656     RELEASE_ASSERT_NOT_REACHED();
657 }
658
659 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
660 {
661     ASSERT(pointer->type() == Int32);
662
663     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
664         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
665         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
666         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
667         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
668             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
669         });
670     } else
671         emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
672
673     return { };
674 }
675
676 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
677 {
678     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
679     return { };
680 }
681
682 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
683 {
684     switch (type) {
685     case Wasm::I32:
686         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
687     case Wasm::I64:
688         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
689     case Wasm::F32:
690         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
691     case Wasm::F64:
692         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
693     case Wasm::Void:
694     case Wasm::Func:
695     case Wasm::Anyfunc:
696         break;
697     }
698     RELEASE_ASSERT_NOT_REACHED();
699     return nullptr;
700 }
701
702 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
703 {
704     return ControlData(m_proc, signature, BlockType::TopLevel, m_proc.addBlock());
705 }
706
707 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
708 {
709     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
710 }
711
712 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
713 {
714     BasicBlock* body = m_proc.addBlock();
715     BasicBlock* continuation = m_proc.addBlock();
716     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
717     body->addPredecessor(m_currentBlock);
718     m_currentBlock = body;
719     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
720 }
721
722 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
723 {
724     // FIXME: This needs to do some kind of stack passing.
725
726     BasicBlock* taken = m_proc.addBlock();
727     BasicBlock* notTaken = m_proc.addBlock();
728     BasicBlock* continuation = m_proc.addBlock();
729
730     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
731     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
732     taken->addPredecessor(m_currentBlock);
733     notTaken->addPredecessor(m_currentBlock);
734
735     m_currentBlock = taken;
736     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
737     return { };
738 }
739
740 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
741 {
742     unifyValuesWithBlock(currentStack, data.result);
743     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
744     return addElseToUnreachable(data);
745 }
746
747 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
748 {
749     ASSERT(data.type() == BlockType::If);
750     m_currentBlock = data.special;
751     data.convertIfToBlock();
752     return { };
753 }
754
755 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
756 {
757     ASSERT(returnValues.size() <= 1);
758     if (returnValues.size())
759         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
760     else
761         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
762     return { };
763 }
764
765 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
766 {
767     if (data.type() != BlockType::Loop)
768         unifyValuesWithBlock(returnValues, data.result);
769
770     BasicBlock* target = data.targetBlockForBranch();
771     if (condition) {
772         BasicBlock* continuation = m_proc.addBlock();
773         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
774         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
775         target->addPredecessor(m_currentBlock);
776         continuation->addPredecessor(m_currentBlock);
777         m_currentBlock = continuation;
778     } else {
779         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
780         target->addPredecessor(m_currentBlock);
781     }
782
783     return { };
784 }
785
786 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
787 {
788     for (size_t i = 0; i < targets.size(); ++i)
789         unifyValuesWithBlock(expressionStack, targets[i]->result);
790     unifyValuesWithBlock(expressionStack, defaultTarget.result);
791
792     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
793     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
794     for (size_t i = 0; i < targets.size(); ++i)
795         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
796
797     return { };
798 }
799
800 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
801 {
802     ControlData& data = entry.controlData;
803
804     unifyValuesWithBlock(expressionStack, data.result);
805     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
806     data.continuation->addPredecessor(m_currentBlock);
807
808     return addEndToUnreachable(entry);
809 }
810
811
812 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
813 {
814     ControlData& data = entry.controlData;
815     m_currentBlock = data.continuation;
816
817     if (data.type() == BlockType::If) {
818         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
819         m_currentBlock->addPredecessor(data.special);
820     }
821
822     for (Variable* result : data.result)
823         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
824
825     // TopLevel does not have any code after this so we need to make sure we emit a return here.
826     if (data.type() == BlockType::TopLevel)
827         return addReturn(entry.controlData, entry.enclosedExpressionStack);
828
829     return { };
830 }
831
832 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
833 {
834     ASSERT(signature->argumentCount() == args.size());
835
836     Type returnType = signature->returnType();
837     Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
838
839     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
840         // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
841         Value* functionImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfImportFunction(functionIndex));
842         Value* jsTypeOfImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Origin(), functionImport, JSCell::typeInfoTypeOffset());
843         Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), jsTypeOfImport, m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), WebAssemblyFunctionType));
844
845         BasicBlock* isWasmBlock = m_proc.addBlock();
846         BasicBlock* isJSBlock = m_proc.addBlock();
847         BasicBlock* continuation = m_proc.addBlock();
848         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, Origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isJSBlock));
849
850         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, Origin(), args, toB3Type(returnType),
851             [&] (PatchpointValue* patchpoint) {
852                 patchpoint->effects.writesPinned = true;
853                 patchpoint->effects.readsPinned = true;
854                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
855                     AllowMacroScratchRegisterUsage allowScratch(jit);
856                     CCallHelpers::Call call = jit.call();
857                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
858                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex, UnlinkedWasmToWasmCall::Target::ToWasm });
859                     });
860                 });
861             });
862         UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, Origin(), wasmCallResult);
863         isWasmBlock->appendNewControlValue(m_proc, Jump, Origin(), continuation);
864
865         Value* jsCallResult = wasmCallingConvention().setupCall(m_proc, isJSBlock, Origin(), args, toB3Type(returnType),
866             [&] (PatchpointValue* patchpoint) {
867                 patchpoint->effects.writesPinned = true;
868                 patchpoint->effects.readsPinned = true;
869                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
870                     AllowMacroScratchRegisterUsage allowScratch(jit);
871                     CCallHelpers::Call call = jit.call();
872                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
873                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex, UnlinkedWasmToWasmCall::Target::ToJs });
874                     });
875                 });
876             });
877         UpsilonValue* jsCallResultUpsilon = returnType == Void ? nullptr : isJSBlock->appendNew<UpsilonValue>(m_proc, Origin(), jsCallResult);
878         isJSBlock->appendNewControlValue(m_proc, Jump, Origin(), continuation);
879
880         m_currentBlock = continuation;
881
882         if (returnType == Void)
883             result = nullptr;
884         else {
885             result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), Origin());
886             wasmCallResultUpsilon->setPhi(result);
887             jsCallResultUpsilon->setPhi(result);
888         }
889
890         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
891         restoreWebAssemblyGlobalState(m_vm, m_info.memory, m_instanceValue, m_proc, continuation);
892     } else {
893         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
894             [&] (PatchpointValue* patchpoint) {
895                 patchpoint->effects.writesPinned = true;
896                 patchpoint->effects.readsPinned = true;
897                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
898                     AllowMacroScratchRegisterUsage allowScratch(jit);
899                     CCallHelpers::Call call = jit.call();
900                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
901                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex, UnlinkedWasmToWasmCall::Target::ToWasm });
902                     });
903                 });
904             });
905     }
906
907     return { };
908 }
909
910 auto B3IRGenerator::addCallIndirect(const Signature* signature, SignatureIndex signatureIndex, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
911 {
912     ASSERT(signatureIndex != Signature::invalidIndex);
913     ExpressionType calleeIndex = args.takeLast();
914     ASSERT(signature->argumentCount() == args.size());
915
916     ExpressionType callableFunctionBuffer;
917     ExpressionType callableFunctionBufferSize;
918     {
919         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
920             m_instanceValue, JSWebAssemblyInstance::offsetOfTable());
921         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
922             table, JSWebAssemblyTable::offsetOfFunctions());
923         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, Origin(),
924             table, JSWebAssemblyTable::offsetOfSize());
925     }
926
927     // Check the index we are looking for is valid.
928     {
929         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
930             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, Origin(), calleeIndex, callableFunctionBufferSize));
931
932         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
933             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
934         });
935     }
936
937     // Compute the offset in the table index space we are looking for.
938     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, Origin(),
939         m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), calleeIndex),
940         m_currentBlock->appendIntConstant(m_proc, Origin(), pointerType(), sizeof(CallableFunction)));
941     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, Origin(), callableFunctionBuffer, offset);
942
943     // Check that the CallableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
944     static_assert(sizeof(CallableFunction::signatureIndex) == sizeof(uint32_t), "Load codegen assumes i32");
945     ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signatureIndex));
946     {
947         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
948             m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(),
949                 calleeSignatureIndex,
950                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), Signature::invalidIndex)));
951
952         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
953             this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
954         });
955     }
956
957     // Check the signature matches the value we expect.
958     {
959         ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), signatureIndex);
960         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
961             m_currentBlock->appendNew<Value>(m_proc, NotEqual, Origin(), calleeSignatureIndex, expectedSignatureIndex));
962
963         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
964             this->emitExceptionCheck(jit, ExceptionType::BadSignature);
965         });
966     }
967
968     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
969
970     Type returnType = signature->returnType();
971     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
972         [&] (PatchpointValue* patchpoint) {
973             patchpoint->effects.writesPinned = true;
974             patchpoint->effects.readsPinned = true;
975
976             patchpoint->append(calleeCode, ValueRep::SomeRegister);
977
978             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
979                 jit.call(params[returnType == Void ? 0 : 1].gpr());
980             });
981         });
982
983     // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
984     restoreWebAssemblyGlobalState(m_vm, m_info.memory, m_instanceValue, m_proc, m_currentBlock);
985
986     return { };
987 }
988
989 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
990 {
991     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
992 }
993
994 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
995 {
996     ASSERT(result.size() <= resultStack.size());
997
998     for (size_t i = 0; i < result.size(); ++i)
999         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1000 }
1001
1002 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1003 {
1004     dataLog(comma, "ExpressionStack:");
1005     for (const auto& expression : expressionStack)
1006         dataLog(comma, *expression);
1007 }
1008
1009 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1010 {
1011     dataLogLn("Processing Graph:");
1012     dataLog(m_proc);
1013     dataLogLn("With current block:", *m_currentBlock);
1014     dataLogLn("Control stack:");
1015     ASSERT(controlStack.size());
1016     for (size_t i = controlStack.size(); i--;) {
1017         dataLog("  ", controlStack[i].controlData, ": ");
1018         CommaPrinter comma(", ", "");
1019         dumpExpressionStack(comma, *expressionStack);
1020         expressionStack = &controlStack[i].enclosedExpressionStack;
1021         dataLogLn();
1022     }
1023     dataLogLn();
1024 }
1025
1026 static void createJSToWasmWrapper(VM& vm, CompilationContext& compilationContext, WasmInternalFunction& function, const Signature* signature, const ModuleInformation& info)
1027 {
1028     Procedure proc;
1029     BasicBlock* block = proc.addBlock();
1030
1031     Origin origin;
1032
1033     jscCallingConvention().setupFrameInPrologue(&function.jsToWasmCalleeMoveLocation, proc, origin, block);
1034
1035     if (!ASSERT_DISABLED) {
1036         // This should be guaranteed by our JS wrapper that handles calls to us.
1037         // Just prevent against crazy when ASSERT is enabled.
1038         Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, origin);
1039         Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, origin, CallFrameSlot::argumentCount * sizeof(Register));
1040         Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, origin,
1041             block->appendNew<Value>(proc, Add, origin, framePointer, offSetOfArgumentCount));
1042
1043         Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, origin, signature->argumentCount());
1044
1045         CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, origin,
1046             block->appendNew<Value>(proc, Above, origin, expectedArgumentCount, argumentCount));
1047
1048         argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1049             jit.breakpoint();
1050         });
1051     }
1052
1053     // FIXME The instance is currently set by the C++ code in WebAssemblyFunction::call. We shouldn't go through the extra C++ hoop. https://bugs.webkit.org/show_bug.cgi?id=166486
1054     Value* instance = block->appendNew<MemoryValue>(proc, Load, pointerType(), Origin(),
1055         block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topJSWebAssemblyInstance));
1056     restoreWebAssemblyGlobalState(vm, info.memory, instance, proc, block);
1057
1058     // Get our arguments.
1059     Vector<Value*> arguments;
1060     jscCallingConvention().loadArguments(signature, proc, block, origin, [&] (Value* argument, unsigned) {
1061         arguments.append(argument);
1062     });
1063
1064     // Move the arguments into place.
1065     Value* result = wasmCallingConvention().setupCall(proc, block, origin, arguments, toB3Type(signature->returnType()), [&] (PatchpointValue* patchpoint) {
1066         CompilationContext* context = &compilationContext;
1067
1068         // wasm -> wasm calls clobber pinned registers unconditionally. This JS -> wasm transition must therefore restore these pinned registers (which are usually callee-saved) to account for this.
1069         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
1070         RegisterSet clobbers;
1071         clobbers.set(pinnedRegs->baseMemoryPointer);
1072         for (auto info : pinnedRegs->sizeRegisters)
1073             clobbers.set(info.sizeRegister);
1074         patchpoint->effects.writesPinned = true;
1075         patchpoint->clobber(clobbers);
1076
1077         patchpoint->setGenerator([context] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1078             AllowMacroScratchRegisterUsage allowScratch(jit);
1079
1080             CCallHelpers::Call call = jit.call();
1081             context->jsEntrypointToWasmEntrypointCall = call;
1082         });
1083     });
1084
1085     // Return the result, if needed.
1086     switch (signature->returnType()) {
1087     case Wasm::Void:
1088         block->appendNewControlValue(proc, B3::Return, origin);
1089         break;
1090     case Wasm::F32:
1091     case Wasm::F64:
1092         result = block->appendNew<Value>(proc, BitwiseCast, origin, result);
1093         FALLTHROUGH;
1094     case Wasm::I32:
1095     case Wasm::I64:
1096         block->appendNewControlValue(proc, B3::Return, origin, result);
1097         break;
1098     case Wasm::Func:
1099     case Wasm::Anyfunc:
1100         RELEASE_ASSERT_NOT_REACHED();
1101     }
1102
1103     B3::prepareForGeneration(proc);
1104     B3::generate(proc, *compilationContext.jsEntrypointJIT);
1105     compilationContext.jsEntrypointByproducts = proc.releaseByproducts();
1106     function.jsToWasmEntrypoint.calleeSaveRegisters = proc.calleeSaveRegisters();
1107 }
1108
1109 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(VM& vm, CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, const Vector<SignatureIndex>& moduleSignatureIndicesToUniquedSignatureIndices, unsigned optLevel)
1110 {
1111     auto result = std::make_unique<WasmInternalFunction>();
1112
1113     compilationContext.jsEntrypointJIT = std::make_unique<CCallHelpers>(&vm);
1114     compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>(&vm);
1115
1116     Procedure procedure;
1117     B3IRGenerator context(vm, info, procedure, result.get(), unlinkedWasmToWasmCalls);
1118     FunctionParser<B3IRGenerator> parser(&vm, context, functionStart, functionLength, signature, info, moduleSignatureIndicesToUniquedSignatureIndices);
1119     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1120
1121     procedure.resetReachability();
1122     validate(procedure, "After parsing:\n");
1123
1124     if (verbose)
1125         dataLog("Pre SSA: ", procedure);
1126     fixSSA(procedure);
1127     if (verbose)
1128         dataLog("Post SSA: ", procedure);
1129
1130     {
1131         B3::prepareForGeneration(procedure, optLevel);
1132         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1133         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1134         result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisters();
1135     }
1136
1137     createJSToWasmWrapper(vm, compilationContext, *result, signature, info);
1138     return WTFMove(result);
1139 }
1140
1141 // Custom wasm ops. These are the ones too messy to do in wasm.json.
1142
1143 void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1144 {
1145     ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1146     const B3::Type type = left->type();
1147
1148     {
1149         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
1150             m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), right,
1151                 m_currentBlock->appendIntConstant(m_proc, Origin(), type, 0)));
1152
1153         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1154             this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1155         });
1156     }
1157
1158     if (operation == Div) {
1159         int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1160
1161         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
1162             m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1163                 m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), left,
1164                     m_currentBlock->appendIntConstant(m_proc, Origin(), type, min)),
1165                 m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), right,
1166                     m_currentBlock->appendIntConstant(m_proc, Origin(), type, -1))));
1167
1168         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1169             this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1170         });
1171     }
1172 }
1173
1174 template<>
1175 auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1176 {
1177     const B3::Opcode op = Div;
1178     emitChecksForModOrDiv(op, left, right);
1179     result = m_currentBlock->appendNew<Value>(m_proc, op, Origin(), left, right);
1180     return { };
1181 }
1182
1183 template<>
1184 auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1185 {
1186     const B3::Opcode op = Mod;
1187     emitChecksForModOrDiv(op, left, right);
1188     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), Origin(), left, right);
1189     return { };
1190 }
1191
1192 template<>
1193 auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1194 {
1195     const B3::Opcode op = UDiv;
1196     emitChecksForModOrDiv(op, left, right);
1197     result = m_currentBlock->appendNew<Value>(m_proc, op, Origin(), left, right);
1198     return { };
1199 }
1200
1201 template<>
1202 auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1203 {
1204     const B3::Opcode op = UMod;
1205     emitChecksForModOrDiv(op, left, right);
1206     result = m_currentBlock->appendNew<Value>(m_proc, op, Origin(), left, right);
1207     return { };
1208 }
1209
1210 template<>
1211 auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1212 {
1213     const B3::Opcode op = Div;
1214     emitChecksForModOrDiv(op, left, right);
1215     result = m_currentBlock->appendNew<Value>(m_proc, op, Origin(), left, right);
1216     return { };
1217 }
1218
1219 template<>
1220 auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1221 {
1222     const B3::Opcode op = Mod;
1223     emitChecksForModOrDiv(op, left, right);
1224     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), Origin(), left, right);
1225     return { };
1226 }
1227
1228 template<>
1229 auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1230 {
1231     const B3::Opcode op = UDiv;
1232     emitChecksForModOrDiv(op, left, right);
1233     result = m_currentBlock->appendNew<Value>(m_proc, op, Origin(), left, right);
1234     return { };
1235 }
1236
1237 template<>
1238 auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1239 {
1240     const B3::Opcode op = UMod;
1241     emitChecksForModOrDiv(op, left, right);
1242     result = m_currentBlock->appendNew<Value>(m_proc, op, Origin(), left, right);
1243     return { };
1244 }
1245
1246 template<>
1247 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1248 {
1249     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1250     patchpoint->append(arg, ValueRep::SomeRegister);
1251     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1252         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1253     });
1254     patchpoint->effects = Effects::none();
1255     result = patchpoint;
1256     return { };
1257 }
1258
1259 template<>
1260 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1261 {
1262     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1263     patchpoint->append(arg, ValueRep::SomeRegister);
1264     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1265         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1266     });
1267     patchpoint->effects = Effects::none();
1268     result = patchpoint;
1269     return { };
1270 }
1271
1272 template<>
1273 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1274 {
1275     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1276     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1277     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1278     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
1279     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
1280     return { };
1281 }
1282
1283 template<>
1284 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1285 {
1286     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1287     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1288     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1289     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
1290     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
1291     return { };
1292 }
1293
1294 template<>
1295 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1296 {
1297     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1298     if (isX86())
1299         patchpoint->numGPScratchRegisters = 1;
1300     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1301     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1302         AllowMacroScratchRegisterUsage allowScratch(jit);
1303 #if CPU(X86_64)
1304         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1305 #else
1306         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1307 #endif
1308     });
1309     patchpoint->effects = Effects::none();
1310     result = patchpoint;
1311     return { };
1312 }
1313
1314 template<>
1315 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1316 {
1317     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1318     if (isX86())
1319         patchpoint->numGPScratchRegisters = 1;
1320     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1321     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1322         AllowMacroScratchRegisterUsage allowScratch(jit);
1323 #if CPU(X86_64)
1324         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1325 #else
1326         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1327 #endif
1328     });
1329     patchpoint->effects = Effects::none();
1330     result = patchpoint;
1331     return { };
1332 }
1333
1334 template<>
1335 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1336 {
1337     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1338     patchpoint->append(arg, ValueRep::SomeRegister);
1339     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1340         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1341     });
1342     patchpoint->effects = Effects::none();
1343     result = patchpoint;
1344     return { };
1345 }
1346
1347 template<>
1348 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1349 {
1350     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1351     patchpoint->append(arg, ValueRep::SomeRegister);
1352     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1353         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1354     });
1355     patchpoint->effects = Effects::none();
1356     result = patchpoint;
1357     return { };
1358 }
1359
1360 template<>
1361 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1362 {
1363     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1364     patchpoint->append(arg, ValueRep::SomeRegister);
1365     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1366         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1367     });
1368     patchpoint->effects = Effects::none();
1369     result = patchpoint;
1370     return { };
1371 }
1372
1373 template<>
1374 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1375 {
1376     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1377     patchpoint->append(arg, ValueRep::SomeRegister);
1378     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1379         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1380     });
1381     patchpoint->effects = Effects::none();
1382     result = patchpoint;
1383     return { };
1384 }
1385
1386 template<>
1387 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1388 {
1389     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
1390     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
1391     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1392         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1393         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1394     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1395     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1396     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1397         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1398     });
1399     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1400     patchpoint->append(arg, ValueRep::SomeRegister);
1401     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1402         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1403     });
1404     patchpoint->effects = Effects::none();
1405     result = patchpoint;
1406     return { };
1407 }
1408
1409 template<>
1410 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1411 {
1412     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
1413     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
1414     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1415         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1416         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1417     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1418     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1419     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1420         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1421     });
1422     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1423     patchpoint->append(arg, ValueRep::SomeRegister);
1424     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1425         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1426     });
1427     patchpoint->effects = Effects::none();
1428     result = patchpoint;
1429     return { };
1430 }
1431
1432
1433 template<>
1434 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1435 {
1436     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
1437     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1438     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1439         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1440         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1441     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1442     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1443     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1444         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1445     });
1446     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1447     patchpoint->append(arg, ValueRep::SomeRegister);
1448     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1449         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1450     });
1451     patchpoint->effects = Effects::none();
1452     result = patchpoint;
1453     return { };
1454 }
1455
1456 template<>
1457 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1458 {
1459     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
1460     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1461     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1462         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1463         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1464     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1465     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1466     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1467         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1468     });
1469     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1470     patchpoint->append(arg, ValueRep::SomeRegister);
1471     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1472         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1473     });
1474     patchpoint->effects = Effects::none();
1475     result = patchpoint;
1476     return { };
1477 }
1478
1479 template<>
1480 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1481 {
1482     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1483     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1484     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1485         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1486         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1487     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1488     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1489     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1490         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1491     });
1492     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1493     patchpoint->append(arg, ValueRep::SomeRegister);
1494     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1495         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1496     });
1497     patchpoint->effects = Effects::none();
1498     result = patchpoint;
1499     return { };
1500 }
1501
1502 template<>
1503 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1504 {
1505     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1506     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1507     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1508         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1509         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1510     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1511     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1512     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1513         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1514     });
1515
1516     Value* constant;
1517     if (isX86()) {
1518         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1519         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1520         // so we can pool them if needed.
1521         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1522     }
1523     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1524     patchpoint->append(arg, ValueRep::SomeRegister);
1525     if (isX86()) {
1526         patchpoint->append(constant, ValueRep::SomeRegister);
1527         patchpoint->numFPScratchRegisters = 1;
1528     }
1529     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1530         AllowMacroScratchRegisterUsage allowScratch(jit);
1531         FPRReg scratch = InvalidFPRReg;
1532         FPRReg constant = InvalidFPRReg;
1533         if (isX86()) {
1534             scratch = params.fpScratch(0);
1535             constant = params[2].fpr();
1536         }
1537         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1538     });
1539     patchpoint->effects = Effects::none();
1540     result = patchpoint;
1541     return { };
1542 }
1543
1544 template<>
1545 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1546 {
1547     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1548     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1549     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1550         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1551         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1552     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1553     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1554     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1555         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1556     });
1557     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1558     patchpoint->append(arg, ValueRep::SomeRegister);
1559     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1560         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1561     });
1562     patchpoint->effects = Effects::none();
1563     result = patchpoint;
1564     return { };
1565 }
1566
1567 template<>
1568 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1569 {
1570     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1571     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1572     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1573         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1574         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1575     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1576     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1577     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1578         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1579     });
1580
1581     Value* constant;
1582     if (isX86()) {
1583         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1584         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1585         // so we can pool them if needed.
1586         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1587     }
1588     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1589     patchpoint->append(arg, ValueRep::SomeRegister);
1590     if (isX86()) {
1591         patchpoint->append(constant, ValueRep::SomeRegister);
1592         patchpoint->numFPScratchRegisters = 1;
1593     }
1594     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1595         AllowMacroScratchRegisterUsage allowScratch(jit);
1596         FPRReg scratch = InvalidFPRReg;
1597         FPRReg constant = InvalidFPRReg;
1598         if (isX86()) {
1599             scratch = params.fpScratch(0);
1600             constant = params[2].fpr();
1601         }
1602         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1603     });
1604     patchpoint->effects = Effects::none();
1605     result = patchpoint;
1606     return { };
1607 }
1608
1609 } } // namespace JSC::Wasm
1610
1611 #include "WasmB3IRGeneratorInlines.h"
1612
1613 #endif // ENABLE(WEBASSEMBLY)