8cb84adcceed8e15499d0831a37c8b94671c0b98
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "AllowMacroScratchRegisterUsageIf.h"
32 #include "B3BasicBlockInlines.h"
33 #include "B3CCallValue.h"
34 #include "B3Compile.h"
35 #include "B3ConstPtrValue.h"
36 #include "B3FixSSA.h"
37 #include "B3Generate.h"
38 #include "B3InsertionSet.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3SwitchValue.h"
42 #include "B3Validate.h"
43 #include "B3ValueInlines.h"
44 #include "B3ValueKey.h"
45 #include "B3Variable.h"
46 #include "B3VariableValue.h"
47 #include "B3WasmAddressValue.h"
48 #include "B3WasmBoundsCheckValue.h"
49 #include "JSCInlines.h"
50 #include "JSWebAssemblyInstance.h"
51 #include "JSWebAssemblyModule.h"
52 #include "JSWebAssemblyRuntimeError.h"
53 #include "VirtualRegister.h"
54 #include "WasmCallingConvention.h"
55 #include "WasmContext.h"
56 #include "WasmExceptionType.h"
57 #include "WasmFunctionParser.h"
58 #include "WasmMemory.h"
59 #include "WasmOpcodeOrigin.h"
60 #include "WasmThunks.h"
61 #include <wtf/Optional.h>
62
63 void dumpProcedure(void* ptr)
64 {
65     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
66     proc->dump(WTF::dataFile());
67 }
68
69 namespace JSC { namespace Wasm {
70
71 using namespace B3;
72
73 namespace {
74 const bool verbose = false;
75 }
76
77 class B3IRGenerator {
78 public:
79     struct ControlData {
80         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
81             : blockType(type)
82             , continuation(continuation)
83             , special(special)
84         {
85             if (signature != Void)
86                 result.append(proc.addVariable(toB3Type(signature)));
87         }
88
89         ControlData()
90         {
91         }
92
93         void dump(PrintStream& out) const
94         {
95             switch (type()) {
96             case BlockType::If:
97                 out.print("If:       ");
98                 break;
99             case BlockType::Block:
100                 out.print("Block:    ");
101                 break;
102             case BlockType::Loop:
103                 out.print("Loop:     ");
104                 break;
105             case BlockType::TopLevel:
106                 out.print("TopLevel: ");
107                 break;
108             }
109             out.print("Continuation: ", *continuation, ", Special: ");
110             if (special)
111                 out.print(*special);
112             else
113                 out.print("None");
114         }
115
116         BlockType type() const { return blockType; }
117
118         bool hasNonVoidSignature() const { return result.size(); }
119
120         BasicBlock* targetBlockForBranch()
121         {
122             if (type() == BlockType::Loop)
123                 return special;
124             return continuation;
125         }
126
127         void convertIfToBlock()
128         {
129             ASSERT(type() == BlockType::If);
130             blockType = BlockType::Block;
131             special = nullptr;
132         }
133
134     private:
135         friend class B3IRGenerator;
136         BlockType blockType;
137         BasicBlock* continuation;
138         BasicBlock* special;
139         Vector<Variable*, 1> result;
140     };
141
142     typedef Value* ExpressionType;
143     typedef ControlData ControlType;
144     typedef Vector<ExpressionType, 1> ExpressionList;
145     typedef Vector<Variable*, 1> ResultList;
146     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
147
148     static constexpr ExpressionType emptyExpression = nullptr;
149
150     typedef String ErrorType;
151     typedef UnexpectedType<ErrorType> UnexpectedResult;
152     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
153     typedef Expected<void, ErrorType> PartialResult;
154     template <typename ...Args>
155     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
156     {
157         using namespace FailureHelper; // See ADL comment in WasmParser.h.
158         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
159     }
160 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
161         if (UNLIKELY(condition))                  \
162             return fail(__VA_ARGS__);             \
163     } while (0)
164
165     B3IRGenerator(const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode);
166
167     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
168     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
169     ExpressionType addConstant(Type, uint64_t);
170
171     // Locals
172     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
173     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
174
175     // Globals
176     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
177     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
178
179     // Memory
180     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
181     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
182     PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
183     PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
184
185     // Basic operators
186     template<OpType>
187     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
188     template<OpType>
189     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
190     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
191
192     // Control flow
193     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
194     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
195     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
196     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
197     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
198     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
199
200     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
201     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
202     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
203     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
204     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
205
206     // Calls
207     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
208     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, SignatureIndex, Vector<ExpressionType>& args, ExpressionType& result);
209     PartialResult WARN_UNUSED_RETURN addUnreachable();
210
211     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
212     void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
213
214     Value* constant(B3::Type, uint64_t bits);
215     void insertConstants();
216
217 private:
218     void emitExceptionCheck(CCallHelpers&, ExceptionType);
219
220     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
221     B3::Kind memoryKind(B3::Opcode memoryOp);
222     ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
223     void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
224
225     void unify(Variable* target, const ExpressionType source);
226     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
227
228     void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
229
230     Value* materializeWasmContext(Procedure&, BasicBlock*);
231     void restoreWasmContext(Procedure&, BasicBlock*, Value*);
232     void restoreWebAssemblyGlobalState(const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
233
234     Origin origin();
235
236     FunctionParser<B3IRGenerator>* m_parser;
237     const ModuleInformation& m_info;
238     MemoryMode m_mode;
239     Procedure& m_proc;
240     BasicBlock* m_currentBlock;
241     Vector<Variable*> m_locals;
242     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
243     HashMap<ValueKey, Value*> m_constantPool;
244     InsertionSet m_constantInsertionValues;
245     GPRReg m_memoryBaseGPR;
246     GPRReg m_memorySizeGPR;
247     GPRReg m_wasmContextGPR;
248     Value* m_instanceValue; // FIXME: make this lazy https://bugs.webkit.org/show_bug.cgi?id=169792
249 };
250
251 Value* B3IRGenerator::materializeWasmContext(Procedure& proc, BasicBlock* block)
252 {
253     if (useFastTLSForContext()) {
254         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, pointerType(), Origin());
255         if (CCallHelpers::loadWasmContextNeedsMacroScratchRegister())
256             patchpoint->clobber(RegisterSet::macroScratchRegisters());
257         patchpoint->setGenerator(
258             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
259                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextNeedsMacroScratchRegister());
260                 jit.loadWasmContext(params[0].gpr());
261             });
262         return patchpoint;
263     }
264
265     // FIXME: Because WasmToWasm call clobbers wasmContext register and does not restore it, we need to restore it in the caller side.
266     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
267     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, pointerType(), Origin());
268     patchpoint->effects.writesPinned = false;
269     patchpoint->effects.readsPinned = true;
270     patchpoint->resultConstraint = ValueRep::reg(m_wasmContextGPR);
271     patchpoint->setGenerator([] (CCallHelpers&, const StackmapGenerationParams&) { });
272     return patchpoint;
273 }
274
275 void B3IRGenerator::restoreWasmContext(Procedure& proc, BasicBlock* block, Value* arg)
276 {
277     if (useFastTLSForContext()) {
278         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
279         if (CCallHelpers::storeWasmContextNeedsMacroScratchRegister())
280             patchpoint->clobber(RegisterSet::macroScratchRegisters());
281         patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
282         patchpoint->setGenerator(
283             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
284                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextNeedsMacroScratchRegister());
285                 jit.storeWasmContext(params[0].gpr());
286             });
287         return;
288     }
289
290     // FIXME: Because WasmToWasm call clobbers wasmContext register and does not restore it, we need to restore it in the caller side.
291     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
292     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
293     Effects effects = Effects::none();
294     effects.writesPinned = true;
295     effects.reads = B3::HeapRange::top();
296     patchpoint->effects = effects;
297     patchpoint->clobberLate(RegisterSet(m_wasmContextGPR));
298     patchpoint->append(m_instanceValue, ValueRep::SomeRegister);
299     GPRReg wasmContextGPR = m_wasmContextGPR;
300     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
301         jit.move(param[0].gpr(), wasmContextGPR);
302     });
303 }
304
305 B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode)
306     : m_info(info)
307     , m_mode(mode)
308     , m_proc(procedure)
309     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
310     , m_constantInsertionValues(m_proc)
311 {
312     m_currentBlock = m_proc.addBlock();
313
314     // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
315     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
316     m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
317     m_wasmContextGPR = pinnedRegs.wasmContextPointer;
318     m_proc.pinRegister(m_memoryBaseGPR);
319     if (!useFastTLSForContext())
320         m_proc.pinRegister(m_wasmContextGPR);
321     ASSERT(!pinnedRegs.sizeRegisters[0].sizeOffset);
322     m_memorySizeGPR = pinnedRegs.sizeRegisters[0].sizeRegister;
323     for (const PinnedSizeRegisterInfo& regInfo : pinnedRegs.sizeRegisters)
324         m_proc.pinRegister(regInfo.sizeRegister);
325
326     if (info.memory) {
327         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
328             AllowMacroScratchRegisterUsage allowScratch(jit);
329             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
330             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
331         });
332     }
333
334     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
335
336     m_instanceValue = materializeWasmContext(m_proc, m_currentBlock);
337 }
338
339 void B3IRGenerator::restoreWebAssemblyGlobalState(const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
340 {
341     restoreWasmContext(proc, block, instance);
342
343     if (!!memory) {
344         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
345         RegisterSet clobbers;
346         clobbers.set(pinnedRegs->baseMemoryPointer);
347         for (auto info : pinnedRegs->sizeRegisters)
348             clobbers.set(info.sizeRegister);
349
350         B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
351         Effects effects = Effects::none();
352         effects.writesPinned = true;
353         effects.reads = B3::HeapRange::top();
354         patchpoint->effects = effects;
355         patchpoint->clobber(clobbers);
356
357         patchpoint->append(instance, ValueRep::SomeRegister);
358
359         patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
360             GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
361             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
362             const auto& sizeRegs = pinnedRegs->sizeRegisters;
363             ASSERT(sizeRegs.size() >= 1);
364             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
365             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfSize()), sizeRegs[0].sizeRegister);
366             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfMemory()), baseMemory);
367             for (unsigned i = 1; i < sizeRegs.size(); ++i)
368                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
369         });
370     }
371 }
372
373 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
374 {
375     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
376     auto jumpToExceptionStub = jit.jump();
377
378     jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
379         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
380     });
381 }
382
383 Value* B3IRGenerator::constant(B3::Type type, uint64_t bits)
384 {
385     auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
386         Value* result = m_proc.addConstant(origin(), type, bits);
387         m_constantInsertionValues.insertValue(0, result);
388         return result;
389     });
390     return result.iterator->value;
391 }
392
393 void B3IRGenerator::insertConstants()
394 {
395     m_constantInsertionValues.execute(m_proc.at(0));
396 }
397
398 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
399 {
400     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
401
402     for (uint32_t i = 0; i < count; ++i) {
403         Variable* local = m_proc.addVariable(toB3Type(type));
404         m_locals.uncheckedAppend(local);
405         m_currentBlock->appendNew<VariableValue>(m_proc, Set, origin(), local, addConstant(type, 0));
406     }
407     return { };
408 }
409
410 auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
411 {
412     ASSERT(!m_locals.size());
413     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
414
415     m_locals.grow(signature.argumentCount());
416     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
417         [=] (ExpressionType argument, unsigned i) {
418             Variable* argumentVariable = m_proc.addVariable(argument->type());
419             m_locals[i] = argumentVariable;
420             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
421         });
422     return { };
423 }
424
425 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
426 {
427     ASSERT(m_locals[index]);
428     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
429     return { };
430 }
431
432 auto B3IRGenerator::addUnreachable() -> PartialResult
433 {
434     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
435     unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
436         this->emitExceptionCheck(jit, ExceptionType::Unreachable);
437     });
438     unreachable->effects.terminal = true;
439     return { };
440 }
441
442 auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
443 {
444     int32_t (*growMemory) (Context*, int32_t) = [] (Context* wasmContext, int32_t delta) -> int32_t {
445         VM& vm = *wasmContext->vm();
446         auto scope = DECLARE_THROW_SCOPE(vm);
447
448         JSWebAssemblyMemory* wasmMemory = wasmContext->memory();
449
450         if (delta < 0)
451             return -1;
452
453         bool shouldThrowExceptionsOnFailure = false;
454         // grow() does not require ExecState* if it doesn't throw exceptions.
455         ExecState* exec = nullptr; 
456         PageCount result = wasmMemory->grow(vm, exec, static_cast<uint32_t>(delta), shouldThrowExceptionsOnFailure);
457         RELEASE_ASSERT(!scope.exception());
458         if (!result)
459             return -1;
460
461         return result.pageCount();
462     };
463
464     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
465         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(growMemory)),
466         m_instanceValue, delta);
467
468     restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, m_currentBlock);
469
470     return { };
471 }
472
473 auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
474 {
475     Value* memoryObject = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfMemory());
476
477     static_assert(sizeof(decltype(static_cast<JSWebAssemblyInstance*>(nullptr)->memory()->memory().size())) == sizeof(uint64_t), "codegen relies on this size");
478     Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), memoryObject, JSWebAssemblyMemory::offsetOfSize());
479     
480     constexpr uint32_t shiftValue = 16;
481     static_assert(PageCount::pageSize == 1 << shiftValue, "This must hold for the code below to be correct.");
482     Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
483         size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
484
485     result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
486
487     return { };
488 }
489
490 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
491 {
492     ASSERT(m_locals[index]);
493     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
494     return { };
495 }
496
497 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
498 {
499     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
500     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, index * sizeof(Register));
501     return { };
502 }
503
504 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
505 {
506     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
507     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
508     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, index * sizeof(Register));
509     return { };
510 }
511
512 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
513 {
514     ASSERT(m_memoryBaseGPR);
515     if (m_mode == MemoryMode::BoundsChecking) {
516         ASSERT(m_memorySizeGPR);
517         ASSERT(sizeOfOperation + offset > offset);
518         m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
519     }
520     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
521     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
522 }
523
524 inline uint32_t sizeOfLoadOp(LoadOpType op)
525 {
526     switch (op) {
527     case LoadOpType::I32Load8S:
528     case LoadOpType::I32Load8U:
529     case LoadOpType::I64Load8S:
530     case LoadOpType::I64Load8U:
531         return 1;
532     case LoadOpType::I32Load16S:
533     case LoadOpType::I64Load16S:
534     case LoadOpType::I32Load16U:
535     case LoadOpType::I64Load16U:
536         return 2;
537     case LoadOpType::I32Load:
538     case LoadOpType::I64Load32S:
539     case LoadOpType::I64Load32U:
540     case LoadOpType::F32Load:
541         return 4;
542     case LoadOpType::I64Load:
543     case LoadOpType::F64Load:
544         return 8;
545     }
546     RELEASE_ASSERT_NOT_REACHED();
547 }
548
549 inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
550 {
551     if (m_mode == MemoryMode::Signaling)
552         return trapping(memoryOp);
553     return memoryOp;
554 }
555
556 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t offset)
557 {
558     switch (op) {
559     case LoadOpType::I32Load8S: {
560         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
561     }
562
563     case LoadOpType::I64Load8S: {
564         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
565         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
566     }
567
568     case LoadOpType::I32Load8U: {
569         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
570     }
571
572     case LoadOpType::I64Load8U: {
573         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
574         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
575     }
576
577     case LoadOpType::I32Load16S: {
578         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
579     }
580     case LoadOpType::I64Load16S: {
581         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
582         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
583     }
584
585     case LoadOpType::I32Load: {
586         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
587     }
588
589     case LoadOpType::I64Load32U: {
590         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
591         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
592     }
593
594     case LoadOpType::I64Load32S: {
595         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
596         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
597     }
598
599     case LoadOpType::I64Load: {
600         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
601     }
602
603     case LoadOpType::F32Load: {
604         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
605     }
606
607     case LoadOpType::F64Load: {
608         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
609     }
610
611     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
612     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
613     case LoadOpType::I32Load16U: {
614         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
615         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
616             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
617     }
618     case LoadOpType::I64Load16U: {
619         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
620         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
621             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
622
623         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), partialResult);
624     }
625     }
626     RELEASE_ASSERT_NOT_REACHED();
627 }
628
629 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
630 {
631     ASSERT(pointer->type() == Int32);
632
633     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
634         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
635         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
636         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
637         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
638             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
639         });
640
641         switch (op) {
642         case LoadOpType::I32Load8S:
643         case LoadOpType::I32Load16S:
644         case LoadOpType::I32Load:
645         case LoadOpType::I32Load16U:
646         case LoadOpType::I32Load8U:
647             result = constant(Int32, 0);
648             break;
649         case LoadOpType::I64Load8S:
650         case LoadOpType::I64Load8U:
651         case LoadOpType::I64Load16S:
652         case LoadOpType::I64Load32U:
653         case LoadOpType::I64Load32S:
654         case LoadOpType::I64Load:
655         case LoadOpType::I64Load16U:
656             result = constant(Int64, 0);
657             break;
658         case LoadOpType::F32Load:
659             result = constant(Float, 0);
660             break;
661         case LoadOpType::F64Load:
662             result = constant(Double, 0);
663             break;
664         }
665
666     } else
667         result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
668
669     return { };
670 }
671
672 inline uint32_t sizeOfStoreOp(StoreOpType op)
673 {
674     switch (op) {
675     case StoreOpType::I32Store8:
676     case StoreOpType::I64Store8:
677         return 1;
678     case StoreOpType::I32Store16:
679     case StoreOpType::I64Store16:
680         return 2;
681     case StoreOpType::I32Store:
682     case StoreOpType::I64Store32:
683     case StoreOpType::F32Store:
684         return 4;
685     case StoreOpType::I64Store:
686     case StoreOpType::F64Store:
687         return 8;
688     }
689     RELEASE_ASSERT_NOT_REACHED();
690 }
691
692
693 inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset)
694 {
695     switch (op) {
696     case StoreOpType::I64Store8:
697         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
698         FALLTHROUGH;
699
700     case StoreOpType::I32Store8:
701         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
702         return;
703
704     case StoreOpType::I64Store16:
705         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
706         FALLTHROUGH;
707
708     case StoreOpType::I32Store16:
709         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
710         return;
711
712     case StoreOpType::I64Store32:
713         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
714         FALLTHROUGH;
715
716     case StoreOpType::I64Store:
717     case StoreOpType::I32Store:
718     case StoreOpType::F32Store:
719     case StoreOpType::F64Store:
720         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
721         return;
722     }
723     RELEASE_ASSERT_NOT_REACHED();
724 }
725
726 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
727 {
728     ASSERT(pointer->type() == Int32);
729
730     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
731         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
732         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
733         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
734         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
735             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
736         });
737     } else
738         emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
739
740     return { };
741 }
742
743 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
744 {
745     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
746     return { };
747 }
748
749 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
750 {
751     return constant(toB3Type(type), value);
752 }
753
754 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
755 {
756     return ControlData(m_proc, signature, BlockType::TopLevel, m_proc.addBlock());
757 }
758
759 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
760 {
761     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
762 }
763
764 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
765 {
766     BasicBlock* body = m_proc.addBlock();
767     BasicBlock* continuation = m_proc.addBlock();
768     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
769     body->addPredecessor(m_currentBlock);
770     m_currentBlock = body;
771     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
772 }
773
774 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
775 {
776     // FIXME: This needs to do some kind of stack passing.
777
778     BasicBlock* taken = m_proc.addBlock();
779     BasicBlock* notTaken = m_proc.addBlock();
780     BasicBlock* continuation = m_proc.addBlock();
781
782     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
783     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
784     taken->addPredecessor(m_currentBlock);
785     notTaken->addPredecessor(m_currentBlock);
786
787     m_currentBlock = taken;
788     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
789     return { };
790 }
791
792 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
793 {
794     unifyValuesWithBlock(currentStack, data.result);
795     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
796     return addElseToUnreachable(data);
797 }
798
799 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
800 {
801     ASSERT(data.type() == BlockType::If);
802     m_currentBlock = data.special;
803     data.convertIfToBlock();
804     return { };
805 }
806
807 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
808 {
809     ASSERT(returnValues.size() <= 1);
810     if (returnValues.size())
811         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
812     else
813         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
814     return { };
815 }
816
817 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
818 {
819     if (data.type() != BlockType::Loop)
820         unifyValuesWithBlock(returnValues, data.result);
821
822     BasicBlock* target = data.targetBlockForBranch();
823     if (condition) {
824         BasicBlock* continuation = m_proc.addBlock();
825         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
826         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
827         target->addPredecessor(m_currentBlock);
828         continuation->addPredecessor(m_currentBlock);
829         m_currentBlock = continuation;
830     } else {
831         m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
832         target->addPredecessor(m_currentBlock);
833     }
834
835     return { };
836 }
837
838 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
839 {
840     for (size_t i = 0; i < targets.size(); ++i)
841         unifyValuesWithBlock(expressionStack, targets[i]->result);
842     unifyValuesWithBlock(expressionStack, defaultTarget.result);
843
844     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
845     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
846     for (size_t i = 0; i < targets.size(); ++i)
847         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
848
849     return { };
850 }
851
852 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
853 {
854     ControlData& data = entry.controlData;
855
856     unifyValuesWithBlock(expressionStack, data.result);
857     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
858     data.continuation->addPredecessor(m_currentBlock);
859
860     return addEndToUnreachable(entry);
861 }
862
863
864 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
865 {
866     ControlData& data = entry.controlData;
867     m_currentBlock = data.continuation;
868
869     if (data.type() == BlockType::If) {
870         data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
871         m_currentBlock->addPredecessor(data.special);
872     }
873
874     for (Variable* result : data.result)
875         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), result));
876
877     // TopLevel does not have any code after this so we need to make sure we emit a return here.
878     if (data.type() == BlockType::TopLevel)
879         return addReturn(entry.controlData, entry.enclosedExpressionStack);
880
881     return { };
882 }
883
884 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
885 {
886     ASSERT(signature.argumentCount() == args.size());
887
888     Type returnType = signature.returnType();
889     Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
890
891     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
892         // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
893         Value* functionImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfImportFunction(functionIndex));
894         Value* jsTypeOfImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin(), functionImport, JSCell::typeInfoTypeOffset());
895         Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), jsTypeOfImport, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), WebAssemblyFunctionType));
896
897         BasicBlock* isWasmBlock = m_proc.addBlock();
898         BasicBlock* isJSBlock = m_proc.addBlock();
899         BasicBlock* continuation = m_proc.addBlock();
900         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isJSBlock));
901
902         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
903             [=] (PatchpointValue* patchpoint) {
904                 patchpoint->effects.writesPinned = true;
905                 patchpoint->effects.readsPinned = true;
906                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
907                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
908                     AllowMacroScratchRegisterUsage allowScratch(jit);
909                     CCallHelpers::Call call = jit.call();
910                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
911                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex });
912                     });
913                 });
914             });
915         UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
916         isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
917
918         // FIXME: Lets remove this indirection by creating a PIC friendly IC
919         // for calls out to JS. This shouldn't be that hard to do. We could probably
920         // implement the IC to be over Wasm::Context*.
921         // https://bugs.webkit.org/show_bug.cgi?id=170375
922         Value* codeBlock = isJSBlock->appendNew<MemoryValue>(m_proc,
923             Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfCodeBlock());
924         Value* jumpDestination = isJSBlock->appendNew<MemoryValue>(m_proc,
925             Load, pointerType(), origin(), codeBlock, JSWebAssemblyCodeBlock::offsetOfImportWasmToJSStub(functionIndex));
926         Value* jsCallResult = wasmCallingConvention().setupCall(m_proc, isJSBlock, origin(), args, toB3Type(returnType),
927             [&] (PatchpointValue* patchpoint) {
928                 patchpoint->effects.writesPinned = true;
929                 patchpoint->effects.readsPinned = true;
930                 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
931                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
932                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex, returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
933                     AllowMacroScratchRegisterUsage allowScratch(jit);
934                     jit.call(params[returnType == Void ? 0 : 1].gpr());
935                 });
936             });
937         UpsilonValue* jsCallResultUpsilon = returnType == Void ? nullptr : isJSBlock->appendNew<UpsilonValue>(m_proc, origin(), jsCallResult);
938         isJSBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
939
940         m_currentBlock = continuation;
941
942         if (returnType == Void)
943             result = nullptr;
944         else {
945             result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
946             wasmCallResultUpsilon->setPhi(result);
947             jsCallResultUpsilon->setPhi(result);
948         }
949
950         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
951         restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, continuation);
952     } else {
953         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
954             [=] (PatchpointValue* patchpoint) {
955                 patchpoint->effects.writesPinned = true;
956                 patchpoint->effects.readsPinned = true;
957
958                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
959                     AllowMacroScratchRegisterUsage allowScratch(jit);
960                     CCallHelpers::Call call = jit.call();
961                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
962                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex });
963                     });
964                 });
965             });
966     }
967
968     return { };
969 }
970
971 auto B3IRGenerator::addCallIndirect(const Signature& signature, SignatureIndex signatureIndex, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
972 {
973     ASSERT(signatureIndex != Signature::invalidIndex);
974     ExpressionType calleeIndex = args.takeLast();
975     ASSERT(signature.argumentCount() == args.size());
976
977     ExpressionType callableFunctionBuffer;
978     ExpressionType callableFunctionBufferSize;
979     {
980         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
981             m_instanceValue, JSWebAssemblyInstance::offsetOfTable());
982         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
983             table, JSWebAssemblyTable::offsetOfFunctions());
984         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
985             table, JSWebAssemblyTable::offsetOfSize());
986     }
987
988     // Check the index we are looking for is valid.
989     {
990         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
991             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferSize));
992
993         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
994             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
995         });
996     }
997
998     // Compute the offset in the table index space we are looking for.
999     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1000         m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex),
1001         constant(pointerType(), sizeof(CallableFunction)));
1002     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1003
1004     // Check that the CallableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1005     static_assert(sizeof(CallableFunction::signatureIndex) == sizeof(uint32_t), "Load codegen assumes i32");
1006     ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signatureIndex));
1007     {
1008         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1009             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1010                 calleeSignatureIndex,
1011                 m_currentBlock->appendNew<Const32Value>(m_proc, origin(), Signature::invalidIndex)));
1012
1013         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1014             this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1015         });
1016     }
1017
1018     // Check the signature matches the value we expect.
1019     {
1020         ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const32Value>(m_proc, origin(), signatureIndex);
1021         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1022             m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1023
1024         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1025             this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1026         });
1027     }
1028
1029     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
1030
1031     Type returnType = signature.returnType();
1032     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1033         [=] (PatchpointValue* patchpoint) {
1034             patchpoint->effects.writesPinned = true;
1035             patchpoint->effects.readsPinned = true;
1036             patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
1037
1038             patchpoint->append(calleeCode, ValueRep::SomeRegister);
1039             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1040                 AllowMacroScratchRegisterUsage allowScratch(jit);
1041                 jit.call(params[returnType == Void ? 0 : 1].gpr());
1042             });
1043         });
1044
1045     // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1046     restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, m_currentBlock);
1047
1048     return { };
1049 }
1050
1051 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
1052 {
1053     m_currentBlock->appendNew<VariableValue>(m_proc, Set, origin(), variable, source);
1054 }
1055
1056 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
1057 {
1058     ASSERT(result.size() <= resultStack.size());
1059
1060     for (size_t i = 0; i < result.size(); ++i)
1061         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1062 }
1063
1064 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1065 {
1066     dataLog(comma, "ExpressionStack:");
1067     for (const auto& expression : expressionStack)
1068         dataLog(comma, *expression);
1069 }
1070
1071 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1072 {
1073     dataLogLn("Constants:");
1074     for (const auto& constant : m_constantPool)
1075         dataLogLn(deepDump(m_proc, constant.value));
1076
1077     dataLogLn("Processing Graph:");
1078     dataLog(m_proc);
1079     dataLogLn("With current block:", *m_currentBlock);
1080     dataLogLn("Control stack:");
1081     ASSERT(controlStack.size());
1082     for (size_t i = controlStack.size(); i--;) {
1083         dataLog("  ", controlStack[i].controlData, ": ");
1084         CommaPrinter comma(", ", "");
1085         dumpExpressionStack(comma, *expressionStack);
1086         expressionStack = &controlStack[i].enclosedExpressionStack;
1087         dataLogLn();
1088     }
1089     dataLogLn();
1090 }
1091
1092 static void createJSToWasmWrapper(CompilationContext& compilationContext, WasmInternalFunction& function, const Signature& signature, const ModuleInformation& info)
1093 {
1094     CCallHelpers& jit = *compilationContext.jsEntrypointJIT;
1095
1096     jit.emitFunctionPrologue();
1097
1098     // FIXME Stop using 0 as codeBlocks. https://bugs.webkit.org/show_bug.cgi?id=165321
1099     jit.store64(CCallHelpers::TrustedImm64(0), CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::codeBlock * static_cast<int>(sizeof(Register))));
1100     MacroAssembler::DataLabelPtr calleeMoveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), GPRInfo::nonPreservedNonReturnGPR);
1101     jit.storePtr(GPRInfo::nonPreservedNonReturnGPR, CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::callee * static_cast<int>(sizeof(Register))));
1102     CodeLocationDataLabelPtr* linkedCalleeMove = &function.jsToWasmCalleeMoveLocation;
1103     jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1104         *linkedCalleeMove = linkBuffer.locationOf(calleeMoveLocation);
1105     });
1106
1107     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1108     RegisterSet toSave = pinnedRegs.toSave();
1109
1110 #if !ASSERT_DISABLED
1111     unsigned toSaveSize = toSave.numberOfSetGPRs();
1112     // They should all be callee saves.
1113     toSave.filter(RegisterSet::calleeSaveRegisters());
1114     ASSERT(toSave.numberOfSetGPRs() == toSaveSize);
1115 #endif
1116
1117     RegisterAtOffsetList registersToSpill(toSave, RegisterAtOffsetList::OffsetBaseType::FramePointerBased);
1118     function.jsToWasmEntrypoint.calleeSaveRegisters = registersToSpill;
1119
1120     unsigned totalFrameSize = registersToSpill.size() * sizeof(void*);
1121     totalFrameSize += WasmCallingConvention::headerSizeInBytes();
1122     totalFrameSize -= sizeof(CallerFrameAndPC);
1123     unsigned numGPRs = 0;
1124     unsigned numFPRs = 0;
1125     for (unsigned i = 0; i < signature.argumentCount(); i++) {
1126         switch (signature.argument(i)) {
1127         case Wasm::I64:
1128         case Wasm::I32:
1129             if (numGPRs >= wasmCallingConvention().m_gprArgs.size())
1130                 totalFrameSize += sizeof(void*);
1131             ++numGPRs;
1132             break;
1133         case Wasm::F32:
1134         case Wasm::F64:
1135             if (numFPRs >= wasmCallingConvention().m_fprArgs.size())
1136                 totalFrameSize += sizeof(void*);
1137             ++numFPRs;
1138             break;
1139         default:
1140             RELEASE_ASSERT_NOT_REACHED();
1141         }
1142     }
1143
1144     totalFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), totalFrameSize);
1145     jit.subPtr(MacroAssembler::TrustedImm32(totalFrameSize), MacroAssembler::stackPointerRegister);
1146
1147     // We save all these registers regardless of having a memory or not.
1148     // The reason is that we use one of these as a scratch. That said,
1149     // almost all real wasm programs use memory, so it's not really
1150     // worth optimizing for the case that they don't.
1151     for (const RegisterAtOffset& regAtOffset : registersToSpill) {
1152         GPRReg reg = regAtOffset.reg().gpr();
1153         ptrdiff_t offset = regAtOffset.offset();
1154         jit.storePtr(reg, CCallHelpers::Address(GPRInfo::callFrameRegister, offset));
1155     }
1156
1157     GPRReg wasmContextGPR = pinnedRegs.wasmContextPointer;
1158
1159     {
1160         CCallHelpers::Address calleeFrame = CCallHelpers::Address(MacroAssembler::stackPointerRegister, -static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC)));
1161         numGPRs = 0;
1162         numFPRs = 0;
1163         // We're going to set the pinned registers after this. So
1164         // we can use this as a scratch for now since we saved it above.
1165         GPRReg scratchReg = pinnedRegs.baseMemoryPointer;
1166
1167         ptrdiff_t jsOffset = CallFrameSlot::thisArgument * sizeof(EncodedJSValue);
1168
1169         // vmEntryToWasm passes Wasm::Context* as the first JS argument when we're
1170         // not using fast TLS to hold the Wasm::Context*.
1171         if (!useFastTLSForContext()) {
1172             jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmContextGPR);
1173             jsOffset += sizeof(EncodedJSValue);
1174         }
1175
1176         ptrdiff_t wasmOffset = CallFrame::headerSizeInRegisters * sizeof(void*);
1177         for (unsigned i = 0; i < signature.argumentCount(); i++) {
1178             switch (signature.argument(i)) {
1179             case Wasm::I32:
1180             case Wasm::I64:
1181                 if (numGPRs >= wasmCallingConvention().m_gprArgs.size()) {
1182                     if (signature.argument(i) == Wasm::I32) {
1183                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1184                         jit.store32(scratchReg, calleeFrame.withOffset(wasmOffset));
1185                     } else {
1186                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1187                         jit.store64(scratchReg, calleeFrame.withOffset(wasmOffset));
1188                     }
1189                     wasmOffset += sizeof(void*);
1190                 } else {
1191                     if (signature.argument(i) == Wasm::I32)
1192                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_gprArgs[numGPRs].gpr());
1193                     else
1194                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_gprArgs[numGPRs].gpr());
1195                 }
1196                 ++numGPRs;
1197                 break;
1198             case Wasm::F32:
1199             case Wasm::F64:
1200                 if (numFPRs >= wasmCallingConvention().m_fprArgs.size()) {
1201                     if (signature.argument(i) == Wasm::F32) {
1202                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1203                         jit.store32(scratchReg, calleeFrame.withOffset(wasmOffset));
1204                     } else {
1205                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1206                         jit.store64(scratchReg, calleeFrame.withOffset(wasmOffset));
1207                     }
1208                     wasmOffset += sizeof(void*);
1209                 } else {
1210                     if (signature.argument(i) == Wasm::F32)
1211                         jit.loadFloat(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_fprArgs[numFPRs].fpr());
1212                     else
1213                         jit.loadDouble(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_fprArgs[numFPRs].fpr());
1214                 }
1215                 ++numFPRs;
1216                 break;
1217             default:
1218                 RELEASE_ASSERT_NOT_REACHED();
1219             }
1220
1221             jsOffset += sizeof(EncodedJSValue);
1222         }
1223     }
1224
1225     if (!!info.memory) {
1226         GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1227
1228         if (!useFastTLSForContext())
1229             jit.loadPtr(CCallHelpers::Address(wasmContextGPR, JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
1230         else {
1231             jit.loadWasmContext(baseMemory);
1232             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
1233         }
1234         const auto& sizeRegs = pinnedRegs.sizeRegisters;
1235         ASSERT(sizeRegs.size() >= 1);
1236         ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
1237         jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfSize()), sizeRegs[0].sizeRegister);
1238         jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfMemory()), baseMemory);
1239         for (unsigned i = 1; i < sizeRegs.size(); ++i)
1240             jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
1241     }
1242
1243     compilationContext.jsEntrypointToWasmEntrypointCall = jit.call();
1244
1245     for (const RegisterAtOffset& regAtOffset : registersToSpill) {
1246         GPRReg reg = regAtOffset.reg().gpr();
1247         ASSERT(reg != GPRInfo::returnValueGPR);
1248         ptrdiff_t offset = regAtOffset.offset();
1249         jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, offset), reg);
1250     }
1251
1252     switch (signature.returnType()) {
1253     case Wasm::F32:
1254         jit.moveFloatTo32(FPRInfo::returnValueFPR, GPRInfo::returnValueGPR);
1255         break;
1256     case Wasm::F64:
1257         jit.moveDoubleTo64(FPRInfo::returnValueFPR, GPRInfo::returnValueGPR);
1258         break;
1259     default:
1260         break;
1261     }
1262
1263     jit.emitFunctionEpilogue();
1264     jit.ret();
1265 }
1266
1267 auto B3IRGenerator::origin() -> Origin
1268 {
1269     return bitwise_cast<Origin>(OpcodeOrigin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset()));
1270 }
1271
1272 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, const Vector<SignatureIndex>& moduleSignatureIndicesToUniquedSignatureIndices, MemoryMode mode, unsigned optLevel)
1273 {
1274     auto result = std::make_unique<WasmInternalFunction>();
1275
1276     compilationContext.jsEntrypointJIT = std::make_unique<CCallHelpers>();
1277     compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1278
1279     Procedure procedure;
1280
1281     procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1282         if (origin.data())
1283             out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1284     });
1285     
1286     // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1287     // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1288     // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1289     // optLevel=1.
1290     procedure.setNeedsUsedRegisters(false);
1291     
1292     procedure.setOptLevel(optLevel);
1293
1294     B3IRGenerator context(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode);
1295     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, info, moduleSignatureIndicesToUniquedSignatureIndices);
1296     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1297
1298     context.insertConstants();
1299
1300     procedure.resetReachability();
1301     if (!ASSERT_DISABLED)
1302         validate(procedure, "After parsing:\n");
1303
1304     dataLogIf(verbose, "Pre SSA: ", procedure);
1305     fixSSA(procedure);
1306     dataLogIf(verbose, "Post SSA: ", procedure);
1307     
1308     {
1309         B3::prepareForGeneration(procedure);
1310         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1311         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1312         result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisters();
1313     }
1314
1315     createJSToWasmWrapper(compilationContext, *result, signature, info);
1316     return WTFMove(result);
1317 }
1318
1319 // Custom wasm ops. These are the ones too messy to do in wasm.json.
1320
1321 void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1322 {
1323     ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1324     const B3::Type type = left->type();
1325
1326     {
1327         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1328             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1329
1330         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1331             this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1332         });
1333     }
1334
1335     if (operation == Div) {
1336         int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1337
1338         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1339             m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1340                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1341                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1342
1343         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1344             this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1345         });
1346     }
1347 }
1348
1349 template<>
1350 auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1351 {
1352     const B3::Opcode op = Div;
1353     emitChecksForModOrDiv(op, left, right);
1354     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1355     return { };
1356 }
1357
1358 template<>
1359 auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1360 {
1361     const B3::Opcode op = Mod;
1362     emitChecksForModOrDiv(op, left, right);
1363     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1364     return { };
1365 }
1366
1367 template<>
1368 auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1369 {
1370     const B3::Opcode op = UDiv;
1371     emitChecksForModOrDiv(op, left, right);
1372     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1373     return { };
1374 }
1375
1376 template<>
1377 auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1378 {
1379     const B3::Opcode op = UMod;
1380     emitChecksForModOrDiv(op, left, right);
1381     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1382     return { };
1383 }
1384
1385 template<>
1386 auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1387 {
1388     const B3::Opcode op = Div;
1389     emitChecksForModOrDiv(op, left, right);
1390     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1391     return { };
1392 }
1393
1394 template<>
1395 auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1396 {
1397     const B3::Opcode op = Mod;
1398     emitChecksForModOrDiv(op, left, right);
1399     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1400     return { };
1401 }
1402
1403 template<>
1404 auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1405 {
1406     const B3::Opcode op = UDiv;
1407     emitChecksForModOrDiv(op, left, right);
1408     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1409     return { };
1410 }
1411
1412 template<>
1413 auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1414 {
1415     const B3::Opcode op = UMod;
1416     emitChecksForModOrDiv(op, left, right);
1417     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1418     return { };
1419 }
1420
1421 template<>
1422 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1423 {
1424     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1425     patchpoint->append(arg, ValueRep::SomeRegister);
1426     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1427         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1428     });
1429     patchpoint->effects = Effects::none();
1430     result = patchpoint;
1431     return { };
1432 }
1433
1434 template<>
1435 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1436 {
1437     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1438     patchpoint->append(arg, ValueRep::SomeRegister);
1439     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1440         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1441     });
1442     patchpoint->effects = Effects::none();
1443     result = patchpoint;
1444     return { };
1445 }
1446
1447 template<>
1448 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1449 {
1450     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1451     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1452     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1453     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1454     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1455     return { };
1456 }
1457
1458 template<>
1459 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1460 {
1461     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1462     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1463     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1464     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1465     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1466     return { };
1467 }
1468
1469 template<>
1470 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1471 {
1472     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1473     if (isX86())
1474         patchpoint->numGPScratchRegisters = 1;
1475     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1476     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1477         AllowMacroScratchRegisterUsage allowScratch(jit);
1478 #if CPU(X86_64)
1479         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1480 #else
1481         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1482 #endif
1483     });
1484     patchpoint->effects = Effects::none();
1485     result = patchpoint;
1486     return { };
1487 }
1488
1489 template<>
1490 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1491 {
1492     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1493     if (isX86())
1494         patchpoint->numGPScratchRegisters = 1;
1495     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1496     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1497         AllowMacroScratchRegisterUsage allowScratch(jit);
1498 #if CPU(X86_64)
1499         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1500 #else
1501         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1502 #endif
1503     });
1504     patchpoint->effects = Effects::none();
1505     result = patchpoint;
1506     return { };
1507 }
1508
1509 template<>
1510 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1511 {
1512     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1513     patchpoint->append(arg, ValueRep::SomeRegister);
1514     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1515         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1516     });
1517     patchpoint->effects = Effects::none();
1518     result = patchpoint;
1519     return { };
1520 }
1521
1522 template<>
1523 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1524 {
1525     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1526     patchpoint->append(arg, ValueRep::SomeRegister);
1527     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1528         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1529     });
1530     patchpoint->effects = Effects::none();
1531     result = patchpoint;
1532     return { };
1533 }
1534
1535 template<>
1536 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1537 {
1538     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1539     patchpoint->append(arg, ValueRep::SomeRegister);
1540     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1541         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1542     });
1543     patchpoint->effects = Effects::none();
1544     result = patchpoint;
1545     return { };
1546 }
1547
1548 template<>
1549 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1550 {
1551     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1552     patchpoint->append(arg, ValueRep::SomeRegister);
1553     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1554         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1555     });
1556     patchpoint->effects = Effects::none();
1557     result = patchpoint;
1558     return { };
1559 }
1560
1561 template<>
1562 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1563 {
1564     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1565     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1566     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1567         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1568         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1569     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1570     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1571     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1572         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1573     });
1574     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1575     patchpoint->append(arg, ValueRep::SomeRegister);
1576     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1577         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1578     });
1579     patchpoint->effects = Effects::none();
1580     result = patchpoint;
1581     return { };
1582 }
1583
1584 template<>
1585 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1586 {
1587     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1588     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1589     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1590         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1591         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1592     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1593     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1594     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1595         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1596     });
1597     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1598     patchpoint->append(arg, ValueRep::SomeRegister);
1599     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1600         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1601     });
1602     patchpoint->effects = Effects::none();
1603     result = patchpoint;
1604     return { };
1605 }
1606
1607
1608 template<>
1609 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1610 {
1611     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1612     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1613     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1614         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1615         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1616     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1617     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1618     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1619         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1620     });
1621     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1622     patchpoint->append(arg, ValueRep::SomeRegister);
1623     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1624         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1625     });
1626     patchpoint->effects = Effects::none();
1627     result = patchpoint;
1628     return { };
1629 }
1630
1631 template<>
1632 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1633 {
1634     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1635     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1636     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1637         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1638         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1639     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1640     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1641     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1642         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1643     });
1644     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1645     patchpoint->append(arg, ValueRep::SomeRegister);
1646     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1647         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1648     });
1649     patchpoint->effects = Effects::none();
1650     result = patchpoint;
1651     return { };
1652 }
1653
1654 template<>
1655 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1656 {
1657     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1658     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1659     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1660         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1661         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1662     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1663     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1664     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1665         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1666     });
1667     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1668     patchpoint->append(arg, ValueRep::SomeRegister);
1669     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1670         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1671     });
1672     patchpoint->effects = Effects::none();
1673     result = patchpoint;
1674     return { };
1675 }
1676
1677 template<>
1678 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1679 {
1680     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1681     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1682     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1683         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1684         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1685     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1686     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1687     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1688         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1689     });
1690
1691     Value* signBitConstant;
1692     if (isX86()) {
1693         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1694         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1695         // so we can pool them if needed.
1696         signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1697     }
1698     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1699     patchpoint->append(arg, ValueRep::SomeRegister);
1700     if (isX86()) {
1701         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1702         patchpoint->numFPScratchRegisters = 1;
1703     }
1704     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1705         AllowMacroScratchRegisterUsage allowScratch(jit);
1706         FPRReg scratch = InvalidFPRReg;
1707         FPRReg constant = InvalidFPRReg;
1708         if (isX86()) {
1709             scratch = params.fpScratch(0);
1710             constant = params[2].fpr();
1711         }
1712         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1713     });
1714     patchpoint->effects = Effects::none();
1715     result = patchpoint;
1716     return { };
1717 }
1718
1719 template<>
1720 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1721 {
1722     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1723     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1724     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1725         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1726         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1727     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1728     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1729     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1730         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1731     });
1732     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1733     patchpoint->append(arg, ValueRep::SomeRegister);
1734     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1735         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1736     });
1737     patchpoint->effects = Effects::none();
1738     result = patchpoint;
1739     return { };
1740 }
1741
1742 template<>
1743 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1744 {
1745     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1746     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1747     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1748         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1749         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1750     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1751     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1752     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1753         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1754     });
1755
1756     Value* signBitConstant;
1757     if (isX86()) {
1758         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1759         // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1760         // so we can pool them if needed.
1761         signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1762     }
1763     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1764     patchpoint->append(arg, ValueRep::SomeRegister);
1765     if (isX86()) {
1766         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1767         patchpoint->numFPScratchRegisters = 1;
1768     }
1769     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1770         AllowMacroScratchRegisterUsage allowScratch(jit);
1771         FPRReg scratch = InvalidFPRReg;
1772         FPRReg constant = InvalidFPRReg;
1773         if (isX86()) {
1774             scratch = params.fpScratch(0);
1775             constant = params[2].fpr();
1776         }
1777         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1778     });
1779     patchpoint->effects = Effects::none();
1780     result = patchpoint;
1781     return { };
1782 }
1783
1784 } } // namespace JSC::Wasm
1785
1786 #include "WasmB3IRGeneratorInlines.h"
1787
1788 #endif // ENABLE(WEBASSEMBLY)