WebAssembly: Make to a compilation API that allows for multi-VM concurrent compilatio...
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "AllowMacroScratchRegisterUsageIf.h"
32 #include "B3BasicBlockInlines.h"
33 #include "B3CCallValue.h"
34 #include "B3Compile.h"
35 #include "B3ConstPtrValue.h"
36 #include "B3FixSSA.h"
37 #include "B3Generate.h"
38 #include "B3InsertionSet.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3SwitchValue.h"
42 #include "B3Validate.h"
43 #include "B3ValueInlines.h"
44 #include "B3ValueKey.h"
45 #include "B3Variable.h"
46 #include "B3VariableValue.h"
47 #include "B3WasmAddressValue.h"
48 #include "B3WasmBoundsCheckValue.h"
49 #include "JSCInlines.h"
50 #include "JSWebAssemblyInstance.h"
51 #include "JSWebAssemblyModule.h"
52 #include "JSWebAssemblyRuntimeError.h"
53 #include "VirtualRegister.h"
54 #include "WasmCallingConvention.h"
55 #include "WasmContext.h"
56 #include "WasmExceptionType.h"
57 #include "WasmFunctionParser.h"
58 #include "WasmMemory.h"
59 #include "WasmOpcodeOrigin.h"
60 #include "WasmThunks.h"
61 #include <wtf/Optional.h>
62
63 void dumpProcedure(void* ptr)
64 {
65     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
66     proc->dump(WTF::dataFile());
67 }
68
69 namespace JSC { namespace Wasm {
70
71 using namespace B3;
72
73 namespace {
74 const bool verbose = false;
75 }
76
77 class B3IRGenerator {
78 public:
79     struct ControlData {
80         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
81             : blockType(type)
82             , continuation(continuation)
83             , special(special)
84         {
85             if (signature != Void)
86                 result.append(proc.addVariable(toB3Type(signature)));
87         }
88
89         ControlData()
90         {
91         }
92
93         void dump(PrintStream& out) const
94         {
95             switch (type()) {
96             case BlockType::If:
97                 out.print("If:       ");
98                 break;
99             case BlockType::Block:
100                 out.print("Block:    ");
101                 break;
102             case BlockType::Loop:
103                 out.print("Loop:     ");
104                 break;
105             case BlockType::TopLevel:
106                 out.print("TopLevel: ");
107                 break;
108             }
109             out.print("Continuation: ", *continuation, ", Special: ");
110             if (special)
111                 out.print(*special);
112             else
113                 out.print("None");
114         }
115
116         BlockType type() const { return blockType; }
117
118         bool hasNonVoidSignature() const { return result.size(); }
119
120         BasicBlock* targetBlockForBranch()
121         {
122             if (type() == BlockType::Loop)
123                 return special;
124             return continuation;
125         }
126
127         void convertIfToBlock()
128         {
129             ASSERT(type() == BlockType::If);
130             blockType = BlockType::Block;
131             special = nullptr;
132         }
133
134     private:
135         friend class B3IRGenerator;
136         BlockType blockType;
137         BasicBlock* continuation;
138         BasicBlock* special;
139         Vector<Variable*, 1> result;
140     };
141
142     typedef Value* ExpressionType;
143     typedef ControlData ControlType;
144     typedef Vector<ExpressionType, 1> ExpressionList;
145     typedef Vector<Variable*, 1> ResultList;
146     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
147
148     static constexpr ExpressionType emptyExpression = nullptr;
149
150     typedef String ErrorType;
151     typedef UnexpectedType<ErrorType> UnexpectedResult;
152     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
153     typedef Expected<void, ErrorType> PartialResult;
154     template <typename ...Args>
155     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
156     {
157         using namespace FailureHelper; // See ADL comment in WasmParser.h.
158         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
159     }
160 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
161         if (UNLIKELY(condition))                  \
162             return fail(__VA_ARGS__);             \
163     } while (0)
164
165     B3IRGenerator(const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode);
166
167     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
168     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
169     ExpressionType addConstant(Type, uint64_t);
170
171     // Locals
172     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
173     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
174
175     // Globals
176     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
177     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
178
179     // Memory
180     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
181     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
182     PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
183     PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
184
185     // Basic operators
186     template<OpType>
187     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
188     template<OpType>
189     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
190     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
191
192     // Control flow
193     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
194     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
195     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
196     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
197     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
198     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
199
200     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
201     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
202     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
203     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
204     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
205
206     // Calls
207     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
208     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
209     PartialResult WARN_UNUSED_RETURN addUnreachable();
210
211     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
212     void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
213
214     Value* constant(B3::Type, uint64_t bits);
215     void insertConstants();
216
217 private:
218     void emitExceptionCheck(CCallHelpers&, ExceptionType);
219
220     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
221     B3::Kind memoryKind(B3::Opcode memoryOp);
222     ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
223     void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
224
225     void unify(Variable* target, const ExpressionType source);
226     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
227
228     void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
229
230     Value* materializeWasmContext(Procedure&, BasicBlock*);
231     void restoreWasmContext(Procedure&, BasicBlock*, Value*);
232     void restoreWebAssemblyGlobalState(const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
233
234     Origin origin();
235
236     FunctionParser<B3IRGenerator>* m_parser;
237     const ModuleInformation& m_info;
238     MemoryMode m_mode;
239     Procedure& m_proc;
240     BasicBlock* m_currentBlock;
241     Vector<Variable*> m_locals;
242     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
243     HashMap<ValueKey, Value*> m_constantPool;
244     InsertionSet m_constantInsertionValues;
245     GPRReg m_memoryBaseGPR;
246     GPRReg m_memorySizeGPR { InvalidGPRReg };
247     GPRReg m_wasmContextGPR;
248     Value* m_instanceValue; // FIXME: make this lazy https://bugs.webkit.org/show_bug.cgi?id=169792
249 };
250
251 Value* B3IRGenerator::materializeWasmContext(Procedure& proc, BasicBlock* block)
252 {
253     if (useFastTLSForContext()) {
254         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, pointerType(), Origin());
255         if (CCallHelpers::loadWasmContextNeedsMacroScratchRegister())
256             patchpoint->clobber(RegisterSet::macroScratchRegisters());
257         patchpoint->setGenerator(
258             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
259                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextNeedsMacroScratchRegister());
260                 jit.loadWasmContext(params[0].gpr());
261             });
262         return patchpoint;
263     }
264
265     // FIXME: Because WasmToWasm call clobbers wasmContext register and does not restore it, we need to restore it in the caller side.
266     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
267     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, pointerType(), Origin());
268     patchpoint->effects.writesPinned = false;
269     patchpoint->effects.readsPinned = true;
270     patchpoint->resultConstraint = ValueRep::reg(m_wasmContextGPR);
271     patchpoint->setGenerator([] (CCallHelpers&, const StackmapGenerationParams&) { });
272     return patchpoint;
273 }
274
275 void B3IRGenerator::restoreWasmContext(Procedure& proc, BasicBlock* block, Value* arg)
276 {
277     if (useFastTLSForContext()) {
278         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
279         if (CCallHelpers::storeWasmContextNeedsMacroScratchRegister())
280             patchpoint->clobber(RegisterSet::macroScratchRegisters());
281         patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
282         patchpoint->setGenerator(
283             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
284                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextNeedsMacroScratchRegister());
285                 jit.storeWasmContext(params[0].gpr());
286             });
287         return;
288     }
289
290     // FIXME: Because WasmToWasm call clobbers wasmContext register and does not restore it, we need to restore it in the caller side.
291     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
292     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
293     Effects effects = Effects::none();
294     effects.writesPinned = true;
295     effects.reads = B3::HeapRange::top();
296     patchpoint->effects = effects;
297     patchpoint->clobberLate(RegisterSet(m_wasmContextGPR));
298     patchpoint->append(m_instanceValue, ValueRep::SomeRegister);
299     GPRReg wasmContextGPR = m_wasmContextGPR;
300     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
301         jit.move(param[0].gpr(), wasmContextGPR);
302     });
303 }
304
305 B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode)
306     : m_info(info)
307     , m_mode(mode)
308     , m_proc(procedure)
309     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
310     , m_constantInsertionValues(m_proc)
311 {
312     m_currentBlock = m_proc.addBlock();
313
314     // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
315     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
316
317     m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
318     m_proc.pinRegister(m_memoryBaseGPR);
319
320     m_wasmContextGPR = pinnedRegs.wasmContextPointer;
321     if (!useFastTLSForContext())
322         m_proc.pinRegister(m_wasmContextGPR);
323
324     if (mode != MemoryMode::Signaling) {
325         ASSERT(!pinnedRegs.sizeRegisters[0].sizeOffset);
326         m_memorySizeGPR = pinnedRegs.sizeRegisters[0].sizeRegister;
327         for (const PinnedSizeRegisterInfo& regInfo : pinnedRegs.sizeRegisters)
328             m_proc.pinRegister(regInfo.sizeRegister);
329     }
330
331     if (info.memory) {
332         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
333             AllowMacroScratchRegisterUsage allowScratch(jit);
334             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
335             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
336         });
337     }
338
339     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
340
341     m_instanceValue = materializeWasmContext(m_proc, m_currentBlock);
342 }
343
344 void B3IRGenerator::restoreWebAssemblyGlobalState(const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
345 {
346     restoreWasmContext(proc, block, instance);
347
348     if (!!memory) {
349         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
350         RegisterSet clobbers;
351         clobbers.set(pinnedRegs->baseMemoryPointer);
352         for (auto info : pinnedRegs->sizeRegisters)
353             clobbers.set(info.sizeRegister);
354
355         B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
356         Effects effects = Effects::none();
357         effects.writesPinned = true;
358         effects.reads = B3::HeapRange::top();
359         patchpoint->effects = effects;
360         patchpoint->clobber(clobbers);
361
362         patchpoint->append(instance, ValueRep::SomeRegister);
363
364         patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
365             GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
366             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
367             const auto& sizeRegs = pinnedRegs->sizeRegisters;
368             ASSERT(sizeRegs.size() >= 1);
369             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
370             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfSize()), sizeRegs[0].sizeRegister);
371             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfMemory()), baseMemory);
372             for (unsigned i = 1; i < sizeRegs.size(); ++i)
373                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
374         });
375     }
376 }
377
378 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
379 {
380     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
381     auto jumpToExceptionStub = jit.jump();
382
383     jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
384         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
385     });
386 }
387
388 Value* B3IRGenerator::constant(B3::Type type, uint64_t bits)
389 {
390     auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
391         Value* result = m_proc.addConstant(origin(), type, bits);
392         m_constantInsertionValues.insertValue(0, result);
393         return result;
394     });
395     return result.iterator->value;
396 }
397
398 void B3IRGenerator::insertConstants()
399 {
400     m_constantInsertionValues.execute(m_proc.at(0));
401 }
402
403 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
404 {
405     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
406
407     for (uint32_t i = 0; i < count; ++i) {
408         Variable* local = m_proc.addVariable(toB3Type(type));
409         m_locals.uncheckedAppend(local);
410         m_currentBlock->appendNew<VariableValue>(m_proc, Set, origin(), local, addConstant(type, 0));
411     }
412     return { };
413 }
414
415 auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
416 {
417     ASSERT(!m_locals.size());
418     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
419
420     m_locals.grow(signature.argumentCount());
421     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
422         [=] (ExpressionType argument, unsigned i) {
423             Variable* argumentVariable = m_proc.addVariable(argument->type());
424             m_locals[i] = argumentVariable;
425             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
426         });
427     return { };
428 }
429
430 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
431 {
432     ASSERT(m_locals[index]);
433     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
434     return { };
435 }
436
437 auto B3IRGenerator::addUnreachable() -> PartialResult
438 {
439     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
440     unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
441         this->emitExceptionCheck(jit, ExceptionType::Unreachable);
442     });
443     unreachable->effects.terminal = true;
444     return { };
445 }
446
447 auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
448 {
449     int32_t (*growMemory) (Context*, int32_t) = [] (Context* wasmContext, int32_t delta) -> int32_t {
450         VM& vm = *wasmContext->vm();
451         auto scope = DECLARE_THROW_SCOPE(vm);
452
453         JSWebAssemblyMemory* wasmMemory = wasmContext->memory();
454
455         if (delta < 0)
456             return -1;
457
458         bool shouldThrowExceptionsOnFailure = false;
459         // grow() does not require ExecState* if it doesn't throw exceptions.
460         ExecState* exec = nullptr; 
461         PageCount result = wasmMemory->grow(vm, exec, static_cast<uint32_t>(delta), shouldThrowExceptionsOnFailure);
462         RELEASE_ASSERT(!scope.exception());
463         if (!result)
464             return -1;
465
466         return result.pageCount();
467     };
468
469     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
470         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(growMemory)),
471         m_instanceValue, delta);
472
473     restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, m_currentBlock);
474
475     return { };
476 }
477
478 auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
479 {
480     Value* memoryObject = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfMemory());
481
482     static_assert(sizeof(decltype(static_cast<JSWebAssemblyInstance*>(nullptr)->memory()->memory().size())) == sizeof(uint64_t), "codegen relies on this size");
483     Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), memoryObject, JSWebAssemblyMemory::offsetOfSize());
484     
485     constexpr uint32_t shiftValue = 16;
486     static_assert(PageCount::pageSize == 1 << shiftValue, "This must hold for the code below to be correct.");
487     Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
488         size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
489
490     result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
491
492     return { };
493 }
494
495 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
496 {
497     ASSERT(m_locals[index]);
498     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
499     return { };
500 }
501
502 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
503 {
504     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
505     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, index * sizeof(Register));
506     return { };
507 }
508
509 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
510 {
511     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
512     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
513     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, index * sizeof(Register));
514     return { };
515 }
516
517 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
518 {
519     ASSERT(m_memoryBaseGPR);
520     if (m_mode == MemoryMode::BoundsChecking) {
521         ASSERT(m_memorySizeGPR);
522         ASSERT(sizeOfOperation + offset > offset);
523         m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
524     }
525     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
526     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
527 }
528
529 inline uint32_t sizeOfLoadOp(LoadOpType op)
530 {
531     switch (op) {
532     case LoadOpType::I32Load8S:
533     case LoadOpType::I32Load8U:
534     case LoadOpType::I64Load8S:
535     case LoadOpType::I64Load8U:
536         return 1;
537     case LoadOpType::I32Load16S:
538     case LoadOpType::I64Load16S:
539     case LoadOpType::I32Load16U:
540     case LoadOpType::I64Load16U:
541         return 2;
542     case LoadOpType::I32Load:
543     case LoadOpType::I64Load32S:
544     case LoadOpType::I64Load32U:
545     case LoadOpType::F32Load:
546         return 4;
547     case LoadOpType::I64Load:
548     case LoadOpType::F64Load:
549         return 8;
550     }
551     RELEASE_ASSERT_NOT_REACHED();
552 }
553
554 inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
555 {
556     if (m_mode == MemoryMode::Signaling)
557         return trapping(memoryOp);
558     return memoryOp;
559 }
560
561 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t offset)
562 {
563     switch (op) {
564     case LoadOpType::I32Load8S: {
565         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
566     }
567
568     case LoadOpType::I64Load8S: {
569         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
570         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
571     }
572
573     case LoadOpType::I32Load8U: {
574         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
575     }
576
577     case LoadOpType::I64Load8U: {
578         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
579         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
580     }
581
582     case LoadOpType::I32Load16S: {
583         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
584     }
585     case LoadOpType::I64Load16S: {
586         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
587         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
588     }
589
590     case LoadOpType::I32Load: {
591         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
592     }
593
594     case LoadOpType::I64Load32U: {
595         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
596         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
597     }
598
599     case LoadOpType::I64Load32S: {
600         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
601         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
602     }
603
604     case LoadOpType::I64Load: {
605         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
606     }
607
608     case LoadOpType::F32Load: {
609         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
610     }
611
612     case LoadOpType::F64Load: {
613         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
614     }
615
616     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
617     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
618     case LoadOpType::I32Load16U: {
619         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
620         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
621             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
622     }
623     case LoadOpType::I64Load16U: {
624         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
625         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), value,
626             m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0x0000ffff));
627
628         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), partialResult);
629     }
630     }
631     RELEASE_ASSERT_NOT_REACHED();
632 }
633
634 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
635 {
636     ASSERT(pointer->type() == Int32);
637
638     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
639         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
640         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
641         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
642         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
643             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
644         });
645
646         switch (op) {
647         case LoadOpType::I32Load8S:
648         case LoadOpType::I32Load16S:
649         case LoadOpType::I32Load:
650         case LoadOpType::I32Load16U:
651         case LoadOpType::I32Load8U:
652             result = constant(Int32, 0);
653             break;
654         case LoadOpType::I64Load8S:
655         case LoadOpType::I64Load8U:
656         case LoadOpType::I64Load16S:
657         case LoadOpType::I64Load32U:
658         case LoadOpType::I64Load32S:
659         case LoadOpType::I64Load:
660         case LoadOpType::I64Load16U:
661             result = constant(Int64, 0);
662             break;
663         case LoadOpType::F32Load:
664             result = constant(Float, 0);
665             break;
666         case LoadOpType::F64Load:
667             result = constant(Double, 0);
668             break;
669         }
670
671     } else
672         result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
673
674     return { };
675 }
676
677 inline uint32_t sizeOfStoreOp(StoreOpType op)
678 {
679     switch (op) {
680     case StoreOpType::I32Store8:
681     case StoreOpType::I64Store8:
682         return 1;
683     case StoreOpType::I32Store16:
684     case StoreOpType::I64Store16:
685         return 2;
686     case StoreOpType::I32Store:
687     case StoreOpType::I64Store32:
688     case StoreOpType::F32Store:
689         return 4;
690     case StoreOpType::I64Store:
691     case StoreOpType::F64Store:
692         return 8;
693     }
694     RELEASE_ASSERT_NOT_REACHED();
695 }
696
697
698 inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset)
699 {
700     switch (op) {
701     case StoreOpType::I64Store8:
702         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
703         FALLTHROUGH;
704
705     case StoreOpType::I32Store8:
706         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
707         return;
708
709     case StoreOpType::I64Store16:
710         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
711         FALLTHROUGH;
712
713     case StoreOpType::I32Store16:
714         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
715         return;
716
717     case StoreOpType::I64Store32:
718         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
719         FALLTHROUGH;
720
721     case StoreOpType::I64Store:
722     case StoreOpType::I32Store:
723     case StoreOpType::F32Store:
724     case StoreOpType::F64Store:
725         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
726         return;
727     }
728     RELEASE_ASSERT_NOT_REACHED();
729 }
730
731 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
732 {
733     ASSERT(pointer->type() == Int32);
734
735     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
736         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
737         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
738         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
739         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
740             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
741         });
742     } else
743         emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
744
745     return { };
746 }
747
748 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
749 {
750     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
751     return { };
752 }
753
754 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
755 {
756     return constant(toB3Type(type), value);
757 }
758
759 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
760 {
761     return ControlData(m_proc, signature, BlockType::TopLevel, m_proc.addBlock());
762 }
763
764 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
765 {
766     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
767 }
768
769 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
770 {
771     BasicBlock* body = m_proc.addBlock();
772     BasicBlock* continuation = m_proc.addBlock();
773     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
774     body->addPredecessor(m_currentBlock);
775     m_currentBlock = body;
776     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
777 }
778
779 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
780 {
781     // FIXME: This needs to do some kind of stack passing.
782
783     BasicBlock* taken = m_proc.addBlock();
784     BasicBlock* notTaken = m_proc.addBlock();
785     BasicBlock* continuation = m_proc.addBlock();
786
787     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
788     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
789     taken->addPredecessor(m_currentBlock);
790     notTaken->addPredecessor(m_currentBlock);
791
792     m_currentBlock = taken;
793     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
794     return { };
795 }
796
797 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
798 {
799     unifyValuesWithBlock(currentStack, data.result);
800     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
801     return addElseToUnreachable(data);
802 }
803
804 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
805 {
806     ASSERT(data.type() == BlockType::If);
807     m_currentBlock = data.special;
808     data.convertIfToBlock();
809     return { };
810 }
811
812 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
813 {
814     ASSERT(returnValues.size() <= 1);
815     if (returnValues.size())
816         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
817     else
818         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
819     return { };
820 }
821
822 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
823 {
824     if (data.type() != BlockType::Loop)
825         unifyValuesWithBlock(returnValues, data.result);
826
827     BasicBlock* target = data.targetBlockForBranch();
828     if (condition) {
829         BasicBlock* continuation = m_proc.addBlock();
830         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
831         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
832         target->addPredecessor(m_currentBlock);
833         continuation->addPredecessor(m_currentBlock);
834         m_currentBlock = continuation;
835     } else {
836         m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
837         target->addPredecessor(m_currentBlock);
838     }
839
840     return { };
841 }
842
843 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
844 {
845     for (size_t i = 0; i < targets.size(); ++i)
846         unifyValuesWithBlock(expressionStack, targets[i]->result);
847     unifyValuesWithBlock(expressionStack, defaultTarget.result);
848
849     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
850     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
851     for (size_t i = 0; i < targets.size(); ++i)
852         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
853
854     return { };
855 }
856
857 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
858 {
859     ControlData& data = entry.controlData;
860
861     unifyValuesWithBlock(expressionStack, data.result);
862     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
863     data.continuation->addPredecessor(m_currentBlock);
864
865     return addEndToUnreachable(entry);
866 }
867
868
869 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
870 {
871     ControlData& data = entry.controlData;
872     m_currentBlock = data.continuation;
873
874     if (data.type() == BlockType::If) {
875         data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
876         m_currentBlock->addPredecessor(data.special);
877     }
878
879     for (Variable* result : data.result)
880         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), result));
881
882     // TopLevel does not have any code after this so we need to make sure we emit a return here.
883     if (data.type() == BlockType::TopLevel)
884         return addReturn(entry.controlData, entry.enclosedExpressionStack);
885
886     return { };
887 }
888
889 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
890 {
891     ASSERT(signature.argumentCount() == args.size());
892
893     Type returnType = signature.returnType();
894     Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
895
896     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
897         // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
898         Value* functionImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfImportFunction(functionIndex));
899         Value* jsTypeOfImport = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin(), functionImport, JSCell::typeInfoTypeOffset());
900         Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), jsTypeOfImport, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), WebAssemblyFunctionType));
901
902         BasicBlock* isWasmBlock = m_proc.addBlock();
903         BasicBlock* isJSBlock = m_proc.addBlock();
904         BasicBlock* continuation = m_proc.addBlock();
905         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isJSBlock));
906
907         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
908             [=] (PatchpointValue* patchpoint) {
909                 patchpoint->effects.writesPinned = true;
910                 patchpoint->effects.readsPinned = true;
911                 // We need to clobber all potential pinned registers since we might be leaving the instance.
912                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
913                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
914                     AllowMacroScratchRegisterUsage allowScratch(jit);
915                     CCallHelpers::Call call = jit.call();
916                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
917                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex });
918                     });
919                 });
920             });
921         UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
922         isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
923
924         // FIXME: Lets remove this indirection by creating a PIC friendly IC
925         // for calls out to JS. This shouldn't be that hard to do. We could probably
926         // implement the IC to be over Wasm::Context*.
927         // https://bugs.webkit.org/show_bug.cgi?id=170375
928         Value* codeBlock = isJSBlock->appendNew<MemoryValue>(m_proc,
929             Load, pointerType(), origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfCodeBlock());
930         Value* jumpDestination = isJSBlock->appendNew<MemoryValue>(m_proc,
931             Load, pointerType(), origin(), codeBlock, JSWebAssemblyCodeBlock::offsetOfImportWasmToJSStub(functionIndex));
932         Value* jsCallResult = wasmCallingConvention().setupCall(m_proc, isJSBlock, origin(), args, toB3Type(returnType),
933             [&] (PatchpointValue* patchpoint) {
934                 patchpoint->effects.writesPinned = true;
935                 patchpoint->effects.readsPinned = true;
936                 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
937                 // We need to clobber all potential pinned registers since we might be leaving the instance.
938                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
939                 patchpoint->setGenerator([functionIndex, returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
940                     AllowMacroScratchRegisterUsage allowScratch(jit);
941                     jit.call(params[returnType == Void ? 0 : 1].gpr());
942                 });
943             });
944         UpsilonValue* jsCallResultUpsilon = returnType == Void ? nullptr : isJSBlock->appendNew<UpsilonValue>(m_proc, origin(), jsCallResult);
945         isJSBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
946
947         m_currentBlock = continuation;
948
949         if (returnType == Void)
950             result = nullptr;
951         else {
952             result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
953             wasmCallResultUpsilon->setPhi(result);
954             jsCallResultUpsilon->setPhi(result);
955         }
956
957         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
958         restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, continuation);
959     } else {
960         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
961             [=] (PatchpointValue* patchpoint) {
962                 patchpoint->effects.writesPinned = true;
963                 patchpoint->effects.readsPinned = true;
964
965                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
966                     AllowMacroScratchRegisterUsage allowScratch(jit);
967                     CCallHelpers::Call call = jit.call();
968                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
969                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex });
970                     });
971                 });
972             });
973     }
974
975     return { };
976 }
977
978 auto B3IRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
979 {
980     ExpressionType calleeIndex = args.takeLast();
981     ASSERT(signature.argumentCount() == args.size());
982
983     ExpressionType callableFunctionBuffer;
984     ExpressionType callableFunctionBufferSize;
985     {
986         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
987             m_instanceValue, JSWebAssemblyInstance::offsetOfTable());
988         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
989             table, JSWebAssemblyTable::offsetOfFunctions());
990         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
991             table, JSWebAssemblyTable::offsetOfSize());
992     }
993
994     // Check the index we are looking for is valid.
995     {
996         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
997             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferSize));
998
999         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1000             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1001         });
1002     }
1003
1004     // Compute the offset in the table index space we are looking for.
1005     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1006         m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex),
1007         constant(pointerType(), sizeof(CallableFunction)));
1008     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1009
1010     // Check that the CallableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1011     static_assert(sizeof(CallableFunction::signatureIndex) == sizeof(uint32_t), "Load codegen assumes i32");
1012     ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signatureIndex));
1013     {
1014         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1015             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1016                 calleeSignatureIndex,
1017                 m_currentBlock->appendNew<Const32Value>(m_proc, origin(), Signature::invalidIndex)));
1018
1019         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1020             this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1021         });
1022     }
1023
1024     // Check the signature matches the value we expect.
1025     {
1026         ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const32Value>(m_proc, origin(), SignatureInformation::get(signature));
1027         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1028             m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1029
1030         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1031             this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1032         });
1033     }
1034
1035     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
1036
1037     Type returnType = signature.returnType();
1038     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1039         [=] (PatchpointValue* patchpoint) {
1040             patchpoint->effects.writesPinned = true;
1041             patchpoint->effects.readsPinned = true;
1042             // We need to clobber all potential pinned registers since we might be leaving the instance.
1043             patchpoint->clobberLate(PinnedRegisterInfo::get().toSave());
1044
1045             patchpoint->append(calleeCode, ValueRep::SomeRegister);
1046             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1047                 AllowMacroScratchRegisterUsage allowScratch(jit);
1048                 jit.call(params[returnType == Void ? 0 : 1].gpr());
1049             });
1050         });
1051
1052     // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1053     restoreWebAssemblyGlobalState(m_info.memory, m_instanceValue, m_proc, m_currentBlock);
1054
1055     return { };
1056 }
1057
1058 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
1059 {
1060     m_currentBlock->appendNew<VariableValue>(m_proc, Set, origin(), variable, source);
1061 }
1062
1063 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
1064 {
1065     ASSERT(result.size() <= resultStack.size());
1066
1067     for (size_t i = 0; i < result.size(); ++i)
1068         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1069 }
1070
1071 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1072 {
1073     dataLog(comma, "ExpressionStack:");
1074     for (const auto& expression : expressionStack)
1075         dataLog(comma, *expression);
1076 }
1077
1078 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1079 {
1080     dataLogLn("Constants:");
1081     for (const auto& constant : m_constantPool)
1082         dataLogLn(deepDump(m_proc, constant.value));
1083
1084     dataLogLn("Processing Graph:");
1085     dataLog(m_proc);
1086     dataLogLn("With current block:", *m_currentBlock);
1087     dataLogLn("Control stack:");
1088     ASSERT(controlStack.size());
1089     for (size_t i = controlStack.size(); i--;) {
1090         dataLog("  ", controlStack[i].controlData, ": ");
1091         CommaPrinter comma(", ", "");
1092         dumpExpressionStack(comma, *expressionStack);
1093         expressionStack = &controlStack[i].enclosedExpressionStack;
1094         dataLogLn();
1095     }
1096     dataLogLn();
1097 }
1098
1099 static void createJSToWasmWrapper(CompilationContext& compilationContext, WasmInternalFunction& function, const Signature& signature, const ModuleInformation& info, MemoryMode mode)
1100 {
1101     CCallHelpers& jit = *compilationContext.jsEntrypointJIT;
1102
1103     jit.emitFunctionPrologue();
1104
1105     // FIXME Stop using 0 as codeBlocks. https://bugs.webkit.org/show_bug.cgi?id=165321
1106     jit.store64(CCallHelpers::TrustedImm64(0), CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::codeBlock * static_cast<int>(sizeof(Register))));
1107     MacroAssembler::DataLabelPtr calleeMoveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), GPRInfo::nonPreservedNonReturnGPR);
1108     jit.storePtr(GPRInfo::nonPreservedNonReturnGPR, CCallHelpers::Address(GPRInfo::callFrameRegister, CallFrameSlot::callee * static_cast<int>(sizeof(Register))));
1109     CodeLocationDataLabelPtr* linkedCalleeMove = &function.jsToWasmCalleeMoveLocation;
1110     jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1111         *linkedCalleeMove = linkBuffer.locationOf(calleeMoveLocation);
1112     });
1113
1114     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1115     RegisterSet toSave = pinnedRegs.toSave(mode);
1116
1117 #if !ASSERT_DISABLED
1118     unsigned toSaveSize = toSave.numberOfSetGPRs();
1119     // They should all be callee saves.
1120     toSave.filter(RegisterSet::calleeSaveRegisters());
1121     ASSERT(toSave.numberOfSetGPRs() == toSaveSize);
1122 #endif
1123
1124     RegisterAtOffsetList registersToSpill(toSave, RegisterAtOffsetList::OffsetBaseType::FramePointerBased);
1125     function.jsToWasmEntrypoint.calleeSaveRegisters = registersToSpill;
1126
1127     unsigned totalFrameSize = registersToSpill.size() * sizeof(void*);
1128     totalFrameSize += WasmCallingConvention::headerSizeInBytes();
1129     totalFrameSize -= sizeof(CallerFrameAndPC);
1130     unsigned numGPRs = 0;
1131     unsigned numFPRs = 0;
1132     for (unsigned i = 0; i < signature.argumentCount(); i++) {
1133         switch (signature.argument(i)) {
1134         case Wasm::I64:
1135         case Wasm::I32:
1136             if (numGPRs >= wasmCallingConvention().m_gprArgs.size())
1137                 totalFrameSize += sizeof(void*);
1138             ++numGPRs;
1139             break;
1140         case Wasm::F32:
1141         case Wasm::F64:
1142             if (numFPRs >= wasmCallingConvention().m_fprArgs.size())
1143                 totalFrameSize += sizeof(void*);
1144             ++numFPRs;
1145             break;
1146         default:
1147             RELEASE_ASSERT_NOT_REACHED();
1148         }
1149     }
1150
1151     totalFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), totalFrameSize);
1152     jit.subPtr(MacroAssembler::TrustedImm32(totalFrameSize), MacroAssembler::stackPointerRegister);
1153
1154     // We save all these registers regardless of having a memory or not.
1155     // The reason is that we use one of these as a scratch. That said,
1156     // almost all real wasm programs use memory, so it's not really
1157     // worth optimizing for the case that they don't.
1158     for (const RegisterAtOffset& regAtOffset : registersToSpill) {
1159         GPRReg reg = regAtOffset.reg().gpr();
1160         ptrdiff_t offset = regAtOffset.offset();
1161         jit.storePtr(reg, CCallHelpers::Address(GPRInfo::callFrameRegister, offset));
1162     }
1163
1164     GPRReg wasmContextGPR = pinnedRegs.wasmContextPointer;
1165
1166     {
1167         CCallHelpers::Address calleeFrame = CCallHelpers::Address(MacroAssembler::stackPointerRegister, -static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC)));
1168         numGPRs = 0;
1169         numFPRs = 0;
1170         // We're going to set the pinned registers after this. So
1171         // we can use this as a scratch for now since we saved it above.
1172         GPRReg scratchReg = pinnedRegs.baseMemoryPointer;
1173
1174         ptrdiff_t jsOffset = CallFrameSlot::thisArgument * sizeof(EncodedJSValue);
1175
1176         // vmEntryToWasm passes Wasm::Context* as the first JS argument when we're
1177         // not using fast TLS to hold the Wasm::Context*.
1178         if (!useFastTLSForContext()) {
1179             jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmContextGPR);
1180             jsOffset += sizeof(EncodedJSValue);
1181         }
1182
1183         ptrdiff_t wasmOffset = CallFrame::headerSizeInRegisters * sizeof(void*);
1184         for (unsigned i = 0; i < signature.argumentCount(); i++) {
1185             switch (signature.argument(i)) {
1186             case Wasm::I32:
1187             case Wasm::I64:
1188                 if (numGPRs >= wasmCallingConvention().m_gprArgs.size()) {
1189                     if (signature.argument(i) == Wasm::I32) {
1190                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1191                         jit.store32(scratchReg, calleeFrame.withOffset(wasmOffset));
1192                     } else {
1193                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1194                         jit.store64(scratchReg, calleeFrame.withOffset(wasmOffset));
1195                     }
1196                     wasmOffset += sizeof(void*);
1197                 } else {
1198                     if (signature.argument(i) == Wasm::I32)
1199                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_gprArgs[numGPRs].gpr());
1200                     else
1201                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_gprArgs[numGPRs].gpr());
1202                 }
1203                 ++numGPRs;
1204                 break;
1205             case Wasm::F32:
1206             case Wasm::F64:
1207                 if (numFPRs >= wasmCallingConvention().m_fprArgs.size()) {
1208                     if (signature.argument(i) == Wasm::F32) {
1209                         jit.load32(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1210                         jit.store32(scratchReg, calleeFrame.withOffset(wasmOffset));
1211                     } else {
1212                         jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), scratchReg);
1213                         jit.store64(scratchReg, calleeFrame.withOffset(wasmOffset));
1214                     }
1215                     wasmOffset += sizeof(void*);
1216                 } else {
1217                     if (signature.argument(i) == Wasm::F32)
1218                         jit.loadFloat(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_fprArgs[numFPRs].fpr());
1219                     else
1220                         jit.loadDouble(CCallHelpers::Address(GPRInfo::callFrameRegister, jsOffset), wasmCallingConvention().m_fprArgs[numFPRs].fpr());
1221                 }
1222                 ++numFPRs;
1223                 break;
1224             default:
1225                 RELEASE_ASSERT_NOT_REACHED();
1226             }
1227
1228             jsOffset += sizeof(EncodedJSValue);
1229         }
1230     }
1231
1232     if (!!info.memory) {
1233         GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1234
1235         if (!useFastTLSForContext())
1236             jit.loadPtr(CCallHelpers::Address(wasmContextGPR, JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
1237         else {
1238             jit.loadWasmContext(baseMemory);
1239             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyInstance::offsetOfMemory()), baseMemory);
1240         }
1241
1242         if (mode != MemoryMode::Signaling) {
1243             const auto& sizeRegs = pinnedRegs.sizeRegisters;
1244             ASSERT(sizeRegs.size() >= 1);
1245             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
1246             jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfSize()), sizeRegs[0].sizeRegister);
1247             for (unsigned i = 1; i < sizeRegs.size(); ++i)
1248                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
1249         }
1250
1251         jit.loadPtr(CCallHelpers::Address(baseMemory, JSWebAssemblyMemory::offsetOfMemory()), baseMemory);
1252     }
1253
1254     compilationContext.jsEntrypointToWasmEntrypointCall = jit.call();
1255
1256     for (const RegisterAtOffset& regAtOffset : registersToSpill) {
1257         GPRReg reg = regAtOffset.reg().gpr();
1258         ASSERT(reg != GPRInfo::returnValueGPR);
1259         ptrdiff_t offset = regAtOffset.offset();
1260         jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, offset), reg);
1261     }
1262
1263     switch (signature.returnType()) {
1264     case Wasm::F32:
1265         jit.moveFloatTo32(FPRInfo::returnValueFPR, GPRInfo::returnValueGPR);
1266         break;
1267     case Wasm::F64:
1268         jit.moveDoubleTo64(FPRInfo::returnValueFPR, GPRInfo::returnValueGPR);
1269         break;
1270     default:
1271         break;
1272     }
1273
1274     jit.emitFunctionEpilogue();
1275     jit.ret();
1276 }
1277
1278 auto B3IRGenerator::origin() -> Origin
1279 {
1280     return bitwise_cast<Origin>(OpcodeOrigin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset()));
1281 }
1282
1283 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, unsigned optLevel)
1284 {
1285     auto result = std::make_unique<WasmInternalFunction>();
1286
1287     compilationContext.jsEntrypointJIT = std::make_unique<CCallHelpers>();
1288     compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1289
1290     Procedure procedure;
1291
1292     procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1293         if (origin.data())
1294             out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1295     });
1296     
1297     // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1298     // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1299     // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1300     // optLevel=1.
1301     procedure.setNeedsUsedRegisters(false);
1302     
1303     procedure.setOptLevel(optLevel);
1304
1305     B3IRGenerator context(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode);
1306     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, info);
1307     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1308
1309     context.insertConstants();
1310
1311     procedure.resetReachability();
1312     if (!ASSERT_DISABLED)
1313         validate(procedure, "After parsing:\n");
1314
1315     dataLogIf(verbose, "Pre SSA: ", procedure);
1316     fixSSA(procedure);
1317     dataLogIf(verbose, "Post SSA: ", procedure);
1318     
1319     {
1320         B3::prepareForGeneration(procedure);
1321         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1322         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1323         result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisters();
1324     }
1325
1326     createJSToWasmWrapper(compilationContext, *result, signature, info, mode);
1327     return WTFMove(result);
1328 }
1329
1330 // Custom wasm ops. These are the ones too messy to do in wasm.json.
1331
1332 void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1333 {
1334     ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1335     const B3::Type type = left->type();
1336
1337     {
1338         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1339             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1340
1341         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1342             this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1343         });
1344     }
1345
1346     if (operation == Div) {
1347         int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1348
1349         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1350             m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1351                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1352                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1353
1354         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1355             this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1356         });
1357     }
1358 }
1359
1360 template<>
1361 auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1362 {
1363     const B3::Opcode op = Div;
1364     emitChecksForModOrDiv(op, left, right);
1365     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1366     return { };
1367 }
1368
1369 template<>
1370 auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1371 {
1372     const B3::Opcode op = Mod;
1373     emitChecksForModOrDiv(op, left, right);
1374     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1375     return { };
1376 }
1377
1378 template<>
1379 auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1380 {
1381     const B3::Opcode op = UDiv;
1382     emitChecksForModOrDiv(op, left, right);
1383     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1384     return { };
1385 }
1386
1387 template<>
1388 auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1389 {
1390     const B3::Opcode op = UMod;
1391     emitChecksForModOrDiv(op, left, right);
1392     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1393     return { };
1394 }
1395
1396 template<>
1397 auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1398 {
1399     const B3::Opcode op = Div;
1400     emitChecksForModOrDiv(op, left, right);
1401     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1402     return { };
1403 }
1404
1405 template<>
1406 auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1407 {
1408     const B3::Opcode op = Mod;
1409     emitChecksForModOrDiv(op, left, right);
1410     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1411     return { };
1412 }
1413
1414 template<>
1415 auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1416 {
1417     const B3::Opcode op = UDiv;
1418     emitChecksForModOrDiv(op, left, right);
1419     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1420     return { };
1421 }
1422
1423 template<>
1424 auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1425 {
1426     const B3::Opcode op = UMod;
1427     emitChecksForModOrDiv(op, left, right);
1428     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1429     return { };
1430 }
1431
1432 template<>
1433 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1434 {
1435     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1436     patchpoint->append(arg, ValueRep::SomeRegister);
1437     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1438         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1439     });
1440     patchpoint->effects = Effects::none();
1441     result = patchpoint;
1442     return { };
1443 }
1444
1445 template<>
1446 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1447 {
1448     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1449     patchpoint->append(arg, ValueRep::SomeRegister);
1450     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1451         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1452     });
1453     patchpoint->effects = Effects::none();
1454     result = patchpoint;
1455     return { };
1456 }
1457
1458 template<>
1459 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1460 {
1461     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1462     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1463     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1464     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1465     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1466     return { };
1467 }
1468
1469 template<>
1470 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1471 {
1472     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
1473     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
1474     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1475     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), bitwise_cast<void*>(popcount));
1476     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1477     return { };
1478 }
1479
1480 template<>
1481 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1482 {
1483     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1484     if (isX86())
1485         patchpoint->numGPScratchRegisters = 1;
1486     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1487     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1488         AllowMacroScratchRegisterUsage allowScratch(jit);
1489 #if CPU(X86_64)
1490         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1491 #else
1492         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1493 #endif
1494     });
1495     patchpoint->effects = Effects::none();
1496     result = patchpoint;
1497     return { };
1498 }
1499
1500 template<>
1501 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1502 {
1503     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1504     if (isX86())
1505         patchpoint->numGPScratchRegisters = 1;
1506     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1507     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1508         AllowMacroScratchRegisterUsage allowScratch(jit);
1509 #if CPU(X86_64)
1510         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1511 #else
1512         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1513 #endif
1514     });
1515     patchpoint->effects = Effects::none();
1516     result = patchpoint;
1517     return { };
1518 }
1519
1520 template<>
1521 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1522 {
1523     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1524     patchpoint->append(arg, ValueRep::SomeRegister);
1525     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1526         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1527     });
1528     patchpoint->effects = Effects::none();
1529     result = patchpoint;
1530     return { };
1531 }
1532
1533 template<>
1534 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1535 {
1536     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1537     patchpoint->append(arg, ValueRep::SomeRegister);
1538     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1539         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1540     });
1541     patchpoint->effects = Effects::none();
1542     result = patchpoint;
1543     return { };
1544 }
1545
1546 template<>
1547 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1548 {
1549     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1550     patchpoint->append(arg, ValueRep::SomeRegister);
1551     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1552         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1553     });
1554     patchpoint->effects = Effects::none();
1555     result = patchpoint;
1556     return { };
1557 }
1558
1559 template<>
1560 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1561 {
1562     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1563     patchpoint->append(arg, ValueRep::SomeRegister);
1564     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1565         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1566     });
1567     patchpoint->effects = Effects::none();
1568     result = patchpoint;
1569     return { };
1570 }
1571
1572 template<>
1573 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1574 {
1575     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1576     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1577     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1578         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1579         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1580     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1581     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1582     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1583         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1584     });
1585     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1586     patchpoint->append(arg, ValueRep::SomeRegister);
1587     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1588         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1589     });
1590     patchpoint->effects = Effects::none();
1591     result = patchpoint;
1592     return { };
1593 }
1594
1595 template<>
1596 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1597 {
1598     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1599     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1600     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1601         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1602         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1603     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1604     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1605     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1606         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1607     });
1608     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1609     patchpoint->append(arg, ValueRep::SomeRegister);
1610     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1611         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1612     });
1613     patchpoint->effects = Effects::none();
1614     result = patchpoint;
1615     return { };
1616 }
1617
1618
1619 template<>
1620 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1621 {
1622     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1623     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1624     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1625         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1626         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1627     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1628     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1629     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1630         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1631     });
1632     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1633     patchpoint->append(arg, ValueRep::SomeRegister);
1634     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1635         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1636     });
1637     patchpoint->effects = Effects::none();
1638     result = patchpoint;
1639     return { };
1640 }
1641
1642 template<>
1643 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1644 {
1645     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1646     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1647     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1648         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1649         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1650     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1651     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1652     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1653         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1654     });
1655     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1656     patchpoint->append(arg, ValueRep::SomeRegister);
1657     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1658         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1659     });
1660     patchpoint->effects = Effects::none();
1661     result = patchpoint;
1662     return { };
1663 }
1664
1665 template<>
1666 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1667 {
1668     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1669     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1670     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1671         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1672         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1673     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1674     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1675     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1676         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1677     });
1678     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1679     patchpoint->append(arg, ValueRep::SomeRegister);
1680     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1681         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1682     });
1683     patchpoint->effects = Effects::none();
1684     result = patchpoint;
1685     return { };
1686 }
1687
1688 template<>
1689 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1690 {
1691     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1692     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1693     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1694         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1695         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1696     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1697     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1698     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1699         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1700     });
1701
1702     Value* signBitConstant;
1703     if (isX86()) {
1704         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1705         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1706         // so we can pool them if needed.
1707         signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1708     }
1709     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1710     patchpoint->append(arg, ValueRep::SomeRegister);
1711     if (isX86()) {
1712         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1713         patchpoint->numFPScratchRegisters = 1;
1714     }
1715     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1716         AllowMacroScratchRegisterUsage allowScratch(jit);
1717         FPRReg scratch = InvalidFPRReg;
1718         FPRReg constant = InvalidFPRReg;
1719         if (isX86()) {
1720             scratch = params.fpScratch(0);
1721             constant = params[2].fpr();
1722         }
1723         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1724     });
1725     patchpoint->effects = Effects::none();
1726     result = patchpoint;
1727     return { };
1728 }
1729
1730 template<>
1731 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1732 {
1733     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1734     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1735     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1736         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1737         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1738     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1739     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1740     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1741         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1742     });
1743     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1744     patchpoint->append(arg, ValueRep::SomeRegister);
1745     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1746         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1747     });
1748     patchpoint->effects = Effects::none();
1749     result = patchpoint;
1750     return { };
1751 }
1752
1753 template<>
1754 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1755 {
1756     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1757     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1758     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1759         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1760         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1761     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1762     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1763     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1764         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1765     });
1766
1767     Value* signBitConstant;
1768     if (isX86()) {
1769         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1770         // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1771         // so we can pool them if needed.
1772         signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1773     }
1774     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1775     patchpoint->append(arg, ValueRep::SomeRegister);
1776     if (isX86()) {
1777         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1778         patchpoint->numFPScratchRegisters = 1;
1779     }
1780     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1781         AllowMacroScratchRegisterUsage allowScratch(jit);
1782         FPRReg scratch = InvalidFPRReg;
1783         FPRReg constant = InvalidFPRReg;
1784         if (isX86()) {
1785             scratch = params.fpScratch(0);
1786             constant = params[2].fpr();
1787         }
1788         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1789     });
1790     patchpoint->effects = Effects::none();
1791     result = patchpoint;
1792     return { };
1793 }
1794
1795 } } // namespace JSC::Wasm
1796
1797 #include "WasmB3IRGeneratorInlines.h"
1798
1799 #endif // ENABLE(WEBASSEMBLY)