5a5f72798ed3e31ed469d2b539cff639940da5f9
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "AllowMacroScratchRegisterUsageIf.h"
32 #include "B3BasicBlockInlines.h"
33 #include "B3CCallValue.h"
34 #include "B3Compile.h"
35 #include "B3ConstPtrValue.h"
36 #include "B3FixSSA.h"
37 #include "B3Generate.h"
38 #include "B3InsertionSet.h"
39 #include "B3SlotBaseValue.h"
40 #include "B3StackmapGenerationParams.h"
41 #include "B3SwitchValue.h"
42 #include "B3UpsilonValue.h"
43 #include "B3Validate.h"
44 #include "B3ValueInlines.h"
45 #include "B3ValueKey.h"
46 #include "B3Variable.h"
47 #include "B3VariableValue.h"
48 #include "B3WasmAddressValue.h"
49 #include "B3WasmBoundsCheckValue.h"
50 #include "JSCInlines.h"
51 #include "ScratchRegisterAllocator.h"
52 #include "VirtualRegister.h"
53 #include "WasmCallingConvention.h"
54 #include "WasmContextInlines.h"
55 #include "WasmExceptionType.h"
56 #include "WasmFunctionParser.h"
57 #include "WasmInstance.h"
58 #include "WasmMemory.h"
59 #include "WasmOMGPlan.h"
60 #include "WasmOpcodeOrigin.h"
61 #include "WasmSignatureInlines.h"
62 #include "WasmThunks.h"
63 #include <limits>
64 #include <wtf/Optional.h>
65 #include <wtf/StdLibExtras.h>
66
67 void dumpProcedure(void* ptr)
68 {
69     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
70     proc->dump(WTF::dataFile());
71 }
72
73 namespace JSC { namespace Wasm {
74
75 using namespace B3;
76
77 namespace {
78 namespace WasmB3IRGeneratorInternal {
79 static const bool verbose = false;
80 }
81 }
82
83 class B3IRGenerator {
84 public:
85     struct ControlData {
86         ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
87             : blockType(type)
88             , continuation(continuation)
89             , special(special)
90         {
91             if (signature != Void)
92                 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
93         }
94
95         ControlData()
96         {
97         }
98
99         void dump(PrintStream& out) const
100         {
101             switch (type()) {
102             case BlockType::If:
103                 out.print("If:       ");
104                 break;
105             case BlockType::Block:
106                 out.print("Block:    ");
107                 break;
108             case BlockType::Loop:
109                 out.print("Loop:     ");
110                 break;
111             case BlockType::TopLevel:
112                 out.print("TopLevel: ");
113                 break;
114             }
115             out.print("Continuation: ", *continuation, ", Special: ");
116             if (special)
117                 out.print(*special);
118             else
119                 out.print("None");
120         }
121
122         BlockType type() const { return blockType; }
123
124         bool hasNonVoidSignature() const { return result.size(); }
125
126         BasicBlock* targetBlockForBranch()
127         {
128             if (type() == BlockType::Loop)
129                 return special;
130             return continuation;
131         }
132
133         void convertIfToBlock()
134         {
135             ASSERT(type() == BlockType::If);
136             blockType = BlockType::Block;
137             special = nullptr;
138         }
139
140         using ResultList = Vector<Value*, 1>; // Value must be a Phi
141
142         ResultList resultForBranch() const
143         {
144             if (type() == BlockType::Loop)
145                 return ResultList();
146             return result;
147         }
148
149     private:
150         friend class B3IRGenerator;
151         BlockType blockType;
152         BasicBlock* continuation;
153         BasicBlock* special;
154         ResultList result;
155     };
156
157     typedef Value* ExpressionType;
158     typedef ControlData ControlType;
159     typedef Vector<ExpressionType, 1> ExpressionList;
160     typedef ControlData::ResultList ResultList;
161     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
162
163     static constexpr ExpressionType emptyExpression() { return nullptr; }
164
165     typedef String ErrorType;
166     typedef Unexpected<ErrorType> UnexpectedResult;
167     typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
168     typedef Expected<void, ErrorType> PartialResult;
169     template <typename ...Args>
170     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
171     {
172         using namespace FailureHelper; // See ADL comment in WasmParser.h.
173         return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
174     }
175 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
176         if (UNLIKELY(condition))                  \
177             return fail(__VA_ARGS__);             \
178     } while (0)
179
180     B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, CompilationMode, unsigned functionIndex, TierUpCount*, ThrowWasmException);
181
182     PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
183     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
184     ExpressionType addConstant(Type, uint64_t);
185
186     // Locals
187     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
188     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
189
190     // Globals
191     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
192     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
193
194     // Memory
195     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
196     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
197     PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
198     PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
199
200     // Basic operators
201     template<OpType>
202     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
203     template<OpType>
204     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
205     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
206
207     // Control flow
208     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
209     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
210     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
211     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
212     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
213     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
214
215     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
216     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
217     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
218     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
219     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
220
221     // Calls
222     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
223     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
224     PartialResult WARN_UNUSED_RETURN addUnreachable();
225
226     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
227     void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
228
229     Value* constant(B3::Type, uint64_t bits, Optional<Origin> = WTF::nullopt);
230     void insertConstants();
231
232     ALWAYS_INLINE void didKill(ExpressionType) { }
233
234 private:
235     void emitExceptionCheck(CCallHelpers&, ExceptionType);
236
237     void emitTierUpCheck(uint32_t decrementCount, Origin);
238
239     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
240     B3::Kind memoryKind(B3::Opcode memoryOp);
241     ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
242     void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
243
244     void unify(const ExpressionType phi, const ExpressionType source);
245     void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
246
247     void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
248
249     int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
250
251     void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*);
252     enum class RestoreCachedStackLimit { No, Yes };
253     void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
254
255     Origin origin();
256
257     FunctionParser<B3IRGenerator>* m_parser { nullptr };
258     const ModuleInformation& m_info;
259     const MemoryMode m_mode { MemoryMode::BoundsChecking };
260     const CompilationMode m_compilationMode { CompilationMode::BBQMode };
261     const unsigned m_functionIndex { UINT_MAX };
262     const TierUpCount* m_tierUp { nullptr };
263
264     Procedure& m_proc;
265     BasicBlock* m_currentBlock { nullptr };
266     Vector<Variable*> m_locals;
267     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
268     HashMap<ValueKey, Value*> m_constantPool;
269     InsertionSet m_constantInsertionValues;
270     GPRReg m_memoryBaseGPR { InvalidGPRReg };
271     GPRReg m_memorySizeGPR { InvalidGPRReg };
272     GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
273     bool m_makesCalls { false };
274
275     Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used.
276     bool m_usesInstanceValue { false };
277     Value* instanceValue()
278     {
279         m_usesInstanceValue = true;
280         return m_instanceValue;
281     }
282
283     uint32_t m_maxNumJSCallArguments { 0 };
284 };
285
286 // Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
287 int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
288 {
289     if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
290         ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
291         return 0;
292     }
293     return offset;
294 }
295
296 void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg)
297 {
298     if (Context::useFastTLS()) {
299         PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
300         if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
301             patchpoint->clobber(RegisterSet::macroScratchRegisters());
302         patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
303         patchpoint->setGenerator(
304             [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
305                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
306                 jit.storeWasmContextInstance(params[0].gpr());
307             });
308         return;
309     }
310
311     // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
312     // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
313     PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
314     Effects effects = Effects::none();
315     effects.writesPinned = true;
316     effects.reads = B3::HeapRange::top();
317     patchpoint->effects = effects;
318     patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
319     patchpoint->append(arg, ValueRep::SomeRegister);
320     GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
321     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
322         jit.move(param[0].gpr(), wasmContextInstanceGPR);
323     });
324 }
325
326 B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
327     : m_info(info)
328     , m_mode(mode)
329     , m_compilationMode(compilationMode)
330     , m_functionIndex(functionIndex)
331     , m_tierUp(tierUp)
332     , m_proc(procedure)
333     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
334     , m_constantInsertionValues(m_proc)
335 {
336     m_currentBlock = m_proc.addBlock();
337
338     // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
339     const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
340
341     m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
342     m_proc.pinRegister(m_memoryBaseGPR);
343
344     m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
345     if (!Context::useFastTLS())
346         m_proc.pinRegister(m_wasmContextInstanceGPR);
347
348     if (mode != MemoryMode::Signaling) {
349         ASSERT(!pinnedRegs.sizeRegisters[0].sizeOffset);
350         m_memorySizeGPR = pinnedRegs.sizeRegisters[0].sizeRegister;
351         for (const PinnedSizeRegisterInfo& regInfo : pinnedRegs.sizeRegisters)
352             m_proc.pinRegister(regInfo.sizeRegister);
353     }
354
355     if (throwWasmException)
356         Thunks::singleton().setThrowWasmException(throwWasmException);
357
358     if (info.memory) {
359         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
360             AllowMacroScratchRegisterUsage allowScratch(jit);
361             switch (m_mode) {
362             case MemoryMode::BoundsChecking:
363                 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
364                 break;
365             case MemoryMode::Signaling:
366                 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
367                 break;
368             }
369             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
370         });
371
372         switch (m_mode) {
373         case MemoryMode::BoundsChecking:
374             break;
375         case MemoryMode::Signaling:
376             // Most memory accesses in signaling mode don't do an explicit
377             // exception check because they can rely on fault handling to detect
378             // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
379             // thunk to exist so that it can jump to that thunk.
380             if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
381                 CRASH();
382             break;
383         }
384     }
385
386     wasmCallingConvention().setupFrameInPrologue(&compilation->calleeMoveLocation, m_proc, Origin(), m_currentBlock);
387
388     {
389         B3::Value* framePointer = m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin());
390         B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, pointerType(), Origin());
391         m_instanceValue = stackOverflowCheck;
392         stackOverflowCheck->appendSomeRegister(framePointer);
393         stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters());
394         if (!Context::useFastTLS()) {
395             // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
396             // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
397             stackOverflowCheck->effects.writesPinned = false;
398             stackOverflowCheck->effects.readsPinned = true;
399             stackOverflowCheck->resultConstraint = ValueRep::reg(m_wasmContextInstanceGPR);
400         }
401         stackOverflowCheck->numGPScratchRegisters = 2;
402         stackOverflowCheck->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
403             const Checked<int32_t> wasmFrameSize = params.proc().frameSize();
404             const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
405             const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
406                 // This allows us to elide stack checks for functions that are terminal nodes in the call
407                 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
408                 // having any such terminal node have its parent caller include some extra size in its
409                 // own check for it. The goal here is twofold:
410                 // 1. Emit less code.
411                 // 2. Try to speed things up by skipping stack checks.
412                 minimumParentCheckSize,
413                 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
414                 // spill all arguments to the stack, we ensure that a stack check here covers the
415                 // stack that such a stub would use.
416                 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
417             ));
418             const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
419             bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
420             bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
421
422             GPRReg contextInstance = Context::useFastTLS() ? params[0].gpr() : m_wasmContextInstanceGPR;
423
424             // This allows leaf functions to not do stack checks if their frame size is within
425             // certain limits since their caller would have already done the check.
426             if (needsOverflowCheck) {
427                 AllowMacroScratchRegisterUsage allowScratch(jit);
428                 GPRReg fp = params[1].gpr();
429                 GPRReg scratch1 = params.gpScratch(0);
430                 GPRReg scratch2 = params.gpScratch(1);
431
432                 if (Context::useFastTLS())
433                     jit.loadWasmContextInstance(contextInstance);
434
435                 jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2);
436                 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1);
437                 MacroAssembler::JumpList overflow;
438                 if (UNLIKELY(needUnderflowCheck))
439                     overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp));
440                 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2));
441                 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
442                     linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
443                 });
444             } else if (m_usesInstanceValue && Context::useFastTLS()) {
445                 // No overflow check is needed, but the instance values still needs to be correct.
446                 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister());
447                 jit.loadWasmContextInstance(contextInstance);
448             } else {
449                 // We said we'd return a pointer. We don't actually need to because it isn't used, but the patchpoint conservatively said it had effects (potential stack check) which prevent it from getting removed.
450             }
451         });
452     }
453
454     emitTierUpCheck(TierUpCount::functionEntryDecrement(), Origin());
455 }
456
457 void B3IRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
458 {
459     restoreWasmContextInstance(proc, block, instance);
460
461     if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
462         // The Instance caches the stack limit, but also knows where its canonical location is.
463         Value* pointerToActualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfPointerToActualStackLimit()));
464         Value* actualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), pointerToActualStackLimit);
465         block->appendNew<MemoryValue>(m_proc, Store, origin(), actualStackLimit, instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedStackLimit()));
466     }
467
468     if (!!memory) {
469         const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
470         RegisterSet clobbers;
471         clobbers.set(pinnedRegs->baseMemoryPointer);
472         for (auto info : pinnedRegs->sizeRegisters)
473             clobbers.set(info.sizeRegister);
474
475         B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
476         Effects effects = Effects::none();
477         effects.writesPinned = true;
478         effects.reads = B3::HeapRange::top();
479         patchpoint->effects = effects;
480         patchpoint->clobber(clobbers);
481
482         patchpoint->append(instance, ValueRep::SomeRegister);
483
484         patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
485             GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
486             const auto& sizeRegs = pinnedRegs->sizeRegisters;
487             ASSERT(sizeRegs.size() >= 1);
488             ASSERT(!sizeRegs[0].sizeOffset); // The following code assumes we start at 0, and calculates subsequent size registers relative to 0.
489             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), sizeRegs[0].sizeRegister);
490             jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
491             for (unsigned i = 1; i < sizeRegs.size(); ++i)
492                 jit.add64(CCallHelpers::TrustedImm32(-sizeRegs[i].sizeOffset), sizeRegs[0].sizeRegister, sizeRegs[i].sizeRegister);
493         });
494     }
495 }
496
497 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
498 {
499     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
500     auto jumpToExceptionStub = jit.jump();
501
502     jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
503         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
504     });
505 }
506
507 Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, Optional<Origin> maybeOrigin)
508 {
509     auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
510         Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits);
511         m_constantInsertionValues.insertValue(0, result);
512         return result;
513     });
514     return result.iterator->value;
515 }
516
517 void B3IRGenerator::insertConstants()
518 {
519     m_constantInsertionValues.execute(m_proc.at(0));
520 }
521
522 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
523 {
524     Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
525     totalBytesChecked += m_locals.size();
526     uint32_t totalBytes;
527     WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
528
529     for (uint32_t i = 0; i < count; ++i) {
530         Variable* local = m_proc.addVariable(toB3Type(type));
531         m_locals.uncheckedAppend(local);
532         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, constant(toB3Type(type), 0, Origin()));
533     }
534     return { };
535 }
536
537 auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
538 {
539     ASSERT(!m_locals.size());
540     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
541
542     m_locals.grow(signature.argumentCount());
543     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
544         [=] (ExpressionType argument, unsigned i) {
545             Variable* argumentVariable = m_proc.addVariable(argument->type());
546             m_locals[i] = argumentVariable;
547             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
548         });
549     return { };
550 }
551
552 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
553 {
554     ASSERT(m_locals[index]);
555     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
556     return { };
557 }
558
559 auto B3IRGenerator::addUnreachable() -> PartialResult
560 {
561     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
562     unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
563         this->emitExceptionCheck(jit, ExceptionType::Unreachable);
564     });
565     unreachable->effects.terminal = true;
566     return { };
567 }
568
569 auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
570 {
571     int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
572         instance->storeTopCallFrame(callFrame);
573
574         if (delta < 0)
575             return -1;
576
577         auto grown = instance->memory()->grow(PageCount(delta));
578         if (!grown) {
579             switch (grown.error()) {
580             case Memory::GrowFailReason::InvalidDelta:
581             case Memory::GrowFailReason::InvalidGrowSize:
582             case Memory::GrowFailReason::WouldExceedMaximum:
583             case Memory::GrowFailReason::OutOfMemory:
584                 return -1;
585             }
586             RELEASE_ASSERT_NOT_REACHED();
587         }
588
589         return grown.value().pageCount();
590     };
591
592     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
593         m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(growMemory, B3CCallPtrTag)),
594         m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, origin()), instanceValue(), delta);
595
596     restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_proc, m_currentBlock);
597
598     return { };
599 }
600
601 auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
602 {
603     static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
604     Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedMemorySize()));
605
606     constexpr uint32_t shiftValue = 16;
607     static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
608     Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
609         size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
610
611     result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
612
613     return { };
614 }
615
616 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
617 {
618     ASSERT(m_locals[index]);
619     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
620     return { };
621 }
622
623 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
624 {
625     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
626     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
627     return { };
628 }
629
630 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
631 {
632     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
633     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
634     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
635     return { };
636 }
637
638 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
639 {
640     ASSERT(m_memoryBaseGPR);
641
642     switch (m_mode) {
643     case MemoryMode::BoundsChecking: {
644         // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
645         ASSERT(m_memorySizeGPR);
646         ASSERT(sizeOfOperation + offset > offset);
647         m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), m_memorySizeGPR, pointer, sizeOfOperation + offset - 1);
648         break;
649     }
650
651     case MemoryMode::Signaling: {
652         // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
653         // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
654         // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
655         // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
656         // register + large offset access. We don't think this will be generated frequently.
657         //
658         // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
659         // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
660         // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
661         // any access equal to or greater than 4GiB will trap, no need to add the redzone.
662         if (offset >= Memory::fastMappedRedzoneBytes()) {
663             size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
664             m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
665         }
666         break;
667     }
668     }
669
670     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
671     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
672 }
673
674 inline uint32_t sizeOfLoadOp(LoadOpType op)
675 {
676     switch (op) {
677     case LoadOpType::I32Load8S:
678     case LoadOpType::I32Load8U:
679     case LoadOpType::I64Load8S:
680     case LoadOpType::I64Load8U:
681         return 1;
682     case LoadOpType::I32Load16S:
683     case LoadOpType::I64Load16S:
684     case LoadOpType::I32Load16U:
685     case LoadOpType::I64Load16U:
686         return 2;
687     case LoadOpType::I32Load:
688     case LoadOpType::I64Load32S:
689     case LoadOpType::I64Load32U:
690     case LoadOpType::F32Load:
691         return 4;
692     case LoadOpType::I64Load:
693     case LoadOpType::F64Load:
694         return 8;
695     }
696     RELEASE_ASSERT_NOT_REACHED();
697 }
698
699 inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
700 {
701     if (m_mode == MemoryMode::Signaling)
702         return trapping(memoryOp);
703     return memoryOp;
704 }
705
706 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
707 {
708     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
709
710     switch (op) {
711     case LoadOpType::I32Load8S: {
712         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
713     }
714
715     case LoadOpType::I64Load8S: {
716         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
717         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
718     }
719
720     case LoadOpType::I32Load8U: {
721         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
722     }
723
724     case LoadOpType::I64Load8U: {
725         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
726         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
727     }
728
729     case LoadOpType::I32Load16S: {
730         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
731     }
732
733     case LoadOpType::I64Load16S: {
734         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
735         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
736     }
737
738     case LoadOpType::I32Load16U: {
739         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
740     }
741
742     case LoadOpType::I64Load16U: {
743         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
744         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
745     }
746
747     case LoadOpType::I32Load: {
748         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
749     }
750
751     case LoadOpType::I64Load32U: {
752         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
753         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
754     }
755
756     case LoadOpType::I64Load32S: {
757         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
758         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
759     }
760
761     case LoadOpType::I64Load: {
762         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
763     }
764
765     case LoadOpType::F32Load: {
766         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
767     }
768
769     case LoadOpType::F64Load: {
770         return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
771     }
772     }
773     RELEASE_ASSERT_NOT_REACHED();
774 }
775
776 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
777 {
778     ASSERT(pointer->type() == Int32);
779
780     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
781         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
782         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
783         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
784         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
785             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
786         });
787
788         switch (op) {
789         case LoadOpType::I32Load8S:
790         case LoadOpType::I32Load16S:
791         case LoadOpType::I32Load:
792         case LoadOpType::I32Load16U:
793         case LoadOpType::I32Load8U:
794             result = constant(Int32, 0);
795             break;
796         case LoadOpType::I64Load8S:
797         case LoadOpType::I64Load8U:
798         case LoadOpType::I64Load16S:
799         case LoadOpType::I64Load32U:
800         case LoadOpType::I64Load32S:
801         case LoadOpType::I64Load:
802         case LoadOpType::I64Load16U:
803             result = constant(Int64, 0);
804             break;
805         case LoadOpType::F32Load:
806             result = constant(Float, 0);
807             break;
808         case LoadOpType::F64Load:
809             result = constant(Double, 0);
810             break;
811         }
812
813     } else
814         result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
815
816     return { };
817 }
818
819 inline uint32_t sizeOfStoreOp(StoreOpType op)
820 {
821     switch (op) {
822     case StoreOpType::I32Store8:
823     case StoreOpType::I64Store8:
824         return 1;
825     case StoreOpType::I32Store16:
826     case StoreOpType::I64Store16:
827         return 2;
828     case StoreOpType::I32Store:
829     case StoreOpType::I64Store32:
830     case StoreOpType::F32Store:
831         return 4;
832     case StoreOpType::I64Store:
833     case StoreOpType::F64Store:
834         return 8;
835     }
836     RELEASE_ASSERT_NOT_REACHED();
837 }
838
839
840 inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
841 {
842     int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
843
844     switch (op) {
845     case StoreOpType::I64Store8:
846         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
847         FALLTHROUGH;
848
849     case StoreOpType::I32Store8:
850         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
851         return;
852
853     case StoreOpType::I64Store16:
854         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
855         FALLTHROUGH;
856
857     case StoreOpType::I32Store16:
858         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
859         return;
860
861     case StoreOpType::I64Store32:
862         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
863         FALLTHROUGH;
864
865     case StoreOpType::I64Store:
866     case StoreOpType::I32Store:
867     case StoreOpType::F32Store:
868     case StoreOpType::F64Store:
869         m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
870         return;
871     }
872     RELEASE_ASSERT_NOT_REACHED();
873 }
874
875 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
876 {
877     ASSERT(pointer->type() == Int32);
878
879     if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
880         // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
881         // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
882         B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
883         throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
884             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
885         });
886     } else
887         emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
888
889     return { };
890 }
891
892 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
893 {
894     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
895     return { };
896 }
897
898 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
899 {
900     return constant(toB3Type(type), value);
901 }
902
903 void B3IRGenerator::emitTierUpCheck(uint32_t decrementCount, Origin origin)
904 {
905     if (!m_tierUp)
906         return;
907
908     ASSERT(m_tierUp);
909     Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(m_tierUp), origin);
910     Value* oldCountDown = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, countDownLocation);
911     Value* newCountDown = m_currentBlock->appendNew<Value>(m_proc, Sub, origin, oldCountDown, constant(Int32, decrementCount, origin));
912     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, newCountDown, countDownLocation);
913
914     PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, origin);
915     Effects effects = Effects::none();
916     // FIXME: we should have a more precise heap range for the tier up count.
917     effects.reads = B3::HeapRange::top();
918     effects.writes = B3::HeapRange::top();
919     patch->effects = effects;
920
921     patch->append(newCountDown, ValueRep::SomeRegister);
922     patch->append(oldCountDown, ValueRep::SomeRegister);
923     patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
924         MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
925         MacroAssembler::Label tierUpResume = jit.label();
926
927         params.addLatePath([=] (CCallHelpers& jit) {
928             tierUp.link(&jit);
929
930             const unsigned extraPaddingBytes = 0;
931             RegisterSet registersToSpill = { };
932             registersToSpill.add(GPRInfo::argumentGPR1);
933             unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
934
935             jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
936             MacroAssembler::Call call = jit.nearCall();
937
938             ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
939             jit.jump(tierUpResume);
940
941             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
942                 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
943
944             });
945         });
946     });
947 }
948
949 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
950 {
951     BasicBlock* body = m_proc.addBlock();
952     BasicBlock* continuation = m_proc.addBlock();
953
954     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
955
956     m_currentBlock = body;
957     emitTierUpCheck(TierUpCount::loopDecrement(), origin());
958
959     return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
960 }
961
962 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
963 {
964     return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
965 }
966
967 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
968 {
969     return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
970 }
971
972 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
973 {
974     // FIXME: This needs to do some kind of stack passing.
975
976     BasicBlock* taken = m_proc.addBlock();
977     BasicBlock* notTaken = m_proc.addBlock();
978     BasicBlock* continuation = m_proc.addBlock();
979
980     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
981     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
982     taken->addPredecessor(m_currentBlock);
983     notTaken->addPredecessor(m_currentBlock);
984
985     m_currentBlock = taken;
986     result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
987     return { };
988 }
989
990 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
991 {
992     unifyValuesWithBlock(currentStack, data.result);
993     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
994     return addElseToUnreachable(data);
995 }
996
997 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
998 {
999     ASSERT(data.type() == BlockType::If);
1000     m_currentBlock = data.special;
1001     data.convertIfToBlock();
1002     return { };
1003 }
1004
1005 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
1006 {
1007     ASSERT(returnValues.size() <= 1);
1008     if (returnValues.size())
1009         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
1010     else
1011         m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
1012     return { };
1013 }
1014
1015 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1016 {
1017     unifyValuesWithBlock(returnValues, data.resultForBranch());
1018
1019     BasicBlock* target = data.targetBlockForBranch();
1020     if (condition) {
1021         BasicBlock* continuation = m_proc.addBlock();
1022         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1023         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
1024         target->addPredecessor(m_currentBlock);
1025         continuation->addPredecessor(m_currentBlock);
1026         m_currentBlock = continuation;
1027     } else {
1028         m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
1029         target->addPredecessor(m_currentBlock);
1030     }
1031
1032     return { };
1033 }
1034
1035 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1036 {
1037     for (size_t i = 0; i < targets.size(); ++i)
1038         unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
1039     unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1040
1041     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
1042     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
1043     for (size_t i = 0; i < targets.size(); ++i)
1044         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
1045
1046     return { };
1047 }
1048
1049 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1050 {
1051     ControlData& data = entry.controlData;
1052
1053     unifyValuesWithBlock(expressionStack, data.result);
1054     m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1055     data.continuation->addPredecessor(m_currentBlock);
1056
1057     return addEndToUnreachable(entry);
1058 }
1059
1060
1061 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1062 {
1063     ControlData& data = entry.controlData;
1064     m_currentBlock = data.continuation;
1065
1066     if (data.type() == BlockType::If) {
1067         data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
1068         m_currentBlock->addPredecessor(data.special);
1069     }
1070
1071     for (Value* result : data.result) {
1072         m_currentBlock->append(result);
1073         entry.enclosedExpressionStack.append(result);
1074     }
1075
1076     // TopLevel does not have any code after this so we need to make sure we emit a return here.
1077     if (data.type() == BlockType::TopLevel)
1078         return addReturn(entry.controlData, entry.enclosedExpressionStack);
1079
1080     return { };
1081 }
1082
1083 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1084 {
1085     ASSERT(signature.argumentCount() == args.size());
1086
1087     m_makesCalls = true;
1088
1089     Type returnType = signature.returnType();
1090     Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1091
1092     if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1093         m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1094
1095         // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
1096         Value* targetInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfTargetInstance(functionIndex)));
1097         // The target instance is 0 unless the call is wasm->wasm.
1098         Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0));
1099
1100         BasicBlock* isWasmBlock = m_proc.addBlock();
1101         BasicBlock* isEmbedderBlock = m_proc.addBlock();
1102         BasicBlock* continuation = m_proc.addBlock();
1103         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
1104
1105         Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
1106             [=] (PatchpointValue* patchpoint) {
1107                 patchpoint->effects.writesPinned = true;
1108                 patchpoint->effects.readsPinned = true;
1109                 // We need to clobber all potential pinned registers since we might be leaving the instance.
1110                 // We pessimistically assume we could be calling to something that is bounds checking.
1111                 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1112                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1113                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1114                     AllowMacroScratchRegisterUsage allowScratch(jit);
1115                     CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1116                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1117                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1118                     });
1119                 });
1120             });
1121         UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
1122         isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1123
1124         // FIXME: Let's remove this indirection by creating a PIC friendly IC
1125         // for calls out to the embedder. This shouldn't be that hard to do. We could probably
1126         // implement the IC to be over Context*.
1127         // https://bugs.webkit.org/show_bug.cgi?id=170375
1128         Value* jumpDestination = isEmbedderBlock->appendNew<MemoryValue>(m_proc,
1129             Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfWasmToEmbedderStub(functionIndex)));
1130
1131         Value* embedderCallResult = wasmCallingConvention().setupCall(m_proc, isEmbedderBlock, origin(), args, toB3Type(returnType),
1132             [=] (PatchpointValue* patchpoint) {
1133                 patchpoint->effects.writesPinned = true;
1134                 patchpoint->effects.readsPinned = true;
1135                 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
1136                 // We need to clobber all potential pinned registers since we might be leaving the instance.
1137                 // We pessimistically assume we could be calling to something that is bounds checking.
1138                 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1139                 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1140                 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1141                     AllowMacroScratchRegisterUsage allowScratch(jit);
1142                     jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1143                 });
1144             });
1145         UpsilonValue* embedderCallResultUpsilon = returnType == Void ? nullptr : isEmbedderBlock->appendNew<UpsilonValue>(m_proc, origin(), embedderCallResult);
1146         isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1147
1148         m_currentBlock = continuation;
1149
1150         if (returnType == Void)
1151             result = nullptr;
1152         else {
1153             result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1154             wasmCallResultUpsilon->setPhi(result);
1155             embedderCallResultUpsilon->setPhi(result);
1156         }
1157
1158         // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1159         restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation);
1160     } else {
1161         result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1162             [=] (PatchpointValue* patchpoint) {
1163                 patchpoint->effects.writesPinned = true;
1164                 patchpoint->effects.readsPinned = true;
1165
1166                 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1167                     AllowMacroScratchRegisterUsage allowScratch(jit);
1168                     CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1169                     jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1170                         unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1171                     });
1172                 });
1173             });
1174     }
1175
1176     return { };
1177 }
1178
1179 auto B3IRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1180 {
1181     ExpressionType calleeIndex = args.takeLast();
1182     ASSERT(signature.argumentCount() == args.size());
1183
1184     m_makesCalls = true;
1185     // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1186     // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1187     // can be to the embedder for our stack check calculation.
1188     m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1189
1190     ExpressionType callableFunctionBuffer;
1191     ExpressionType instancesBuffer;
1192     ExpressionType callableFunctionBufferLength;
1193     ExpressionType mask;
1194     {
1195         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1196             instanceValue(), safeCast<int32_t>(Instance::offsetOfTable()));
1197         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1198             table, safeCast<int32_t>(Table::offsetOfFunctions()));
1199         instancesBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1200             table, safeCast<int32_t>(Table::offsetOfInstances()));
1201         callableFunctionBufferLength = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1202             table, safeCast<int32_t>(Table::offsetOfLength()));
1203         mask = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(),
1204             m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1205                 table, safeCast<int32_t>(Table::offsetOfMask())));
1206     }
1207
1208     // Check the index we are looking for is valid.
1209     {
1210         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1211             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferLength));
1212
1213         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1214             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1215         });
1216     }
1217
1218     calleeIndex = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex);
1219
1220     if (Options::enableSpectreMitigations())
1221         calleeIndex = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), mask, calleeIndex);
1222
1223     ExpressionType callableFunction;
1224     {
1225         // Compute the offset in the table index space we are looking for.
1226         ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1227             calleeIndex, constant(pointerType(), sizeof(WasmToWasmImportableFunction)));
1228         callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1229
1230         // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1231         // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1232         static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1233         ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), callableFunction, safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfSignatureIndex()));
1234         {
1235             CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1236                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1237                     calleeSignatureIndex,
1238                     m_currentBlock->appendNew<Const64Value>(m_proc, origin(), Signature::invalidIndex)));
1239
1240             check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1241                 this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1242             });
1243         }
1244
1245         // Check the signature matches the value we expect.
1246         {
1247             ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const64Value>(m_proc, origin(), SignatureInformation::get(signature));
1248             CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1249                 m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1250
1251             check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1252                 this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1253             });
1254         }
1255     }
1256
1257     // Do a context switch if needed.
1258     {
1259         Value* offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1260             calleeIndex, constant(pointerType(), sizeof(Instance*)));
1261         Value* newContextInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1262             m_currentBlock->appendNew<Value>(m_proc, Add, origin(), instancesBuffer, offset));
1263
1264         BasicBlock* continuation = m_proc.addBlock();
1265         BasicBlock* doContextSwitch = m_proc.addBlock();
1266
1267         Value* isSameContextInstance = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1268             newContextInstance, instanceValue());
1269         m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1270             isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch));
1271
1272         PatchpointValue* patchpoint = doContextSwitch->appendNew<PatchpointValue>(m_proc, B3::Void, origin());
1273         patchpoint->effects.writesPinned = true;
1274         // We pessimistically assume we're calling something with BoundsChecking memory.
1275         // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1276         patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1277         patchpoint->clobber(RegisterSet::macroScratchRegisters());
1278         patchpoint->append(newContextInstance, ValueRep::SomeRegister);
1279         patchpoint->append(instanceValue(), ValueRep::SomeRegister);
1280         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1281             AllowMacroScratchRegisterUsage allowScratch(jit);
1282             GPRReg newContextInstance = params[0].gpr();
1283             GPRReg oldContextInstance = params[1].gpr();
1284             const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1285             const auto& sizeRegs = pinnedRegs.sizeRegisters;
1286             GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1287             ASSERT(newContextInstance != baseMemory);
1288             jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1289             jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1290             jit.storeWasmContextInstance(newContextInstance);
1291             ASSERT(sizeRegs[0].sizeRegister != baseMemory);
1292             // FIXME: We should support more than one memory size register
1293             //   see: https://bugs.webkit.org/show_bug.cgi?id=162952
1294             ASSERT(sizeRegs.size() == 1);
1295             ASSERT(sizeRegs[0].sizeRegister != newContextInstance);
1296             ASSERT(!sizeRegs[0].sizeOffset);
1297             jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), sizeRegs[0].sizeRegister); // Memory size.
1298             jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1299         });
1300         doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation);
1301
1302         m_currentBlock = continuation;
1303     }
1304
1305     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1306         m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
1307             safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation())));
1308
1309     Type returnType = signature.returnType();
1310     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1311         [=] (PatchpointValue* patchpoint) {
1312             patchpoint->effects.writesPinned = true;
1313             patchpoint->effects.readsPinned = true;
1314             // We need to clobber all potential pinned registers since we might be leaving the instance.
1315             // We pessimistically assume we're always calling something that is bounds checking so
1316             // because the wasm->wasm thunk unconditionally overrides the size registers.
1317             // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1318             // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1319             patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1320
1321             patchpoint->append(calleeCode, ValueRep::SomeRegister);
1322             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1323                 AllowMacroScratchRegisterUsage allowScratch(jit);
1324                 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1325             });
1326         });
1327
1328     // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1329     restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock);
1330
1331     return { };
1332 }
1333
1334 void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1335 {
1336     m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1337 }
1338
1339 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1340 {
1341     ASSERT(result.size() <= resultStack.size());
1342
1343     for (size_t i = 0; i < result.size(); ++i)
1344         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1345 }
1346
1347 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1348 {
1349     dataLog(comma, "ExpressionStack:");
1350     for (const auto& expression : expressionStack)
1351         dataLog(comma, *expression);
1352 }
1353
1354 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1355 {
1356     dataLogLn("Constants:");
1357     for (const auto& constant : m_constantPool)
1358         dataLogLn(deepDump(m_proc, constant.value));
1359
1360     dataLogLn("Processing Graph:");
1361     dataLog(m_proc);
1362     dataLogLn("With current block:", *m_currentBlock);
1363     dataLogLn("Control stack:");
1364     ASSERT(controlStack.size());
1365     for (size_t i = controlStack.size(); i--;) {
1366         dataLog("  ", controlStack[i].controlData, ": ");
1367         CommaPrinter comma(", ", "");
1368         dumpExpressionStack(comma, *expressionStack);
1369         expressionStack = &controlStack[i].enclosedExpressionStack;
1370         dataLogLn();
1371     }
1372     dataLogLn();
1373 }
1374
1375 auto B3IRGenerator::origin() -> Origin
1376 {
1377     OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset());
1378     ASSERT(isValidOpType(static_cast<uint8_t>(origin.opcode())));
1379     return bitwise_cast<Origin>(origin);
1380 }
1381
1382 Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1383 {
1384     auto result = std::make_unique<InternalFunction>();
1385
1386     compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
1387     compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1388
1389     Procedure procedure;
1390
1391     procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1392         if (origin.data())
1393             out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1394     });
1395     
1396     // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1397     // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1398     // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1399     // optLevel=1.
1400     procedure.setNeedsUsedRegisters(false);
1401     
1402     procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
1403         ? Options::webAssemblyBBQOptimizationLevel()
1404         : Options::webAssemblyOMGOptimizationLevel());
1405
1406     B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, compilationMode, functionIndex, tierUp, throwWasmException);
1407     FunctionParser<B3IRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1408     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1409
1410     irGenerator.insertConstants();
1411
1412     procedure.resetReachability();
1413     if (!ASSERT_DISABLED)
1414         validate(procedure, "After parsing:\n");
1415
1416     dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure);
1417     fixSSA(procedure);
1418     dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure);
1419     
1420     {
1421         B3::prepareForGeneration(procedure);
1422         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1423         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1424         result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1425     }
1426
1427     return result;
1428 }
1429
1430 // Custom wasm ops. These are the ones too messy to do in wasm.json.
1431
1432 void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1433 {
1434     ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1435     const B3::Type type = left->type();
1436
1437     {
1438         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1439             m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1440
1441         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1442             this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1443         });
1444     }
1445
1446     if (operation == Div) {
1447         int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1448
1449         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1450             m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1451                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1452                 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1453
1454         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1455             this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1456         });
1457     }
1458 }
1459
1460 template<>
1461 auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1462 {
1463     const B3::Opcode op = Div;
1464     emitChecksForModOrDiv(op, left, right);
1465     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1466     return { };
1467 }
1468
1469 template<>
1470 auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1471 {
1472     const B3::Opcode op = Mod;
1473     emitChecksForModOrDiv(op, left, right);
1474     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1475     return { };
1476 }
1477
1478 template<>
1479 auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1480 {
1481     const B3::Opcode op = UDiv;
1482     emitChecksForModOrDiv(op, left, right);
1483     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1484     return { };
1485 }
1486
1487 template<>
1488 auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1489 {
1490     const B3::Opcode op = UMod;
1491     emitChecksForModOrDiv(op, left, right);
1492     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1493     return { };
1494 }
1495
1496 template<>
1497 auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1498 {
1499     const B3::Opcode op = Div;
1500     emitChecksForModOrDiv(op, left, right);
1501     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1502     return { };
1503 }
1504
1505 template<>
1506 auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1507 {
1508     const B3::Opcode op = Mod;
1509     emitChecksForModOrDiv(op, left, right);
1510     result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1511     return { };
1512 }
1513
1514 template<>
1515 auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1516 {
1517     const B3::Opcode op = UDiv;
1518     emitChecksForModOrDiv(op, left, right);
1519     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1520     return { };
1521 }
1522
1523 template<>
1524 auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1525 {
1526     const B3::Opcode op = UMod;
1527     emitChecksForModOrDiv(op, left, right);
1528     result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1529     return { };
1530 }
1531
1532 template<>
1533 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1534 {
1535     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1536     patchpoint->append(arg, ValueRep::SomeRegister);
1537     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1538         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1539     });
1540     patchpoint->effects = Effects::none();
1541     result = patchpoint;
1542     return { };
1543 }
1544
1545 template<>
1546 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1547 {
1548     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1549     patchpoint->append(arg, ValueRep::SomeRegister);
1550     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1551         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1552     });
1553     patchpoint->effects = Effects::none();
1554     result = patchpoint;
1555     return { };
1556 }
1557
1558 template<>
1559 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1560 {
1561 #if CPU(X86_64)
1562     if (MacroAssembler::supportsCountPopulation()) {
1563         PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1564         patchpoint->append(arg, ValueRep::SomeRegister);
1565         patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1566             jit.countPopulation32(params[1].gpr(), params[0].gpr());
1567         });
1568         patchpoint->effects = Effects::none();
1569         result = patchpoint;
1570         return { };
1571     }
1572 #endif
1573
1574     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1575     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1576     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1577     return { };
1578 }
1579
1580 template<>
1581 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1582 {
1583 #if CPU(X86_64)
1584     if (MacroAssembler::supportsCountPopulation()) {
1585         PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1586         patchpoint->append(arg, ValueRep::SomeRegister);
1587         patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1588             jit.countPopulation64(params[1].gpr(), params[0].gpr());
1589         });
1590         patchpoint->effects = Effects::none();
1591         result = patchpoint;
1592         return { };
1593     }
1594 #endif
1595
1596     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1597     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1598     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1599     return { };
1600 }
1601
1602 template<>
1603 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1604 {
1605     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1606     if (isX86())
1607         patchpoint->numGPScratchRegisters = 1;
1608     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1609     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1610     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1611         AllowMacroScratchRegisterUsage allowScratch(jit);
1612 #if CPU(X86_64)
1613         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1614 #else
1615         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1616 #endif
1617     });
1618     patchpoint->effects = Effects::none();
1619     result = patchpoint;
1620     return { };
1621 }
1622
1623 template<>
1624 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1625 {
1626     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1627     if (isX86())
1628         patchpoint->numGPScratchRegisters = 1;
1629     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1630     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1631     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1632         AllowMacroScratchRegisterUsage allowScratch(jit);
1633 #if CPU(X86_64)
1634         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1635 #else
1636         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1637 #endif
1638     });
1639     patchpoint->effects = Effects::none();
1640     result = patchpoint;
1641     return { };
1642 }
1643
1644 template<>
1645 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1646 {
1647     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1648     patchpoint->append(arg, ValueRep::SomeRegister);
1649     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1650         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1651     });
1652     patchpoint->effects = Effects::none();
1653     result = patchpoint;
1654     return { };
1655 }
1656
1657 template<>
1658 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1659 {
1660     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1661     patchpoint->append(arg, ValueRep::SomeRegister);
1662     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1663         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1664     });
1665     patchpoint->effects = Effects::none();
1666     result = patchpoint;
1667     return { };
1668 }
1669
1670 template<>
1671 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1672 {
1673     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1674     patchpoint->append(arg, ValueRep::SomeRegister);
1675     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1676         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1677     });
1678     patchpoint->effects = Effects::none();
1679     result = patchpoint;
1680     return { };
1681 }
1682
1683 template<>
1684 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1685 {
1686     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1687     patchpoint->append(arg, ValueRep::SomeRegister);
1688     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1689         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1690     });
1691     patchpoint->effects = Effects::none();
1692     result = patchpoint;
1693     return { };
1694 }
1695
1696 template<>
1697 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1698 {
1699     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1700     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1701     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1702         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1703         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1704     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1705     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1706     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1707         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1708     });
1709     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1710     patchpoint->append(arg, ValueRep::SomeRegister);
1711     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1712         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1713     });
1714     patchpoint->effects = Effects::none();
1715     result = patchpoint;
1716     return { };
1717 }
1718
1719 template<>
1720 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1721 {
1722     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1723     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1724     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1725         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1726         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1727     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1728     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1729     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1730         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1731     });
1732     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1733     patchpoint->append(arg, ValueRep::SomeRegister);
1734     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1735         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1736     });
1737     patchpoint->effects = Effects::none();
1738     result = patchpoint;
1739     return { };
1740 }
1741
1742
1743 template<>
1744 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1745 {
1746     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1747     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1748     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1749         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1750         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1751     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1752     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1753     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1754         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1755     });
1756     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1757     patchpoint->append(arg, ValueRep::SomeRegister);
1758     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1759         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1760     });
1761     patchpoint->effects = Effects::none();
1762     result = patchpoint;
1763     return { };
1764 }
1765
1766 template<>
1767 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1768 {
1769     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1770     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1771     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1772         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1773         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1774     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1775     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1776     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1777         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1778     });
1779     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1780     patchpoint->append(arg, ValueRep::SomeRegister);
1781     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1782         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1783     });
1784     patchpoint->effects = Effects::none();
1785     result = patchpoint;
1786     return { };
1787 }
1788
1789 template<>
1790 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1791 {
1792     Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1793     Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1794     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1795         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1796         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1797     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1798     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1799     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1800         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1801     });
1802     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1803     patchpoint->append(arg, ValueRep::SomeRegister);
1804     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1805         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1806     });
1807     patchpoint->effects = Effects::none();
1808     result = patchpoint;
1809     return { };
1810 }
1811
1812 template<>
1813 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1814 {
1815     Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1816     Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1817     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1818         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1819         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1820     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1821     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1822     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1823         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1824     });
1825
1826     Value* signBitConstant;
1827     if (isX86()) {
1828         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1829         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1830         // so we can pool them if needed.
1831         signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1832     }
1833     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1834     patchpoint->append(arg, ValueRep::SomeRegister);
1835     if (isX86()) {
1836         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1837         patchpoint->numFPScratchRegisters = 1;
1838     }
1839     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1840     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1841         AllowMacroScratchRegisterUsage allowScratch(jit);
1842         FPRReg scratch = InvalidFPRReg;
1843         FPRReg constant = InvalidFPRReg;
1844         if (isX86()) {
1845             scratch = params.fpScratch(0);
1846             constant = params[2].fpr();
1847         }
1848         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1849     });
1850     patchpoint->effects = Effects::none();
1851     result = patchpoint;
1852     return { };
1853 }
1854
1855 template<>
1856 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1857 {
1858     Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1859     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1860     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1861         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1862         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1863     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1864     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1865     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1866         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1867     });
1868     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1869     patchpoint->append(arg, ValueRep::SomeRegister);
1870     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1871         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1872     });
1873     patchpoint->effects = Effects::none();
1874     result = patchpoint;
1875     return { };
1876 }
1877
1878 template<>
1879 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1880 {
1881     Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1882     Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1883     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1884         m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1885         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1886     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1887     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1888     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1889         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1890     });
1891
1892     Value* signBitConstant;
1893     if (isX86()) {
1894         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1895         // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1896         // so we can pool them if needed.
1897         signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1898     }
1899     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1900     patchpoint->append(arg, ValueRep::SomeRegister);
1901     if (isX86()) {
1902         patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1903         patchpoint->numFPScratchRegisters = 1;
1904     }
1905     patchpoint->clobber(RegisterSet::macroScratchRegisters());
1906     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1907         AllowMacroScratchRegisterUsage allowScratch(jit);
1908         FPRReg scratch = InvalidFPRReg;
1909         FPRReg constant = InvalidFPRReg;
1910         if (isX86()) {
1911             scratch = params.fpScratch(0);
1912             constant = params[2].fpr();
1913         }
1914         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1915     });
1916     patchpoint->effects = Effects::none();
1917     result = patchpoint;
1918     return { };
1919 }
1920
1921 } } // namespace JSC::Wasm
1922
1923 #include "WasmB3IRGeneratorInlines.h"
1924
1925 #endif // ENABLE(WEBASSEMBLY)