WebAssembly: Allow br, br_if, and br_table to act as a return
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "B3BasicBlockInlines.h"
32 #include "B3CCallValue.h"
33 #include "B3Compile.h"
34 #include "B3ConstPtrValue.h"
35 #include "B3FixSSA.h"
36 #include "B3Generate.h"
37 #include "B3StackmapGenerationParams.h"
38 #include "B3SwitchValue.h"
39 #include "B3Validate.h"
40 #include "B3ValueInlines.h"
41 #include "B3Variable.h"
42 #include "B3VariableValue.h"
43 #include "B3WasmAddressValue.h"
44 #include "B3WasmBoundsCheckValue.h"
45 #include "JSCInlines.h"
46 #include "JSWebAssemblyInstance.h"
47 #include "JSWebAssemblyModule.h"
48 #include "JSWebAssemblyRuntimeError.h"
49 #include "VirtualRegister.h"
50 #include "WasmCallingConvention.h"
51 #include "WasmExceptionType.h"
52 #include "WasmFunctionParser.h"
53 #include "WasmMemory.h"
54 #include <wtf/Optional.h>
55
56 void dumpProcedure(void* ptr)
57 {
58     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
59     proc->dump(WTF::dataFile());
60 }
61
62 namespace JSC { namespace Wasm {
63
64 using namespace B3;
65
66 namespace {
67 const bool verbose = false;
68 }
69
70 class B3IRGenerator {
71 public:
72     struct ControlData {
73         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
74             : blockType(type)
75             , continuation(continuation)
76             , special(special)
77         {
78             if (signature != Void)
79                 result.append(proc.addVariable(toB3Type(signature)));
80         }
81
82         ControlData()
83         {
84         }
85
86         void dump(PrintStream& out) const
87         {
88             switch (type()) {
89             case BlockType::If:
90                 out.print("If:       ");
91                 break;
92             case BlockType::Block:
93                 out.print("Block:    ");
94                 break;
95             case BlockType::Loop:
96                 out.print("Loop:     ");
97                 break;
98             case BlockType::TopLevel:
99                 out.print("TopLevel: ");
100                 break;
101             }
102             out.print("Continuation: ", *continuation, ", Special: ");
103             if (special)
104                 out.print(*special);
105             else
106                 out.print("None");
107         }
108
109         BlockType type() const { return blockType; }
110
111         bool hasNonVoidSignature() const { return result.size(); }
112
113         BasicBlock* targetBlockForBranch()
114         {
115             if (type() == BlockType::Loop)
116                 return special;
117             return continuation;
118         }
119
120         void convertIfToBlock()
121         {
122             ASSERT(type() == BlockType::If);
123             blockType = BlockType::Block;
124             special = nullptr;
125         }
126
127     private:
128         friend class B3IRGenerator;
129         BlockType blockType;
130         BasicBlock* continuation;
131         BasicBlock* special;
132         Vector<Variable*, 1> result;
133     };
134
135     typedef Value* ExpressionType;
136     typedef ControlData ControlType;
137     typedef Vector<ExpressionType, 1> ExpressionList;
138     typedef Vector<Variable*, 1> ResultList;
139     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
140
141     static constexpr ExpressionType emptyExpression = nullptr;
142
143     typedef String ErrorType;
144     typedef UnexpectedType<ErrorType> UnexpectedResult;
145     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
146     typedef Expected<void, ErrorType> PartialResult;
147     template <typename ...Args>
148     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
149     {
150         using namespace FailureHelper; // See ADL comment in WasmParser.h.
151         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
152     }
153 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
154         if (UNLIKELY(condition))                  \
155             return fail(__VA_ARGS__);             \
156     } while (0)
157
158     B3IRGenerator(VM&, const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, const ImmutableFunctionIndexSpace&);
159
160     PartialResult WARN_UNUSED_RETURN addArguments(const Signature*);
161     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
162     ExpressionType addConstant(Type, uint64_t);
163
164     // Locals
165     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
166     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
167
168     // Globals
169     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
170     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
171
172     // Memory
173     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
174     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
175
176     // Basic operators
177     template<OpType>
178     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
179     template<OpType>
180     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
181     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
182
183     // Control flow
184     ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
185     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
186     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
187     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
188     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
189     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
190
191     PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
192     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
193     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
194     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
195     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
196
197     // Calls
198     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
199     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature*, SignatureIndex, Vector<ExpressionType>& args, ExpressionType& result);
200     PartialResult WARN_UNUSED_RETURN addUnreachable();
201
202     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack);
203
204     void emitExceptionCheck(CCallHelpers&, ExceptionType);
205
206 private:
207     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
208     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
209     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
210
211     void unify(Variable* target, const ExpressionType source);
212     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
213     Value* zeroForType(Type);
214
215     VM& m_vm;
216     const ImmutableFunctionIndexSpace& m_functionIndexSpace;
217     const ModuleInformation& m_info;
218     Procedure& m_proc;
219     BasicBlock* m_currentBlock;
220     Vector<Variable*> m_locals;
221     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
222     GPRReg m_memoryBaseGPR;
223     GPRReg m_memorySizeGPR;
224     Value* m_zeroValues[numTypes];
225     Value* m_instanceValue;
226 };
227
228 B3IRGenerator::B3IRGenerator(VM& vm, const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace)
229     : m_vm(vm)
230     , m_functionIndexSpace(functionIndexSpace)
231     , m_info(info)
232     , m_proc(procedure)
233     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
234 {
235     m_currentBlock = m_proc.addBlock();
236
237     for (unsigned i = 0; i < numTypes; ++i) {
238         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
239         case B3::Int32:
240         case B3::Int64:
241         case B3::Float:
242         case B3::Double:
243             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
244             break;
245         case B3::Void:
246             m_zeroValues[i] = nullptr;
247             break;
248         }
249     }
250
251     if (!!info.memory) {
252         m_memoryBaseGPR = info.memory.pinnedRegisters().baseMemoryPointer;
253         m_proc.pinRegister(m_memoryBaseGPR);
254         ASSERT(!info.memory.pinnedRegisters().sizeRegisters[0].sizeOffset);
255         m_memorySizeGPR = info.memory.pinnedRegisters().sizeRegisters[0].sizeRegister;
256         for (const PinnedSizeRegisterInfo& regInfo : info.memory.pinnedRegisters().sizeRegisters)
257             m_proc.pinRegister(regInfo.sizeRegister);
258
259         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
260             AllowMacroScratchRegisterUsage allowScratch(jit);
261             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
262             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
263         });
264     }
265
266     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
267
268     m_instanceValue = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
269         m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), &m_vm.topJSWebAssemblyInstance));
270 }
271
272 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
273 {
274     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
275     auto jumpToExceptionStub = jit.jump();
276
277     VM* vm = &m_vm;
278     jit.addLinkTask([vm, jumpToExceptionStub] (LinkBuffer& linkBuffer) {
279         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(vm->getCTIStub(throwExceptionFromWasmThunkGenerator).code()));
280     });
281 }
282
283 Value* B3IRGenerator::zeroForType(Type type)
284 {
285     ASSERT(type != Void);
286     Value* zeroValue = m_zeroValues[linearizeType(type)];
287     ASSERT(zeroValue);
288     return zeroValue;
289 }
290
291 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
292 {
293     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
294
295     for (uint32_t i = 0; i < count; ++i) {
296         Variable* local = m_proc.addVariable(toB3Type(type));
297         m_locals.uncheckedAppend(local);
298         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
299     }
300     return { };
301 }
302
303 auto B3IRGenerator::addArguments(const Signature* signature) -> PartialResult
304 {
305     ASSERT(!m_locals.size());
306     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature->argumentCount()), "can't allocate memory for ", signature->argumentCount(), " arguments");
307
308     m_locals.grow(signature->argumentCount());
309     wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
310         [&] (ExpressionType argument, unsigned i) {
311             Variable* argumentVariable = m_proc.addVariable(argument->type());
312             m_locals[i] = argumentVariable;
313             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
314         });
315     return { };
316 }
317
318 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
319 {
320     ASSERT(m_locals[index]);
321     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
322     return { };
323 }
324
325 auto B3IRGenerator::addUnreachable() -> PartialResult
326 {
327     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
328     unreachable->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
329         jit.breakpoint();
330     });
331     unreachable->effects.terminal = true;
332     return { };
333 }
334
335 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
336 {
337     ASSERT(m_locals[index]);
338     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
339     return { };
340 }
341
342 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
343 {
344     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
345     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), Origin(), globalsArray, index * sizeof(Register));
346     return { };
347 }
348
349 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
350 {
351     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
352     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
353     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, Origin(), value, globalsArray, index * sizeof(Register));
354     return { };
355 }
356
357 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
358 {
359     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
360     ASSERT(sizeOfOperation + offset > offset);
361     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
362     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
363     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
364 }
365
366 inline uint32_t sizeOfLoadOp(LoadOpType op)
367 {
368     switch (op) {
369     case LoadOpType::I32Load8S:
370     case LoadOpType::I32Load8U:
371     case LoadOpType::I64Load8S:
372     case LoadOpType::I64Load8U:
373         return 1;
374     case LoadOpType::I32Load16S:
375     case LoadOpType::I64Load16S:
376     case LoadOpType::I32Load16U:
377     case LoadOpType::I64Load16U:
378         return 2;
379     case LoadOpType::I32Load:
380     case LoadOpType::I64Load32S:
381     case LoadOpType::I64Load32U:
382     case LoadOpType::F32Load:
383         return 4;
384     case LoadOpType::I64Load:
385     case LoadOpType::F64Load:
386         return 8;
387     }
388     RELEASE_ASSERT_NOT_REACHED();
389 }
390
391 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
392 {
393     switch (op) {
394     case LoadOpType::I32Load8S: {
395         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
396     }
397
398     case LoadOpType::I64Load8S: {
399         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
400         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
401     }
402
403     case LoadOpType::I32Load8U: {
404         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
405     }
406
407     case LoadOpType::I64Load8U: {
408         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
409         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
410     }
411
412     case LoadOpType::I32Load16S: {
413         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
414     }
415     case LoadOpType::I64Load16S: {
416         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
417         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
418     }
419
420     case LoadOpType::I32Load: {
421         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
422     }
423
424     case LoadOpType::I64Load32U: {
425         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
426         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
427     }
428
429     case LoadOpType::I64Load32S: {
430         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
431         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
432     }
433
434     case LoadOpType::I64Load: {
435         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer);
436     }
437
438     case LoadOpType::F32Load: {
439         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer);
440     }
441
442     case LoadOpType::F64Load: {
443         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer);
444     }
445
446     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
447     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
448     case LoadOpType::I32Load16U: {
449         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
450         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
451                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
452     }
453     case LoadOpType::I64Load16U: {
454         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
455         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
456                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
457
458         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), partialResult);
459     }
460     }
461     RELEASE_ASSERT_NOT_REACHED();
462 }
463
464 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
465 {
466     ASSERT(pointer->type() == Int32);
467
468     result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
469     return { };
470 }
471
472 inline uint32_t sizeOfStoreOp(StoreOpType op)
473 {
474     switch (op) {
475     case StoreOpType::I32Store8:
476     case StoreOpType::I64Store8:
477         return 1;
478     case StoreOpType::I32Store16:
479     case StoreOpType::I64Store16:
480         return 2;
481     case StoreOpType::I32Store:
482     case StoreOpType::I64Store32:
483     case StoreOpType::F32Store:
484         return 4;
485     case StoreOpType::I64Store:
486     case StoreOpType::F64Store:
487         return 8;
488     }
489     RELEASE_ASSERT_NOT_REACHED();
490 }
491
492
493 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
494 {
495     switch (op) {
496     case StoreOpType::I64Store8:
497         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
498         FALLTHROUGH;
499
500     case StoreOpType::I32Store8:
501         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
502         return;
503
504     case StoreOpType::I64Store16:
505         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
506         FALLTHROUGH;
507
508     case StoreOpType::I32Store16:
509         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
510         return;
511
512     case StoreOpType::I64Store32:
513         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
514         FALLTHROUGH;
515
516     case StoreOpType::I64Store:
517     case StoreOpType::I32Store:
518     case StoreOpType::F32Store:
519     case StoreOpType::F64Store:
520         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
521         return;
522     }
523     RELEASE_ASSERT_NOT_REACHED();
524 }
525
526 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
527 {
528     ASSERT(pointer->type() == Int32);
529
530     emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
531     return { };
532 }
533
534 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
535 {
536     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
537     return { };
538 }
539
540 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
541 {
542     switch (type) {
543     case Wasm::I32:
544         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
545     case Wasm::I64:
546         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
547     case Wasm::F32:
548         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
549     case Wasm::F64:
550         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
551     case Wasm::Void:
552     case Wasm::Func:
553     case Wasm::Anyfunc:
554         break;
555     }
556     RELEASE_ASSERT_NOT_REACHED();
557     return nullptr;
558 }
559
560 B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
561 {
562     return ControlData(m_proc, signature, BlockType::TopLevel, m_proc.addBlock());
563 }
564
565 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
566 {
567     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
568 }
569
570 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
571 {
572     BasicBlock* body = m_proc.addBlock();
573     BasicBlock* continuation = m_proc.addBlock();
574     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
575     body->addPredecessor(m_currentBlock);
576     m_currentBlock = body;
577     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
578 }
579
580 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
581 {
582     // FIXME: This needs to do some kind of stack passing.
583
584     BasicBlock* taken = m_proc.addBlock();
585     BasicBlock* notTaken = m_proc.addBlock();
586     BasicBlock* continuation = m_proc.addBlock();
587
588     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
589     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
590     taken->addPredecessor(m_currentBlock);
591     notTaken->addPredecessor(m_currentBlock);
592
593     m_currentBlock = taken;
594     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
595     return { };
596 }
597
598 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
599 {
600     unifyValuesWithBlock(currentStack, data.result);
601     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
602     return addElseToUnreachable(data);
603 }
604
605 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
606 {
607     ASSERT(data.type() == BlockType::If);
608     m_currentBlock = data.special;
609     data.convertIfToBlock();
610     return { };
611 }
612
613 auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
614 {
615     ASSERT(returnValues.size() <= 1);
616     if (returnValues.size())
617         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
618     else
619         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
620     return { };
621 }
622
623 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
624 {
625     if (data.type() != BlockType::Loop)
626         unifyValuesWithBlock(returnValues, data.result);
627
628     BasicBlock* target = data.targetBlockForBranch();
629     if (condition) {
630         BasicBlock* continuation = m_proc.addBlock();
631         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
632         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
633         target->addPredecessor(m_currentBlock);
634         continuation->addPredecessor(m_currentBlock);
635         m_currentBlock = continuation;
636     } else {
637         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
638         target->addPredecessor(m_currentBlock);
639     }
640
641     return { };
642 }
643
644 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
645 {
646     for (size_t i = 0; i < targets.size(); ++i)
647         unifyValuesWithBlock(expressionStack, targets[i]->result);
648     unifyValuesWithBlock(expressionStack, defaultTarget.result);
649
650     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
651     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
652     for (size_t i = 0; i < targets.size(); ++i)
653         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
654
655     return { };
656 }
657
658 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
659 {
660     ControlData& data = entry.controlData;
661
662     unifyValuesWithBlock(expressionStack, data.result);
663     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
664     data.continuation->addPredecessor(m_currentBlock);
665
666     return addEndToUnreachable(entry);
667 }
668
669
670 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
671 {
672     ControlData& data = entry.controlData;
673     m_currentBlock = data.continuation;
674
675     if (data.type() == BlockType::If) {
676         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
677         m_currentBlock->addPredecessor(data.special);
678     }
679
680     for (Variable* result : data.result)
681         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
682
683     // TopLevel does not have any code after this so we need to make sure we emit a return here.
684     if (data.type() == BlockType::TopLevel)
685         return addReturn(entry.controlData, entry.enclosedExpressionStack);
686
687     return { };
688 }
689
690 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
691 {
692     ASSERT(signature->argumentCount() == args.size());
693
694     Type returnType = signature->returnType();
695
696     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
697         [&] (PatchpointValue* patchpoint) {
698             patchpoint->effects.writesPinned = true;
699             patchpoint->effects.readsPinned = true;
700
701             Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
702             patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
703                 AllowMacroScratchRegisterUsage allowScratch(jit);
704
705                 CCallHelpers::Call call = jit.call();
706
707                 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
708                     unlinkedWasmToWasmCalls->append({ linkBuffer.locationOf(call), functionIndex });
709                 });
710             });
711         });
712     return { };
713 }
714
715 auto B3IRGenerator::addCallIndirect(const Signature* signature, SignatureIndex signatureIndex, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
716 {
717     ASSERT(signatureIndex != Signature::invalidIndex);
718     ExpressionType calleeIndex = args.takeLast();
719     ASSERT(signature->argumentCount() == args.size());
720
721     ExpressionType callableFunctionBuffer;
722     ExpressionType callableFunctionBufferSize;
723     {
724         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
725             m_instanceValue, JSWebAssemblyInstance::offsetOfTable());
726         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
727             table, JSWebAssemblyTable::offsetOfFunctions());
728         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, Origin(),
729             table, JSWebAssemblyTable::offsetOfSize());
730     }
731
732     // Check the index we are looking for is valid.
733     {
734         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
735             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, Origin(), calleeIndex, callableFunctionBufferSize));
736
737         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
738             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
739         });
740     }
741
742     // Compute the offset in the table index space we are looking for.
743     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, Origin(),
744         m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), calleeIndex),
745         m_currentBlock->appendIntConstant(m_proc, Origin(), pointerType(), sizeof(CallableFunction)));
746     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, Origin(), callableFunctionBuffer, offset);
747
748     // Check that the CallableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
749     static_assert(sizeof(CallableFunction::signatureIndex) == sizeof(uint32_t), "Load codegen assumes i32");
750     ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signatureIndex));
751     {
752         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
753             m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(),
754                 calleeSignatureIndex,
755                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), Signature::invalidIndex)));
756
757         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
758             this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
759         });
760     }
761
762     // Check the signature matches the value we expect.
763     {
764         ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), signatureIndex);
765         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
766             m_currentBlock->appendNew<Value>(m_proc, NotEqual, Origin(), calleeSignatureIndex, expectedSignatureIndex));
767
768         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
769             this->emitExceptionCheck(jit, ExceptionType::BadSignature);
770         });
771     }
772
773     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
774
775     Type returnType = signature->returnType();
776     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
777         [&] (PatchpointValue* patchpoint) {
778             patchpoint->effects.writesPinned = true;
779             patchpoint->effects.readsPinned = true;
780
781             patchpoint->append(calleeCode, ValueRep::SomeRegister);
782
783             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
784                 jit.call(params[returnType == Void ? 0 : 1].gpr());
785             });
786         });
787
788     return { };
789 }
790
791 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
792 {
793     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
794 }
795
796 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
797 {
798     ASSERT(result.size() <= resultStack.size());
799
800     for (size_t i = 0; i < result.size(); ++i)
801         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
802 }
803
804 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
805 {
806     dataLogLn(comma, "ExpressionStack:");
807     for (const auto& expression : expressionStack)
808         dataLogLn(comma, *expression);
809 }
810
811 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack)
812 {
813     dataLogLn("Processing Graph:");
814     dataLog(m_proc);
815     dataLogLn("With current block:", *m_currentBlock);
816     dataLogLn("Control stack:");
817     for (auto& data : controlStack) {
818         dataLogLn("  ", data.controlData);
819         if (data.enclosedExpressionStack.size()) {
820             CommaPrinter comma("    ", "  with ");
821             dumpExpressionStack(comma, data.enclosedExpressionStack);
822         }
823     }
824
825     CommaPrinter comma("  ", "");
826     dumpExpressionStack(comma, expressionStack);
827     dataLogLn("\n");
828 }
829
830 static void createJSToWasmWrapper(VM& vm, CompilationContext& compilationContext, WasmInternalFunction& function, const Signature* signature, const MemoryInformation& memory)
831 {
832     Procedure proc;
833     BasicBlock* block = proc.addBlock();
834
835     Origin origin;
836
837     jscCallingConvention().setupFrameInPrologue(&function.jsToWasmCalleeMoveLocation, proc, origin, block);
838
839     if (!ASSERT_DISABLED) {
840         // This should be guaranteed by our JS wrapper that handles calls to us.
841         // Just prevent against crazy when ASSERT is enabled.
842         Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, origin);
843         Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, origin, CallFrameSlot::argumentCount * sizeof(Register));
844         Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, origin,
845             block->appendNew<Value>(proc, Add, origin, framePointer, offSetOfArgumentCount));
846
847         Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, origin, signature->argumentCount());
848
849         CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, origin,
850             block->appendNew<Value>(proc, Above, origin, expectedArgumentCount, argumentCount));
851
852         argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
853             jit.breakpoint();
854         });
855     }
856
857     // Move memory values to the approriate places, if needed.
858     Value* baseMemory = nullptr;
859     Vector<Value*> sizes;
860     if (!!memory) {
861         baseMemory = block->appendNew<MemoryValue>(proc, Load, Int64, Origin(),
862             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemoryPointer));
863         Value* size = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
864             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemorySize));
865         sizes.reserveCapacity(memory.pinnedRegisters().sizeRegisters.size());
866         for (auto info : memory.pinnedRegisters().sizeRegisters) {
867             sizes.append(block->appendNew<Value>(proc, Sub, origin, size,
868                 block->appendNew<Const32Value>(proc, origin, info.sizeOffset)));
869         }
870     }
871
872     // Get our arguments.
873     Vector<Value*> arguments;
874     jscCallingConvention().loadArguments(signature, proc, block, origin, [&] (Value* argument, unsigned) {
875         arguments.append(argument);
876     });
877
878     // Move the arguments into place.
879     Value* result = wasmCallingConvention().setupCall(proc, block, origin, arguments, toB3Type(signature->returnType()), [&] (PatchpointValue* patchpoint) {
880         if (!!memory) {
881             ASSERT(sizes.size() == memory.pinnedRegisters().sizeRegisters.size());
882             patchpoint->append(ConstrainedValue(baseMemory, ValueRep::reg(memory.pinnedRegisters().baseMemoryPointer)));
883             for (unsigned i = 0; i < sizes.size(); ++i)
884                 patchpoint->append(ConstrainedValue(sizes[i], ValueRep::reg(memory.pinnedRegisters().sizeRegisters[i].sizeRegister)));
885         }
886
887         CompilationContext* context = &compilationContext;
888         patchpoint->setGenerator([context] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
889             AllowMacroScratchRegisterUsage allowScratch(jit);
890
891             CCallHelpers::Call call = jit.call();
892             context->jsEntrypointToWasmEntrypointCall = call;
893         });
894     });
895
896     // Return the result, if needed.
897     switch (signature->returnType()) {
898     case Wasm::Void:
899         block->appendNewControlValue(proc, B3::Return, origin);
900         break;
901     case Wasm::F32:
902     case Wasm::F64:
903         result = block->appendNew<Value>(proc, BitwiseCast, origin, result);
904         FALLTHROUGH;
905     case Wasm::I32:
906     case Wasm::I64:
907         block->appendNewControlValue(proc, B3::Return, origin, result);
908         break;
909     case Wasm::Func:
910     case Wasm::Anyfunc:
911         RELEASE_ASSERT_NOT_REACHED();
912     }
913
914     B3::prepareForGeneration(proc);
915     B3::generate(proc, *compilationContext.jsEntrypointJIT);
916     compilationContext.jsEntrypointByproducts = proc.releaseByproducts();
917     function.jsToWasmEntrypoint.calleeSaveRegisters = proc.calleeSaveRegisters();
918 }
919
920 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(VM& vm, CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace, const ModuleInformation& info, unsigned optLevel)
921 {
922     auto result = std::make_unique<WasmInternalFunction>();
923
924     compilationContext.jsEntrypointJIT = std::make_unique<CCallHelpers>(&vm);
925     compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>(&vm);
926
927     Procedure procedure;
928     B3IRGenerator context(vm, info, procedure, result.get(), unlinkedWasmToWasmCalls, functionIndexSpace);
929     FunctionParser<B3IRGenerator> parser(&vm, context, functionStart, functionLength, signature, functionIndexSpace, info);
930     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
931
932     procedure.resetReachability();
933     validate(procedure, "After parsing:\n");
934
935     if (verbose)
936         dataLog("Pre SSA: ", procedure);
937     fixSSA(procedure);
938     if (verbose)
939         dataLog("Post SSA: ", procedure);
940
941     {
942         B3::prepareForGeneration(procedure, optLevel);
943         B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
944         compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
945         result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisters();
946     }
947
948     createJSToWasmWrapper(vm, compilationContext, *result, signature, info.memory);
949     return WTFMove(result);
950 }
951
952 // Custom wasm ops. These are the ones too messy to do in wasm.json.
953
954 template<>
955 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
956 {
957     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
958     patchpoint->append(arg, ValueRep::SomeRegister);
959     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
960         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
961     });
962     patchpoint->effects = Effects::none();
963     result = patchpoint;
964     return { };
965 }
966
967 template<>
968 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
969 {
970     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
971     patchpoint->append(arg, ValueRep::SomeRegister);
972     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
973         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
974     });
975     patchpoint->effects = Effects::none();
976     result = patchpoint;
977     return { };
978 }
979
980 template<>
981 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
982 {
983     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
984     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
985     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
986     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
987     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
988     return { };
989 }
990
991 template<>
992 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
993 {
994     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
995     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
996     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
997     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
998     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
999     return { };
1000 }
1001
1002 template<>
1003 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1004 {
1005     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1006     if (isX86())
1007         patchpoint->numGPScratchRegisters = 1;
1008     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1009     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1010         AllowMacroScratchRegisterUsage allowScratch(jit);
1011 #if CPU(X86_64)
1012         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1013 #else
1014         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1015 #endif
1016     });
1017     patchpoint->effects = Effects::none();
1018     result = patchpoint;
1019     return { };
1020 }
1021
1022 template<>
1023 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1024 {
1025     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1026     if (isX86())
1027         patchpoint->numGPScratchRegisters = 1;
1028     patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1029     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1030         AllowMacroScratchRegisterUsage allowScratch(jit);
1031 #if CPU(X86_64)
1032         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1033 #else
1034         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1035 #endif
1036     });
1037     patchpoint->effects = Effects::none();
1038     result = patchpoint;
1039     return { };
1040 }
1041
1042 template<>
1043 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1044 {
1045     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1046     patchpoint->append(arg, ValueRep::SomeRegister);
1047     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1048         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1049     });
1050     patchpoint->effects = Effects::none();
1051     result = patchpoint;
1052     return { };
1053 }
1054
1055 template<>
1056 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1057 {
1058     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1059     patchpoint->append(arg, ValueRep::SomeRegister);
1060     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1061         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1062     });
1063     patchpoint->effects = Effects::none();
1064     result = patchpoint;
1065     return { };
1066 }
1067
1068 template<>
1069 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1070 {
1071     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1072     patchpoint->append(arg, ValueRep::SomeRegister);
1073     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1074         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1075     });
1076     patchpoint->effects = Effects::none();
1077     result = patchpoint;
1078     return { };
1079 }
1080
1081 template<>
1082 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1083 {
1084     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1085     patchpoint->append(arg, ValueRep::SomeRegister);
1086     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1087         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1088     });
1089     patchpoint->effects = Effects::none();
1090     result = patchpoint;
1091     return { };
1092 }
1093
1094 template<>
1095 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1096 {
1097     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
1098     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
1099     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1100         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1101         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1102     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1103     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1104     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1105         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1106     });
1107     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1108     patchpoint->append(arg, ValueRep::SomeRegister);
1109     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1110         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1111     });
1112     patchpoint->effects = Effects::none();
1113     result = patchpoint;
1114     return { };
1115 }
1116
1117 template<>
1118 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1119 {
1120     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
1121     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
1122     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1123         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1124         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1125     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1126     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1127     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1128         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1129     });
1130     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1131     patchpoint->append(arg, ValueRep::SomeRegister);
1132     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1133         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1134     });
1135     patchpoint->effects = Effects::none();
1136     result = patchpoint;
1137     return { };
1138 }
1139
1140
1141 template<>
1142 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1143 {
1144     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
1145     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1146     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1147         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1148         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1149     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1150     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1151     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1152         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1153     });
1154     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1155     patchpoint->append(arg, ValueRep::SomeRegister);
1156     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1157         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1158     });
1159     patchpoint->effects = Effects::none();
1160     result = patchpoint;
1161     return { };
1162 }
1163
1164 template<>
1165 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1166 {
1167     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
1168     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1169     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1170         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1171         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1172     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1173     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1174     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1175         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1176     });
1177     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1178     patchpoint->append(arg, ValueRep::SomeRegister);
1179     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1180         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1181     });
1182     patchpoint->effects = Effects::none();
1183     result = patchpoint;
1184     return { };
1185 }
1186
1187 template<>
1188 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1189 {
1190     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1191     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1192     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1193         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1194         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1195     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1196     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1197     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1198         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1199     });
1200     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1201     patchpoint->append(arg, ValueRep::SomeRegister);
1202     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1203         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1204     });
1205     patchpoint->effects = Effects::none();
1206     result = patchpoint;
1207     return { };
1208 }
1209
1210 template<>
1211 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1212 {
1213     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1214     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1215     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1216         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1217         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1218     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1219     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1220     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1221         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1222     });
1223
1224     Value* constant;
1225     if (isX86()) {
1226         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1227         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1228         // so we can pool them if needed.
1229         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1230     }
1231     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1232     patchpoint->append(arg, ValueRep::SomeRegister);
1233     if (isX86()) {
1234         patchpoint->append(constant, ValueRep::SomeRegister);
1235         patchpoint->numFPScratchRegisters = 1;
1236     }
1237     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1238         AllowMacroScratchRegisterUsage allowScratch(jit);
1239         FPRReg scratch = InvalidFPRReg;
1240         FPRReg constant = InvalidFPRReg;
1241         if (isX86()) {
1242             scratch = params.fpScratch(0);
1243             constant = params[2].fpr();
1244         }
1245         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1246     });
1247     patchpoint->effects = Effects::none();
1248     result = patchpoint;
1249     return { };
1250 }
1251
1252 template<>
1253 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1254 {
1255     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1256     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1257     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1258         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1259         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1260     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1261     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1262     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1263         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1264     });
1265     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1266     patchpoint->append(arg, ValueRep::SomeRegister);
1267     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1268         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1269     });
1270     patchpoint->effects = Effects::none();
1271     result = patchpoint;
1272     return { };
1273 }
1274
1275 template<>
1276 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1277 {
1278     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1279     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1280     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1281         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1282         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1283     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1284     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1285     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1286         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1287     });
1288
1289     Value* constant;
1290     if (isX86()) {
1291         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1292         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1293         // so we can pool them if needed.
1294         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1295     }
1296     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1297     patchpoint->append(arg, ValueRep::SomeRegister);
1298     if (isX86()) {
1299         patchpoint->append(constant, ValueRep::SomeRegister);
1300         patchpoint->numFPScratchRegisters = 1;
1301     }
1302     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1303         AllowMacroScratchRegisterUsage allowScratch(jit);
1304         FPRReg scratch = InvalidFPRReg;
1305         FPRReg constant = InvalidFPRReg;
1306         if (isX86()) {
1307             scratch = params.fpScratch(0);
1308             constant = params[2].fpr();
1309         }
1310         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1311     });
1312     patchpoint->effects = Effects::none();
1313     result = patchpoint;
1314     return { };
1315 }
1316
1317 } } // namespace JSC::Wasm
1318
1319 #include "WasmB3IRGeneratorInlines.h"
1320
1321 #endif // ENABLE(WEBASSEMBLY)