WebAssembly: WasmB3IRGenerator should throw exceptions instead of crash
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "B3BasicBlockInlines.h"
32 #include "B3CCallValue.h"
33 #include "B3Compile.h"
34 #include "B3ConstPtrValue.h"
35 #include "B3FixSSA.h"
36 #include "B3Generate.h"
37 #include "B3StackmapGenerationParams.h"
38 #include "B3SwitchValue.h"
39 #include "B3Validate.h"
40 #include "B3ValueInlines.h"
41 #include "B3Variable.h"
42 #include "B3VariableValue.h"
43 #include "B3WasmAddressValue.h"
44 #include "B3WasmBoundsCheckValue.h"
45 #include "JSCInlines.h"
46 #include "JSWebAssemblyInstance.h"
47 #include "JSWebAssemblyModule.h"
48 #include "JSWebAssemblyRuntimeError.h"
49 #include "VirtualRegister.h"
50 #include "WasmCallingConvention.h"
51 #include "WasmExceptionType.h"
52 #include "WasmFunctionParser.h"
53 #include "WasmMemory.h"
54 #include <wtf/Optional.h>
55
56 void dumpProcedure(void* ptr)
57 {
58     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
59     proc->dump(WTF::dataFile());
60 }
61
62 namespace JSC { namespace Wasm {
63
64 using namespace B3;
65
66 namespace {
67 const bool verbose = false;
68 }
69
70 class B3IRGenerator {
71 public:
72     struct ControlData {
73         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
74             : blockType(type)
75             , continuation(continuation)
76             , special(special)
77         {
78             if (signature != Void)
79                 result.append(proc.addVariable(toB3Type(signature)));
80         }
81
82         ControlData()
83         {
84         }
85
86         void dump(PrintStream& out) const
87         {
88             switch (type()) {
89             case BlockType::If:
90                 out.print("If:    ");
91                 break;
92             case BlockType::Block:
93                 out.print("Block: ");
94                 break;
95             case BlockType::Loop:
96                 out.print("Loop:  ");
97                 break;
98             }
99             out.print("Continuation: ", *continuation, ", Special: ");
100             if (special)
101                 out.print(*special);
102             else
103                 out.print("None");
104         }
105
106         BlockType type() const { return blockType; }
107
108         bool hasNonVoidSignature() const { return result.size(); }
109
110         BasicBlock* targetBlockForBranch()
111         {
112             if (type() == BlockType::Loop)
113                 return special;
114             return continuation;
115         }
116
117         void convertIfToBlock()
118         {
119             ASSERT(type() == BlockType::If);
120             blockType = BlockType::Block;
121             special = nullptr;
122         }
123
124     private:
125         friend class B3IRGenerator;
126         BlockType blockType;
127         BasicBlock* continuation;
128         BasicBlock* special;
129         Vector<Variable*, 1> result;
130     };
131
132     typedef Value* ExpressionType;
133     typedef ControlData ControlType;
134     typedef Vector<ExpressionType, 1> ExpressionList;
135     typedef Vector<Variable*, 1> ResultList;
136     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
137
138     static constexpr ExpressionType emptyExpression = nullptr;
139
140     typedef String ErrorType;
141     typedef UnexpectedType<ErrorType> UnexpectedResult;
142     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
143     typedef Expected<void, ErrorType> PartialResult;
144     template <typename ...Args>
145     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
146     {
147         using namespace FailureHelper; // See ADL comment in WasmParser.h.
148         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
149     }
150 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
151         if (UNLIKELY(condition))                  \
152             return fail(__VA_ARGS__);             \
153     } while (0)
154
155     B3IRGenerator(VM&, const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, const ImmutableFunctionIndexSpace&);
156
157     PartialResult WARN_UNUSED_RETURN addArguments(const Vector<Type>&);
158     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
159     ExpressionType addConstant(Type, uint64_t);
160
161     // Locals
162     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
163     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
164
165     // Globals
166     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
167     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
168
169     // Memory
170     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
171     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
172
173     // Basic operators
174     template<OpType>
175     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
176     template<OpType>
177     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
178     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
179
180     // Control flow
181     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
182     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
183     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
184     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
185     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
186
187     PartialResult WARN_UNUSED_RETURN addReturn(const ExpressionList& returnValues);
188     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
189     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
190     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
191     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
192
193     // Calls
194     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
195     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
196     PartialResult WARN_UNUSED_RETURN addUnreachable();
197
198     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack);
199
200     void emitExceptionCheck(CCallHelpers&, ExceptionType);
201
202 private:
203     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
204     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
205     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
206
207     void unify(Variable* target, const ExpressionType source);
208     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
209     Value* zeroForType(Type);
210
211     VM& m_vm;
212     const ImmutableFunctionIndexSpace& m_functionIndexSpace;
213     const ModuleInformation& m_info;
214     Procedure& m_proc;
215     BasicBlock* m_currentBlock;
216     Vector<Variable*> m_locals;
217     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
218     GPRReg m_memoryBaseGPR;
219     GPRReg m_memorySizeGPR;
220     Value* m_zeroValues[numTypes];
221     Value* m_instanceValue;
222 };
223
224 B3IRGenerator::B3IRGenerator(VM& vm, const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace)
225     : m_vm(vm)
226     , m_functionIndexSpace(functionIndexSpace)
227     , m_info(info)
228     , m_proc(procedure)
229     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
230 {
231     m_currentBlock = m_proc.addBlock();
232
233     for (unsigned i = 0; i < numTypes; ++i) {
234         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
235         case B3::Int32:
236         case B3::Int64:
237         case B3::Float:
238         case B3::Double:
239             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
240             break;
241         case B3::Void:
242             m_zeroValues[i] = nullptr;
243             break;
244         }
245     }
246
247     if (!!info.memory) {
248         m_memoryBaseGPR = info.memory.pinnedRegisters().baseMemoryPointer;
249         m_proc.pinRegister(m_memoryBaseGPR);
250         ASSERT(!info.memory.pinnedRegisters().sizeRegisters[0].sizeOffset);
251         m_memorySizeGPR = info.memory.pinnedRegisters().sizeRegisters[0].sizeRegister;
252         for (const PinnedSizeRegisterInfo& regInfo : info.memory.pinnedRegisters().sizeRegisters)
253             m_proc.pinRegister(regInfo.sizeRegister);
254
255         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
256             AllowMacroScratchRegisterUsage allowScratch(jit);
257             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
258             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
259         });
260
261         B3::PatchpointValue* foo = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
262         foo->setGenerator(
263             [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
264                 AllowMacroScratchRegisterUsage allowScratch(jit);
265             });
266     }
267
268     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
269
270     m_instanceValue = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
271         m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), &m_vm.topJSWebAssemblyInstance));
272 }
273
274 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
275 {
276     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
277     auto jumpToExceptionStub = jit.jump();
278
279     VM* vm = &m_vm;
280     jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
281         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(vm->getCTIStub(throwExceptionFromWasmThunkGenerator).code()));
282     });
283 }
284
285 Value* B3IRGenerator::zeroForType(Type type)
286 {
287     ASSERT(type != Void);
288     Value* zeroValue = m_zeroValues[linearizeType(type)];
289     ASSERT(zeroValue);
290     return zeroValue;
291 }
292
293 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
294 {
295     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
296
297     for (uint32_t i = 0; i < count; ++i) {
298         Variable* local = m_proc.addVariable(toB3Type(type));
299         m_locals.uncheckedAppend(local);
300         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
301     }
302     return { };
303 }
304
305 auto B3IRGenerator::addArguments(const Vector<Type>& types) -> PartialResult
306 {
307     ASSERT(!m_locals.size());
308     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(types.size()), "can't allocate memory for ", types.size(), " arguments");
309
310     m_locals.grow(types.size());
311     wasmCallingConvention().loadArguments(types, m_proc, m_currentBlock, Origin(),
312         [&] (ExpressionType argument, unsigned i) {
313             Variable* argumentVariable = m_proc.addVariable(argument->type());
314             m_locals[i] = argumentVariable;
315             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
316         });
317     return { };
318 }
319
320 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
321 {
322     ASSERT(m_locals[index]);
323     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
324     return { };
325 }
326
327 auto B3IRGenerator::addUnreachable() -> PartialResult
328 {
329     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
330     unreachable->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
331         jit.breakpoint();
332     });
333     unreachable->effects.terminal = true;
334     return { };
335 }
336
337 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
338 {
339     ASSERT(m_locals[index]);
340     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
341     return { };
342 }
343
344 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
345 {
346     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
347     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), Origin(), globalsArray, index * sizeof(Register));
348     return { };
349 }
350
351 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
352 {
353     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
354     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
355     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, Origin(), value, globalsArray, index * sizeof(Register));
356     return { };
357 }
358
359 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
360 {
361     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
362     ASSERT(sizeOfOperation + offset > offset);
363     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
364     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
365     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
366 }
367
368 inline uint32_t sizeOfLoadOp(LoadOpType op)
369 {
370     switch (op) {
371     case LoadOpType::I32Load8S:
372     case LoadOpType::I32Load8U:
373     case LoadOpType::I64Load8S:
374     case LoadOpType::I64Load8U:
375         return 1;
376     case LoadOpType::I32Load16S:
377     case LoadOpType::I64Load16S:
378     case LoadOpType::I32Load16U:
379     case LoadOpType::I64Load16U:
380         return 2;
381     case LoadOpType::I32Load:
382     case LoadOpType::I64Load32S:
383     case LoadOpType::I64Load32U:
384     case LoadOpType::F32Load:
385         return 4;
386     case LoadOpType::I64Load:
387     case LoadOpType::F64Load:
388         return 8;
389     }
390     RELEASE_ASSERT_NOT_REACHED();
391 }
392
393 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
394 {
395     switch (op) {
396     case LoadOpType::I32Load8S: {
397         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
398     }
399
400     case LoadOpType::I64Load8S: {
401         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
402         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
403     }
404
405     case LoadOpType::I32Load8U: {
406         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
407     }
408
409     case LoadOpType::I64Load8U: {
410         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
411         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
412     }
413
414     case LoadOpType::I32Load16S: {
415         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
416     }
417     case LoadOpType::I64Load16S: {
418         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
419         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
420     }
421
422     case LoadOpType::I32Load: {
423         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
424     }
425
426     case LoadOpType::I64Load32U: {
427         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
428         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
429     }
430
431     case LoadOpType::I64Load32S: {
432         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
433         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
434     }
435
436     case LoadOpType::I64Load: {
437         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer);
438     }
439
440     case LoadOpType::F32Load: {
441         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer);
442     }
443
444     case LoadOpType::F64Load: {
445         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer);
446     }
447
448     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
449     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
450     case LoadOpType::I32Load16U: {
451         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
452         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
453                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
454     }
455     case LoadOpType::I64Load16U: {
456         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
457         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
458                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
459
460         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), partialResult);
461     }
462     }
463     RELEASE_ASSERT_NOT_REACHED();
464 }
465
466 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
467 {
468     ASSERT(pointer->type() == Int32);
469
470     result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
471     return { };
472 }
473
474 inline uint32_t sizeOfStoreOp(StoreOpType op)
475 {
476     switch (op) {
477     case StoreOpType::I32Store8:
478     case StoreOpType::I64Store8:
479         return 1;
480     case StoreOpType::I32Store16:
481     case StoreOpType::I64Store16:
482         return 2;
483     case StoreOpType::I32Store:
484     case StoreOpType::I64Store32:
485     case StoreOpType::F32Store:
486         return 4;
487     case StoreOpType::I64Store:
488     case StoreOpType::F64Store:
489         return 8;
490     }
491     RELEASE_ASSERT_NOT_REACHED();
492 }
493
494
495 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
496 {
497     switch (op) {
498     case StoreOpType::I64Store8:
499         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
500         FALLTHROUGH;
501
502     case StoreOpType::I32Store8:
503         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
504         return;
505
506     case StoreOpType::I64Store16:
507         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
508         FALLTHROUGH;
509
510     case StoreOpType::I32Store16:
511         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
512         return;
513
514     case StoreOpType::I64Store32:
515         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
516         FALLTHROUGH;
517
518     case StoreOpType::I64Store:
519     case StoreOpType::I32Store:
520     case StoreOpType::F32Store:
521     case StoreOpType::F64Store:
522         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
523         return;
524     }
525     RELEASE_ASSERT_NOT_REACHED();
526 }
527
528 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
529 {
530     ASSERT(pointer->type() == Int32);
531
532     emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
533     return { };
534 }
535
536 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
537 {
538     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
539     return { };
540 }
541
542 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
543 {
544     switch (type) {
545     case Wasm::I32:
546         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
547     case Wasm::I64:
548         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
549     case Wasm::F32:
550         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
551     case Wasm::F64:
552         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
553     case Wasm::Void:
554     case Wasm::Func:
555     case Wasm::Anyfunc:
556         break;
557     }
558     RELEASE_ASSERT_NOT_REACHED();
559     return nullptr;
560 }
561
562 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
563 {
564     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
565 }
566
567 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
568 {
569     BasicBlock* body = m_proc.addBlock();
570     BasicBlock* continuation = m_proc.addBlock();
571     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
572     body->addPredecessor(m_currentBlock);
573     m_currentBlock = body;
574     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
575 }
576
577 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
578 {
579     // FIXME: This needs to do some kind of stack passing.
580
581     BasicBlock* taken = m_proc.addBlock();
582     BasicBlock* notTaken = m_proc.addBlock();
583     BasicBlock* continuation = m_proc.addBlock();
584
585     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
586     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
587     taken->addPredecessor(m_currentBlock);
588     notTaken->addPredecessor(m_currentBlock);
589
590     m_currentBlock = taken;
591     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
592     return { };
593 }
594
595 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
596 {
597     unifyValuesWithBlock(currentStack, data.result);
598     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
599     return addElseToUnreachable(data);
600 }
601
602 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
603 {
604     ASSERT(data.type() == BlockType::If);
605     m_currentBlock = data.special;
606     data.convertIfToBlock();
607     return { };
608 }
609
610 auto B3IRGenerator::addReturn(const ExpressionList& returnValues) -> PartialResult
611 {
612     ASSERT(returnValues.size() <= 1);
613     if (returnValues.size())
614         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
615     else
616         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
617     return { };
618 }
619
620 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
621 {
622     if (data.type() != BlockType::Loop)
623         unifyValuesWithBlock(returnValues, data.result);
624
625     BasicBlock* target = data.targetBlockForBranch();
626     if (condition) {
627         BasicBlock* continuation = m_proc.addBlock();
628         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
629         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
630         target->addPredecessor(m_currentBlock);
631         continuation->addPredecessor(m_currentBlock);
632         m_currentBlock = continuation;
633     } else {
634         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
635         target->addPredecessor(m_currentBlock);
636     }
637
638     return { };
639 }
640
641 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
642 {
643     for (size_t i = 0; i < targets.size(); ++i)
644         unifyValuesWithBlock(expressionStack, targets[i]->result);
645     unifyValuesWithBlock(expressionStack, defaultTarget.result);
646
647     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
648     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
649     for (size_t i = 0; i < targets.size(); ++i)
650         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
651
652     return { };
653 }
654
655 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
656 {
657     ControlData& data = entry.controlData;
658
659     unifyValuesWithBlock(expressionStack, data.result);
660     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
661     data.continuation->addPredecessor(m_currentBlock);
662
663     return addEndToUnreachable(entry);
664 }
665
666
667 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
668 {
669     ControlData& data = entry.controlData;
670     m_currentBlock = data.continuation;
671
672     if (data.type() == BlockType::If) {
673         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
674         m_currentBlock->addPredecessor(data.special);
675     }
676
677     for (Variable* result : data.result)
678         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
679
680     return { };
681 }
682
683 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
684 {
685     ASSERT(signature->arguments.size() == args.size());
686
687     Type returnType = signature->returnType;
688
689     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
690         [&] (PatchpointValue* patchpoint) {
691             patchpoint->effects.writesPinned = true;
692             patchpoint->effects.readsPinned = true;
693
694             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
695                 AllowMacroScratchRegisterUsage allowScratch(jit);
696
697                 CCallHelpers::Call call = jit.call();
698
699                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
700                     m_unlinkedWasmToWasmCalls.append({ linkBuffer.locationOf(call), functionIndex });
701                 });
702             });
703         });
704     return { };
705 }
706
707 auto B3IRGenerator::addCallIndirect(const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
708 {
709     ExpressionType calleeIndex = args.takeLast();
710     ASSERT(signature->arguments.size() == args.size());
711
712     ExpressionType callableFunctionBuffer;
713     ExpressionType callableFunctionBufferSize;
714     {
715         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
716             m_instanceValue, JSWebAssemblyInstance::offsetOfTable());
717         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
718             table, JSWebAssemblyTable::offsetOfFunctions());
719         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, Origin(),
720             table, JSWebAssemblyTable::offsetOfSize());
721     }
722
723     // Check the index we are looking for is valid.
724     {
725         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
726             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, Origin(), calleeIndex, callableFunctionBufferSize));
727
728         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
729             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
730         });
731     }
732
733     // Compute the offset in the table index space we are looking for.
734     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, Origin(),
735         m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), calleeIndex),
736         m_currentBlock->appendIntConstant(m_proc, Origin(), pointerType(), sizeof(CallableFunction)));
737     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, Origin(), callableFunctionBuffer, offset);
738
739     // Check that the CallableFunction is initialized. We trap if it isn't. A null Signature* indicates it's not initialized.
740     ExpressionType calleeSignature = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signature));
741     {
742         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
743             m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), 
744                 calleeSignature, 
745                 m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), 0)));
746
747         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
748             this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
749         });
750     }
751
752     // Check the signature matches the value we expect.
753     {
754         ExpressionType expectedSignature = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), signature);
755         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
756             m_currentBlock->appendNew<Value>(m_proc, NotEqual, Origin(), calleeSignature, expectedSignature));
757
758         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
759             this->emitExceptionCheck(jit, ExceptionType::BadSignature);
760         });
761     }
762
763     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
764
765     Type returnType = signature->returnType;
766     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
767         [&] (PatchpointValue* patchpoint) {
768             patchpoint->effects.writesPinned = true;
769             patchpoint->effects.readsPinned = true;
770
771             patchpoint->append(calleeCode, ValueRep::SomeRegister);
772
773             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
774                 jit.call(params[returnType == Void ? 0 : 1].gpr());
775             });
776         });
777
778     return { };
779 }
780
781 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
782 {
783     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
784 }
785
786 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
787 {
788     ASSERT(result.size() <= resultStack.size());
789
790     for (size_t i = 0; i < result.size(); ++i)
791         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
792 }
793
794 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
795 {
796     dataLogLn(comma, "ExpressionStack:");
797     for (const auto& expression : expressionStack)
798         dataLogLn(comma, *expression);
799 }
800
801 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack)
802 {
803     dataLogLn("Processing Graph:");
804     dataLog(m_proc);
805     dataLogLn("With current block:", *m_currentBlock);
806     dataLogLn("Control stack:");
807     for (auto& data : controlStack) {
808         dataLogLn("  ", data.controlData);
809         if (data.enclosedExpressionStack.size()) {
810             CommaPrinter comma("    ", "  with ");
811             dumpExpressionStack(comma, data.enclosedExpressionStack);
812         }
813     }
814
815     CommaPrinter comma("  ", "");
816     dumpExpressionStack(comma, expressionStack);
817     dataLogLn("\n");
818 }
819
820 static std::unique_ptr<B3::Compilation> createJSToWasmWrapper(VM& vm, WasmInternalFunction& function, const Signature* signature, MacroAssemblerCodePtr mainFunction, const MemoryInformation& memory)
821 {
822     Procedure proc;
823     BasicBlock* block = proc.addBlock();
824
825     Origin origin;
826
827     jscCallingConvention().setupFrameInPrologue(&function.jsToWasmCalleeMoveLocation, proc, origin, block);
828
829     if (!ASSERT_DISABLED) {
830         // This should be guaranteed by our JS wrapper that handles calls to us.
831         // Just prevent against crazy when ASSERT is enabled.
832         Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, origin);
833         Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, origin, CallFrameSlot::argumentCount * sizeof(Register));
834         Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, origin,
835             block->appendNew<Value>(proc, Add, origin, framePointer, offSetOfArgumentCount));
836
837         Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, origin, signature->arguments.size());
838
839         CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, origin,
840             block->appendNew<Value>(proc, Above, origin, expectedArgumentCount, argumentCount));
841
842         argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
843             jit.breakpoint();
844         });
845     }
846
847     // Move memory values to the approriate places, if needed.
848     Value* baseMemory = nullptr;
849     Vector<Value*> sizes;
850     if (!!memory) {
851         baseMemory = block->appendNew<MemoryValue>(proc, Load, Int64, Origin(),
852             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemoryPointer));
853         Value* size = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
854             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemorySize));
855         sizes.reserveCapacity(memory.pinnedRegisters().sizeRegisters.size());
856         for (auto info : memory.pinnedRegisters().sizeRegisters) {
857             sizes.append(block->appendNew<Value>(proc, Sub, origin, size,
858                 block->appendNew<Const32Value>(proc, origin, info.sizeOffset)));
859         }
860     }
861
862     // Get our arguments.
863     Vector<Value*> arguments;
864     jscCallingConvention().loadArguments(signature->arguments, proc, block, origin, [&] (Value* argument, unsigned) {
865         arguments.append(argument);
866     });
867
868     // Move the arguments into place.
869     Value* result = wasmCallingConvention().setupCall(proc, block, origin, arguments, toB3Type(signature->returnType), [&] (PatchpointValue* patchpoint) {
870         if (!!memory) {
871             ASSERT(sizes.size() == memory.pinnedRegisters().sizeRegisters.size());
872             patchpoint->append(ConstrainedValue(baseMemory, ValueRep::reg(memory.pinnedRegisters().baseMemoryPointer)));
873             for (unsigned i = 0; i < sizes.size(); ++i)
874                 patchpoint->append(ConstrainedValue(sizes[i], ValueRep::reg(memory.pinnedRegisters().sizeRegisters[i].sizeRegister)));
875         }
876
877         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
878             AllowMacroScratchRegisterUsage allowScratch(jit);
879
880             CCallHelpers::Call call = jit.call();
881             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
882                 linkBuffer.link(call, FunctionPtr(mainFunction.executableAddress()));
883             });
884         });
885     });
886
887     // Return the result, if needed.
888     switch (signature->returnType) {
889     case Wasm::Void:
890         block->appendNewControlValue(proc, B3::Return, origin);
891         break;
892     case Wasm::F32:
893     case Wasm::F64:
894         result = block->appendNew<Value>(proc, BitwiseCast, origin, result);
895         FALLTHROUGH;
896     case Wasm::I32:
897     case Wasm::I64:
898         block->appendNewControlValue(proc, B3::Return, origin, result);
899         break;
900     case Wasm::Func:
901     case Wasm::Anyfunc:
902         RELEASE_ASSERT_NOT_REACHED();
903     }
904
905     auto jsEntrypoint = std::make_unique<Compilation>(B3::compile(vm, proc));
906     function.jsToWasmEntrypoint.calleeSaveRegisters = proc.calleeSaveRegisters();
907     return jsEntrypoint;
908 }
909
910 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(VM& vm, const uint8_t* functionStart, size_t functionLength, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace, const ModuleInformation& info, unsigned optLevel)
911 {
912     auto result = std::make_unique<WasmInternalFunction>();
913
914     Procedure procedure;
915     B3IRGenerator context(vm, info, procedure, result.get(), unlinkedWasmToWasmCalls, functionIndexSpace);
916     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, functionIndexSpace, info);
917     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
918
919     procedure.resetReachability();
920     validate(procedure, "After parsing:\n");
921
922     if (verbose)
923         dataLog("Pre SSA: ", procedure);
924     fixSSA(procedure);
925     if (verbose)
926         dataLog("Post SSA: ", procedure);
927
928     result->wasmEntrypoint.compilation = std::make_unique<B3::Compilation>(B3::compile(vm, procedure, optLevel));
929     result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisters();
930
931     result->jsToWasmEntrypoint.compilation = createJSToWasmWrapper(vm, *result, signature, result->wasmEntrypoint.compilation->code(), info.memory);
932     return WTFMove(result);
933 }
934
935 // Custom wasm ops. These are the ones too messy to do in wasm.json.
936
937 template<>
938 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
939 {
940     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
941     patchpoint->append(arg, ValueRep::SomeRegister);
942     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
943         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
944     });
945     patchpoint->effects = Effects::none();
946     result = patchpoint;
947     return { };
948 }
949
950 template<>
951 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
952 {
953     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
954     patchpoint->append(arg, ValueRep::SomeRegister);
955     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
956         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
957     });
958     patchpoint->effects = Effects::none();
959     result = patchpoint;
960     return { };
961 }
962
963 template<>
964 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
965 {
966     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
967     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
968     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
969     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
970     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
971     return { };
972 }
973
974 template<>
975 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
976 {
977     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
978     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
979     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
980     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
981     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
982     return { };
983 }
984
985 template<>
986 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
987 {
988     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
989     if (isX86())
990         patchpoint->numGPScratchRegisters = 1;
991     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
992     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
993         AllowMacroScratchRegisterUsage allowScratch(jit);
994 #if CPU(X86_64)
995         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
996 #else
997         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
998 #endif
999     });
1000     patchpoint->effects = Effects::none();
1001     result = patchpoint;
1002     return { };
1003 }
1004
1005 template<>
1006 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1007 {
1008     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1009     if (isX86())
1010         patchpoint->numGPScratchRegisters = 1;
1011     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
1012     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1013         AllowMacroScratchRegisterUsage allowScratch(jit);
1014 #if CPU(X86_64)
1015         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1016 #else
1017         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1018 #endif
1019     });
1020     patchpoint->effects = Effects::none();
1021     result = patchpoint;
1022     return { };
1023 }
1024
1025 template<>
1026 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1027 {
1028     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1029     patchpoint->append(arg, ValueRep::SomeRegister);
1030     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1031         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1032     });
1033     patchpoint->effects = Effects::none();
1034     result = patchpoint;
1035     return { };
1036 }
1037
1038 template<>
1039 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1040 {
1041     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1042     patchpoint->append(arg, ValueRep::SomeRegister);
1043     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1044         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1045     });
1046     patchpoint->effects = Effects::none();
1047     result = patchpoint;
1048     return { };
1049 }
1050
1051 template<>
1052 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1053 {
1054     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1055     patchpoint->append(arg, ValueRep::SomeRegister);
1056     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1057         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1058     });
1059     patchpoint->effects = Effects::none();
1060     result = patchpoint;
1061     return { };
1062 }
1063
1064 template<>
1065 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1066 {
1067     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1068     patchpoint->append(arg, ValueRep::SomeRegister);
1069     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1070         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1071     });
1072     patchpoint->effects = Effects::none();
1073     result = patchpoint;
1074     return { };
1075 }
1076
1077 template<>
1078 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1079 {
1080     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
1081     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
1082     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1083         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1084         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1085     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1086     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1087     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1088         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1089     });
1090     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1091     patchpoint->append(arg, ValueRep::SomeRegister);
1092     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1093         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1094     });
1095     patchpoint->effects = Effects::none();
1096     result = patchpoint;
1097     return { };
1098 }
1099
1100 template<>
1101 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1102 {
1103     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
1104     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
1105     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1106         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1107         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1108     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1109     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1110     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1111         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1112     });
1113     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1114     patchpoint->append(arg, ValueRep::SomeRegister);
1115     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1116         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1117     });
1118     patchpoint->effects = Effects::none();
1119     result = patchpoint;
1120     return { };
1121 }
1122
1123
1124 template<>
1125 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1126 {
1127     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
1128     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1129     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1130         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1131         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1132     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1133     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1134     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1135         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1136     });
1137     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1138     patchpoint->append(arg, ValueRep::SomeRegister);
1139     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1140         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1141     });
1142     patchpoint->effects = Effects::none();
1143     result = patchpoint;
1144     return { };
1145 }
1146
1147 template<>
1148 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1149 {
1150     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
1151     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1152     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1153         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1154         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1155     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1156     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1157     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1158         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1159     });
1160     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1161     patchpoint->append(arg, ValueRep::SomeRegister);
1162     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1163         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1164     });
1165     patchpoint->effects = Effects::none();
1166     result = patchpoint;
1167     return { };
1168 }
1169
1170 template<>
1171 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1172 {
1173     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1174     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1175     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1176         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1177         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1178     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1179     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1180     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1181         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1182     });
1183     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1184     patchpoint->append(arg, ValueRep::SomeRegister);
1185     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1186         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1187     });
1188     patchpoint->effects = Effects::none();
1189     result = patchpoint;
1190     return { };
1191 }
1192
1193 template<>
1194 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1195 {
1196     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1197     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1198     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1199         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1200         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1201     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1202     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1203     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1204         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1205     });
1206
1207     Value* constant;
1208     if (isX86()) {
1209         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1210         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1211         // so we can pool them if needed.
1212         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1213     }
1214     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1215     patchpoint->append(arg, ValueRep::SomeRegister);
1216     if (isX86()) {
1217         patchpoint->append(constant, ValueRep::SomeRegister);
1218         patchpoint->numFPScratchRegisters = 1;
1219     }
1220     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1221         AllowMacroScratchRegisterUsage allowScratch(jit);
1222         FPRReg scratch = InvalidFPRReg;
1223         FPRReg constant = InvalidFPRReg;
1224         if (isX86()) {
1225             scratch = params.fpScratch(0);
1226             constant = params[2].fpr();
1227         }
1228         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1229     });
1230     patchpoint->effects = Effects::none();
1231     result = patchpoint;
1232     return { };
1233 }
1234
1235 template<>
1236 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1237 {
1238     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1239     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1240     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1241         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1242         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1243     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1244     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1245     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1246         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1247     });
1248     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1249     patchpoint->append(arg, ValueRep::SomeRegister);
1250     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1251         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1252     });
1253     patchpoint->effects = Effects::none();
1254     result = patchpoint;
1255     return { };
1256 }
1257
1258 template<>
1259 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1260 {
1261     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1262     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1263     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1264         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1265         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1266     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1267     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1268     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1269         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1270     });
1271
1272     Value* constant;
1273     if (isX86()) {
1274         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1275         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1276         // so we can pool them if needed.
1277         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1278     }
1279     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1280     patchpoint->append(arg, ValueRep::SomeRegister);
1281     if (isX86()) {
1282         patchpoint->append(constant, ValueRep::SomeRegister);
1283         patchpoint->numFPScratchRegisters = 1;
1284     }
1285     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1286         AllowMacroScratchRegisterUsage allowScratch(jit);
1287         FPRReg scratch = InvalidFPRReg;
1288         FPRReg constant = InvalidFPRReg;
1289         if (isX86()) {
1290             scratch = params.fpScratch(0);
1291             constant = params[2].fpr();
1292         }
1293         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1294     });
1295     patchpoint->effects = Effects::none();
1296     result = patchpoint;
1297     return { };
1298 }
1299
1300 } } // namespace JSC::Wasm
1301
1302 #include "WasmB3IRGeneratorInlines.h"
1303
1304 #endif // ENABLE(WEBASSEMBLY)