c2de9b338bbb6fc866f8549b74fada8d932584f2
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "B3BasicBlockInlines.h"
32 #include "B3CCallValue.h"
33 #include "B3ConstPtrValue.h"
34 #include "B3FixSSA.h"
35 #include "B3StackmapGenerationParams.h"
36 #include "B3SwitchValue.h"
37 #include "B3Validate.h"
38 #include "B3ValueInlines.h"
39 #include "B3Variable.h"
40 #include "B3VariableValue.h"
41 #include "B3WasmAddressValue.h"
42 #include "B3WasmBoundsCheckValue.h"
43 #include "ExceptionScope.h"
44 #include "FrameTracers.h"
45 #include "JITExceptions.h"
46 #include "JSCInlines.h"
47 #include "JSWebAssemblyInstance.h"
48 #include "JSWebAssemblyModule.h"
49 #include "JSWebAssemblyRuntimeError.h"
50 #include "VirtualRegister.h"
51 #include "WasmCallingConvention.h"
52 #include "WasmFunctionParser.h"
53 #include "WasmMemory.h"
54 #include <wtf/Optional.h>
55
56 void dumpProcedure(void* ptr)
57 {
58     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
59     proc->dump(WTF::dataFile());
60 }
61
62 namespace JSC { namespace Wasm {
63
64 using namespace B3;
65
66 namespace {
67 const bool verbose = false;
68 }
69
70 class B3IRGenerator {
71 public:
72     struct ControlData {
73         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
74             : blockType(type)
75             , continuation(continuation)
76             , special(special)
77         {
78             if (signature != Void)
79                 result.append(proc.addVariable(toB3Type(signature)));
80         }
81
82         ControlData()
83         {
84         }
85
86         void dump(PrintStream& out) const
87         {
88             switch (type()) {
89             case BlockType::If:
90                 out.print("If:    ");
91                 break;
92             case BlockType::Block:
93                 out.print("Block: ");
94                 break;
95             case BlockType::Loop:
96                 out.print("Loop:  ");
97                 break;
98             }
99             out.print("Continuation: ", *continuation, ", Special: ");
100             if (special)
101                 out.print(*special);
102             else
103                 out.print("None");
104         }
105
106         BlockType type() const { return blockType; }
107
108         bool hasNonVoidSignature() const { return result.size(); }
109
110         BasicBlock* targetBlockForBranch()
111         {
112             if (type() == BlockType::Loop)
113                 return special;
114             return continuation;
115         }
116
117         void convertIfToBlock()
118         {
119             ASSERT(type() == BlockType::If);
120             blockType = BlockType::Block;
121             special = nullptr;
122         }
123
124     private:
125         friend class B3IRGenerator;
126         BlockType blockType;
127         BasicBlock* continuation;
128         BasicBlock* special;
129         Vector<Variable*, 1> result;
130     };
131
132     typedef Value* ExpressionType;
133     typedef ControlData ControlType;
134     typedef Vector<ExpressionType, 1> ExpressionList;
135     typedef Vector<Variable*, 1> ResultList;
136     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
137
138     static constexpr ExpressionType emptyExpression = nullptr;
139
140     typedef String ErrorType;
141     typedef UnexpectedType<ErrorType> UnexpectedResult;
142     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
143     typedef Expected<void, ErrorType> PartialResult;
144     template <typename ...Args>
145     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
146     {
147         using namespace FailureHelper; // See ADL comment in WasmParser.h.
148         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
149     }
150 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
151         if (UNLIKELY(condition))                  \
152             return fail(__VA_ARGS__);             \
153     } while (0)
154
155     B3IRGenerator(VM&, const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, const ImmutableFunctionIndexSpace&);
156
157     PartialResult WARN_UNUSED_RETURN addArguments(const Vector<Type>&);
158     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
159     ExpressionType addConstant(Type, uint64_t);
160
161     // Locals
162     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
163     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
164
165     // Globals
166     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
167     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
168
169     // Memory
170     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
171     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
172
173     // Basic operators
174     template<OpType>
175     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
176     template<OpType>
177     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
178     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
179
180     // Control flow
181     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
182     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
183     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
184     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
185     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
186
187     PartialResult WARN_UNUSED_RETURN addReturn(const ExpressionList& returnValues);
188     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
189     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
190     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
191     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
192
193     // Calls
194     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
195     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
196     PartialResult WARN_UNUSED_RETURN addUnreachable();
197
198     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack);
199
200 private:
201     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
202     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
203     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
204
205     void unify(Variable* target, const ExpressionType source);
206     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
207     Value* zeroForType(Type);
208
209     VM& m_vm;
210     const ImmutableFunctionIndexSpace& m_functionIndexSpace;
211     const ModuleInformation& m_info;
212     Procedure& m_proc;
213     BasicBlock* m_currentBlock;
214     Vector<Variable*> m_locals;
215     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
216     GPRReg m_memoryBaseGPR;
217     GPRReg m_memorySizeGPR;
218     Value* m_zeroValues[numTypes];
219     Value* m_instanceValue;
220
221 };
222
223 B3IRGenerator::B3IRGenerator(VM& vm, const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace)
224     : m_vm(vm)
225     , m_functionIndexSpace(functionIndexSpace)
226     , m_info(info)
227     , m_proc(procedure)
228     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
229 {
230     m_currentBlock = m_proc.addBlock();
231
232     for (unsigned i = 0; i < numTypes; ++i) {
233         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
234         case B3::Int32:
235         case B3::Int64:
236         case B3::Float:
237         case B3::Double:
238             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
239             break;
240         case B3::Void:
241             m_zeroValues[i] = nullptr;
242             break;
243         }
244     }
245
246     if (!!info.memory) {
247         m_memoryBaseGPR = info.memory.pinnedRegisters().baseMemoryPointer;
248         m_proc.pinRegister(m_memoryBaseGPR);
249         ASSERT(!info.memory.pinnedRegisters().sizeRegisters[0].sizeOffset);
250         m_memorySizeGPR = info.memory.pinnedRegisters().sizeRegisters[0].sizeRegister;
251         for (const PinnedSizeRegisterInfo& regInfo : info.memory.pinnedRegisters().sizeRegisters)
252             m_proc.pinRegister(regInfo.sizeRegister);
253
254         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
255             AllowMacroScratchRegisterUsage allowScratch(jit);
256             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
257             jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer();
258
259             jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
260
261             CCallHelpers::Call call = jit.call();
262             jit.jumpToExceptionHandler();
263
264             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
265                 void (*throwMemoryException)(ExecState*) = [] (ExecState* exec) {
266                     VM* vm = &exec->vm();
267                     NativeCallFrameTracer tracer(vm, exec);
268
269                     {
270                         auto throwScope = DECLARE_THROW_SCOPE(*vm);
271                         JSGlobalObject* globalObject = vm->topJSWebAssemblyInstance->globalObject();
272
273                         JSWebAssemblyRuntimeError* error = JSWebAssemblyRuntimeError::create(
274                             exec, globalObject->WebAssemblyRuntimeErrorStructure(), "Out of bounds memory access");
275                         throwException(exec, throwScope, error);
276                     }
277
278                     genericUnwind(vm, exec);
279                     ASSERT(!!vm->callFrameForCatch);
280                 };
281
282                 linkBuffer.link(call, throwMemoryException);
283             });
284         });
285
286         B3::PatchpointValue* foo = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
287         foo->setGenerator(
288             [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
289                 AllowMacroScratchRegisterUsage allowScratch(jit);
290             });
291     }
292
293     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
294
295     m_instanceValue = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
296         m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), &m_vm.topJSWebAssemblyInstance));
297 }
298
299 Value* B3IRGenerator::zeroForType(Type type)
300 {
301     ASSERT(type != Void);
302     Value* zeroValue = m_zeroValues[linearizeType(type)];
303     ASSERT(zeroValue);
304     return zeroValue;
305 }
306
307 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
308 {
309     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
310
311     for (uint32_t i = 0; i < count; ++i) {
312         Variable* local = m_proc.addVariable(toB3Type(type));
313         m_locals.uncheckedAppend(local);
314         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
315     }
316     return { };
317 }
318
319 auto B3IRGenerator::addArguments(const Vector<Type>& types) -> PartialResult
320 {
321     ASSERT(!m_locals.size());
322     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(types.size()), "can't allocate memory for ", types.size(), " arguments");
323
324     m_locals.grow(types.size());
325     wasmCallingConvention().loadArguments(types, m_proc, m_currentBlock, Origin(),
326         [&] (ExpressionType argument, unsigned i) {
327             Variable* argumentVariable = m_proc.addVariable(argument->type());
328             m_locals[i] = argumentVariable;
329             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
330         });
331     return { };
332 }
333
334 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
335 {
336     ASSERT(m_locals[index]);
337     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
338     return { };
339 }
340
341 auto B3IRGenerator::addUnreachable() -> PartialResult
342 {
343     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
344     unreachable->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
345         jit.breakpoint();
346     });
347     unreachable->effects.terminal = true;
348     return { };
349 }
350
351 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
352 {
353     ASSERT(m_locals[index]);
354     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
355     return { };
356 }
357
358 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
359 {
360     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
361     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), Origin(), globalsArray, index * sizeof(Register));
362     return { };
363 }
364
365 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
366 {
367     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
368     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
369     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, Origin(), value, globalsArray, index * sizeof(Register));
370     return { };
371 }
372
373 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
374 {
375     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
376     ASSERT(sizeOfOperation + offset > offset);
377     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
378     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
379     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
380 }
381
382 inline uint32_t sizeOfLoadOp(LoadOpType op)
383 {
384     switch (op) {
385     case LoadOpType::I32Load8S:
386     case LoadOpType::I32Load8U:
387     case LoadOpType::I64Load8S:
388     case LoadOpType::I64Load8U:
389         return 1;
390     case LoadOpType::I32Load16S:
391     case LoadOpType::I64Load16S:
392     case LoadOpType::I32Load16U:
393     case LoadOpType::I64Load16U:
394         return 2;
395     case LoadOpType::I32Load:
396     case LoadOpType::I64Load32S:
397     case LoadOpType::I64Load32U:
398     case LoadOpType::F32Load:
399         return 4;
400     case LoadOpType::I64Load:
401     case LoadOpType::F64Load:
402         return 8;
403     }
404     RELEASE_ASSERT_NOT_REACHED();
405 }
406
407 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
408 {
409     switch (op) {
410     case LoadOpType::I32Load8S: {
411         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
412     }
413
414     case LoadOpType::I64Load8S: {
415         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
416         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
417     }
418
419     case LoadOpType::I32Load8U: {
420         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
421     }
422
423     case LoadOpType::I64Load8U: {
424         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
425         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
426     }
427
428     case LoadOpType::I32Load16S: {
429         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
430     }
431     case LoadOpType::I64Load16S: {
432         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
433         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
434     }
435
436     case LoadOpType::I32Load: {
437         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
438     }
439
440     case LoadOpType::I64Load32U: {
441         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
442         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
443     }
444
445     case LoadOpType::I64Load32S: {
446         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
447         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
448     }
449
450     case LoadOpType::I64Load: {
451         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer);
452     }
453
454     case LoadOpType::F32Load: {
455         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer);
456     }
457
458     case LoadOpType::F64Load: {
459         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer);
460     }
461
462     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
463     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
464     case LoadOpType::I32Load16U: {
465         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
466         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
467                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
468     }
469     case LoadOpType::I64Load16U: {
470         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
471         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
472                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
473
474         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), partialResult);
475     }
476     }
477     RELEASE_ASSERT_NOT_REACHED();
478 }
479
480 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
481 {
482     ASSERT(pointer->type() == Int32);
483
484     result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
485     return { };
486 }
487
488 inline uint32_t sizeOfStoreOp(StoreOpType op)
489 {
490     switch (op) {
491     case StoreOpType::I32Store8:
492     case StoreOpType::I64Store8:
493         return 1;
494     case StoreOpType::I32Store16:
495     case StoreOpType::I64Store16:
496         return 2;
497     case StoreOpType::I32Store:
498     case StoreOpType::I64Store32:
499     case StoreOpType::F32Store:
500         return 4;
501     case StoreOpType::I64Store:
502     case StoreOpType::F64Store:
503         return 8;
504     }
505     RELEASE_ASSERT_NOT_REACHED();
506 }
507
508
509 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
510 {
511     switch (op) {
512     case StoreOpType::I64Store8:
513         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
514         FALLTHROUGH;
515
516     case StoreOpType::I32Store8:
517         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
518         return;
519
520     case StoreOpType::I64Store16:
521         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
522         FALLTHROUGH;
523
524     case StoreOpType::I32Store16:
525         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
526         return;
527
528     case StoreOpType::I64Store32:
529         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
530         FALLTHROUGH;
531
532     case StoreOpType::I64Store:
533     case StoreOpType::I32Store:
534     case StoreOpType::F32Store:
535     case StoreOpType::F64Store:
536         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
537         return;
538     }
539     RELEASE_ASSERT_NOT_REACHED();
540 }
541
542 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
543 {
544     ASSERT(pointer->type() == Int32);
545
546     emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
547     return { };
548 }
549
550 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
551 {
552     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
553     return { };
554 }
555
556 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
557 {
558     switch (type) {
559     case Wasm::I32:
560         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
561     case Wasm::I64:
562         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
563     case Wasm::F32:
564         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
565     case Wasm::F64:
566         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
567     case Wasm::Void:
568     case Wasm::Func:
569     case Wasm::Anyfunc:
570         break;
571     }
572     RELEASE_ASSERT_NOT_REACHED();
573     return nullptr;
574 }
575
576 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
577 {
578     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
579 }
580
581 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
582 {
583     BasicBlock* body = m_proc.addBlock();
584     BasicBlock* continuation = m_proc.addBlock();
585     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
586     body->addPredecessor(m_currentBlock);
587     m_currentBlock = body;
588     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
589 }
590
591 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
592 {
593     // FIXME: This needs to do some kind of stack passing.
594
595     BasicBlock* taken = m_proc.addBlock();
596     BasicBlock* notTaken = m_proc.addBlock();
597     BasicBlock* continuation = m_proc.addBlock();
598
599     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
600     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
601     taken->addPredecessor(m_currentBlock);
602     notTaken->addPredecessor(m_currentBlock);
603
604     m_currentBlock = taken;
605     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
606     return { };
607 }
608
609 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
610 {
611     unifyValuesWithBlock(currentStack, data.result);
612     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
613     return addElseToUnreachable(data);
614 }
615
616 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
617 {
618     ASSERT(data.type() == BlockType::If);
619     m_currentBlock = data.special;
620     data.convertIfToBlock();
621     return { };
622 }
623
624 auto B3IRGenerator::addReturn(const ExpressionList& returnValues) -> PartialResult
625 {
626     ASSERT(returnValues.size() <= 1);
627     if (returnValues.size())
628         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
629     else
630         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
631     return { };
632 }
633
634 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
635 {
636     if (data.type() != BlockType::Loop)
637         unifyValuesWithBlock(returnValues, data.result);
638
639     BasicBlock* target = data.targetBlockForBranch();
640     if (condition) {
641         BasicBlock* continuation = m_proc.addBlock();
642         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
643         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
644         target->addPredecessor(m_currentBlock);
645         continuation->addPredecessor(m_currentBlock);
646         m_currentBlock = continuation;
647     } else {
648         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
649         target->addPredecessor(m_currentBlock);
650     }
651
652     return { };
653 }
654
655 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
656 {
657     for (size_t i = 0; i < targets.size(); ++i)
658         unifyValuesWithBlock(expressionStack, targets[i]->result);
659     unifyValuesWithBlock(expressionStack, defaultTarget.result);
660
661     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
662     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
663     for (size_t i = 0; i < targets.size(); ++i)
664         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
665
666     return { };
667 }
668
669 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
670 {
671     ControlData& data = entry.controlData;
672
673     unifyValuesWithBlock(expressionStack, data.result);
674     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
675     data.continuation->addPredecessor(m_currentBlock);
676
677     return addEndToUnreachable(entry);
678 }
679
680
681 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
682 {
683     ControlData& data = entry.controlData;
684     m_currentBlock = data.continuation;
685
686     if (data.type() == BlockType::If) {
687         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
688         m_currentBlock->addPredecessor(data.special);
689     }
690
691     for (Variable* result : data.result)
692         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
693
694     return { };
695 }
696
697 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
698 {
699     ASSERT(signature->arguments.size() == args.size());
700
701     Type returnType = signature->returnType;
702
703     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
704         [&] (PatchpointValue* patchpoint) {
705             patchpoint->effects.writesPinned = true;
706             patchpoint->effects.readsPinned = true;
707
708             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
709                 AllowMacroScratchRegisterUsage allowScratch(jit);
710
711                 CCallHelpers::Call call = jit.call();
712
713                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
714                     m_unlinkedWasmToWasmCalls.append({ linkBuffer.locationOf(call), functionIndex });
715                 });
716             });
717         });
718     return { };
719 }
720
721 auto B3IRGenerator::addCallIndirect(const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
722 {
723     ExpressionType calleeIndex = args.takeLast();
724     ASSERT(signature->arguments.size() == args.size());
725
726     ExpressionType callableFunctionBuffer;
727     ExpressionType callableFunctionBufferSize;
728     {
729         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
730             m_instanceValue, JSWebAssemblyInstance::offsetOfTable());
731         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
732             table, JSWebAssemblyTable::offsetOfFunctions());
733         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, Origin(),
734             table, JSWebAssemblyTable::offsetOfSize());
735     }
736
737     // Check the index we are looking for is valid.
738     {
739         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
740             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, Origin(), calleeIndex, callableFunctionBufferSize));
741
742         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
743             jit.breakpoint();
744         });
745     }
746
747     // Compute the offset in the table index space we are looking for.
748     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, Origin(),
749         m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), calleeIndex),
750         m_currentBlock->appendIntConstant(m_proc, Origin(), pointerType(), sizeof(CallableFunction)));
751     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, Origin(), callableFunctionBuffer, offset);
752
753     // Check that the CallableFunction is initialized. We trap if it isn't. A null Signature* indicates it's not initialized.
754     ExpressionType calleeSignature = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signature));
755     {
756         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
757             m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), 
758                 calleeSignature, 
759                 m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), 0)));
760
761         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
762             jit.breakpoint();
763         });
764     }
765
766     // Check the signature matches the value we expect.
767     {
768         ExpressionType expectedSignature = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), signature);
769         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
770             m_currentBlock->appendNew<Value>(m_proc, NotEqual, Origin(), calleeSignature, expectedSignature));
771
772         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
773             jit.breakpoint();
774         });
775     }
776
777     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
778
779     Type returnType = signature->returnType;
780     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
781         [&] (PatchpointValue* patchpoint) {
782             patchpoint->effects.writesPinned = true;
783             patchpoint->effects.readsPinned = true;
784
785             patchpoint->append(calleeCode, ValueRep::SomeRegister);
786
787             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
788                 jit.call(params[returnType == Void ? 0 : 1].gpr());
789             });
790         });
791
792     return { };
793 }
794
795 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
796 {
797     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
798 }
799
800 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
801 {
802     ASSERT(result.size() <= resultStack.size());
803
804     for (size_t i = 0; i < result.size(); ++i)
805         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
806 }
807
808 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
809 {
810     dataLogLn(comma, "ExpressionStack:");
811     for (const auto& expression : expressionStack)
812         dataLogLn(comma, *expression);
813 }
814
815 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack)
816 {
817     dataLogLn("Processing Graph:");
818     dataLog(m_proc);
819     dataLogLn("With current block:", *m_currentBlock);
820     dataLogLn("Control stack:");
821     for (auto& data : controlStack) {
822         dataLogLn("  ", data.controlData);
823         if (data.enclosedExpressionStack.size()) {
824             CommaPrinter comma("    ", "  with ");
825             dumpExpressionStack(comma, data.enclosedExpressionStack);
826         }
827     }
828
829     CommaPrinter comma("  ", "");
830     dumpExpressionStack(comma, expressionStack);
831     dataLogLn("\n");
832 }
833
834 static std::unique_ptr<Compilation> createJSToWasmWrapper(VM& vm, WasmInternalFunction& function, const Signature* signature, MacroAssemblerCodePtr mainFunction, const MemoryInformation& memory)
835 {
836     Procedure proc;
837     BasicBlock* block = proc.addBlock();
838
839     Origin origin;
840
841     jscCallingConvention().setupFrameInPrologue(&function.jsToWasmCalleeMoveLocation, proc, origin, block);
842
843     Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, origin);
844     Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, origin, CallFrameSlot::argumentCount * sizeof(Register));
845     Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, origin,
846         block->appendNew<Value>(proc, Add, origin, framePointer, offSetOfArgumentCount));
847
848     Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, origin, signature->arguments.size());
849
850     CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, origin,
851         block->appendNew<Value>(proc, Above, origin, expectedArgumentCount, argumentCount));
852
853     argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
854         jit.breakpoint();
855     });
856
857     // Move memory values to the approriate places, if needed.
858     Value* baseMemory = nullptr;
859     Vector<Value*> sizes;
860     if (!!memory) {
861         baseMemory = block->appendNew<MemoryValue>(proc, Load, Int64, Origin(),
862             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemoryPointer));
863         Value* size = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
864             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemorySize));
865         sizes.reserveCapacity(memory.pinnedRegisters().sizeRegisters.size());
866         for (auto info : memory.pinnedRegisters().sizeRegisters) {
867             sizes.append(block->appendNew<Value>(proc, Sub, origin, size,
868                 block->appendNew<Const32Value>(proc, origin, info.sizeOffset)));
869         }
870     }
871
872     // Get our arguments.
873     Vector<Value*> arguments;
874     jscCallingConvention().loadArguments(signature->arguments, proc, block, origin, [&] (Value* argument, unsigned) {
875         arguments.append(argument);
876     });
877
878     // Move the arguments into place.
879     Value* result = wasmCallingConvention().setupCall(proc, block, origin, arguments, toB3Type(signature->returnType), [&] (PatchpointValue* patchpoint) {
880         if (!!memory) {
881             ASSERT(sizes.size() == memory.pinnedRegisters().sizeRegisters.size());
882             patchpoint->append(ConstrainedValue(baseMemory, ValueRep::reg(memory.pinnedRegisters().baseMemoryPointer)));
883             for (unsigned i = 0; i < sizes.size(); ++i)
884                 patchpoint->append(ConstrainedValue(sizes[i], ValueRep::reg(memory.pinnedRegisters().sizeRegisters[i].sizeRegister)));
885         }
886
887         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
888             AllowMacroScratchRegisterUsage allowScratch(jit);
889
890             CCallHelpers::Call call = jit.call();
891             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
892                 linkBuffer.link(call, FunctionPtr(mainFunction.executableAddress()));
893             });
894         });
895     });
896
897     // Return the result, if needed.
898     switch (signature->returnType) {
899     case Wasm::Void:
900         block->appendNewControlValue(proc, B3::Return, origin);
901         break;
902     case Wasm::F32:
903     case Wasm::F64:
904         result = block->appendNew<Value>(proc, BitwiseCast, origin, result);
905         FALLTHROUGH;
906     case Wasm::I32:
907     case Wasm::I64:
908         block->appendNewControlValue(proc, B3::Return, origin, result);
909         break;
910     case Wasm::Func:
911     case Wasm::Anyfunc:
912         RELEASE_ASSERT_NOT_REACHED();
913     }
914
915     auto jsEntrypoint = std::make_unique<Compilation>(vm, proc);
916     function.jsToWasmEntrypoint.calleeSaveRegisters = proc.calleeSaveRegisters();
917     return jsEntrypoint;
918 }
919
920 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(VM& vm, const uint8_t* functionStart, size_t functionLength, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace, const ModuleInformation& info, unsigned optLevel)
921 {
922     auto result = std::make_unique<WasmInternalFunction>();
923
924     Procedure procedure;
925     B3IRGenerator context(vm, info, procedure, result.get(), unlinkedWasmToWasmCalls, functionIndexSpace);
926     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, functionIndexSpace, info);
927     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
928
929     procedure.resetReachability();
930     validate(procedure, "After parsing:\n");
931
932     if (verbose)
933         dataLog("Pre SSA: ", procedure);
934     fixSSA(procedure);
935     if (verbose)
936         dataLog("Post SSA: ", procedure);
937
938     result->wasmEntrypoint.compilation = std::make_unique<Compilation>(vm, procedure, optLevel);
939     result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisters();
940     result->jsToWasmEntrypoint.compilation = createJSToWasmWrapper(vm, *result, signature, result->wasmEntrypoint.compilation->code(), info.memory);
941     return WTFMove(result);
942 }
943
944 // Custom wasm ops. These are the ones too messy to do in wasm.json.
945
946 template<>
947 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
948 {
949     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
950     patchpoint->append(arg, ValueRep::SomeRegister);
951     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
952         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
953     });
954     patchpoint->effects = Effects::none();
955     result = patchpoint;
956     return { };
957 }
958
959 template<>
960 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
961 {
962     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
963     patchpoint->append(arg, ValueRep::SomeRegister);
964     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
965         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
966     });
967     patchpoint->effects = Effects::none();
968     result = patchpoint;
969     return { };
970 }
971
972 template<>
973 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
974 {
975     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
976     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
977     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
978     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
979     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
980     return { };
981 }
982
983 template<>
984 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
985 {
986     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
987     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
988     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
989     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
990     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
991     return { };
992 }
993
994 template<>
995 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
996 {
997     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
998     if (isX86())
999         patchpoint->numGPScratchRegisters = 1;
1000     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
1001     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1002         AllowMacroScratchRegisterUsage allowScratch(jit);
1003 #if CPU(X86_64)
1004         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1005 #else
1006         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1007 #endif
1008     });
1009     patchpoint->effects = Effects::none();
1010     result = patchpoint;
1011     return { };
1012 }
1013
1014 template<>
1015 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1016 {
1017     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1018     if (isX86())
1019         patchpoint->numGPScratchRegisters = 1;
1020     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
1021     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1022         AllowMacroScratchRegisterUsage allowScratch(jit);
1023 #if CPU(X86_64)
1024         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1025 #else
1026         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1027 #endif
1028     });
1029     patchpoint->effects = Effects::none();
1030     result = patchpoint;
1031     return { };
1032 }
1033
1034 template<>
1035 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1036 {
1037     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1038     patchpoint->append(arg, ValueRep::SomeRegister);
1039     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1040         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1041     });
1042     patchpoint->effects = Effects::none();
1043     result = patchpoint;
1044     return { };
1045 }
1046
1047 template<>
1048 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1049 {
1050     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1051     patchpoint->append(arg, ValueRep::SomeRegister);
1052     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1053         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1054     });
1055     patchpoint->effects = Effects::none();
1056     result = patchpoint;
1057     return { };
1058 }
1059
1060 template<>
1061 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1062 {
1063     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1064     patchpoint->append(arg, ValueRep::SomeRegister);
1065     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1066         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1067     });
1068     patchpoint->effects = Effects::none();
1069     result = patchpoint;
1070     return { };
1071 }
1072
1073 template<>
1074 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1075 {
1076     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1077     patchpoint->append(arg, ValueRep::SomeRegister);
1078     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1079         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1080     });
1081     patchpoint->effects = Effects::none();
1082     result = patchpoint;
1083     return { };
1084 }
1085
1086 template<>
1087 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1088 {
1089     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
1090     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
1091     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1092         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1093         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1094     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1095     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1096     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1097         jit.breakpoint();
1098     });
1099     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1100     patchpoint->append(arg, ValueRep::SomeRegister);
1101     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1102         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1103     });
1104     patchpoint->effects = Effects::none();
1105     result = patchpoint;
1106     return { };
1107 }
1108
1109 template<>
1110 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1111 {
1112     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
1113     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
1114     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1115         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1116         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1117     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1118     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1119     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1120         jit.breakpoint();
1121     });
1122     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1123     patchpoint->append(arg, ValueRep::SomeRegister);
1124     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1125         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1126     });
1127     patchpoint->effects = Effects::none();
1128     result = patchpoint;
1129     return { };
1130 }
1131
1132
1133 template<>
1134 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1135 {
1136     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
1137     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1138     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1139         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1140         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1141     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1142     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1143     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1144         jit.breakpoint();
1145     });
1146     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1147     patchpoint->append(arg, ValueRep::SomeRegister);
1148     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1149         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1150     });
1151     patchpoint->effects = Effects::none();
1152     result = patchpoint;
1153     return { };
1154 }
1155
1156 template<>
1157 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1158 {
1159     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
1160     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1161     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1162         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1163         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1164     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1165     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1166     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1167         jit.breakpoint();
1168     });
1169     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1170     patchpoint->append(arg, ValueRep::SomeRegister);
1171     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1172         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1173     });
1174     patchpoint->effects = Effects::none();
1175     result = patchpoint;
1176     return { };
1177 }
1178
1179 template<>
1180 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1181 {
1182     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1183     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1184     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1185         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1186         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1187     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1188     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1189     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1190         jit.breakpoint();
1191     });
1192     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1193     patchpoint->append(arg, ValueRep::SomeRegister);
1194     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1195         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1196     });
1197     patchpoint->effects = Effects::none();
1198     result = patchpoint;
1199     return { };
1200 }
1201
1202 template<>
1203 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1204 {
1205     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1206     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1207     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1208         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1209         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1210     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1211     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1212     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1213         jit.breakpoint();
1214     });
1215
1216     Value* constant;
1217     if (isX86()) {
1218         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1219         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1220         // so we can pool them if needed.
1221         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1222     }
1223     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1224     patchpoint->append(arg, ValueRep::SomeRegister);
1225     if (isX86()) {
1226         patchpoint->append(constant, ValueRep::SomeRegister);
1227         patchpoint->numFPScratchRegisters = 1;
1228     }
1229     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1230         AllowMacroScratchRegisterUsage allowScratch(jit);
1231         FPRReg scratch = InvalidFPRReg;
1232         FPRReg constant = InvalidFPRReg;
1233         if (isX86()) {
1234             scratch = params.fpScratch(0);
1235             constant = params[2].fpr();
1236         }
1237         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1238     });
1239     patchpoint->effects = Effects::none();
1240     result = patchpoint;
1241     return { };
1242 }
1243
1244 template<>
1245 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1246 {
1247     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1248     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1249     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1250         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1251         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1252     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1253     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1254     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1255         jit.breakpoint();
1256     });
1257     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1258     patchpoint->append(arg, ValueRep::SomeRegister);
1259     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1260         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1261     });
1262     patchpoint->effects = Effects::none();
1263     result = patchpoint;
1264     return { };
1265 }
1266
1267 template<>
1268 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1269 {
1270     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1271     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1272     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1273         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1274         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1275     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1276     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1277     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1278         jit.breakpoint();
1279     });
1280
1281     Value* constant;
1282     if (isX86()) {
1283         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1284         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1285         // so we can pool them if needed.
1286         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1287     }
1288     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1289     patchpoint->append(arg, ValueRep::SomeRegister);
1290     if (isX86()) {
1291         patchpoint->append(constant, ValueRep::SomeRegister);
1292         patchpoint->numFPScratchRegisters = 1;
1293     }
1294     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1295         AllowMacroScratchRegisterUsage allowScratch(jit);
1296         FPRReg scratch = InvalidFPRReg;
1297         FPRReg constant = InvalidFPRReg;
1298         if (isX86()) {
1299             scratch = params.fpScratch(0);
1300             constant = params[2].fpr();
1301         }
1302         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1303     });
1304     patchpoint->effects = Effects::none();
1305     result = patchpoint;
1306     return { };
1307 }
1308
1309 } } // namespace JSC::Wasm
1310
1311 #include "WasmB3IRGeneratorInlines.h"
1312
1313 #endif // ENABLE(WEBASSEMBLY)