Wasm should support call_indirect
[WebKit-https.git] / Source / JavaScriptCore / wasm / WasmB3IRGenerator.cpp
1 /*
2  * Copyright (C) 2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WasmB3IRGenerator.h"
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "B3BasicBlockInlines.h"
32 #include "B3CCallValue.h"
33 #include "B3ConstPtrValue.h"
34 #include "B3FixSSA.h"
35 #include "B3StackmapGenerationParams.h"
36 #include "B3SwitchValue.h"
37 #include "B3Validate.h"
38 #include "B3ValueInlines.h"
39 #include "B3Variable.h"
40 #include "B3VariableValue.h"
41 #include "B3WasmAddressValue.h"
42 #include "B3WasmBoundsCheckValue.h"
43 #include "JSWebAssemblyInstance.h"
44 #include "JSWebAssemblyModule.h"
45 #include "VirtualRegister.h"
46 #include "WasmCallingConvention.h"
47 #include "WasmFunctionParser.h"
48 #include "WasmMemory.h"
49 #include <wtf/Optional.h>
50
51 void dumpProcedure(void* ptr)
52 {
53     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
54     proc->dump(WTF::dataFile());
55 }
56
57 namespace JSC { namespace Wasm {
58
59 using namespace B3;
60
61 namespace {
62 const bool verbose = false;
63 }
64
65 class B3IRGenerator {
66 public:
67     struct ControlData {
68         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
69             : blockType(type)
70             , continuation(continuation)
71             , special(special)
72         {
73             if (signature != Void)
74                 result.append(proc.addVariable(toB3Type(signature)));
75         }
76
77         ControlData()
78         {
79         }
80
81         void dump(PrintStream& out) const
82         {
83             switch (type()) {
84             case BlockType::If:
85                 out.print("If:    ");
86                 break;
87             case BlockType::Block:
88                 out.print("Block: ");
89                 break;
90             case BlockType::Loop:
91                 out.print("Loop:  ");
92                 break;
93             }
94             out.print("Continuation: ", *continuation, ", Special: ");
95             if (special)
96                 out.print(*special);
97             else
98                 out.print("None");
99         }
100
101         BlockType type() const { return blockType; }
102
103         bool hasNonVoidSignature() const { return result.size(); }
104
105         BasicBlock* targetBlockForBranch()
106         {
107             if (type() == BlockType::Loop)
108                 return special;
109             return continuation;
110         }
111
112         void convertIfToBlock()
113         {
114             ASSERT(type() == BlockType::If);
115             blockType = BlockType::Block;
116             special = nullptr;
117         }
118
119     private:
120         friend class B3IRGenerator;
121         BlockType blockType;
122         BasicBlock* continuation;
123         BasicBlock* special;
124         Vector<Variable*, 1> result;
125     };
126
127     typedef Value* ExpressionType;
128     typedef ControlData ControlType;
129     typedef Vector<ExpressionType, 1> ExpressionList;
130     typedef Vector<Variable*, 1> ResultList;
131     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
132
133     static constexpr ExpressionType emptyExpression = nullptr;
134
135     B3IRGenerator(const MemoryInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, const ImmutableFunctionIndexSpace&);
136
137     bool WARN_UNUSED_RETURN addArguments(const Vector<Type>&);
138     bool WARN_UNUSED_RETURN addLocal(Type, uint32_t);
139     ExpressionType addConstant(Type, uint64_t);
140
141     // Locals
142     bool WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
143     bool WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
144
145     // Memory
146     bool WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
147     bool WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
148
149     // Basic operators
150     template<OpType>
151     bool WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
152     template<OpType>
153     bool WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
154     bool WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
155
156     // Control flow
157     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
158     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
159     bool WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
160     bool WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
161     bool WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
162
163     bool WARN_UNUSED_RETURN addReturn(const ExpressionList& returnValues);
164     bool WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
165     bool WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
166     bool WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
167     bool WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
168
169     // Calls
170     bool WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
171     bool WARN_UNUSED_RETURN addCallIndirect(const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
172
173     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack);
174
175     void setErrorMessage(String&&) { UNREACHABLE_FOR_PLATFORM(); }
176
177 private:
178     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
179     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
180     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
181
182     void unify(Variable* target, const ExpressionType source);
183     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
184     Value* zeroForType(Type);
185
186     const ImmutableFunctionIndexSpace& m_functionIndexSpace;
187     Procedure& m_proc;
188     BasicBlock* m_currentBlock;
189     Vector<Variable*> m_locals;
190     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
191     GPRReg m_memoryBaseGPR;
192     GPRReg m_memorySizeGPR;
193     Value* m_zeroValues[numTypes];
194     Value* m_functionIndexSpaceValue;
195 };
196
197 B3IRGenerator::B3IRGenerator(const MemoryInformation& memory, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace)
198     : m_functionIndexSpace(functionIndexSpace)
199     , m_proc(procedure)
200     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
201 {
202     m_currentBlock = m_proc.addBlock();
203
204     for (unsigned i = 0; i < numTypes; ++i) {
205         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
206         case B3::Int32:
207         case B3::Int64:
208         case B3::Float:
209         case B3::Double:
210             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
211             break;
212         case B3::Void:
213             m_zeroValues[i] = nullptr;
214             break;
215         }
216     }
217
218     if (!!memory) {
219         m_memoryBaseGPR = memory.pinnedRegisters().baseMemoryPointer;
220         m_proc.pinRegister(m_memoryBaseGPR);
221         ASSERT(!memory.pinnedRegisters().sizeRegisters[0].sizeOffset);
222         m_memorySizeGPR = memory.pinnedRegisters().sizeRegisters[0].sizeRegister;
223         for (const PinnedSizeRegisterInfo& info : memory.pinnedRegisters().sizeRegisters)
224             m_proc.pinRegister(info.sizeRegister);
225
226         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
227             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
228             // FIXME: This should unwind the stack and throw a JS exception. See: https://bugs.webkit.org/show_bug.cgi?id=163351
229             jit.breakpoint();
230         });
231     }
232
233     wasmCallingConvention().setupFrameInPrologue(compilation, m_proc, Origin(), m_currentBlock);
234
235     m_functionIndexSpaceValue = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), functionIndexSpace.buffer.get());
236 }
237
238 Value* B3IRGenerator::zeroForType(Type type)
239 {
240     ASSERT(type != Void);
241     Value* zeroValue = m_zeroValues[linearizeType(type)];
242     ASSERT(zeroValue);
243     return zeroValue;
244 }
245
246 bool B3IRGenerator::addLocal(Type type, uint32_t count)
247 {
248     if (!m_locals.tryReserveCapacity(m_locals.size() + count))
249         return false;
250
251     for (uint32_t i = 0; i < count; ++i) {
252         Variable* local = m_proc.addVariable(toB3Type(type));
253         m_locals.uncheckedAppend(local);
254         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
255     }
256     return true;
257 }
258
259 bool B3IRGenerator::addArguments(const Vector<Type>& types)
260 {
261     ASSERT(!m_locals.size());
262     if (!m_locals.tryReserveCapacity(types.size()))
263         return false;
264
265     m_locals.grow(types.size());
266     wasmCallingConvention().loadArguments(types, m_proc, m_currentBlock, Origin(),
267         [&] (ExpressionType argument, unsigned i) {
268             Variable* argumentVariable = m_proc.addVariable(argument->type());
269             m_locals[i] = argumentVariable;
270             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
271         });
272     return true;
273 }
274
275 bool B3IRGenerator::getLocal(uint32_t index, ExpressionType& result)
276 {
277     ASSERT(m_locals[index]);
278     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
279     return true;
280 }
281
282 bool B3IRGenerator::setLocal(uint32_t index, ExpressionType value)
283 {
284     ASSERT(m_locals[index]);
285     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
286     return true;
287 }
288
289 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
290 {
291     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
292     ASSERT(sizeOfOperation + offset > offset);
293     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
294     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
295     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
296 }
297
298 inline uint32_t sizeOfLoadOp(LoadOpType op)
299 {
300     switch (op) {
301     case LoadOpType::I32Load8S:
302     case LoadOpType::I32Load8U:
303     case LoadOpType::I64Load8S:
304     case LoadOpType::I64Load8U:
305         return 1;
306     case LoadOpType::I32Load16S:
307     case LoadOpType::I64Load16S:
308         return 2;
309     case LoadOpType::I32Load:
310     case LoadOpType::I64Load32S:
311     case LoadOpType::I64Load32U:
312     case LoadOpType::F32Load:
313         return 4;
314     case LoadOpType::I64Load:
315     case LoadOpType::F64Load:
316         return 8;
317     case LoadOpType::I32Load16U:
318     case LoadOpType::I64Load16U:
319         break;
320     }
321     RELEASE_ASSERT_NOT_REACHED();
322 }
323
324 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
325 {
326     switch (op) {
327     case LoadOpType::I32Load8S: {
328         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
329     }
330
331     case LoadOpType::I64Load8S: {
332         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
333         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
334     }
335
336     case LoadOpType::I32Load8U: {
337         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
338     }
339
340     case LoadOpType::I64Load8U: {
341         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
342         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
343     }
344
345     case LoadOpType::I32Load16S: {
346         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
347     }
348     case LoadOpType::I64Load16S: {
349         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
350         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
351     }
352
353     case LoadOpType::I32Load: {
354         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
355     }
356
357     case LoadOpType::I64Load32U: {
358         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
359         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
360     }
361
362     case LoadOpType::I64Load32S: {
363         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
364         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
365     }
366
367     case LoadOpType::I64Load: {
368         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer);
369     }
370
371     case LoadOpType::F32Load: {
372         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer);
373     }
374
375     case LoadOpType::F64Load: {
376         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer);
377     }
378
379     // B3 doesn't support Load16Z yet.
380     case LoadOpType::I32Load16U:
381     case LoadOpType::I64Load16U:
382         break;
383     }
384     RELEASE_ASSERT_NOT_REACHED();
385 }
386
387 bool B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset)
388 {
389     ASSERT(pointer->type() == Int32);
390
391     result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
392     return true;
393 }
394
395 inline uint32_t sizeOfStoreOp(StoreOpType op)
396 {
397     switch (op) {
398     case StoreOpType::I32Store8:
399     case StoreOpType::I64Store8:
400         return 1;
401     case StoreOpType::I32Store16:
402     case StoreOpType::I64Store16:
403         return 2;
404     case StoreOpType::I32Store:
405     case StoreOpType::I64Store32:
406     case StoreOpType::F32Store:
407         return 4;
408     case StoreOpType::I64Store:
409     case StoreOpType::F64Store:
410         return 8;
411     }
412     RELEASE_ASSERT_NOT_REACHED();
413 }
414
415
416 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
417 {
418     switch (op) {
419     case StoreOpType::I64Store8:
420         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
421         FALLTHROUGH;
422
423     case StoreOpType::I32Store8:
424         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
425         return;
426
427     case StoreOpType::I64Store16:
428         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
429         FALLTHROUGH;
430
431     case StoreOpType::I32Store16:
432         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
433         return;
434
435     case StoreOpType::I64Store32:
436         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
437         FALLTHROUGH;
438
439     case StoreOpType::I64Store:
440     case StoreOpType::I32Store:
441     case StoreOpType::F32Store:
442     case StoreOpType::F64Store:
443         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
444         return;
445     }
446     RELEASE_ASSERT_NOT_REACHED();
447 }
448
449 bool B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset)
450 {
451     ASSERT(pointer->type() == Int32);
452
453     emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
454     return true;
455 }
456
457 bool B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result)
458 {
459     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
460     return true;
461 }
462
463 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
464 {
465     switch (type) {
466     case Wasm::I32:
467         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
468     case Wasm::I64:
469         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
470     case Wasm::F32:
471         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
472     case Wasm::F64:
473         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
474     case Wasm::Void:
475     case Wasm::Func:
476     case Wasm::Anyfunc:
477         break;
478     }
479     RELEASE_ASSERT_NOT_REACHED();
480     return nullptr;
481 }
482
483 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
484 {
485     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
486 }
487
488 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
489 {
490     BasicBlock* body = m_proc.addBlock();
491     BasicBlock* continuation = m_proc.addBlock();
492     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
493     body->addPredecessor(m_currentBlock);
494     m_currentBlock = body;
495     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
496 }
497
498 bool B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result)
499 {
500     // FIXME: This needs to do some kind of stack passing.
501
502     BasicBlock* taken = m_proc.addBlock();
503     BasicBlock* notTaken = m_proc.addBlock();
504     BasicBlock* continuation = m_proc.addBlock();
505
506     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
507     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
508     taken->addPredecessor(m_currentBlock);
509     notTaken->addPredecessor(m_currentBlock);
510
511     m_currentBlock = taken;
512     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
513     return true;
514 }
515
516 bool B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack)
517 {
518     unifyValuesWithBlock(currentStack, data.result);
519     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
520     return addElseToUnreachable(data);
521 }
522
523 bool B3IRGenerator::addElseToUnreachable(ControlData& data)
524 {
525     ASSERT(data.type() == BlockType::If);
526     m_currentBlock = data.special;
527     data.convertIfToBlock();
528     return true;
529 }
530
531 bool B3IRGenerator::addReturn(const ExpressionList& returnValues)
532 {
533     ASSERT(returnValues.size() <= 1);
534     if (returnValues.size())
535         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
536     else
537         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
538     return true;
539 }
540
541 bool B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues)
542 {
543     if (data.type() != BlockType::Loop)
544         unifyValuesWithBlock(returnValues, data.result);
545
546     BasicBlock* target = data.targetBlockForBranch();
547     if (condition) {
548         BasicBlock* continuation = m_proc.addBlock();
549         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
550         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
551         target->addPredecessor(m_currentBlock);
552         continuation->addPredecessor(m_currentBlock);
553         m_currentBlock = continuation;
554     } else {
555         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
556         target->addPredecessor(m_currentBlock);
557     }
558
559     return true;
560 }
561
562 bool B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack)
563 {
564     for (size_t i = 0; i < targets.size(); ++i)
565         unifyValuesWithBlock(expressionStack, targets[i]->result);
566     unifyValuesWithBlock(expressionStack, defaultTarget.result);
567
568     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
569     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
570     for (size_t i = 0; i < targets.size(); ++i)
571         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
572
573     return true;
574 }
575
576 bool B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack)
577 {
578     ControlData& data = entry.controlData;
579
580     unifyValuesWithBlock(expressionStack, data.result);
581     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
582     data.continuation->addPredecessor(m_currentBlock);
583
584     return addEndToUnreachable(entry);
585 }
586
587
588 bool B3IRGenerator::addEndToUnreachable(ControlEntry& entry)
589 {
590     ControlData& data = entry.controlData;
591     m_currentBlock = data.continuation;
592
593     if (data.type() == BlockType::If) {
594         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
595         m_currentBlock->addPredecessor(data.special);
596     }
597
598     for (Variable* result : data.result)
599         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
600
601     return true;
602 }
603
604 bool B3IRGenerator::addCall(uint32_t functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result)
605 {
606     ASSERT(signature->arguments.size() == args.size());
607
608     Type returnType = signature->returnType;
609
610     size_t callIndex = m_unlinkedWasmToWasmCalls.size();
611     m_unlinkedWasmToWasmCalls.grow(callIndex + 1);
612     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
613         [&] (PatchpointValue* patchpoint) {
614             patchpoint->effects.writesPinned = true;
615             patchpoint->effects.readsPinned = true;
616
617             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
618                 AllowMacroScratchRegisterUsage allowScratch(jit);
619
620                 CCallHelpers::Call call = jit.call();
621
622                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
623                     m_unlinkedWasmToWasmCalls[callIndex] = { linkBuffer.locationOf(call), functionIndex };
624                 });
625             });
626         });
627     return true;
628 }
629
630 bool B3IRGenerator::addCallIndirect(const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result)
631 {
632     ExpressionType calleeIndex = args.takeLast();
633     ASSERT(signature->arguments.size() == args.size());
634
635     // Check the index we are looking for is valid.
636     {
637         ExpressionType maxValidIndex = m_currentBlock->appendIntConstant(m_proc, Origin(), Int32, m_functionIndexSpace.size);
638         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
639             m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), m_zeroValues[linearizeType(I32)],
640                 m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), calleeIndex, maxValidIndex)));
641
642         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
643             jit.breakpoint();
644         });
645     }
646
647     // Compute the offset in the function index space we are looking for.
648     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, Origin(),
649         m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), calleeIndex),
650         m_currentBlock->appendIntConstant(m_proc, Origin(), pointerType(), sizeof(CallableFunction)));
651     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, Origin(), m_functionIndexSpaceValue, offset);
652
653     // Check the signature matches the value we expect.
654     {
655         ExpressionType calleeSignature = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signature));
656         ExpressionType expectedSignature = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), signature);
657         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
658             m_currentBlock->appendNew<Value>(m_proc, NotEqual, Origin(), calleeSignature, expectedSignature));
659
660         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
661             jit.breakpoint();
662         });
663     }
664
665     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
666
667     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(signature->returnType),
668         [&] (PatchpointValue* patchpoint) {
669             patchpoint->effects.writesPinned = true;
670             patchpoint->effects.readsPinned = true;
671
672             patchpoint->append(calleeCode, ValueRep::SomeRegister);
673
674             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
675                 jit.call(params[0].gpr());
676             });
677         });
678     return true;
679 }
680
681 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
682 {
683     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
684 }
685
686 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
687 {
688     ASSERT(result.size() <= resultStack.size());
689
690     for (size_t i = 0; i < result.size(); ++i)
691         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
692 }
693
694 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
695 {
696     dataLogLn(comma, "ExpressionStack:");
697     for (const auto& expression : expressionStack)
698         dataLogLn(comma, *expression);
699 }
700
701 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack)
702 {
703     dataLogLn("Processing Graph:");
704     dataLog(m_proc);
705     dataLogLn("With current block:", *m_currentBlock);
706     dataLogLn("Control stack:");
707     for (auto& data : controlStack) {
708         dataLogLn("  ", data.controlData);
709         if (data.enclosedExpressionStack.size()) {
710             CommaPrinter comma("    ", "  with ");
711             dumpExpressionStack(comma, data.enclosedExpressionStack);
712         }
713     }
714
715     CommaPrinter comma("  ", "");
716     dumpExpressionStack(comma, expressionStack);
717     dataLogLn("\n");
718 }
719
720 static std::unique_ptr<Compilation> createJSToWasmWrapper(VM& vm, const Signature* signature, MacroAssemblerCodePtr mainFunction, const MemoryInformation& memory)
721 {
722     Procedure proc;
723     BasicBlock* block = proc.addBlock();
724
725     // Check argument count is sane.
726     Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, Origin());
727     Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, Origin(), CallFrameSlot::argumentCount * sizeof(Register));
728     Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
729         block->appendNew<Value>(proc, Add, Origin(), framePointer, offSetOfArgumentCount));
730
731     Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, Origin(), signature->arguments.size());
732
733     CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, Origin(),
734         block->appendNew<Value>(proc, Above, Origin(), expectedArgumentCount, argumentCount));
735     argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
736         jit.breakpoint();
737     });
738
739     // Move memory values to the approriate places, if needed.
740     Value* baseMemory = nullptr;
741     Vector<Value*> sizes;
742     if (!!memory) {
743         baseMemory = block->appendNew<MemoryValue>(proc, Load, Int64, Origin(),
744             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemoryPointer));
745         Value* size = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
746             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemorySize));
747         sizes.reserveCapacity(memory.pinnedRegisters().sizeRegisters.size());
748         for (auto info : memory.pinnedRegisters().sizeRegisters) {
749             sizes.append(block->appendNew<Value>(proc, Sub, Origin(), size,
750                 block->appendNew<Const32Value>(proc, Origin(), info.sizeOffset)));
751         }
752     }
753
754     // Get our arguments.
755     Vector<Value*> arguments;
756     jscCallingConvention().loadArguments(signature->arguments, proc, block, Origin(), [&] (Value* argument, unsigned) {
757         arguments.append(argument);
758     });
759
760     // Move the arguments into place.
761     Value* result = wasmCallingConvention().setupCall(proc, block, Origin(), arguments, toB3Type(signature->returnType), [&] (PatchpointValue* patchpoint) {
762         if (!!memory) {
763             ASSERT(sizes.size() == memory.pinnedRegisters().sizeRegisters.size());
764             patchpoint->append(ConstrainedValue(baseMemory, ValueRep::reg(memory.pinnedRegisters().baseMemoryPointer)));
765             for (unsigned i = 0; i < sizes.size(); ++i)
766                 patchpoint->append(ConstrainedValue(sizes[i], ValueRep::reg(memory.pinnedRegisters().sizeRegisters[i].sizeRegister)));
767         }
768
769         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
770             AllowMacroScratchRegisterUsage allowScratch(jit);
771
772             CCallHelpers::Call call = jit.call();
773             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
774                 linkBuffer.link(call, FunctionPtr(mainFunction.executableAddress()));
775             });
776         });
777     });
778
779     // Return the result, if needed.
780     switch (signature->returnType) {
781     case Wasm::Void:
782         block->appendNewControlValue(proc, B3::Return, Origin());
783         break;
784     case Wasm::F32:
785     case Wasm::F64:
786         result = block->appendNew<Value>(proc, BitwiseCast, Origin(), result);
787         FALLTHROUGH;
788     case Wasm::I32:
789     case Wasm::I64:
790         block->appendNewControlValue(proc, B3::Return, Origin(), result);
791         break;
792     case Wasm::Func:
793     case Wasm::Anyfunc:
794         RELEASE_ASSERT_NOT_REACHED();
795     }
796
797     return std::make_unique<Compilation>(vm, proc);
798 }
799
800 std::unique_ptr<WasmInternalFunction> parseAndCompile(VM& vm, const uint8_t* functionStart, size_t functionLength, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace, const ModuleInformation& info, unsigned optLevel)
801 {
802     auto result = std::make_unique<WasmInternalFunction>();
803
804     Procedure procedure;
805     B3IRGenerator context(info.memory, procedure, result.get(), unlinkedWasmToWasmCalls, functionIndexSpace);
806     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, functionIndexSpace, info);
807     if (!parser.parse())
808         RELEASE_ASSERT_NOT_REACHED();
809
810     procedure.resetReachability();
811     validate(procedure, "After parsing:\n");
812
813     if (verbose)
814         dataLog("Pre SSA: ", procedure);
815     fixSSA(procedure);
816     if (verbose)
817         dataLog("Post SSA: ", procedure);
818
819     result->code = std::make_unique<Compilation>(vm, procedure, optLevel);
820     result->jsToWasmEntryPoint = createJSToWasmWrapper(vm, signature, result->code->code(), info.memory);
821     return result;
822 }
823
824 // Custom wasm ops. These are the ones too messy to do in wasm.json.
825
826 template<>
827 bool B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result)
828 {
829     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
830     patchpoint->append(arg, ValueRep::SomeRegister);
831     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
832         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
833     });
834     patchpoint->effects = Effects::none();
835     result = patchpoint;
836     return true;
837 }
838
839 template<>
840 bool B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result)
841 {
842     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
843     patchpoint->append(arg, ValueRep::SomeRegister);
844     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
845         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
846     });
847     patchpoint->effects = Effects::none();
848     result = patchpoint;
849     return true;
850 }
851
852 template<>
853 bool B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result)
854 {
855     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
856     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
857     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
858     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
859     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
860     return true;
861 }
862
863 template<>
864 bool B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result)
865 {
866     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
867     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
868     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
869     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
870     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
871     return true;
872 }
873
874 template<>
875 bool B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result)
876 {
877     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
878     if (isX86())
879         patchpoint->numGPScratchRegisters = 1;
880     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
881     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
882         AllowMacroScratchRegisterUsage allowScratch(jit);
883 #if CPU(X86_64)
884         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
885 #else
886         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
887 #endif
888     });
889     patchpoint->effects = Effects::none();
890     result = patchpoint;
891     return true;
892 }
893
894 template<>
895 bool B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result)
896 {
897     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
898     if (isX86())
899         patchpoint->numGPScratchRegisters = 1;
900     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
901     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
902         AllowMacroScratchRegisterUsage allowScratch(jit);
903 #if CPU(X86_64)
904         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
905 #else
906         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
907 #endif
908     });
909     patchpoint->effects = Effects::none();
910     result = patchpoint;
911     return true;
912 }
913
914 template<>
915 bool B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result)
916 {
917     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
918     patchpoint->append(arg, ValueRep::SomeRegister);
919     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
920         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
921     });
922     patchpoint->effects = Effects::none();
923     result = patchpoint;
924     return true;
925 }
926
927 template<>
928 bool B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result)
929 {
930     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
931     patchpoint->append(arg, ValueRep::SomeRegister);
932     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
933         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
934     });
935     patchpoint->effects = Effects::none();
936     result = patchpoint;
937     return true;
938 }
939
940 template<>
941 bool B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result)
942 {
943     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
944     patchpoint->append(arg, ValueRep::SomeRegister);
945     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
946         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
947     });
948     patchpoint->effects = Effects::none();
949     result = patchpoint;
950     return true;
951 }
952
953 template<>
954 bool B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result)
955 {
956     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
957     patchpoint->append(arg, ValueRep::SomeRegister);
958     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
959         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
960     });
961     patchpoint->effects = Effects::none();
962     result = patchpoint;
963     return true;
964 }
965
966 template<>
967 bool B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result)
968 {
969     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
970     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
971     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
972         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
973         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
974     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
975     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
976     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
977         jit.breakpoint();
978     });
979     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
980     patchpoint->append(arg, ValueRep::SomeRegister);
981     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
982         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
983     });
984     patchpoint->effects = Effects::none();
985     result = patchpoint;
986     return true;
987 }
988
989 template<>
990 bool B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result)
991 {
992     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
993     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
994     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
995         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
996         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
997     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
998     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
999     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1000         jit.breakpoint();
1001     });
1002     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1003     patchpoint->append(arg, ValueRep::SomeRegister);
1004     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1005         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1006     });
1007     patchpoint->effects = Effects::none();
1008     result = patchpoint;
1009     return true;
1010 }
1011
1012
1013 template<>
1014 bool B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result)
1015 {
1016     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
1017     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1018     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1019         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1020         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1021     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1022     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1023     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1024         jit.breakpoint();
1025     });
1026     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1027     patchpoint->append(arg, ValueRep::SomeRegister);
1028     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1029         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1030     });
1031     patchpoint->effects = Effects::none();
1032     result = patchpoint;
1033     return true;
1034 }
1035
1036 template<>
1037 bool B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result)
1038 {
1039     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
1040     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1041     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1042         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1043         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1044     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1045     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1046     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1047         jit.breakpoint();
1048     });
1049     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1050     patchpoint->append(arg, ValueRep::SomeRegister);
1051     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1052         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1053     });
1054     patchpoint->effects = Effects::none();
1055     result = patchpoint;
1056     return true;
1057 }
1058
1059 template<>
1060 bool B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result)
1061 {
1062     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1063     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1064     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1065         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1066         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1067     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1068     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1069     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1070         jit.breakpoint();
1071     });
1072     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1073     patchpoint->append(arg, ValueRep::SomeRegister);
1074     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1075         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1076     });
1077     patchpoint->effects = Effects::none();
1078     result = patchpoint;
1079     return true;
1080 }
1081
1082 template<>
1083 bool B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result)
1084 {
1085     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1086     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1087     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1088         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1089         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1090     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1091     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1092     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1093         jit.breakpoint();
1094     });
1095
1096     Value* constant;
1097     if (isX86()) {
1098         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1099         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1100         // so we can pool them if needed.
1101         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1102     }
1103     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1104     patchpoint->append(arg, ValueRep::SomeRegister);
1105     if (isX86()) {
1106         patchpoint->append(constant, ValueRep::SomeRegister);
1107         patchpoint->numFPScratchRegisters = 1;
1108     }
1109     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1110         AllowMacroScratchRegisterUsage allowScratch(jit);
1111         FPRReg scratch = InvalidFPRReg;
1112         FPRReg constant = InvalidFPRReg;
1113         if (isX86()) {
1114             scratch = params.fpScratch(0);
1115             constant = params[2].fpr();
1116         }
1117         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1118     });
1119     patchpoint->effects = Effects::none();
1120     result = patchpoint;
1121     return true;
1122 }
1123
1124 template<>
1125 bool B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result)
1126 {
1127     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1128     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1129     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1130         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1131         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1132     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1133     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1134     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1135         jit.breakpoint();
1136     });
1137     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1138     patchpoint->append(arg, ValueRep::SomeRegister);
1139     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1140         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1141     });
1142     patchpoint->effects = Effects::none();
1143     result = patchpoint;
1144     return true;
1145 }
1146
1147 template<>
1148 bool B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result)
1149 {
1150     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1151     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1152     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1153         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1154         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1155     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1156     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1157     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1158         jit.breakpoint();
1159     });
1160
1161     Value* constant;
1162     if (isX86()) {
1163         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1164         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1165         // so we can pool them if needed.
1166         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1167     }
1168     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1169     patchpoint->append(arg, ValueRep::SomeRegister);
1170     if (isX86()) {
1171         patchpoint->append(constant, ValueRep::SomeRegister);
1172         patchpoint->numFPScratchRegisters = 1;
1173     }
1174     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1175         AllowMacroScratchRegisterUsage allowScratch(jit);
1176         FPRReg scratch = InvalidFPRReg;
1177         FPRReg constant = InvalidFPRReg;
1178         if (isX86()) {
1179             scratch = params.fpScratch(0);
1180             constant = params[2].fpr();
1181         }
1182         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1183     });
1184     patchpoint->effects = Effects::none();
1185     result = patchpoint;
1186     return true;
1187 }
1188
1189 } } // namespace JSC::Wasm
1190
1191 #include "WasmB3IRGeneratorInlines.h"
1192
1193 #endif // ENABLE(WEBASSEMBLY)