Save and restore callee save registers in WebAssembly
[WebKit-https.git] / Source / JavaScriptCore / wasm / WASMFunctionCompiler.h
1 /*
2  * Copyright (C) 2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #ifndef WASMFunctionCompiler_h
27 #define WASMFunctionCompiler_h
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "BinarySwitch.h"
32 #include "CCallHelpers.h"
33 #include "JIT.h"
34 #include "JITOperations.h"
35 #include "LinkBuffer.h"
36 #include "MaxFrameExtentForSlowPathCall.h"
37
38 #define UNUSED 0
39
40 namespace JSC {
41
42 static int32_t JIT_OPERATION operationConvertJSValueToInt32(ExecState* exec, EncodedJSValue value)
43 {
44     return JSValue::decode(value).toInt32(exec);
45 }
46
47 static double JIT_OPERATION operationConvertJSValueToDouble(ExecState* exec, EncodedJSValue value)
48 {
49     return JSValue::decode(value).toNumber(exec);
50 }
51
52 #if !CPU(X86) && !CPU(X86_64)
53 static int32_t JIT_OPERATION operationDiv(int32_t left, int32_t right)
54 {
55     return left / right;
56 }
57
58 static int32_t JIT_OPERATION operationMod(int32_t left, int32_t right)
59 {
60     return left % right;
61 }
62
63 static uint32_t JIT_OPERATION operationUnsignedDiv(uint32_t left, uint32_t right)
64 {
65     return left / right;
66 }
67
68 static uint32_t JIT_OPERATION operationUnsignedMod(uint32_t left, uint32_t right)
69 {
70     return left % right;
71 }
72 #endif
73
74 class WASMFunctionCompiler : private CCallHelpers {
75 public:
76     typedef int Expression;
77     typedef int Statement;
78     typedef int ExpressionList;
79     struct JumpTarget {
80         Label label;
81         JumpList jumpList;
82     };
83     enum class JumpCondition { Zero, NonZero };
84
85     WASMFunctionCompiler(VM& vm, CodeBlock* codeBlock, JSWASMModule* module, unsigned stackHeight)
86         : CCallHelpers(&vm, codeBlock)
87         , m_module(module)
88         , m_stackHeight(stackHeight)
89     {
90     }
91
92     void startFunction(const Vector<WASMType>& arguments, uint32_t numberOfI32LocalVariables, uint32_t numberOfF32LocalVariables, uint32_t numberOfF64LocalVariables)
93     {
94         m_calleeSaveSpace = WTF::roundUpToMultipleOf(sizeof(StackSlot), RegisterSet::webAssemblyCalleeSaveRegisters().numberOfSetRegisters() * sizeof(void*));
95         m_codeBlock->setCalleeSaveRegisters(RegisterSet::webAssemblyCalleeSaveRegisters());
96
97         emitFunctionPrologue();
98         emitPutImmediateToCallFrameHeader(m_codeBlock, JSStack::CodeBlock);
99
100         m_beginLabel = label();
101
102         addPtr(TrustedImm32(-m_calleeSaveSpace - WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_stackHeight) * sizeof(StackSlot) - maxFrameExtentForSlowPathCall), GPRInfo::callFrameRegister, GPRInfo::regT1);
103         m_stackOverflow = branchPtr(Above, AbsoluteAddress(m_vm->addressOfStackLimit()), GPRInfo::regT1);
104
105         move(GPRInfo::regT1, stackPointerRegister);
106         checkStackPointerAlignment();
107
108         emitSaveCalleeSaves();
109         emitMaterializeTagCheckRegisters();
110
111         m_numberOfLocals = arguments.size() + numberOfI32LocalVariables + numberOfF32LocalVariables + numberOfF64LocalVariables;
112
113         unsigned localIndex = 0;
114         for (size_t i = 0; i < arguments.size(); ++i) {
115             Address address(GPRInfo::callFrameRegister, CallFrame::argumentOffset(i) * sizeof(Register));
116             switch (arguments[i]) {
117             case WASMType::I32:
118 #if USE(JSVALUE64)
119                 loadValueAndConvertToInt32(address, GPRInfo::regT0);
120 #else
121                 loadValueAndConvertToInt32(address, GPRInfo::regT0, GPRInfo::regT1);
122 #endif
123                 store32(GPRInfo::regT0, localAddress(localIndex++));
124                 break;
125             case WASMType::F32:
126             case WASMType::F64:
127 #if USE(JSVALUE64)
128                 loadValueAndConvertToDouble(address, FPRInfo::fpRegT0, GPRInfo::regT0, GPRInfo::regT1);
129 #else
130                 loadValueAndConvertToDouble(address, FPRInfo::fpRegT0, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2, FPRInfo::fpRegT1);
131 #endif
132                 if (arguments[i] == WASMType::F32)
133                     convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
134                 storeDouble(FPRInfo::fpRegT0, localAddress(localIndex++));
135                 break;
136             default:
137                 ASSERT_NOT_REACHED();
138             }
139         }
140         for (uint32_t i = 0; i < numberOfI32LocalVariables; ++i)
141             store32(TrustedImm32(0), localAddress(localIndex++));
142         for (uint32_t i = 0; i < numberOfF32LocalVariables; ++i)
143             store32(TrustedImm32(0), localAddress(localIndex++));
144         for (uint32_t i = 0; i < numberOfF64LocalVariables; ++i) {
145 #if USE(JSVALUE64)
146             store64(TrustedImm64(0), localAddress(localIndex++));
147 #else
148             store32(TrustedImm32(0), localAddress(localIndex));
149             store32(TrustedImm32(0), localAddress(localIndex).withOffset(4));
150             localIndex++;
151 #endif
152         }
153
154         m_codeBlock->setNumParameters(1 + arguments.size());
155     }
156
157     void endFunction()
158     {
159         ASSERT(!m_tempStackTop);
160
161         // FIXME: Remove these if the last statement is a return statement.
162 #if USE(JSVALUE64)
163         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR);
164 #else
165         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
166 #endif
167         moveTrustedValue(jsUndefined(), returnValueRegs);
168         emitRestoreCalleeSaves();
169         emitFunctionEpilogue();
170         ret();
171
172         m_stackOverflow.link(this);
173         if (maxFrameExtentForSlowPathCall)
174             addPtr(TrustedImm32(-maxFrameExtentForSlowPathCall), stackPointerRegister);
175         setupArgumentsWithExecState(TrustedImmPtr(m_codeBlock));
176         appendCallWithExceptionCheck(operationThrowStackOverflowError);
177
178         // FIXME: Implement arity check.
179         Label arityCheck = label();
180         emitFunctionPrologue();
181         emitPutImmediateToCallFrameHeader(m_codeBlock, JSStack::CodeBlock);
182         jump(m_beginLabel);
183
184         if (!m_divideErrorJumpList.empty()) {
185             m_divideErrorJumpList.link(this);
186
187             setupArgumentsExecState();
188             appendCallWithExceptionCheck(operationThrowDivideError);
189         }
190
191         if (!m_exceptionChecks.empty()) {
192             m_exceptionChecks.link(this);
193
194             copyCalleeSavesToVMCalleeSavesBuffer();
195
196             // lookupExceptionHandler is passed two arguments, the VM and the exec (the CallFrame*).
197             move(TrustedImmPtr(vm()), GPRInfo::argumentGPR0);
198             move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
199
200 #if CPU(X86)
201             // FIXME: should use the call abstraction, but this is currently in the SpeculativeJIT layer!
202             poke(GPRInfo::argumentGPR0);
203             poke(GPRInfo::argumentGPR1, 1);
204 #endif
205             m_calls.append(std::make_pair(call(), FunctionPtr(lookupExceptionHandlerFromCallerFrame).value()));
206             jumpToExceptionHandler();
207         }
208
209         LinkBuffer patchBuffer(*m_vm, *this, m_codeBlock, JITCompilationMustSucceed);
210
211         for (const auto& iterator : m_calls)
212             patchBuffer.link(iterator.first, FunctionPtr(iterator.second));
213
214         for (size_t i = 0; i < m_callCompilationInfo.size(); ++i) {
215             CallCompilationInfo& compilationInfo = m_callCompilationInfo[i];
216             CallLinkInfo& info = *compilationInfo.callLinkInfo;
217             info.setCallLocations(patchBuffer.locationOfNearCall(compilationInfo.callReturnLocation),
218                 patchBuffer.locationOf(compilationInfo.hotPathBegin),
219                 patchBuffer.locationOfNearCall(compilationInfo.hotPathOther));
220         }
221
222         MacroAssemblerCodePtr withArityCheck = patchBuffer.locationOf(arityCheck);
223         CodeRef result = FINALIZE_CODE(patchBuffer, ("Baseline JIT code for WebAssembly"));
224         m_codeBlock->setJITCode(adoptRef(new DirectJITCode(result, withArityCheck, JITCode::BaselineJIT)));
225         m_codeBlock->capabilityLevel();
226     }
227
228     void buildSetLocal(uint32_t localIndex, int, WASMType type)
229     {
230         switch (type) {
231         case WASMType::I32:
232         case WASMType::F32:
233             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
234             m_tempStackTop--;
235             store32(GPRInfo::regT0, localAddress(localIndex));
236             break;
237         case WASMType::F64:
238             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
239             m_tempStackTop--;
240             storeDouble(FPRInfo::fpRegT0, localAddress(localIndex));
241             break;
242         default:
243             ASSERT_NOT_REACHED();
244         }
245     }
246
247     void buildSetGlobal(uint32_t globalIndex, int, WASMType type)
248     {
249         move(TrustedImmPtr(&m_module->globalVariables()[globalIndex]), GPRInfo::regT0);
250         switch (type) {
251         case WASMType::I32:
252         case WASMType::F32:
253             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
254             store32(GPRInfo::regT1, GPRInfo::regT0);
255             break;
256         case WASMType::F64:
257             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
258             storeDouble(FPRInfo::fpRegT0, GPRInfo::regT0);
259             break;
260         default:
261             ASSERT_NOT_REACHED();
262         }
263         m_tempStackTop--;
264     }
265
266     void buildReturn(int, WASMExpressionType returnType)
267     {
268 #if USE(JSVALUE64)
269         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR);
270 #else
271         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
272 #endif
273         switch (returnType) {
274         case WASMExpressionType::I32:
275             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::returnValueGPR);
276 #if USE(JSVALUE64)
277             or64(GPRInfo::tagTypeNumberRegister, GPRInfo::returnValueGPR);
278 #else
279             move(TrustedImm32(JSValue::Int32Tag), GPRInfo::returnValueGPR2);
280 #endif
281             m_tempStackTop--;
282             break;
283         case WASMExpressionType::F32:
284         case WASMExpressionType::F64:
285             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
286             if (returnType == WASMExpressionType::F32)
287                 convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
288 #if USE(JSVALUE64)
289             boxDouble(FPRInfo::fpRegT0, GPRInfo::returnValueGPR);
290 #else
291             boxDouble(FPRInfo::fpRegT0, GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
292 #endif
293             m_tempStackTop--;
294             break;
295         case WASMExpressionType::Void:
296             moveTrustedValue(jsUndefined(), returnValueRegs);
297             break;
298         default:
299             ASSERT_NOT_REACHED();
300         }
301         emitRestoreCalleeSaves();
302         emitFunctionEpilogue();
303         ret();
304     }
305
306     int buildImmediateI32(uint32_t immediate)
307     {
308         store32(Imm32(immediate), temporaryAddress(m_tempStackTop++));
309         return UNUSED;
310     }
311
312     int buildImmediateF32(float immediate)
313     {
314         store32(Imm32(bitwise_cast<int32_t>(immediate)), temporaryAddress(m_tempStackTop++));
315         return UNUSED;
316     }
317
318     int buildImmediateF64(double immediate)
319     {
320 #if USE(JSVALUE64)
321         store64(Imm64(bitwise_cast<int64_t>(immediate)), temporaryAddress(m_tempStackTop++));
322 #else
323         union {
324             double doubleValue;
325             int32_t int32Values[2];
326         } u = { immediate };
327         m_tempStackTop++;
328         store32(Imm32(u.int32Values[0]), temporaryAddress(m_tempStackTop - 1));
329         store32(Imm32(u.int32Values[1]), temporaryAddress(m_tempStackTop - 1).withOffset(4));
330 #endif
331         return UNUSED;
332     }
333
334     int buildGetLocal(uint32_t localIndex, WASMType type)
335     {
336         switch (type) {
337         case WASMType::I32:
338         case WASMType::F32:
339             load32(localAddress(localIndex), GPRInfo::regT0);
340             store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop++));
341             break;
342         case WASMType::F64:
343             loadDouble(localAddress(localIndex), FPRInfo::fpRegT0);
344             storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop++));
345             break;
346         default:
347             ASSERT_NOT_REACHED();
348         }
349         return UNUSED;
350     }
351
352     int buildGetGlobal(uint32_t globalIndex, WASMType type)
353     {
354         move(TrustedImmPtr(&m_module->globalVariables()[globalIndex]), GPRInfo::regT0);
355         switch (type) {
356         case WASMType::I32:
357         case WASMType::F32:
358             load32(GPRInfo::regT0, GPRInfo::regT0);
359             store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop++));
360             break;
361         case WASMType::F64:
362             loadDouble(GPRInfo::regT0, FPRInfo::fpRegT0);
363             storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop++));
364             break;
365         default:
366             ASSERT_NOT_REACHED();
367         }
368         return UNUSED;
369     }
370
371     int buildUnaryI32(int, WASMOpExpressionI32 op)
372     {
373         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
374         switch (op) {
375         case WASMOpExpressionI32::Negate:
376             neg32(GPRInfo::regT0);
377             break;
378         case WASMOpExpressionI32::BitNot:
379             xor32(TrustedImm32(-1), GPRInfo::regT0);
380             break;
381         case WASMOpExpressionI32::CountLeadingZeros:
382             countLeadingZeros32(GPRInfo::regT0, GPRInfo::regT0);
383             break;
384         case WASMOpExpressionI32::LogicalNot: {
385             // FIXME: Don't use branches.
386             Jump zero = branchTest32(Zero, GPRInfo::regT0);
387             move(TrustedImm32(0), GPRInfo::regT0);
388             Jump end = jump();
389             zero.link(this);
390             move(TrustedImm32(1), GPRInfo::regT0);
391             end.link(this);
392             break;
393         }
394         case WASMOpExpressionI32::Abs: {
395             // FIXME: Don't use branches.
396             Jump end = branchTest32(PositiveOrZero, GPRInfo::regT0);
397             neg32(GPRInfo::regT0);
398             end.link(this);
399             break;
400         }
401         default:
402             ASSERT_NOT_REACHED();
403         }
404         store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
405         return UNUSED;
406     }
407
408     int buildUnaryF32(int, WASMOpExpressionF32 op)
409     {
410         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
411         switch (op) {
412         case WASMOpExpressionF32::Negate:
413             convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
414             negateDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
415             convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
416             break;
417         case WASMOpExpressionF32::Abs:
418             convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
419             absDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
420             convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
421             break;
422         case WASMOpExpressionF32::Ceil:
423             callOperation(ceilf, FPRInfo::fpRegT1, FPRInfo::fpRegT0);
424             break;
425         case WASMOpExpressionF32::Floor:
426             callOperation(floorf, FPRInfo::fpRegT1, FPRInfo::fpRegT0);
427             break;
428         case WASMOpExpressionF32::Sqrt:
429             convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
430             sqrtDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
431             convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
432             break;
433         default:
434             ASSERT_NOT_REACHED();
435         }
436         storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
437         return UNUSED;
438     }
439
440     int buildBinaryI32(int, int, WASMOpExpressionI32 op)
441     {
442         load32(temporaryAddress(m_tempStackTop - 2), GPRInfo::regT0);
443         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
444         switch (op) {
445         case WASMOpExpressionI32::Add:
446             add32(GPRInfo::regT1, GPRInfo::regT0);
447             break;
448         case WASMOpExpressionI32::Sub:
449             sub32(GPRInfo::regT1, GPRInfo::regT0);
450             break;
451         case WASMOpExpressionI32::Mul:
452             mul32(GPRInfo::regT1, GPRInfo::regT0);
453             break;
454         case WASMOpExpressionI32::SDiv:
455         case WASMOpExpressionI32::UDiv:
456         case WASMOpExpressionI32::SMod:
457         case WASMOpExpressionI32::UMod: {
458             m_divideErrorJumpList.append(branchTest32(Zero, GPRInfo::regT1));
459             if (op == WASMOpExpressionI32::SDiv || op == WASMOpExpressionI32::SMod) {
460                 Jump denominatorNotNeg1 = branch32(NotEqual, GPRInfo::regT1, TrustedImm32(-1));
461                 m_divideErrorJumpList.append(branch32(Equal, GPRInfo::regT0, TrustedImm32(-2147483647-1)));
462                 denominatorNotNeg1.link(this);
463             }
464 #if CPU(X86) || CPU(X86_64)
465             ASSERT(GPRInfo::regT0 == X86Registers::eax);
466             move(GPRInfo::regT1, X86Registers::ecx);
467             if (op == WASMOpExpressionI32::SDiv || op == WASMOpExpressionI32::SMod) {
468                 m_assembler.cdq();
469                 m_assembler.idivl_r(X86Registers::ecx);
470             } else {
471                 ASSERT(op == WASMOpExpressionI32::UDiv || op == WASMOpExpressionI32::UMod);
472                 xor32(X86Registers::edx, X86Registers::edx);
473                 m_assembler.divl_r(X86Registers::ecx);
474             }
475             if (op == WASMOpExpressionI32::SMod || op == WASMOpExpressionI32::UMod)
476                 move(X86Registers::edx, GPRInfo::regT0);
477 #else
478             // FIXME: We should be able to do an inline div on ARMv7 and ARM64.
479             switch (op) {
480             case WASMOpExpressionI32::SDiv:
481                 callOperation(operationDiv, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
482                 break;
483             case WASMOpExpressionI32::UDiv:
484                 callOperation(operationUnsignedDiv, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
485                 break;
486             case WASMOpExpressionI32::SMod:
487                 callOperation(operationMod, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
488                 break;
489             case WASMOpExpressionI32::UMod:
490                 callOperation(operationUnsignedMod, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
491                 break;
492             default:
493                 ASSERT_NOT_REACHED();
494             }
495 #endif
496             break;
497         }
498         case WASMOpExpressionI32::BitOr:
499             or32(GPRInfo::regT1, GPRInfo::regT0);
500             break;
501         case WASMOpExpressionI32::BitAnd:
502             and32(GPRInfo::regT1, GPRInfo::regT0);
503             break;
504         case WASMOpExpressionI32::BitXor:
505             xor32(GPRInfo::regT1, GPRInfo::regT0);
506             break;
507         case WASMOpExpressionI32::LeftShift:
508             lshift32(GPRInfo::regT1, GPRInfo::regT0);
509             break;
510         case WASMOpExpressionI32::ArithmeticRightShift:
511             rshift32(GPRInfo::regT1, GPRInfo::regT0);
512             break;
513         case WASMOpExpressionI32::LogicalRightShift:
514             urshift32(GPRInfo::regT1, GPRInfo::regT0);
515             break;
516         default:
517             ASSERT_NOT_REACHED();
518         }
519         m_tempStackTop--;
520         store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
521         return UNUSED;
522     }
523
524     int buildBinaryF32(int, int, WASMOpExpressionF32 op)
525     {
526         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
527         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
528         convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
529         convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
530         switch (op) {
531         case WASMOpExpressionF32::Add:
532             addDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
533             break;
534         case WASMOpExpressionF32::Sub:
535             subDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
536             break;
537         case WASMOpExpressionF32::Mul:
538             mulDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
539             break;
540         case WASMOpExpressionF32::Div:
541             divDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
542             break;
543         default:
544             RELEASE_ASSERT_NOT_REACHED();
545         }
546         convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
547         m_tempStackTop--;
548         storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
549         return UNUSED;
550     }
551
552     int buildRelationalI32(int, int, WASMOpExpressionI32 op)
553     {
554         load32(temporaryAddress(m_tempStackTop - 2), GPRInfo::regT0);
555         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
556         RelationalCondition condition;
557         switch (op) {
558         case WASMOpExpressionI32::EqualI32:
559             condition = Equal;
560             break;
561         case WASMOpExpressionI32::NotEqualI32:
562             condition = NotEqual;
563             break;
564         case WASMOpExpressionI32::SLessThanI32:
565             condition = LessThan;
566             break;
567         case WASMOpExpressionI32::ULessThanI32:
568             condition = Below;
569             break;
570         case WASMOpExpressionI32::SLessThanOrEqualI32:
571             condition = LessThanOrEqual;
572             break;
573         case WASMOpExpressionI32::ULessThanOrEqualI32:
574             condition = BelowOrEqual;
575             break;
576         case WASMOpExpressionI32::SGreaterThanI32:
577             condition = GreaterThan;
578             break;
579         case WASMOpExpressionI32::UGreaterThanI32:
580             condition = Above;
581             break;
582         case WASMOpExpressionI32::SGreaterThanOrEqualI32:
583             condition = GreaterThanOrEqual;
584             break;
585         case WASMOpExpressionI32::UGreaterThanOrEqualI32:
586             condition = AboveOrEqual;
587             break;
588         default:
589             RELEASE_ASSERT_NOT_REACHED();
590         }
591         compare32(condition, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
592         m_tempStackTop--;
593         store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
594         return UNUSED;
595     }
596
597     int buildRelationalF32(int, int, WASMOpExpressionI32 op)
598     {
599         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
600         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
601         convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
602         convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
603         DoubleCondition condition;
604         switch (op) {
605         case WASMOpExpressionI32::EqualF32:
606             condition = DoubleEqual;
607             break;
608         case WASMOpExpressionI32::NotEqualF32:
609             condition = DoubleNotEqual;
610             break;
611         case WASMOpExpressionI32::LessThanF32:
612             condition = DoubleLessThan;
613             break;
614         case WASMOpExpressionI32::LessThanOrEqualF32:
615             condition = DoubleLessThanOrEqual;
616             break;
617         case WASMOpExpressionI32::GreaterThanF32:
618             condition = DoubleGreaterThan;
619             break;
620         case WASMOpExpressionI32::GreaterThanOrEqualF32:
621             condition = DoubleGreaterThanOrEqual;
622             break;
623         default:
624             RELEASE_ASSERT_NOT_REACHED();
625         }
626         m_tempStackTop--;
627         Jump trueCase = branchDouble(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT1);
628         store32(TrustedImm32(0), temporaryAddress(m_tempStackTop - 1));
629         Jump end = jump();
630         trueCase.link(this);
631         store32(TrustedImm32(1), temporaryAddress(m_tempStackTop - 1));
632         end.link(this);
633         return UNUSED;
634     }
635
636     int buildRelationalF64(int, int, WASMOpExpressionI32 op)
637     {
638         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
639         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
640         DoubleCondition condition;
641         switch (op) {
642         case WASMOpExpressionI32::EqualF64:
643             condition = DoubleEqual;
644             break;
645         case WASMOpExpressionI32::NotEqualF64:
646             condition = DoubleNotEqual;
647             break;
648         case WASMOpExpressionI32::LessThanF64:
649             condition = DoubleLessThan;
650             break;
651         case WASMOpExpressionI32::LessThanOrEqualF64:
652             condition = DoubleLessThanOrEqual;
653             break;
654         case WASMOpExpressionI32::GreaterThanF64:
655             condition = DoubleGreaterThan;
656             break;
657         case WASMOpExpressionI32::GreaterThanOrEqualF64:
658             condition = DoubleGreaterThanOrEqual;
659             break;
660         default:
661             RELEASE_ASSERT_NOT_REACHED();
662         }
663         m_tempStackTop--;
664         Jump trueCase = branchDouble(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT1);
665         store32(TrustedImm32(0), temporaryAddress(m_tempStackTop - 1));
666         Jump end = jump();
667         trueCase.link(this);
668         store32(TrustedImm32(1), temporaryAddress(m_tempStackTop - 1));
669         end.link(this);
670         return UNUSED;
671     }
672
673     int buildCallInternal(uint32_t functionIndex, int, const WASMSignature& signature, WASMExpressionType returnType)
674     {
675         boxArgumentsAndAdjustStackPointer(signature.arguments);
676
677         JSFunction* function = m_module->functions()[functionIndex].get();
678         move(TrustedImmPtr(function), GPRInfo::regT0);
679
680         callAndUnboxResult(returnType);
681         return UNUSED;
682     }
683
684     int buildCallIndirect(uint32_t functionPointerTableIndex, int, int, const WASMSignature& signature, WASMExpressionType returnType)
685     {
686         boxArgumentsAndAdjustStackPointer(signature.arguments);
687
688         const Vector<JSFunction*>& functions = m_module->functionPointerTables()[functionPointerTableIndex].functions;
689         move(TrustedImmPtr(functions.data()), GPRInfo::regT0);
690         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
691         m_tempStackTop--;
692         and32(TrustedImm32(functions.size() - 1), GPRInfo::regT1);
693         loadPtr(BaseIndex(GPRInfo::regT0, GPRInfo::regT1, timesPtr()), GPRInfo::regT0);
694
695         callAndUnboxResult(returnType);
696         return UNUSED;
697     }
698
699     int buildCallImport(uint32_t functionImportIndex, int, const WASMSignature& signature, WASMExpressionType returnType)
700     {
701         boxArgumentsAndAdjustStackPointer(signature.arguments);
702
703         JSFunction* function = m_module->importedFunctions()[functionImportIndex].get();
704         move(TrustedImmPtr(function), GPRInfo::regT0);
705
706         callAndUnboxResult(returnType);
707         return UNUSED;
708     }
709
710     void appendExpressionList(int&, int) { }
711
712     void linkTarget(JumpTarget& target)
713     {
714         target.label = label();
715         target.jumpList.link(this);
716     }
717
718     void jumpToTarget(JumpTarget& target)
719     {
720         if (target.label.isSet())
721             jump(target.label);
722         else
723             target.jumpList.append(jump());
724     }
725
726     void jumpToTargetIf(JumpCondition condition, int, JumpTarget& target)
727     {
728         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
729         m_tempStackTop--;
730         Jump taken = branchTest32((condition == JumpCondition::Zero) ? Zero : NonZero, GPRInfo::regT0);
731         if (target.label.isSet())
732             taken.linkTo(target.label, this);
733         else
734             target.jumpList.append(taken);
735     }
736
737     void startLoop()
738     {
739         m_breakTargets.append(JumpTarget());
740         m_continueTargets.append(JumpTarget());
741     }
742
743     void endLoop()
744     {
745         m_breakTargets.removeLast();
746         m_continueTargets.removeLast();
747     }
748
749     void startSwitch()
750     {
751         m_breakTargets.append(JumpTarget());
752     }
753
754     void endSwitch()
755     {
756         m_breakTargets.removeLast();
757     }
758
759     void startLabel()
760     {
761         m_breakLabelTargets.append(JumpTarget());
762         m_continueLabelTargets.append(JumpTarget());
763
764         linkTarget(m_continueLabelTargets.last());
765     }
766
767     void endLabel()
768     {
769         linkTarget(m_breakLabelTargets.last());
770
771         m_breakLabelTargets.removeLast();
772         m_continueLabelTargets.removeLast();
773     }
774
775     JumpTarget& breakTarget()
776     {
777         return m_breakTargets.last();
778     }
779
780     JumpTarget& continueTarget()
781     {
782         return m_continueTargets.last();
783     }
784
785     JumpTarget& breakLabelTarget(uint32_t labelIndex)
786     {
787         return m_breakLabelTargets[labelIndex];
788     }
789
790     JumpTarget& continueLabelTarget(uint32_t labelIndex)
791     {
792         return m_continueLabelTargets[labelIndex];
793     }
794
795     void buildSwitch(int, const Vector<int64_t>& cases, Vector<JumpTarget>& targets, JumpTarget defaultTarget)
796     {
797         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
798         m_tempStackTop--;
799         BinarySwitch binarySwitch(GPRInfo::regT0, cases, BinarySwitch::Int32);
800         while (binarySwitch.advance(*this)) {
801             unsigned index = binarySwitch.caseIndex();
802             jump(targets[index].label);
803         }
804         binarySwitch.fallThrough().linkTo(defaultTarget.label, this);
805     }
806
807 private:
808     union StackSlot {
809         int32_t intValue;
810         float floatValue;
811         double doubleValue;
812     };
813
814     Address localAddress(unsigned localIndex) const
815     {
816         ASSERT(localIndex < m_numberOfLocals);
817         return Address(GPRInfo::callFrameRegister, -m_calleeSaveSpace - (localIndex + 1) * sizeof(StackSlot));
818     }
819
820     Address temporaryAddress(unsigned temporaryIndex) const
821     {
822         ASSERT(m_numberOfLocals + temporaryIndex < m_stackHeight);
823         return Address(GPRInfo::callFrameRegister, -m_calleeSaveSpace - (m_numberOfLocals + temporaryIndex + 1) * sizeof(StackSlot));
824     }
825
826     void appendCall(const FunctionPtr& function)
827     {
828         m_calls.append(std::make_pair(call(), function.value()));
829     }
830
831     void appendCallWithExceptionCheck(const FunctionPtr& function)
832     {
833         appendCall(function);
834         m_exceptionChecks.append(emitExceptionCheck());
835     }
836
837     Call emitNakedCall(CodePtr function)
838     {
839         Call nakedCall = nearCall();
840         m_calls.append(std::make_pair(nakedCall, function.executableAddress()));
841         return nakedCall;
842     }
843
844     void appendCallSetResult(const FunctionPtr& function, GPRReg result)
845     {
846         appendCall(function);
847         move(GPRInfo::returnValueGPR, result);
848     }
849
850 #if CPU(X86)
851     void appendCallSetResult(const FunctionPtr& function, FPRReg result)
852     {
853         appendCall(function);
854         m_assembler.fstpl(0, stackPointerRegister);
855         loadDouble(stackPointerRegister, result);
856     }
857 #elif CPU(ARM) && !CPU(ARM_HARDFP)
858     void appendCallSetResult(const FunctionPtr& function, FPRReg result)
859     {
860         appendCall(function);
861         m_assembler.vmov(result, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR2);
862     }
863 #else // CPU(X86_64) || (CPU(ARM) && CPU(ARM_HARDFP)) || CPU(ARM64) || CPU(MIPS) || CPU(SH4)
864     void appendCallSetResult(const FunctionPtr& function, FPRReg result)
865     {
866         appendCall(function);
867         moveDouble(FPRInfo::returnValueFPR, result);
868     }
869 #endif
870
871 #if USE(JSVALUE64)
872     void callOperation(Z_JITOperation_EJ operation, GPRReg src, GPRReg dst)
873     {
874         setupArgumentsWithExecState(src);
875         appendCallSetResult(operation, dst);
876     }
877
878     void callOperation(D_JITOperation_EJ operation, GPRReg src, FPRReg dst)
879     {
880         setupArgumentsWithExecState(src);
881         appendCallSetResult(operation, dst);
882     }
883 #else
884     // EncodedJSValue in JSVALUE32_64 is a 64-bit integer. When being compiled in ARM EABI, it must be aligned even-numbered register (r0, r2 or [sp]).
885     // To avoid assemblies from using wrong registers, let's occupy r1 or r3 with a dummy argument when necessary.
886 #if (COMPILER_SUPPORTS(EABI) && CPU(ARM)) || CPU(MIPS)
887 #define EABI_32BIT_DUMMY_ARG      TrustedImm32(0),
888 #else
889 #define EABI_32BIT_DUMMY_ARG
890 #endif
891
892     void callOperation(Z_JITOperation_EJ operation, GPRReg srcTag, GPRReg srcPayload, GPRReg dst)
893     {
894         setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG srcPayload, srcTag);
895         appendCallSetResult(operation, dst);
896     }
897
898     void callOperation(D_JITOperation_EJ operation, GPRReg srcTag, GPRReg srcPayload, FPRReg dst)
899     {
900         setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG srcPayload, srcTag);
901         appendCallSetResult(operation, dst);
902     }
903 #endif
904
905     void callOperation(float JIT_OPERATION (*operation)(float), FPRegisterID src, FPRegisterID dst)
906     {
907         setupArguments(src);
908         appendCallSetResult(operation, dst);
909     }
910
911     void callOperation(int32_t JIT_OPERATION (*operation)(int32_t, int32_t), GPRReg src1, GPRReg src2, GPRReg dst)
912     {
913         setupArguments(src1, src2);
914         appendCallSetResult(operation, dst);
915     }
916
917     void callOperation(uint32_t JIT_OPERATION (*operation)(uint32_t, uint32_t), GPRReg src1, GPRReg src2, GPRReg dst)
918     {
919         setupArguments(src1, src2);
920         appendCallSetResult(operation, dst);
921     }
922
923     void boxArgumentsAndAdjustStackPointer(const Vector<WASMType>& arguments)
924     {
925         size_t argumentCount = arguments.size();
926         int stackOffset = -m_calleeSaveSpace - WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_numberOfLocals + m_tempStackTop + argumentCount + 1 + JSStack::CallFrameHeaderSize);
927
928         storeTrustedValue(jsUndefined(), Address(GPRInfo::callFrameRegister, (stackOffset + CallFrame::thisArgumentOffset()) * sizeof(Register)));
929
930         for (size_t i = 0; i < argumentCount; ++i) {
931             Address address(GPRInfo::callFrameRegister, (stackOffset + CallFrame::argumentOffset(i)) * sizeof(Register));
932             switch (arguments[i]) {
933             case WASMType::I32:
934                 load32(temporaryAddress(m_tempStackTop - argumentCount + i), GPRInfo::regT0);
935 #if USE(JSVALUE64)
936                 or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
937                 store64(GPRInfo::regT0, address);
938 #else
939                 store32(GPRInfo::regT0, address.withOffset(PayloadOffset));
940                 store32(TrustedImm32(JSValue::Int32Tag), address.withOffset(TagOffset));
941 #endif
942                 break;
943             default:
944                 ASSERT_NOT_REACHED();
945             }
946         }
947         m_tempStackTop -= argumentCount;
948
949         addPtr(TrustedImm32(stackOffset * sizeof(Register) + sizeof(CallerFrameAndPC)), GPRInfo::callFrameRegister, stackPointerRegister);
950         store32(TrustedImm32(argumentCount + 1), Address(stackPointerRegister, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)));
951     }
952
953     void callAndUnboxResult(WASMExpressionType returnType)
954     {
955         // regT0 holds callee.
956 #if USE(JSVALUE64)
957         store64(GPRInfo::regT0, Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) - sizeof(CallerFrameAndPC)));
958 #else
959         store32(GPRInfo::regT0, Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)));
960         store32(TrustedImm32(JSValue::CellTag), Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) + TagOffset - sizeof(CallerFrameAndPC)));
961 #endif
962
963         DataLabelPtr addressOfLinkedFunctionCheck;
964         Jump slowCase = branchPtrWithPatch(NotEqual, GPRInfo::regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
965
966         CallLinkInfo* info = m_codeBlock->addCallLinkInfo();
967         info->setUpCall(CallLinkInfo::Call, CodeOrigin(), GPRInfo::regT0);
968         m_callCompilationInfo.append(CallCompilationInfo());
969         m_callCompilationInfo.last().hotPathBegin = addressOfLinkedFunctionCheck;
970         m_callCompilationInfo.last().callLinkInfo = info;
971         m_callCompilationInfo.last().hotPathOther = nearCall();
972         Jump end = jump();
973
974         slowCase.link(this);
975         move(TrustedImmPtr(info), GPRInfo::regT2);
976         m_callCompilationInfo.last().callReturnLocation = emitNakedCall(m_vm->getCTIStub(linkCallThunkGenerator).code());
977
978         end.link(this);
979         addPtr(TrustedImm32(-m_calleeSaveSpace - WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_stackHeight) * sizeof(StackSlot) - maxFrameExtentForSlowPathCall), GPRInfo::callFrameRegister, stackPointerRegister);
980         checkStackPointerAlignment();
981
982         switch (returnType) {
983         case WASMExpressionType::I32:
984             store32(GPRInfo::returnValueGPR, temporaryAddress(m_tempStackTop++));
985             break;
986         case WASMExpressionType::Void:
987             break;
988         default:
989             ASSERT_NOT_REACHED();
990         }
991     }
992
993 #if USE(JSVALUE64)
994     void loadValueAndConvertToInt32(Address address, GPRReg dst)
995     {
996         JSValueRegs tempRegs(dst);
997         loadValue(address, tempRegs);
998         Jump checkJSInt32 = branchIfInt32(tempRegs);
999
1000         callOperation(operationConvertJSValueToInt32, dst, dst);
1001
1002         checkJSInt32.link(this);
1003     }
1004
1005     void loadValueAndConvertToDouble(Address address, FPRReg dst, GPRReg scratch1, GPRReg scratch2)
1006     {
1007         JSValueRegs tempRegs(scratch1);
1008         loadValue(address, tempRegs);
1009         Jump checkJSInt32 = branchIfInt32(tempRegs);
1010         Jump checkJSNumber = branchIfNumber(tempRegs, scratch2);
1011         JumpList end;
1012
1013         callOperation(operationConvertJSValueToDouble, tempRegs.gpr(), dst);
1014         end.append(jump());
1015
1016         checkJSInt32.link(this);
1017         convertInt32ToDouble(tempRegs.gpr(), dst);
1018         end.append(jump());
1019
1020         checkJSNumber.link(this);
1021         unboxDoubleWithoutAssertions(tempRegs.gpr(), dst);
1022         end.link(this);
1023     }
1024 #else
1025     void loadValueAndConvertToInt32(Address address, GPRReg dst, GPRReg scratch)
1026     {
1027         JSValueRegs tempRegs(scratch, dst);
1028         loadValue(address, tempRegs);
1029         Jump checkJSInt32 = branchIfInt32(tempRegs);
1030
1031         callOperation(operationConvertJSValueToInt32, tempRegs.tagGPR(), tempRegs.payloadGPR(), dst);
1032
1033         checkJSInt32.link(this);
1034     }
1035
1036     void loadValueAndConvertToDouble(Address address, FPRReg dst, GPRReg scratch1, GPRReg scratch2, GPRReg scratch3, FPRReg fpScratch)
1037     {
1038         JSValueRegs tempRegs(scratch2, scratch1);
1039         loadValue(address, tempRegs);
1040         Jump checkJSInt32 = branchIfInt32(tempRegs);
1041         Jump checkJSNumber = branchIfNumber(tempRegs, scratch3);
1042         JumpList end;
1043
1044         callOperation(operationConvertJSValueToDouble, tempRegs.tagGPR(), tempRegs.payloadGPR(), dst);
1045         end.append(jump());
1046
1047         checkJSInt32.link(this);
1048         convertInt32ToDouble(tempRegs.payloadGPR(), dst);
1049         end.append(jump());
1050
1051         checkJSNumber.link(this);
1052         unboxDouble(tempRegs.tagGPR(), tempRegs.payloadGPR(), dst, fpScratch);
1053         end.link(this);
1054     }
1055 #endif
1056
1057     JSWASMModule* m_module;
1058     unsigned m_stackHeight;
1059     unsigned m_numberOfLocals;
1060     unsigned m_tempStackTop { 0 };
1061     unsigned m_calleeSaveSpace;
1062
1063     Vector<JumpTarget> m_breakTargets;
1064     Vector<JumpTarget> m_continueTargets;
1065     Vector<JumpTarget> m_breakLabelTargets;
1066     Vector<JumpTarget> m_continueLabelTargets;
1067
1068     Label m_beginLabel;
1069     Jump m_stackOverflow;
1070     JumpList m_divideErrorJumpList;
1071     JumpList m_exceptionChecks;
1072
1073     Vector<std::pair<Call, void*>> m_calls;
1074     Vector<CallCompilationInfo> m_callCompilationInfo;
1075 };
1076
1077 } // namespace JSC
1078
1079 #endif // ENABLE(WEBASSEMBLY)
1080
1081 #endif // WASMFunctionCompiler_h