5bac43d9bab6f9380e8fe3d3743644e5effbd049
[WebKit-https.git] / Source / JavaScriptCore / wasm / WASMFunctionCompiler.h
1 /*
2  * Copyright (C) 2015-2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #ifndef WASMFunctionCompiler_h
27 #define WASMFunctionCompiler_h
28
29 #if ENABLE(WEBASSEMBLY)
30
31 #include "BinarySwitch.h"
32 #include "CCallHelpers.h"
33 #include "JIT.h"
34 #include "JITOperations.h"
35 #include "JSArrayBuffer.h"
36 #include "LinkBuffer.h"
37 #include "MaxFrameExtentForSlowPathCall.h"
38
39 #define UNUSED 0
40
41 namespace JSC {
42
43 static int32_t JIT_OPERATION operationConvertJSValueToInt32(ExecState* exec, EncodedJSValue value)
44 {
45     return JSValue::decode(value).toInt32(exec);
46 }
47
48 static double JIT_OPERATION operationConvertJSValueToDouble(ExecState* exec, EncodedJSValue value)
49 {
50     return JSValue::decode(value).toNumber(exec);
51 }
52
53 #if !CPU(X86) && !CPU(X86_64)
54 static int32_t JIT_OPERATION operationDiv(int32_t left, int32_t right)
55 {
56     return left / right;
57 }
58
59 static int32_t JIT_OPERATION operationMod(int32_t left, int32_t right)
60 {
61     return left % right;
62 }
63
64 static uint32_t JIT_OPERATION operationUnsignedDiv(uint32_t left, uint32_t right)
65 {
66     return left / right;
67 }
68
69 static uint32_t JIT_OPERATION operationUnsignedMod(uint32_t left, uint32_t right)
70 {
71     return left % right;
72 }
73 #endif
74
75 #if !USE(JSVALUE64)
76 static double JIT_OPERATION operationConvertUnsignedInt32ToDouble(uint32_t value)
77 {
78     return static_cast<double>(value);
79 }
80 #endif
81
82 static size_t sizeOfMemoryType(WASMMemoryType memoryType)
83 {
84     switch (memoryType) {
85     case WASMMemoryType::I8:
86         return 1;
87     case WASMMemoryType::I16:
88         return 2;
89     case WASMMemoryType::I32:
90     case WASMMemoryType::F32:
91         return 4;
92     case WASMMemoryType::F64:
93         return 8;
94     default:
95         ASSERT_NOT_REACHED();
96     }
97     RELEASE_ASSERT_NOT_REACHED();
98     return 0;
99 }
100
101 class WASMFunctionCompiler : private CCallHelpers {
102 public:
103     typedef int Expression;
104     typedef int Statement;
105     typedef int ExpressionList;
106     struct MemoryAddress {
107         MemoryAddress(void*) { }
108         MemoryAddress(int, uint32_t offset)
109             : offset(offset)
110         {
111         }
112         uint32_t offset;
113     };
114     struct JumpTarget {
115         Label label;
116         JumpList jumpList;
117     };
118     enum class JumpCondition { Zero, NonZero };
119
120     WASMFunctionCompiler(VM& vm, CodeBlock* codeBlock, JSWASMModule* module, unsigned stackHeight)
121         : CCallHelpers(&vm, codeBlock)
122         , m_module(module)
123         , m_stackHeight(stackHeight)
124     {
125     }
126
127     void startFunction(const Vector<WASMType>& arguments, uint32_t numberOfI32LocalVariables, uint32_t numberOfF32LocalVariables, uint32_t numberOfF64LocalVariables)
128     {
129         m_calleeSaveSpace = WTF::roundUpToMultipleOf(sizeof(StackSlot), RegisterSet::webAssemblyCalleeSaveRegisters().numberOfSetRegisters() * sizeof(void*));
130         m_codeBlock->setCalleeSaveRegisters(RegisterSet::webAssemblyCalleeSaveRegisters());
131
132         emitFunctionPrologue();
133         emitPutToCallFrameHeader(m_codeBlock, CallFrameSlot::codeBlock);
134
135         m_beginLabel = label();
136
137         addPtr(TrustedImm32(-m_calleeSaveSpace - WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_stackHeight) * sizeof(StackSlot) - maxFrameExtentForSlowPathCall), GPRInfo::callFrameRegister, GPRInfo::regT1);
138         m_stackOverflow = branchPtr(Above, AbsoluteAddress(m_vm->addressOfOSStackLimitWithReserve()), GPRInfo::regT1);
139
140         move(GPRInfo::regT1, stackPointerRegister);
141         checkStackPointerAlignment();
142
143         emitSaveCalleeSaves();
144         emitMaterializeTagCheckRegisters();
145
146         m_numberOfLocals = arguments.size() + numberOfI32LocalVariables + numberOfF32LocalVariables + numberOfF64LocalVariables;
147
148         unsigned localIndex = 0;
149         for (size_t i = 0; i < arguments.size(); ++i) {
150             // FIXME: No need to do type conversion if the caller is a WebAssembly function.
151             // https://bugs.webkit.org/show_bug.cgi?id=149310
152             Address address(GPRInfo::callFrameRegister, CallFrame::argumentOffset(i) * sizeof(Register));
153 #if USE(JSVALUE64)
154             JSValueRegs valueRegs(GPRInfo::regT0);
155 #else
156             JSValueRegs valueRegs(GPRInfo::regT1, GPRInfo::regT0);
157 #endif
158             loadValue(address, valueRegs);
159             switch (arguments[i]) {
160             case WASMType::I32:
161                 convertValueToInt32(valueRegs, GPRInfo::regT0);
162                 store32(GPRInfo::regT0, localAddress(localIndex++));
163                 break;
164             case WASMType::F32:
165             case WASMType::F64:
166 #if USE(JSVALUE64)
167                 convertValueToDouble(valueRegs, FPRInfo::fpRegT0, GPRInfo::regT1);
168 #else
169                 convertValueToDouble(valueRegs, FPRInfo::fpRegT0, GPRInfo::regT2, FPRInfo::fpRegT1);
170 #endif
171                 if (arguments[i] == WASMType::F32)
172                     convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
173                 storeDouble(FPRInfo::fpRegT0, localAddress(localIndex++));
174                 break;
175             default:
176                 ASSERT_NOT_REACHED();
177             }
178         }
179         for (uint32_t i = 0; i < numberOfI32LocalVariables; ++i)
180             store32(TrustedImm32(0), localAddress(localIndex++));
181         for (uint32_t i = 0; i < numberOfF32LocalVariables; ++i)
182             store32(TrustedImm32(0), localAddress(localIndex++));
183         for (uint32_t i = 0; i < numberOfF64LocalVariables; ++i) {
184 #if USE(JSVALUE64)
185             store64(TrustedImm64(0), localAddress(localIndex++));
186 #else
187             store32(TrustedImm32(0), localAddress(localIndex));
188             store32(TrustedImm32(0), localAddress(localIndex).withOffset(4));
189             localIndex++;
190 #endif
191         }
192
193         m_codeBlock->setNumParameters(1 + arguments.size());
194     }
195
196     void endFunction()
197     {
198         ASSERT(!m_tempStackTop);
199
200         // FIXME: Remove these if the last statement is a return statement.
201 #if USE(JSVALUE64)
202         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR);
203 #else
204         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
205 #endif
206         moveTrustedValue(jsUndefined(), returnValueRegs);
207         emitRestoreCalleeSaves();
208         emitFunctionEpilogue();
209         ret();
210
211         m_stackOverflow.link(this);
212         if (maxFrameExtentForSlowPathCall)
213             addPtr(TrustedImm32(-maxFrameExtentForSlowPathCall), stackPointerRegister);
214         setupArgumentsWithExecState(TrustedImmPtr(m_codeBlock));
215         appendCallWithExceptionCheck(operationThrowStackOverflowError);
216
217         // FIXME: Implement arity check.
218         Label arityCheck = label();
219         emitFunctionPrologue();
220         emitPutToCallFrameHeader(m_codeBlock, CallFrameSlot::codeBlock);
221         jump(m_beginLabel);
222
223         if (!m_divideErrorJumpList.empty()) {
224             m_divideErrorJumpList.link(this);
225
226             setupArgumentsExecState();
227             appendCallWithExceptionCheck(operationThrowDivideError);
228         }
229
230         if (!m_outOfBoundsErrorJumpList.empty()) {
231             m_outOfBoundsErrorJumpList.link(this);
232
233             setupArgumentsExecState();
234             appendCallWithExceptionCheck(operationThrowOutOfBoundsAccessError);
235         }
236
237         if (!m_exceptionChecks.empty()) {
238             m_exceptionChecks.link(this);
239
240             copyCalleeSavesToVMEntryFrameCalleeSavesBuffer();
241
242             // lookupExceptionHandler is passed two arguments, the VM and the exec (the CallFrame*).
243             move(TrustedImmPtr(vm()), GPRInfo::argumentGPR0);
244             move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
245
246 #if CPU(X86)
247             // FIXME: should use the call abstraction, but this is currently in the SpeculativeJIT layer!
248             poke(GPRInfo::argumentGPR0);
249             poke(GPRInfo::argumentGPR1, 1);
250 #endif
251             m_calls.append(std::make_pair(call(), FunctionPtr(lookupExceptionHandlerFromCallerFrame).value()));
252             jumpToExceptionHandler();
253         }
254
255         LinkBuffer patchBuffer(*m_vm, *this, m_codeBlock, JITCompilationMustSucceed);
256
257         for (const auto& iterator : m_calls)
258             patchBuffer.link(iterator.first, FunctionPtr(iterator.second));
259
260         for (size_t i = 0; i < m_callCompilationInfo.size(); ++i) {
261             CallCompilationInfo& compilationInfo = m_callCompilationInfo[i];
262             CallLinkInfo& info = *compilationInfo.callLinkInfo;
263             info.setCallLocations(patchBuffer.locationOfNearCall(compilationInfo.callReturnLocation),
264                 patchBuffer.locationOf(compilationInfo.hotPathBegin),
265                 patchBuffer.locationOfNearCall(compilationInfo.hotPathOther));
266         }
267
268         MacroAssemblerCodePtr withArityCheck = patchBuffer.locationOf(arityCheck);
269         CodeRef result = FINALIZE_CODE(patchBuffer, ("Baseline JIT code for WebAssembly"));
270         m_codeBlock->setJITCode(adoptRef(new DirectJITCode(result, withArityCheck, JITCode::BaselineJIT)));
271         m_codeBlock->capabilityLevel();
272     }
273
274     int buildSetLocal(WASMOpKind opKind, uint32_t localIndex, int, WASMType type)
275     {
276         switch (type) {
277         case WASMType::I32:
278         case WASMType::F32:
279             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
280             store32(GPRInfo::regT0, localAddress(localIndex));
281             break;
282         case WASMType::F64:
283             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
284             storeDouble(FPRInfo::fpRegT0, localAddress(localIndex));
285             break;
286         default:
287             ASSERT_NOT_REACHED();
288         }
289         if (opKind == WASMOpKind::Statement)
290             m_tempStackTop--;
291         return UNUSED;
292     }
293
294     int buildSetGlobal(WASMOpKind opKind, uint32_t globalIndex, int, WASMType type)
295     {
296         move(TrustedImmPtr(&m_module->globalVariables()[globalIndex]), GPRInfo::regT0);
297         switch (type) {
298         case WASMType::I32:
299         case WASMType::F32:
300             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
301             store32(GPRInfo::regT1, GPRInfo::regT0);
302             break;
303         case WASMType::F64:
304             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
305             storeDouble(FPRInfo::fpRegT0, GPRInfo::regT0);
306             break;
307         default:
308             ASSERT_NOT_REACHED();
309         }
310         if (opKind == WASMOpKind::Statement)
311             m_tempStackTop--;
312         return UNUSED;
313     }
314
315     void buildReturn(int, WASMExpressionType returnType)
316     {
317 #if USE(JSVALUE64)
318         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR);
319 #else
320         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
321 #endif
322         switch (returnType) {
323         case WASMExpressionType::I32:
324             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::returnValueGPR);
325 #if USE(JSVALUE64)
326             or64(GPRInfo::tagTypeNumberRegister, GPRInfo::returnValueGPR);
327 #else
328             move(TrustedImm32(JSValue::Int32Tag), GPRInfo::returnValueGPR2);
329 #endif
330             m_tempStackTop--;
331             break;
332         case WASMExpressionType::F32:
333         case WASMExpressionType::F64:
334             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
335             if (returnType == WASMExpressionType::F32)
336                 convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
337             convertDoubleToValue(FPRInfo::fpRegT0, returnValueRegs);
338             m_tempStackTop--;
339             break;
340         case WASMExpressionType::Void:
341             moveTrustedValue(jsUndefined(), returnValueRegs);
342             break;
343         default:
344             ASSERT_NOT_REACHED();
345         }
346         emitRestoreCalleeSaves();
347         emitFunctionEpilogue();
348         ret();
349     }
350
351     int buildImmediateI32(uint32_t immediate)
352     {
353         store32(Imm32(immediate), temporaryAddress(m_tempStackTop++));
354         return UNUSED;
355     }
356
357     int buildImmediateF32(float immediate)
358     {
359         store32(Imm32(bitwise_cast<int32_t>(immediate)), temporaryAddress(m_tempStackTop++));
360         return UNUSED;
361     }
362
363     int buildImmediateF64(double immediate)
364     {
365 #if USE(JSVALUE64)
366         store64(Imm64(bitwise_cast<int64_t>(immediate)), temporaryAddress(m_tempStackTop++));
367 #else
368         union {
369             double doubleValue;
370             int32_t int32Values[2];
371         } u = { immediate };
372         m_tempStackTop++;
373         store32(Imm32(u.int32Values[0]), temporaryAddress(m_tempStackTop - 1));
374         store32(Imm32(u.int32Values[1]), temporaryAddress(m_tempStackTop - 1).withOffset(4));
375 #endif
376         return UNUSED;
377     }
378
379     int buildGetLocal(uint32_t localIndex, WASMType type)
380     {
381         switch (type) {
382         case WASMType::I32:
383         case WASMType::F32:
384             load32(localAddress(localIndex), GPRInfo::regT0);
385             store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop++));
386             break;
387         case WASMType::F64:
388             loadDouble(localAddress(localIndex), FPRInfo::fpRegT0);
389             storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop++));
390             break;
391         default:
392             ASSERT_NOT_REACHED();
393         }
394         return UNUSED;
395     }
396
397     int buildGetGlobal(uint32_t globalIndex, WASMType type)
398     {
399         move(TrustedImmPtr(&m_module->globalVariables()[globalIndex]), GPRInfo::regT0);
400         switch (type) {
401         case WASMType::I32:
402         case WASMType::F32:
403             load32(GPRInfo::regT0, GPRInfo::regT0);
404             store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop++));
405             break;
406         case WASMType::F64:
407             loadDouble(GPRInfo::regT0, FPRInfo::fpRegT0);
408             storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop++));
409             break;
410         default:
411             ASSERT_NOT_REACHED();
412         }
413         return UNUSED;
414     }
415
416     int buildConvertType(int, WASMExpressionType fromType, WASMExpressionType toType, WASMTypeConversion conversion)
417     {
418         switch (fromType) {
419         case WASMExpressionType::I32:
420             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
421             ASSERT(toType == WASMExpressionType::F32 || toType == WASMExpressionType::F64);
422             if (conversion == WASMTypeConversion::ConvertSigned)
423                 convertInt32ToDouble(GPRInfo::regT0, FPRInfo::fpRegT0);
424             else {
425                 ASSERT(conversion == WASMTypeConversion::ConvertUnsigned);
426 #if USE(JSVALUE64)
427                 convertInt64ToDouble(GPRInfo::regT0, FPRInfo::fpRegT0);
428 #else
429                 callOperation(operationConvertUnsignedInt32ToDouble, GPRInfo::regT0, FPRInfo::fpRegT0);
430 #endif
431             }
432             if (toType == WASMExpressionType::F32)
433                 convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
434             storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
435             break;
436         case WASMExpressionType::F32:
437             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
438             switch (toType) {
439             case WASMExpressionType::I32:
440                 ASSERT(conversion == WASMTypeConversion::ConvertSigned);
441                 convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
442                 truncateDoubleToInt32(FPRInfo::fpRegT0, GPRInfo::regT0);
443                 store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
444                 break;
445             case WASMExpressionType::F64:
446                 ASSERT(conversion == WASMTypeConversion::Promote);
447                 convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
448                 storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
449                 break;
450             default:
451                 ASSERT_NOT_REACHED();
452             }
453             break;
454         case WASMExpressionType::F64:
455             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
456             switch (toType) {
457             case WASMExpressionType::I32:
458                 ASSERT(conversion == WASMTypeConversion::ConvertSigned);
459                 truncateDoubleToInt32(FPRInfo::fpRegT0, GPRInfo::regT0);
460                 store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
461                 break;
462             case WASMExpressionType::F32:
463                 ASSERT(conversion == WASMTypeConversion::Demote);
464                 convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
465                 storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
466                 break;
467             default:
468                 ASSERT_NOT_REACHED();
469             }
470             break;
471         default:
472             ASSERT_NOT_REACHED();
473         }
474         return UNUSED;
475     }
476
477     int buildLoad(const MemoryAddress& memoryAddress, WASMExpressionType expressionType, WASMMemoryType memoryType, MemoryAccessConversion conversion)
478     {
479         const ArrayBuffer* arrayBuffer = m_module->arrayBuffer()->impl();
480         move(TrustedImmPtr(arrayBuffer->data()), GPRInfo::regT0);
481         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
482         if (memoryAddress.offset)
483             add32(TrustedImm32(memoryAddress.offset), GPRInfo::regT1, GPRInfo::regT1);
484         and32(TrustedImm32(~(sizeOfMemoryType(memoryType) - 1)), GPRInfo::regT1);
485
486         ASSERT(arrayBuffer->byteLength() < (unsigned)(1 << 31));
487         if (arrayBuffer->byteLength() >= sizeOfMemoryType(memoryType))
488             m_outOfBoundsErrorJumpList.append(branch32(Above, GPRInfo::regT1, TrustedImm32(arrayBuffer->byteLength() - sizeOfMemoryType(memoryType))));
489         else
490             m_outOfBoundsErrorJumpList.append(jump());
491
492         BaseIndex address = BaseIndex(GPRInfo::regT0, GPRInfo::regT1, TimesOne);
493
494         switch (expressionType) {
495         case WASMExpressionType::I32:
496             switch (memoryType) {
497             case WASMMemoryType::I8:
498                 if (conversion == MemoryAccessConversion::SignExtend)
499                     load8SignedExtendTo32(address, GPRInfo::regT0);
500                 else {
501                     ASSERT(conversion == MemoryAccessConversion::ZeroExtend);
502                     load8(address, GPRInfo::regT0);
503                 }
504                 break;
505             case WASMMemoryType::I16:
506                 if (conversion == MemoryAccessConversion::SignExtend)
507                     load16SignedExtendTo32(address, GPRInfo::regT0);
508                 else {
509                     ASSERT(conversion == MemoryAccessConversion::ZeroExtend);
510                     load16(address, GPRInfo::regT0);
511                 }
512                 break;
513             case WASMMemoryType::I32:
514                 load32(address, GPRInfo::regT0);
515                 break;
516             default:
517                 ASSERT_NOT_REACHED();
518             }
519             store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
520             break;
521         case WASMExpressionType::F32:
522             ASSERT(memoryType == WASMMemoryType::F32 && conversion == MemoryAccessConversion::NoConversion);
523             load32(address, GPRInfo::regT0);
524             store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
525             break;
526         case WASMExpressionType::F64:
527             ASSERT(memoryType == WASMMemoryType::F64 && conversion == MemoryAccessConversion::NoConversion);
528             loadDouble(address, FPRInfo::fpRegT0);
529             storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
530             break;
531         default:
532             ASSERT_NOT_REACHED();
533         }
534         return UNUSED;
535     }
536
537     int buildStore(WASMOpKind opKind, const MemoryAddress& memoryAddress, WASMExpressionType expressionType, WASMMemoryType memoryType, int)
538     {
539         const ArrayBuffer* arrayBuffer = m_module->arrayBuffer()->impl();
540         move(TrustedImmPtr(arrayBuffer->data()), GPRInfo::regT0);
541         load32(temporaryAddress(m_tempStackTop - 2), GPRInfo::regT1);
542         if (memoryAddress.offset)
543             add32(TrustedImm32(memoryAddress.offset), GPRInfo::regT1, GPRInfo::regT1);
544         and32(TrustedImm32(~(sizeOfMemoryType(memoryType) - 1)), GPRInfo::regT1);
545
546         ASSERT(arrayBuffer->byteLength() < (1u << 31));
547         if (arrayBuffer->byteLength() >= sizeOfMemoryType(memoryType))
548             m_outOfBoundsErrorJumpList.append(branch32(Above, GPRInfo::regT1, TrustedImm32(arrayBuffer->byteLength() - sizeOfMemoryType(memoryType))));
549         else
550             m_outOfBoundsErrorJumpList.append(jump());
551
552         BaseIndex address = BaseIndex(GPRInfo::regT0, GPRInfo::regT1, TimesOne);
553
554         switch (expressionType) {
555         case WASMExpressionType::I32:
556             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT2);
557             switch (memoryType) {
558             case WASMMemoryType::I8:
559                 store8(GPRInfo::regT2, address);
560                 break;
561             case WASMMemoryType::I16:
562                 store16(GPRInfo::regT2, address);
563                 break;
564             case WASMMemoryType::I32:
565                 store32(GPRInfo::regT2, address);
566                 break;
567             default:
568                 ASSERT_NOT_REACHED();
569             }
570             break;
571         case WASMExpressionType::F32:
572             ASSERT(memoryType == WASMMemoryType::F32);
573             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT2);
574             store32(GPRInfo::regT2, address);
575             break;
576         case WASMExpressionType::F64:
577             ASSERT(memoryType == WASMMemoryType::F64);
578             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
579             storeDouble(FPRInfo::fpRegT0, address);
580             break;
581         default:
582             ASSERT_NOT_REACHED();
583         }
584         m_tempStackTop -= 2;
585
586         if (opKind == WASMOpKind::Expression) {
587             switch (expressionType) {
588             case WASMExpressionType::I32:
589             case WASMExpressionType::F32:
590                 store32(GPRInfo::regT2, temporaryAddress(m_tempStackTop++));
591                 break;
592             case WASMExpressionType::F64:
593                 storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop++));
594                 break;
595             default:
596                 ASSERT_NOT_REACHED();
597             }
598         }
599         return UNUSED;
600     }
601
602     int buildUnaryI32(int, WASMOpExpressionI32 op)
603     {
604         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
605         switch (op) {
606         case WASMOpExpressionI32::Negate:
607             neg32(GPRInfo::regT0);
608             break;
609         case WASMOpExpressionI32::BitNot:
610             xor32(TrustedImm32(-1), GPRInfo::regT0);
611             break;
612         case WASMOpExpressionI32::CountLeadingZeros:
613             countLeadingZeros32(GPRInfo::regT0, GPRInfo::regT0);
614             break;
615         case WASMOpExpressionI32::LogicalNot: {
616             // FIXME: Don't use branches.
617             Jump zero = branchTest32(Zero, GPRInfo::regT0);
618             move(TrustedImm32(0), GPRInfo::regT0);
619             Jump end = jump();
620             zero.link(this);
621             move(TrustedImm32(1), GPRInfo::regT0);
622             end.link(this);
623             break;
624         }
625         case WASMOpExpressionI32::Abs: {
626             // FIXME: Don't use branches.
627             Jump end = branchTest32(PositiveOrZero, GPRInfo::regT0);
628             neg32(GPRInfo::regT0);
629             end.link(this);
630             break;
631         }
632         default:
633             ASSERT_NOT_REACHED();
634         }
635         store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
636         return UNUSED;
637     }
638
639     int buildUnaryF32(int, WASMOpExpressionF32 op)
640     {
641         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
642         switch (op) {
643         case WASMOpExpressionF32::Negate:
644             convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
645             negateDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
646             convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
647             break;
648         case WASMOpExpressionF32::Abs:
649             convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
650             absDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
651             convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
652             break;
653         case WASMOpExpressionF32::Ceil:
654             callOperation(ceilf, FPRInfo::fpRegT1, FPRInfo::fpRegT0);
655             break;
656         case WASMOpExpressionF32::Floor:
657             callOperation(floorf, FPRInfo::fpRegT1, FPRInfo::fpRegT0);
658             break;
659         case WASMOpExpressionF32::Sqrt:
660             convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
661             sqrtDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
662             convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
663             break;
664         default:
665             ASSERT_NOT_REACHED();
666         }
667         storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
668         return UNUSED;
669     }
670
671     int buildUnaryF64(int, WASMOpExpressionF64 op)
672     {
673         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
674         switch (op) {
675         case WASMOpExpressionF64::Negate:
676             negateDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
677             break;
678         case WASMOpExpressionF64::Abs:
679             absDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
680             break;
681         case WASMOpExpressionF64::Sqrt:
682             sqrtDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
683             break;
684         case WASMOpExpressionF64::Ceil:
685         case WASMOpExpressionF64::Floor:
686         case WASMOpExpressionF64::Cos:
687         case WASMOpExpressionF64::Sin:
688         case WASMOpExpressionF64::Tan:
689         case WASMOpExpressionF64::ACos:
690         case WASMOpExpressionF64::ASin:
691         case WASMOpExpressionF64::ATan:
692         case WASMOpExpressionF64::Exp:
693         case WASMOpExpressionF64::Ln:
694             D_JITOperation_D operation;
695             switch (op) {
696             case WASMOpExpressionF64::Ceil:
697                 operation = ceil;
698                 break;
699             case WASMOpExpressionF64::Floor:
700                 operation = floor;
701                 break;
702             case WASMOpExpressionF64::Cos:
703                 operation = cos;
704                 break;
705             case WASMOpExpressionF64::Sin:
706                 operation = sin;
707                 break;
708             case WASMOpExpressionF64::Tan:
709                 operation = tan;
710                 break;
711             case WASMOpExpressionF64::ACos:
712                 operation = acos;
713                 break;
714             case WASMOpExpressionF64::ASin:
715                 operation = asin;
716                 break;
717             case WASMOpExpressionF64::ATan:
718                 operation = atan;
719                 break;
720             case WASMOpExpressionF64::Exp:
721                 operation = exp;
722                 break;
723             case WASMOpExpressionF64::Ln:
724                 operation = log;
725                 break;
726             default:
727                 RELEASE_ASSERT_NOT_REACHED();
728             }
729             callOperation(operation, FPRInfo::fpRegT1, FPRInfo::fpRegT0);
730             break;
731         default:
732             ASSERT_NOT_REACHED();
733         }
734         storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
735         return UNUSED;
736     }
737
738     int buildBinaryI32(int, int, WASMOpExpressionI32 op)
739     {
740         load32(temporaryAddress(m_tempStackTop - 2), GPRInfo::regT0);
741         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
742         switch (op) {
743         case WASMOpExpressionI32::Add:
744             add32(GPRInfo::regT1, GPRInfo::regT0);
745             break;
746         case WASMOpExpressionI32::Sub:
747             sub32(GPRInfo::regT1, GPRInfo::regT0);
748             break;
749         case WASMOpExpressionI32::Mul:
750             mul32(GPRInfo::regT1, GPRInfo::regT0);
751             break;
752         case WASMOpExpressionI32::SDiv:
753         case WASMOpExpressionI32::UDiv:
754         case WASMOpExpressionI32::SMod:
755         case WASMOpExpressionI32::UMod: {
756             m_divideErrorJumpList.append(branchTest32(Zero, GPRInfo::regT1));
757             if (op == WASMOpExpressionI32::SDiv || op == WASMOpExpressionI32::SMod) {
758                 Jump denominatorNotNeg1 = branch32(NotEqual, GPRInfo::regT1, TrustedImm32(-1));
759                 m_divideErrorJumpList.append(branch32(Equal, GPRInfo::regT0, TrustedImm32(-2147483647-1)));
760                 denominatorNotNeg1.link(this);
761             }
762 #if CPU(X86) || CPU(X86_64)
763             ASSERT(GPRInfo::regT0 == X86Registers::eax);
764             move(GPRInfo::regT1, X86Registers::ecx);
765             if (op == WASMOpExpressionI32::SDiv || op == WASMOpExpressionI32::SMod) {
766                 x86ConvertToDoubleWord32();
767                 x86Div32(X86Registers::ecx);
768             } else {
769                 ASSERT(op == WASMOpExpressionI32::UDiv || op == WASMOpExpressionI32::UMod);
770                 xor32(X86Registers::edx, X86Registers::edx);
771                 m_assembler.divl_r(X86Registers::ecx);
772             }
773             if (op == WASMOpExpressionI32::SMod || op == WASMOpExpressionI32::UMod)
774                 move(X86Registers::edx, GPRInfo::regT0);
775 #else
776             // FIXME: We should be able to do an inline div on ARMv7 and ARM64.
777             switch (op) {
778             case WASMOpExpressionI32::SDiv:
779                 callOperation(operationDiv, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
780                 break;
781             case WASMOpExpressionI32::UDiv:
782                 callOperation(operationUnsignedDiv, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
783                 break;
784             case WASMOpExpressionI32::SMod:
785                 callOperation(operationMod, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
786                 break;
787             case WASMOpExpressionI32::UMod:
788                 callOperation(operationUnsignedMod, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
789                 break;
790             default:
791                 ASSERT_NOT_REACHED();
792             }
793 #endif
794             break;
795         }
796         case WASMOpExpressionI32::BitOr:
797             or32(GPRInfo::regT1, GPRInfo::regT0);
798             break;
799         case WASMOpExpressionI32::BitAnd:
800             and32(GPRInfo::regT1, GPRInfo::regT0);
801             break;
802         case WASMOpExpressionI32::BitXor:
803             xor32(GPRInfo::regT1, GPRInfo::regT0);
804             break;
805         case WASMOpExpressionI32::LeftShift:
806             lshift32(GPRInfo::regT1, GPRInfo::regT0);
807             break;
808         case WASMOpExpressionI32::ArithmeticRightShift:
809             rshift32(GPRInfo::regT1, GPRInfo::regT0);
810             break;
811         case WASMOpExpressionI32::LogicalRightShift:
812             urshift32(GPRInfo::regT1, GPRInfo::regT0);
813             break;
814         default:
815             ASSERT_NOT_REACHED();
816         }
817         m_tempStackTop--;
818         store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
819         return UNUSED;
820     }
821
822     int buildBinaryF32(int, int, WASMOpExpressionF32 op)
823     {
824         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
825         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
826         convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
827         convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
828         switch (op) {
829         case WASMOpExpressionF32::Add:
830             addDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
831             break;
832         case WASMOpExpressionF32::Sub:
833             subDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
834             break;
835         case WASMOpExpressionF32::Mul:
836             mulDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
837             break;
838         case WASMOpExpressionF32::Div:
839             divDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
840             break;
841         default:
842             RELEASE_ASSERT_NOT_REACHED();
843         }
844         convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
845         m_tempStackTop--;
846         storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
847         return UNUSED;
848     }
849
850     int buildBinaryF64(int, int, WASMOpExpressionF64 op)
851     {
852         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
853         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
854         switch (op) {
855         case WASMOpExpressionF64::Add:
856             addDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
857             break;
858         case WASMOpExpressionF64::Sub:
859             subDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
860             break;
861         case WASMOpExpressionF64::Mul:
862             mulDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
863             break;
864         case WASMOpExpressionF64::Div:
865             divDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
866             break;
867         case WASMOpExpressionF64::Mod:
868         case WASMOpExpressionF64::ATan2:
869         case WASMOpExpressionF64::Pow:
870             D_JITOperation_DD operation;
871             switch (op) {
872             case WASMOpExpressionF64::Mod:
873                 operation = fmod;
874                 break;
875             case WASMOpExpressionF64::ATan2:
876                 operation = atan2;
877                 break;
878             case WASMOpExpressionF64::Pow:
879                 operation = pow;
880                 break;
881             default:
882                 RELEASE_ASSERT_NOT_REACHED();
883             }
884             callOperation(operation, FPRInfo::fpRegT0, FPRInfo::fpRegT1, FPRInfo::fpRegT0);
885             break;
886         default:
887             ASSERT_NOT_REACHED();
888         }
889         m_tempStackTop--;
890         storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
891         return UNUSED;
892     }
893
894     int buildRelationalI32(int, int, WASMOpExpressionI32 op)
895     {
896         load32(temporaryAddress(m_tempStackTop - 2), GPRInfo::regT0);
897         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
898         RelationalCondition condition;
899         switch (op) {
900         case WASMOpExpressionI32::EqualI32:
901             condition = Equal;
902             break;
903         case WASMOpExpressionI32::NotEqualI32:
904             condition = NotEqual;
905             break;
906         case WASMOpExpressionI32::SLessThanI32:
907             condition = LessThan;
908             break;
909         case WASMOpExpressionI32::ULessThanI32:
910             condition = Below;
911             break;
912         case WASMOpExpressionI32::SLessThanOrEqualI32:
913             condition = LessThanOrEqual;
914             break;
915         case WASMOpExpressionI32::ULessThanOrEqualI32:
916             condition = BelowOrEqual;
917             break;
918         case WASMOpExpressionI32::SGreaterThanI32:
919             condition = GreaterThan;
920             break;
921         case WASMOpExpressionI32::UGreaterThanI32:
922             condition = Above;
923             break;
924         case WASMOpExpressionI32::SGreaterThanOrEqualI32:
925             condition = GreaterThanOrEqual;
926             break;
927         case WASMOpExpressionI32::UGreaterThanOrEqualI32:
928             condition = AboveOrEqual;
929             break;
930         default:
931             RELEASE_ASSERT_NOT_REACHED();
932         }
933         compare32(condition, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
934         m_tempStackTop--;
935         store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
936         return UNUSED;
937     }
938
939     int buildRelationalF32(int, int, WASMOpExpressionI32 op)
940     {
941         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
942         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
943         convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
944         convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
945         DoubleCondition condition;
946         switch (op) {
947         case WASMOpExpressionI32::EqualF32:
948             condition = DoubleEqual;
949             break;
950         case WASMOpExpressionI32::NotEqualF32:
951             condition = DoubleNotEqual;
952             break;
953         case WASMOpExpressionI32::LessThanF32:
954             condition = DoubleLessThan;
955             break;
956         case WASMOpExpressionI32::LessThanOrEqualF32:
957             condition = DoubleLessThanOrEqual;
958             break;
959         case WASMOpExpressionI32::GreaterThanF32:
960             condition = DoubleGreaterThan;
961             break;
962         case WASMOpExpressionI32::GreaterThanOrEqualF32:
963             condition = DoubleGreaterThanOrEqual;
964             break;
965         default:
966             RELEASE_ASSERT_NOT_REACHED();
967         }
968         m_tempStackTop--;
969         Jump trueCase = branchDouble(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT1);
970         store32(TrustedImm32(0), temporaryAddress(m_tempStackTop - 1));
971         Jump end = jump();
972         trueCase.link(this);
973         store32(TrustedImm32(1), temporaryAddress(m_tempStackTop - 1));
974         end.link(this);
975         return UNUSED;
976     }
977
978     int buildRelationalF64(int, int, WASMOpExpressionI32 op)
979     {
980         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
981         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
982         DoubleCondition condition;
983         switch (op) {
984         case WASMOpExpressionI32::EqualF64:
985             condition = DoubleEqual;
986             break;
987         case WASMOpExpressionI32::NotEqualF64:
988             condition = DoubleNotEqual;
989             break;
990         case WASMOpExpressionI32::LessThanF64:
991             condition = DoubleLessThan;
992             break;
993         case WASMOpExpressionI32::LessThanOrEqualF64:
994             condition = DoubleLessThanOrEqual;
995             break;
996         case WASMOpExpressionI32::GreaterThanF64:
997             condition = DoubleGreaterThan;
998             break;
999         case WASMOpExpressionI32::GreaterThanOrEqualF64:
1000             condition = DoubleGreaterThanOrEqual;
1001             break;
1002         default:
1003             RELEASE_ASSERT_NOT_REACHED();
1004         }
1005         m_tempStackTop--;
1006         Jump trueCase = branchDouble(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT1);
1007         store32(TrustedImm32(0), temporaryAddress(m_tempStackTop - 1));
1008         Jump end = jump();
1009         trueCase.link(this);
1010         store32(TrustedImm32(1), temporaryAddress(m_tempStackTop - 1));
1011         end.link(this);
1012         return UNUSED;
1013     }
1014
1015     int buildMinOrMaxI32(int, int, WASMOpExpressionI32 op)
1016     {
1017         load32(temporaryAddress(m_tempStackTop - 2), GPRInfo::regT0);
1018         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
1019         RelationalCondition condition;
1020         switch (op) {
1021         case WASMOpExpressionI32::SMin:
1022             condition = LessThanOrEqual;
1023             break;
1024         case WASMOpExpressionI32::UMin:
1025             condition = BelowOrEqual;
1026             break;
1027         case WASMOpExpressionI32::SMax:
1028             condition = GreaterThanOrEqual;
1029             break;
1030         case WASMOpExpressionI32::UMax:
1031             condition = AboveOrEqual;
1032             break;
1033         default:
1034             RELEASE_ASSERT_NOT_REACHED();
1035         }
1036         Jump useLeft = branch32(condition, GPRInfo::regT0, GPRInfo::regT1);
1037         store32(GPRInfo::regT1, temporaryAddress(m_tempStackTop - 2));
1038         useLeft.link(this);
1039         m_tempStackTop--;
1040         return UNUSED;
1041     }
1042
1043     int buildMinOrMaxF64(int, int, WASMOpExpressionF64 op)
1044     {
1045         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
1046         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
1047         DoubleCondition condition;
1048         switch (op) {
1049         case WASMOpExpressionF64::Min:
1050             condition = DoubleLessThanOrEqual;
1051             break;
1052         case WASMOpExpressionF64::Max:
1053             condition = DoubleGreaterThanOrEqual;
1054             break;
1055         default:
1056             RELEASE_ASSERT_NOT_REACHED();
1057         }
1058         Jump useLeft = branchDouble(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT1);
1059         storeDouble(FPRInfo::fpRegT1, temporaryAddress(m_tempStackTop - 2));
1060         useLeft.link(this);
1061         m_tempStackTop--;
1062         return UNUSED;
1063     }
1064
1065     int buildCallInternal(uint32_t functionIndex, int, const WASMSignature& signature, WASMExpressionType returnType)
1066     {
1067         boxArgumentsAndAdjustStackPointer(signature.arguments);
1068
1069         JSFunction* function = m_module->functions()[functionIndex].get();
1070         move(TrustedImmPtr(function), GPRInfo::regT0);
1071
1072         callAndUnboxResult(returnType);
1073         return UNUSED;
1074     }
1075
1076     int buildCallIndirect(uint32_t functionPointerTableIndex, int, int, const WASMSignature& signature, WASMExpressionType returnType)
1077     {
1078         boxArgumentsAndAdjustStackPointer(signature.arguments);
1079
1080         const Vector<JSFunction*>& functions = m_module->functionPointerTables()[functionPointerTableIndex].functions;
1081         move(TrustedImmPtr(functions.data()), GPRInfo::regT0);
1082         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
1083         m_tempStackTop--;
1084         and32(TrustedImm32(functions.size() - 1), GPRInfo::regT1);
1085         loadPtr(BaseIndex(GPRInfo::regT0, GPRInfo::regT1, timesPtr()), GPRInfo::regT0);
1086
1087         callAndUnboxResult(returnType);
1088         return UNUSED;
1089     }
1090
1091     int buildCallImport(uint32_t functionImportIndex, int, const WASMSignature& signature, WASMExpressionType returnType)
1092     {
1093         boxArgumentsAndAdjustStackPointer(signature.arguments);
1094
1095         JSFunction* function = m_module->importedFunctions()[functionImportIndex].get();
1096         move(TrustedImmPtr(function), GPRInfo::regT0);
1097
1098         callAndUnboxResult(returnType);
1099         return UNUSED;
1100     }
1101
1102     void appendExpressionList(int&, int) { }
1103
1104     void discard(int)
1105     {
1106         m_tempStackTop--;
1107     }
1108
1109     void linkTarget(JumpTarget& target)
1110     {
1111         target.label = label();
1112         target.jumpList.link(this);
1113     }
1114
1115     void jumpToTarget(JumpTarget& target)
1116     {
1117         if (target.label.isSet())
1118             jump(target.label);
1119         else
1120             target.jumpList.append(jump());
1121     }
1122
1123     void jumpToTargetIf(JumpCondition condition, int, JumpTarget& target)
1124     {
1125         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
1126         m_tempStackTop--;
1127         Jump taken = branchTest32((condition == JumpCondition::Zero) ? Zero : NonZero, GPRInfo::regT0);
1128         if (target.label.isSet())
1129             taken.linkTo(target.label, this);
1130         else
1131             target.jumpList.append(taken);
1132     }
1133
1134     void startLoop()
1135     {
1136         m_breakTargets.append(JumpTarget());
1137         m_continueTargets.append(JumpTarget());
1138     }
1139
1140     void endLoop()
1141     {
1142         m_breakTargets.removeLast();
1143         m_continueTargets.removeLast();
1144     }
1145
1146     void startSwitch()
1147     {
1148         m_breakTargets.append(JumpTarget());
1149     }
1150
1151     void endSwitch()
1152     {
1153         m_breakTargets.removeLast();
1154     }
1155
1156     void startLabel()
1157     {
1158         m_breakLabelTargets.append(JumpTarget());
1159         m_continueLabelTargets.append(JumpTarget());
1160
1161         linkTarget(m_continueLabelTargets.last());
1162     }
1163
1164     void endLabel()
1165     {
1166         linkTarget(m_breakLabelTargets.last());
1167
1168         m_breakLabelTargets.removeLast();
1169         m_continueLabelTargets.removeLast();
1170     }
1171
1172     JumpTarget& breakTarget()
1173     {
1174         return m_breakTargets.last();
1175     }
1176
1177     JumpTarget& continueTarget()
1178     {
1179         return m_continueTargets.last();
1180     }
1181
1182     JumpTarget& breakLabelTarget(uint32_t labelIndex)
1183     {
1184         return m_breakLabelTargets[labelIndex];
1185     }
1186
1187     JumpTarget& continueLabelTarget(uint32_t labelIndex)
1188     {
1189         return m_continueLabelTargets[labelIndex];
1190     }
1191
1192     void buildSwitch(int, const Vector<int64_t>& cases, Vector<JumpTarget>& targets, JumpTarget defaultTarget)
1193     {
1194         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
1195         m_tempStackTop--;
1196         BinarySwitch binarySwitch(GPRInfo::regT0, cases, BinarySwitch::Int32);
1197         while (binarySwitch.advance(*this)) {
1198             unsigned index = binarySwitch.caseIndex();
1199             jump(targets[index].label);
1200         }
1201         binarySwitch.fallThrough().linkTo(defaultTarget.label, this);
1202     }
1203
1204 private:
1205     union StackSlot {
1206         int32_t intValue;
1207         float floatValue;
1208         double doubleValue;
1209     };
1210
1211     enum class FloatingPointPrecision { Single, Double };
1212
1213     Address localAddress(unsigned localIndex) const
1214     {
1215         ASSERT(localIndex < m_numberOfLocals);
1216         return Address(GPRInfo::callFrameRegister, -m_calleeSaveSpace - (localIndex + 1) * sizeof(StackSlot));
1217     }
1218
1219     Address temporaryAddress(unsigned temporaryIndex) const
1220     {
1221         ASSERT(m_numberOfLocals + temporaryIndex < m_stackHeight);
1222         return Address(GPRInfo::callFrameRegister, -m_calleeSaveSpace - (m_numberOfLocals + temporaryIndex + 1) * sizeof(StackSlot));
1223     }
1224
1225     void appendCall(const FunctionPtr& function)
1226     {
1227         m_calls.append(std::make_pair(call(), function.value()));
1228     }
1229
1230     void appendCallWithExceptionCheck(const FunctionPtr& function)
1231     {
1232         appendCall(function);
1233         m_exceptionChecks.append(emitExceptionCheck());
1234     }
1235
1236     Call emitNakedCall(CodePtr function)
1237     {
1238         Call nakedCall = nearCall();
1239         m_calls.append(std::make_pair(nakedCall, function.executableAddress()));
1240         return nakedCall;
1241     }
1242
1243     void appendCallSetResult(const FunctionPtr& function, GPRReg result)
1244     {
1245         appendCall(function);
1246         move(GPRInfo::returnValueGPR, result);
1247     }
1248
1249 #if CPU(X86)
1250     void appendCallSetResult(const FunctionPtr& function, FPRReg result, FloatingPointPrecision precision)
1251     {
1252         appendCall(function);
1253         switch (precision) {
1254         case FloatingPointPrecision::Single:
1255             m_assembler.fstps(0, stackPointerRegister);
1256             break;
1257         case FloatingPointPrecision::Double:
1258             m_assembler.fstpl(0, stackPointerRegister);
1259             break;
1260         }
1261         loadDouble(stackPointerRegister, result);
1262     }
1263 #elif CPU(ARM) && !CPU(ARM_HARDFP)
1264     void appendCallSetResult(const FunctionPtr& function, FPRReg result, FloatingPointPrecision)
1265     {
1266         appendCall(function);
1267         m_assembler.vmov(result, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR2);
1268     }
1269 #else // CPU(X86_64) || (CPU(ARM) && CPU(ARM_HARDFP)) || CPU(ARM64) || CPU(MIPS) || CPU(SH4)
1270     void appendCallSetResult(const FunctionPtr& function, FPRReg result, FloatingPointPrecision)
1271     {
1272         appendCall(function);
1273         moveDouble(FPRInfo::returnValueFPR, result);
1274     }
1275 #endif
1276
1277 #if USE(JSVALUE64)
1278     void callOperation(Z_JITOperation_EJ operation, GPRReg src, GPRReg dst)
1279     {
1280         setupArgumentsWithExecState(src);
1281         appendCallSetResult(operation, dst);
1282     }
1283
1284     void callOperation(D_JITOperation_EJ operation, GPRReg src, FPRReg dst)
1285     {
1286         setupArgumentsWithExecState(src);
1287         appendCallSetResult(operation, dst, FloatingPointPrecision::Double);
1288     }
1289 #else
1290 // EncodedJSValue in JSVALUE32_64 is a 64-bit integer. When being compiled in ARM EABI, it must be aligned on an even-numbered register (r0, r2 or [sp]).
1291 // To prevent the assembler from using wrong registers, let's occupy r1 or r3 with a dummy argument when necessary.
1292 #if (COMPILER_SUPPORTS(EABI) && CPU(ARM)) || CPU(MIPS)
1293 #define EABI_32BIT_DUMMY_ARG      TrustedImm32(0),
1294 #else
1295 #define EABI_32BIT_DUMMY_ARG
1296 #endif
1297
1298     void callOperation(Z_JITOperation_EJ operation, GPRReg srcTag, GPRReg srcPayload, GPRReg dst)
1299     {
1300         setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG srcPayload, srcTag);
1301         appendCallSetResult(operation, dst);
1302     }
1303
1304     void callOperation(D_JITOperation_EJ operation, GPRReg srcTag, GPRReg srcPayload, FPRReg dst)
1305     {
1306         setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG srcPayload, srcTag);
1307         appendCallSetResult(operation, dst, FloatingPointPrecision::Double);
1308     }
1309 #endif
1310
1311     void callOperation(float JIT_OPERATION (*operation)(float), FPRegisterID src, FPRegisterID dst)
1312     {
1313         setupArguments(src);
1314         appendCallSetResult(operation, dst, FloatingPointPrecision::Single);
1315     }
1316
1317     void callOperation(D_JITOperation_D operation, FPRegisterID src, FPRegisterID dst)
1318     {
1319         setupArguments(src);
1320         appendCallSetResult(operation, dst, FloatingPointPrecision::Double);
1321     }
1322
1323     void callOperation(int32_t JIT_OPERATION (*operation)(int32_t, int32_t), GPRReg src1, GPRReg src2, GPRReg dst)
1324     {
1325         setupArguments(src1, src2);
1326         appendCallSetResult(operation, dst);
1327     }
1328
1329     void callOperation(uint32_t JIT_OPERATION (*operation)(uint32_t, uint32_t), GPRReg src1, GPRReg src2, GPRReg dst)
1330     {
1331         setupArguments(src1, src2);
1332         appendCallSetResult(operation, dst);
1333     }
1334
1335     void callOperation(D_JITOperation_DD operation, FPRegisterID src1, FPRegisterID src2, FPRegisterID dst)
1336     {
1337         setupArguments(src1, src2);
1338         appendCallSetResult(operation, dst, FloatingPointPrecision::Double);
1339     }
1340
1341     void callOperation(double JIT_OPERATION (*operation)(uint32_t), GPRReg src, FPRegisterID dst)
1342     {
1343         setupArguments(src);
1344         appendCallSetResult(operation, dst, FloatingPointPrecision::Double);
1345     }
1346
1347     void boxArgumentsAndAdjustStackPointer(const Vector<WASMType>& arguments)
1348     {
1349         size_t argumentCount = arguments.size();
1350         int stackOffset = -m_calleeSaveSpace - WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_numberOfLocals + m_tempStackTop + argumentCount + 1 + CallFrame::headerSizeInRegisters);
1351
1352         storeTrustedValue(jsUndefined(), Address(GPRInfo::callFrameRegister, (stackOffset + CallFrame::thisArgumentOffset()) * sizeof(Register)));
1353
1354         for (size_t i = 0; i < argumentCount; ++i) {
1355             Address address(GPRInfo::callFrameRegister, (stackOffset + CallFrame::argumentOffset(i)) * sizeof(Register));
1356 #if USE(JSVALUE64)
1357             JSValueRegs valueRegs(GPRInfo::regT0);
1358 #else
1359             JSValueRegs valueRegs(GPRInfo::regT1, GPRInfo::regT0);
1360 #endif
1361             switch (arguments[i]) {
1362             case WASMType::I32:
1363                 load32(temporaryAddress(m_tempStackTop - argumentCount + i), GPRInfo::regT0);
1364 #if USE(JSVALUE64)
1365                 or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
1366                 store64(GPRInfo::regT0, address);
1367 #else
1368                 store32(GPRInfo::regT0, address.withOffset(PayloadOffset));
1369                 store32(TrustedImm32(JSValue::Int32Tag), address.withOffset(TagOffset));
1370 #endif
1371                 break;
1372             case WASMType::F32:
1373             case WASMType::F64:
1374                 loadDouble(temporaryAddress(m_tempStackTop - argumentCount + i), FPRInfo::fpRegT0);
1375                 if (arguments[i] == WASMType::F32)
1376                     convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
1377                 convertDoubleToValue(FPRInfo::fpRegT0, valueRegs);
1378                 storeValue(valueRegs, address);
1379                 break;
1380             default:
1381                 ASSERT_NOT_REACHED();
1382             }
1383         }
1384         m_tempStackTop -= argumentCount;
1385
1386         addPtr(TrustedImm32(stackOffset * sizeof(Register) + sizeof(CallerFrameAndPC)), GPRInfo::callFrameRegister, stackPointerRegister);
1387         store32(TrustedImm32(argumentCount + 1), Address(stackPointerRegister, CallFrameSlot::argumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)));
1388     }
1389
1390     void callAndUnboxResult(WASMExpressionType returnType)
1391     {
1392         // regT0 holds callee.
1393 #if USE(JSVALUE64)
1394         store64(GPRInfo::regT0, Address(stackPointerRegister, CallFrameSlot::callee * static_cast<int>(sizeof(Register)) - sizeof(CallerFrameAndPC)));
1395 #else
1396         store32(GPRInfo::regT0, Address(stackPointerRegister, CallFrameSlot::callee * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)));
1397         store32(TrustedImm32(JSValue::CellTag), Address(stackPointerRegister, CallFrameSlot::callee * static_cast<int>(sizeof(Register)) + TagOffset - sizeof(CallerFrameAndPC)));
1398 #endif
1399
1400         DataLabelPtr addressOfLinkedFunctionCheck;
1401         Jump slowCase = branchPtrWithPatch(NotEqual, GPRInfo::regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
1402
1403         CallLinkInfo* info = m_codeBlock->addCallLinkInfo();
1404         info->setUpCall(CallLinkInfo::Call, CodeOrigin(), GPRInfo::regT0);
1405         m_callCompilationInfo.append(CallCompilationInfo());
1406         m_callCompilationInfo.last().hotPathBegin = addressOfLinkedFunctionCheck;
1407         m_callCompilationInfo.last().callLinkInfo = info;
1408         m_callCompilationInfo.last().hotPathOther = nearCall();
1409         Jump end = jump();
1410
1411         slowCase.link(this);
1412         move(TrustedImmPtr(info), GPRInfo::regT2);
1413         m_callCompilationInfo.last().callReturnLocation = emitNakedCall(m_vm->getCTIStub(linkCallThunkGenerator).code());
1414
1415         end.link(this);
1416         addPtr(TrustedImm32(-m_calleeSaveSpace - WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_stackHeight) * sizeof(StackSlot) - maxFrameExtentForSlowPathCall), GPRInfo::callFrameRegister, stackPointerRegister);
1417         checkStackPointerAlignment();
1418
1419         // FIXME: No need to do type conversion if the callee is a WebAssembly function.
1420         // https://bugs.webkit.org/show_bug.cgi?id=149310
1421 #if USE(JSVALUE64)
1422         JSValueRegs valueRegs(GPRInfo::returnValueGPR);
1423 #else
1424         JSValueRegs valueRegs(GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
1425 #endif
1426         switch (returnType) {
1427         case WASMExpressionType::I32:
1428             convertValueToInt32(valueRegs, GPRInfo::regT0);
1429             store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop++));
1430             break;
1431         case WASMExpressionType::F32:
1432         case WASMExpressionType::F64:
1433 #if USE(JSVALUE64)
1434             convertValueToDouble(valueRegs, FPRInfo::fpRegT0, GPRInfo::nonPreservedNonReturnGPR);
1435 #else
1436             convertValueToDouble(valueRegs, FPRInfo::fpRegT0, GPRInfo::nonPreservedNonReturnGPR, FPRInfo::fpRegT1);
1437 #endif
1438             if (returnType == WASMExpressionType::F32)
1439                 convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
1440             storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop++));
1441             break;
1442         case WASMExpressionType::Void:
1443             break;
1444         default:
1445             ASSERT_NOT_REACHED();
1446         }
1447     }
1448
1449 #if USE(JSVALUE64)
1450     void convertValueToInt32(JSValueRegs valueRegs, GPRReg dst)
1451     {
1452         Jump checkJSInt32 = branchIfInt32(valueRegs);
1453
1454         callOperation(operationConvertJSValueToInt32, valueRegs.gpr(), valueRegs.gpr());
1455
1456         checkJSInt32.link(this);
1457         move(valueRegs.gpr(), dst);
1458     }
1459
1460     void convertValueToDouble(JSValueRegs valueRegs, FPRReg dst, GPRReg scratch)
1461     {
1462         Jump checkJSInt32 = branchIfInt32(valueRegs);
1463         Jump checkJSNumber = branchIfNumber(valueRegs, scratch);
1464         JumpList end;
1465
1466         callOperation(operationConvertJSValueToDouble, valueRegs.gpr(), dst);
1467         end.append(jump());
1468
1469         checkJSInt32.link(this);
1470         convertInt32ToDouble(valueRegs.gpr(), dst);
1471         end.append(jump());
1472
1473         checkJSNumber.link(this);
1474         unboxDoubleWithoutAssertions(valueRegs.gpr(), scratch, dst);
1475         end.link(this);
1476     }
1477 #else
1478     void convertValueToInt32(JSValueRegs valueRegs, GPRReg dst)
1479     {
1480         Jump checkJSInt32 = branchIfInt32(valueRegs);
1481
1482         callOperation(operationConvertJSValueToInt32, valueRegs.tagGPR(), valueRegs.payloadGPR(), valueRegs.payloadGPR());
1483
1484         checkJSInt32.link(this);
1485         move(valueRegs.payloadGPR(), dst);
1486     }
1487
1488     void convertValueToDouble(JSValueRegs valueRegs, FPRReg dst, GPRReg scratch, FPRReg fpScratch)
1489     {
1490         Jump checkJSInt32 = branchIfInt32(valueRegs);
1491         Jump checkJSNumber = branchIfNumber(valueRegs, scratch);
1492         JumpList end;
1493
1494         callOperation(operationConvertJSValueToDouble, valueRegs.tagGPR(), valueRegs.payloadGPR(), dst);
1495         end.append(jump());
1496
1497         checkJSInt32.link(this);
1498         convertInt32ToDouble(valueRegs.payloadGPR(), dst);
1499         end.append(jump());
1500
1501         checkJSNumber.link(this);
1502         unboxDouble(valueRegs.tagGPR(), valueRegs.payloadGPR(), dst, fpScratch);
1503         end.link(this);
1504     }
1505 #endif
1506
1507     void convertDoubleToValue(FPRReg fpr, JSValueRegs valueRegs)
1508     {
1509 #if USE(JSVALUE64)
1510         boxDouble(fpr, valueRegs.gpr());
1511 #else
1512         boxDouble(fpr, valueRegs.tagGPR(), valueRegs.payloadGPR());
1513 #endif
1514     }
1515
1516     JSWASMModule* m_module;
1517     unsigned m_stackHeight;
1518     unsigned m_numberOfLocals;
1519     unsigned m_tempStackTop { 0 };
1520     unsigned m_calleeSaveSpace;
1521
1522     Vector<JumpTarget> m_breakTargets;
1523     Vector<JumpTarget> m_continueTargets;
1524     Vector<JumpTarget> m_breakLabelTargets;
1525     Vector<JumpTarget> m_continueLabelTargets;
1526
1527     Label m_beginLabel;
1528     Jump m_stackOverflow;
1529     JumpList m_divideErrorJumpList;
1530     JumpList m_outOfBoundsErrorJumpList;
1531     JumpList m_exceptionChecks;
1532
1533     Vector<std::pair<Call, void*>> m_calls;
1534     Vector<CallCompilationInfo> m_callCompilationInfo;
1535 };
1536
1537 } // namespace JSC
1538
1539 #endif // ENABLE(WEBASSEMBLY)
1540
1541 #endif // WASMFunctionCompiler_h