Source/JavaScriptCore/wasm/WASMFunctionCompiler.h

   1 /*
   2  * Copyright (C) 2015 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #ifndef WASMFunctionCompiler_h
  27 #define WASMFunctionCompiler_h
  28
  29 #if ENABLE(WEBASSEMBLY)
  30
  31 #include "BinarySwitch.h"
  32 #include "CCallHelpers.h"
  33 #include "JIT.h"
  34 #include "JITOperations.h"
  35 #include "LinkBuffer.h"
  36 #include "MaxFrameExtentForSlowPathCall.h"
  37
  38 #define UNUSED 0
  39
  40 namespace JSC {
  41
  42 static int32_t JIT_OPERATION operationConvertJSValueToInt32(ExecState* exec, EncodedJSValue value)
  43 {
  44     return JSValue::decode(value).toInt32(exec);
  45 }
  46
  47 static double JIT_OPERATION operationConvertJSValueToDouble(ExecState* exec, EncodedJSValue value)
  48 {
  49     return JSValue::decode(value).toNumber(exec);
  50 }
  51
  52 #if !CPU(X86) && !CPU(X86_64)
  53 static int32_t JIT_OPERATION operationDiv(int32_t left, int32_t right)
  54 {
  55     return left / right;
  56 }
  57
  58 static int32_t JIT_OPERATION operationMod(int32_t left, int32_t right)
  59 {
  60     return left % right;
  61 }
  62
  63 static uint32_t JIT_OPERATION operationUnsignedDiv(uint32_t left, uint32_t right)
  64 {
  65     return left / right;
  66 }
  67
  68 static uint32_t JIT_OPERATION operationUnsignedMod(uint32_t left, uint32_t right)
  69 {
  70     return left % right;
  71 }
  72 #endif
  73
  74 class WASMFunctionCompiler : private CCallHelpers {
  75 public:
  76     typedef int Expression;
  77     typedef int Statement;
  78     typedef int ExpressionList;
  79     struct JumpTarget {
  80         Label label;
  81         JumpList jumpList;
  82     };
  83     enum class JumpCondition { Zero, NonZero };
  84
  85     WASMFunctionCompiler(VM& vm, CodeBlock* codeBlock, JSWASMModule* module, unsigned stackHeight)
  86         : CCallHelpers(&vm, codeBlock)
  87         , m_module(module)
  88         , m_stackHeight(stackHeight)
  89     {
  90     }
  91
  92     void startFunction(const Vector<WASMType>& arguments, uint32_t numberOfI32LocalVariables, uint32_t numberOfF32LocalVariables, uint32_t numberOfF64LocalVariables)
  93     {
  94         emitFunctionPrologue();
  95         emitPutImmediateToCallFrameHeader(m_codeBlock, JSStack::CodeBlock);
  96
  97         m_beginLabel = label();
  98
  99         addPtr(TrustedImm32(-WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_stackHeight) * sizeof(StackSlot) - maxFrameExtentForSlowPathCall), GPRInfo::callFrameRegister, GPRInfo::regT1);
 100         m_stackOverflow = branchPtr(Above, AbsoluteAddress(m_vm->addressOfStackLimit()), GPRInfo::regT1);
 101
 102         move(GPRInfo::regT1, stackPointerRegister);
 103         checkStackPointerAlignment();
 104
 105         m_numberOfLocals = arguments.size() + numberOfI32LocalVariables + numberOfF32LocalVariables + numberOfF64LocalVariables;
 106
 107         unsigned localIndex = 0;
 108         for (size_t i = 0; i < arguments.size(); ++i) {
 109             Address address(GPRInfo::callFrameRegister, CallFrame::argumentOffset(i) * sizeof(Register));
 110             switch (arguments[i]) {
 111             case WASMType::I32:
 112 #if USE(JSVALUE64)
 113                 loadValueAndConvertToInt32(address, GPRInfo::regT0);
 114 #else
 115                 loadValueAndConvertToInt32(address, GPRInfo::regT0, GPRInfo::regT1);
 116 #endif
 117                 store32(GPRInfo::regT0, localAddress(localIndex++));
 118                 break;
 119             case WASMType::F32:
 120             case WASMType::F64:
 121 #if USE(JSVALUE64)
 122                 loadValueAndConvertToDouble(address, FPRInfo::fpRegT0, GPRInfo::regT0, GPRInfo::regT1);
 123 #else
 124                 loadValueAndConvertToDouble(address, FPRInfo::fpRegT0, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2, FPRInfo::fpRegT1);
 125 #endif
 126                 if (arguments[i] == WASMType::F32)
 127                     convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
 128                 storeDouble(FPRInfo::fpRegT0, localAddress(localIndex++));
 129                 break;
 130             default:
 131                 ASSERT_NOT_REACHED();
 132             }
 133         }
 134         for (uint32_t i = 0; i < numberOfI32LocalVariables; ++i)
 135             store32(TrustedImm32(0), localAddress(localIndex++));
 136         for (uint32_t i = 0; i < numberOfF32LocalVariables; ++i)
 137             store32(TrustedImm32(0), localAddress(localIndex++));
 138         for (uint32_t i = 0; i < numberOfF64LocalVariables; ++i) {
 139 #if USE(JSVALUE64)
 140             store64(TrustedImm64(0), localAddress(localIndex++));
 141 #else
 142             store32(TrustedImm32(0), localAddress(localIndex));
 143             store32(TrustedImm32(0), localAddress(localIndex).withOffset(4));
 144             localIndex++;
 145 #endif
 146         }
 147
 148         m_codeBlock->setNumParameters(1 + arguments.size());
 149     }
 150
 151     void endFunction()
 152     {
 153         ASSERT(!m_tempStackTop);
 154
 155         // FIXME: Remove these if the last statement is a return statement.
 156 #if USE(JSVALUE64)
 157         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR);
 158 #else
 159         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
 160 #endif
 161         moveTrustedValue(jsUndefined(), returnValueRegs);
 162         emitFunctionEpilogue();
 163         ret();
 164
 165         m_stackOverflow.link(this);
 166         if (maxFrameExtentForSlowPathCall)
 167             addPtr(TrustedImm32(-maxFrameExtentForSlowPathCall), stackPointerRegister);
 168         setupArgumentsWithExecState(TrustedImmPtr(m_codeBlock));
 169         appendCallWithExceptionCheck(operationThrowStackOverflowError);
 170
 171         // FIXME: Implement arity check.
 172         Label arityCheck = label();
 173         emitFunctionPrologue();
 174         emitPutImmediateToCallFrameHeader(m_codeBlock, JSStack::CodeBlock);
 175         jump(m_beginLabel);
 176
 177         if (!m_divideErrorJumpList.empty()) {
 178             m_divideErrorJumpList.link(this);
 179
 180             setupArgumentsExecState();
 181             appendCallWithExceptionCheck(operationThrowDivideError);
 182         }
 183
 184         if (!m_exceptionChecks.empty()) {
 185             m_exceptionChecks.link(this);
 186
 187             // lookupExceptionHandler is passed two arguments, the VM and the exec (the CallFrame*).
 188             move(TrustedImmPtr(vm()), GPRInfo::argumentGPR0);
 189             move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
 190
 191 #if CPU(X86)
 192             // FIXME: should use the call abstraction, but this is currently in the SpeculativeJIT layer!
 193             poke(GPRInfo::argumentGPR0);
 194             poke(GPRInfo::argumentGPR1, 1);
 195 #endif
 196             m_calls.append(std::make_pair(call(), FunctionPtr(lookupExceptionHandlerFromCallerFrame).value()));
 197             jumpToExceptionHandler();
 198         }
 199
 200         LinkBuffer patchBuffer(*m_vm, *this, m_codeBlock, JITCompilationMustSucceed);
 201
 202         for (const auto& iterator : m_calls)
 203             patchBuffer.link(iterator.first, FunctionPtr(iterator.second));
 204
 205         for (size_t i = 0; i < m_callCompilationInfo.size(); ++i) {
 206             CallCompilationInfo& compilationInfo = m_callCompilationInfo[i];
 207             CallLinkInfo& info = *compilationInfo.callLinkInfo;
 208             info.setCallLocations(patchBuffer.locationOfNearCall(compilationInfo.callReturnLocation),
 209                 patchBuffer.locationOf(compilationInfo.hotPathBegin),
 210                 patchBuffer.locationOfNearCall(compilationInfo.hotPathOther));
 211         }
 212
 213         MacroAssemblerCodePtr withArityCheck = patchBuffer.locationOf(arityCheck);
 214         CodeRef result = FINALIZE_CODE(patchBuffer, ("Baseline JIT code for WebAssembly"));
 215         m_codeBlock->setJITCode(adoptRef(new DirectJITCode(result, withArityCheck, JITCode::BaselineJIT)));
 216         m_codeBlock->capabilityLevel();
 217     }
 218
 219     void buildSetLocal(uint32_t localIndex, int, WASMType type)
 220     {
 221         switch (type) {
 222         case WASMType::I32:
 223         case WASMType::F32:
 224             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
 225             m_tempStackTop--;
 226             store32(GPRInfo::regT0, localAddress(localIndex));
 227             break;
 228         case WASMType::F64:
 229             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
 230             m_tempStackTop--;
 231             storeDouble(FPRInfo::fpRegT0, localAddress(localIndex));
 232             break;
 233         default:
 234             ASSERT_NOT_REACHED();
 235         }
 236     }
 237
 238     void buildSetGlobal(uint32_t globalIndex, int, WASMType type)
 239     {
 240         move(TrustedImmPtr(&m_module->globalVariables()[globalIndex]), GPRInfo::regT0);
 241         switch (type) {
 242         case WASMType::I32:
 243         case WASMType::F32:
 244             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
 245             store32(GPRInfo::regT1, GPRInfo::regT0);
 246             break;
 247         case WASMType::F64:
 248             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
 249             storeDouble(FPRInfo::fpRegT0, GPRInfo::regT0);
 250             break;
 251         default:
 252             ASSERT_NOT_REACHED();
 253         }
 254         m_tempStackTop--;
 255     }
 256
 257     void buildReturn(int, WASMExpressionType returnType)
 258     {
 259 #if USE(JSVALUE64)
 260         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR);
 261 #else
 262         JSValueRegs returnValueRegs(GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
 263 #endif
 264         switch (returnType) {
 265         case WASMExpressionType::I32:
 266             load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::returnValueGPR);
 267 #if USE(JSVALUE64)
 268             or64(GPRInfo::tagTypeNumberRegister, GPRInfo::returnValueGPR);
 269 #else
 270             move(TrustedImm32(JSValue::Int32Tag), GPRInfo::returnValueGPR2);
 271 #endif
 272             m_tempStackTop--;
 273             break;
 274         case WASMExpressionType::F32:
 275         case WASMExpressionType::F64:
 276             loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT0);
 277             if (returnType == WASMExpressionType::F32)
 278                 convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
 279 #if USE(JSVALUE64)
 280             boxDouble(FPRInfo::fpRegT0, GPRInfo::returnValueGPR);
 281 #else
 282             boxDouble(FPRInfo::fpRegT0, GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
 283 #endif
 284             m_tempStackTop--;
 285             break;
 286         case WASMExpressionType::Void:
 287             moveTrustedValue(jsUndefined(), returnValueRegs);
 288             break;
 289         default:
 290             ASSERT_NOT_REACHED();
 291         }
 292         emitFunctionEpilogue();
 293         ret();
 294     }
 295
 296     int buildImmediateI32(uint32_t immediate)
 297     {
 298         store32(Imm32(immediate), temporaryAddress(m_tempStackTop++));
 299         return UNUSED;
 300     }
 301
 302     int buildImmediateF32(float immediate)
 303     {
 304         store32(Imm32(bitwise_cast<int32_t>(immediate)), temporaryAddress(m_tempStackTop++));
 305         return UNUSED;
 306     }
 307
 308     int buildImmediateF64(double immediate)
 309     {
 310 #if USE(JSVALUE64)
 311         store64(Imm64(bitwise_cast<int64_t>(immediate)), temporaryAddress(m_tempStackTop++));
 312 #else
 313         union {
 314             double doubleValue;
 315             int32_t int32Values[2];
 316         } u = { immediate };
 317         m_tempStackTop++;
 318         store32(Imm32(u.int32Values[0]), temporaryAddress(m_tempStackTop - 1));
 319         store32(Imm32(u.int32Values[1]), temporaryAddress(m_tempStackTop - 1).withOffset(4));
 320 #endif
 321         return UNUSED;
 322     }
 323
 324     int buildGetLocal(uint32_t localIndex, WASMType type)
 325     {
 326         switch (type) {
 327         case WASMType::I32:
 328         case WASMType::F32:
 329             load32(localAddress(localIndex), GPRInfo::regT0);
 330             store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop++));
 331             break;
 332         case WASMType::F64:
 333             loadDouble(localAddress(localIndex), FPRInfo::fpRegT0);
 334             storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop++));
 335             break;
 336         default:
 337             ASSERT_NOT_REACHED();
 338         }
 339         return UNUSED;
 340     }
 341
 342     int buildGetGlobal(uint32_t globalIndex, WASMType type)
 343     {
 344         move(TrustedImmPtr(&m_module->globalVariables()[globalIndex]), GPRInfo::regT0);
 345         switch (type) {
 346         case WASMType::I32:
 347         case WASMType::F32:
 348             load32(GPRInfo::regT0, GPRInfo::regT0);
 349             store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop++));
 350             break;
 351         case WASMType::F64:
 352             loadDouble(GPRInfo::regT0, FPRInfo::fpRegT0);
 353             storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop++));
 354             break;
 355         default:
 356             ASSERT_NOT_REACHED();
 357         }
 358         return UNUSED;
 359     }
 360
 361     int buildUnaryI32(int, WASMOpExpressionI32 op)
 362     {
 363         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
 364         switch (op) {
 365         case WASMOpExpressionI32::Negate:
 366             neg32(GPRInfo::regT0);
 367             break;
 368         case WASMOpExpressionI32::BitNot:
 369             xor32(TrustedImm32(-1), GPRInfo::regT0);
 370             break;
 371         case WASMOpExpressionI32::CountLeadingZeros:
 372             countLeadingZeros32(GPRInfo::regT0, GPRInfo::regT0);
 373             break;
 374         case WASMOpExpressionI32::LogicalNot: {
 375             // FIXME: Don't use branches.
 376             Jump zero = branchTest32(Zero, GPRInfo::regT0);
 377             move(TrustedImm32(0), GPRInfo::regT0);
 378             Jump end = jump();
 379             zero.link(this);
 380             move(TrustedImm32(1), GPRInfo::regT0);
 381             end.link(this);
 382             break;
 383         }
 384         case WASMOpExpressionI32::Abs: {
 385             // FIXME: Don't use branches.
 386             Jump end = branchTest32(PositiveOrZero, GPRInfo::regT0);
 387             neg32(GPRInfo::regT0);
 388             end.link(this);
 389             break;
 390         }
 391         default:
 392             ASSERT_NOT_REACHED();
 393         }
 394         store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
 395         return UNUSED;
 396     }
 397
 398     int buildUnaryF32(int, WASMOpExpressionF32 op)
 399     {
 400         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
 401         switch (op) {
 402         case WASMOpExpressionF32::Negate:
 403             convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
 404             negateDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
 405             convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
 406             break;
 407         case WASMOpExpressionF32::Abs:
 408             convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
 409             absDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
 410             convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
 411             break;
 412         case WASMOpExpressionF32::Ceil:
 413             callOperation(ceilf, FPRInfo::fpRegT1, FPRInfo::fpRegT0);
 414             break;
 415         case WASMOpExpressionF32::Floor:
 416             callOperation(floorf, FPRInfo::fpRegT1, FPRInfo::fpRegT0);
 417             break;
 418         case WASMOpExpressionF32::Sqrt:
 419             convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
 420             sqrtDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
 421             convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
 422             break;
 423         default:
 424             ASSERT_NOT_REACHED();
 425         }
 426         storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
 427         return UNUSED;
 428     }
 429
 430     int buildBinaryI32(int, int, WASMOpExpressionI32 op)
 431     {
 432         load32(temporaryAddress(m_tempStackTop - 2), GPRInfo::regT0);
 433         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
 434         switch (op) {
 435         case WASMOpExpressionI32::Add:
 436             add32(GPRInfo::regT1, GPRInfo::regT0);
 437             break;
 438         case WASMOpExpressionI32::Sub:
 439             sub32(GPRInfo::regT1, GPRInfo::regT0);
 440             break;
 441         case WASMOpExpressionI32::Mul:
 442             mul32(GPRInfo::regT1, GPRInfo::regT0);
 443             break;
 444         case WASMOpExpressionI32::SDiv:
 445         case WASMOpExpressionI32::UDiv:
 446         case WASMOpExpressionI32::SMod:
 447         case WASMOpExpressionI32::UMod: {
 448             m_divideErrorJumpList.append(branchTest32(Zero, GPRInfo::regT1));
 449             if (op == WASMOpExpressionI32::SDiv || op == WASMOpExpressionI32::SMod) {
 450                 Jump denominatorNotNeg1 = branch32(NotEqual, GPRInfo::regT1, TrustedImm32(-1));
 451                 m_divideErrorJumpList.append(branch32(Equal, GPRInfo::regT0, TrustedImm32(-2147483647-1)));
 452                 denominatorNotNeg1.link(this);
 453             }
 454 #if CPU(X86) || CPU(X86_64)
 455             ASSERT(GPRInfo::regT0 == X86Registers::eax);
 456             move(GPRInfo::regT1, X86Registers::ecx);
 457             if (op == WASMOpExpressionI32::SDiv || op == WASMOpExpressionI32::SMod) {
 458                 m_assembler.cdq();
 459                 m_assembler.idivl_r(X86Registers::ecx);
 460             } else {
 461                 ASSERT(op == WASMOpExpressionI32::UDiv || op == WASMOpExpressionI32::UMod);
 462                 xor32(X86Registers::edx, X86Registers::edx);
 463                 m_assembler.divl_r(X86Registers::ecx);
 464             }
 465             if (op == WASMOpExpressionI32::SMod || op == WASMOpExpressionI32::UMod)
 466                 move(X86Registers::edx, GPRInfo::regT0);
 467 #else
 468             // FIXME: We should be able to do an inline div on ARMv7 and ARM64.
 469             switch (op) {
 470             case WASMOpExpressionI32::SDiv:
 471                 callOperation(operationDiv, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
 472                 break;
 473             case WASMOpExpressionI32::UDiv:
 474                 callOperation(operationUnsignedDiv, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
 475                 break;
 476             case WASMOpExpressionI32::SMod:
 477                 callOperation(operationMod, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
 478                 break;
 479             case WASMOpExpressionI32::UMod:
 480                 callOperation(operationUnsignedMod, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
 481                 break;
 482             default:
 483                 ASSERT_NOT_REACHED();
 484             }
 485 #endif
 486             break;
 487         }
 488         case WASMOpExpressionI32::BitOr:
 489             or32(GPRInfo::regT1, GPRInfo::regT0);
 490             break;
 491         case WASMOpExpressionI32::BitAnd:
 492             and32(GPRInfo::regT1, GPRInfo::regT0);
 493             break;
 494         case WASMOpExpressionI32::BitXor:
 495             xor32(GPRInfo::regT1, GPRInfo::regT0);
 496             break;
 497         case WASMOpExpressionI32::LeftShift:
 498             lshift32(GPRInfo::regT1, GPRInfo::regT0);
 499             break;
 500         case WASMOpExpressionI32::ArithmeticRightShift:
 501             rshift32(GPRInfo::regT1, GPRInfo::regT0);
 502             break;
 503         case WASMOpExpressionI32::LogicalRightShift:
 504             urshift32(GPRInfo::regT1, GPRInfo::regT0);
 505             break;
 506         default:
 507             ASSERT_NOT_REACHED();
 508         }
 509         m_tempStackTop--;
 510         store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
 511         return UNUSED;
 512     }
 513
 514     int buildBinaryF32(int, int, WASMOpExpressionF32 op)
 515     {
 516         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
 517         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
 518         convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
 519         convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
 520         switch (op) {
 521         case WASMOpExpressionF32::Add:
 522             addDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
 523             break;
 524         case WASMOpExpressionF32::Sub:
 525             subDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
 526             break;
 527         case WASMOpExpressionF32::Mul:
 528             mulDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
 529             break;
 530         case WASMOpExpressionF32::Div:
 531             divDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT0);
 532             break;
 533         default:
 534             RELEASE_ASSERT_NOT_REACHED();
 535         }
 536         convertDoubleToFloat(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
 537         m_tempStackTop--;
 538         storeDouble(FPRInfo::fpRegT0, temporaryAddress(m_tempStackTop - 1));
 539         return UNUSED;
 540     }
 541
 542     int buildRelationalI32(int, int, WASMOpExpressionI32 op)
 543     {
 544         load32(temporaryAddress(m_tempStackTop - 2), GPRInfo::regT0);
 545         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
 546         RelationalCondition condition;
 547         switch (op) {
 548         case WASMOpExpressionI32::EqualI32:
 549             condition = Equal;
 550             break;
 551         case WASMOpExpressionI32::NotEqualI32:
 552             condition = NotEqual;
 553             break;
 554         case WASMOpExpressionI32::SLessThanI32:
 555             condition = LessThan;
 556             break;
 557         case WASMOpExpressionI32::ULessThanI32:
 558             condition = Below;
 559             break;
 560         case WASMOpExpressionI32::SLessThanOrEqualI32:
 561             condition = LessThanOrEqual;
 562             break;
 563         case WASMOpExpressionI32::ULessThanOrEqualI32:
 564             condition = BelowOrEqual;
 565             break;
 566         case WASMOpExpressionI32::SGreaterThanI32:
 567             condition = GreaterThan;
 568             break;
 569         case WASMOpExpressionI32::UGreaterThanI32:
 570             condition = Above;
 571             break;
 572         case WASMOpExpressionI32::SGreaterThanOrEqualI32:
 573             condition = GreaterThanOrEqual;
 574             break;
 575         case WASMOpExpressionI32::UGreaterThanOrEqualI32:
 576             condition = AboveOrEqual;
 577             break;
 578         default:
 579             RELEASE_ASSERT_NOT_REACHED();
 580         }
 581         compare32(condition, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT0);
 582         m_tempStackTop--;
 583         store32(GPRInfo::regT0, temporaryAddress(m_tempStackTop - 1));
 584         return UNUSED;
 585     }
 586
 587     int buildRelationalF32(int, int, WASMOpExpressionI32 op)
 588     {
 589         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
 590         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
 591         convertFloatToDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT0);
 592         convertFloatToDouble(FPRInfo::fpRegT1, FPRInfo::fpRegT1);
 593         DoubleCondition condition;
 594         switch (op) {
 595         case WASMOpExpressionI32::EqualF32:
 596             condition = DoubleEqual;
 597             break;
 598         case WASMOpExpressionI32::NotEqualF32:
 599             condition = DoubleNotEqual;
 600             break;
 601         case WASMOpExpressionI32::LessThanF32:
 602             condition = DoubleLessThan;
 603             break;
 604         case WASMOpExpressionI32::LessThanOrEqualF32:
 605             condition = DoubleLessThanOrEqual;
 606             break;
 607         case WASMOpExpressionI32::GreaterThanF32:
 608             condition = DoubleGreaterThan;
 609             break;
 610         case WASMOpExpressionI32::GreaterThanOrEqualF32:
 611             condition = DoubleGreaterThanOrEqual;
 612             break;
 613         default:
 614             RELEASE_ASSERT_NOT_REACHED();
 615         }
 616         m_tempStackTop--;
 617         Jump trueCase = branchDouble(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT1);
 618         store32(TrustedImm32(0), temporaryAddress(m_tempStackTop - 1));
 619         Jump end = jump();
 620         trueCase.link(this);
 621         store32(TrustedImm32(1), temporaryAddress(m_tempStackTop - 1));
 622         end.link(this);
 623         return UNUSED;
 624     }
 625
 626     int buildRelationalF64(int, int, WASMOpExpressionI32 op)
 627     {
 628         loadDouble(temporaryAddress(m_tempStackTop - 2), FPRInfo::fpRegT0);
 629         loadDouble(temporaryAddress(m_tempStackTop - 1), FPRInfo::fpRegT1);
 630         DoubleCondition condition;
 631         switch (op) {
 632         case WASMOpExpressionI32::EqualF64:
 633             condition = DoubleEqual;
 634             break;
 635         case WASMOpExpressionI32::NotEqualF64:
 636             condition = DoubleNotEqual;
 637             break;
 638         case WASMOpExpressionI32::LessThanF64:
 639             condition = DoubleLessThan;
 640             break;
 641         case WASMOpExpressionI32::LessThanOrEqualF64:
 642             condition = DoubleLessThanOrEqual;
 643             break;
 644         case WASMOpExpressionI32::GreaterThanF64:
 645             condition = DoubleGreaterThan;
 646             break;
 647         case WASMOpExpressionI32::GreaterThanOrEqualF64:
 648             condition = DoubleGreaterThanOrEqual;
 649             break;
 650         default:
 651             RELEASE_ASSERT_NOT_REACHED();
 652         }
 653         m_tempStackTop--;
 654         Jump trueCase = branchDouble(condition, FPRInfo::fpRegT0, FPRInfo::fpRegT1);
 655         store32(TrustedImm32(0), temporaryAddress(m_tempStackTop - 1));
 656         Jump end = jump();
 657         trueCase.link(this);
 658         store32(TrustedImm32(1), temporaryAddress(m_tempStackTop - 1));
 659         end.link(this);
 660         return UNUSED;
 661     }
 662
 663     int buildCallInternal(uint32_t functionIndex, int, const WASMSignature& signature, WASMExpressionType returnType)
 664     {
 665         boxArgumentsAndAdjustStackPointer(signature.arguments);
 666
 667         JSFunction* function = m_module->functions()[functionIndex].get();
 668         move(TrustedImmPtr(function), GPRInfo::regT0);
 669
 670         callAndUnboxResult(returnType);
 671         return UNUSED;
 672     }
 673
 674     int buildCallIndirect(uint32_t functionPointerTableIndex, int, int, const WASMSignature& signature, WASMExpressionType returnType)
 675     {
 676         boxArgumentsAndAdjustStackPointer(signature.arguments);
 677
 678         const Vector<JSFunction*>& functions = m_module->functionPointerTables()[functionPointerTableIndex].functions;
 679         move(TrustedImmPtr(functions.data()), GPRInfo::regT0);
 680         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT1);
 681         m_tempStackTop--;
 682         and32(TrustedImm32(functions.size() - 1), GPRInfo::regT1);
 683         loadPtr(BaseIndex(GPRInfo::regT0, GPRInfo::regT1, timesPtr()), GPRInfo::regT0);
 684
 685         callAndUnboxResult(returnType);
 686         return UNUSED;
 687     }
 688
 689     int buildCallImport(uint32_t functionImportIndex, int, const WASMSignature& signature, WASMExpressionType returnType)
 690     {
 691         boxArgumentsAndAdjustStackPointer(signature.arguments);
 692
 693         JSFunction* function = m_module->importedFunctions()[functionImportIndex].get();
 694         move(TrustedImmPtr(function), GPRInfo::regT0);
 695
 696         callAndUnboxResult(returnType);
 697         return UNUSED;
 698     }
 699
 700     void appendExpressionList(int&, int) { }
 701
 702     void linkTarget(JumpTarget& target)
 703     {
 704         target.label = label();
 705         target.jumpList.link(this);
 706     }
 707
 708     void jumpToTarget(JumpTarget& target)
 709     {
 710         if (target.label.isSet())
 711             jump(target.label);
 712         else
 713             target.jumpList.append(jump());
 714     }
 715
 716     void jumpToTargetIf(JumpCondition condition, int, JumpTarget& target)
 717     {
 718         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
 719         m_tempStackTop--;
 720         Jump taken = branchTest32((condition == JumpCondition::Zero) ? Zero : NonZero, GPRInfo::regT0);
 721         if (target.label.isSet())
 722             taken.linkTo(target.label, this);
 723         else
 724             target.jumpList.append(taken);
 725     }
 726
 727     void startLoop()
 728     {
 729         m_breakTargets.append(JumpTarget());
 730         m_continueTargets.append(JumpTarget());
 731     }
 732
 733     void endLoop()
 734     {
 735         m_breakTargets.removeLast();
 736         m_continueTargets.removeLast();
 737     }
 738
 739     void startSwitch()
 740     {
 741         m_breakTargets.append(JumpTarget());
 742     }
 743
 744     void endSwitch()
 745     {
 746         m_breakTargets.removeLast();
 747     }
 748
 749     void startLabel()
 750     {
 751         m_breakLabelTargets.append(JumpTarget());
 752         m_continueLabelTargets.append(JumpTarget());
 753
 754         linkTarget(m_continueLabelTargets.last());
 755     }
 756
 757     void endLabel()
 758     {
 759         linkTarget(m_breakLabelTargets.last());
 760
 761         m_breakLabelTargets.removeLast();
 762         m_continueLabelTargets.removeLast();
 763     }
 764
 765     JumpTarget& breakTarget()
 766     {
 767         return m_breakTargets.last();
 768     }
 769
 770     JumpTarget& continueTarget()
 771     {
 772         return m_continueTargets.last();
 773     }
 774
 775     JumpTarget& breakLabelTarget(uint32_t labelIndex)
 776     {
 777         return m_breakLabelTargets[labelIndex];
 778     }
 779
 780     JumpTarget& continueLabelTarget(uint32_t labelIndex)
 781     {
 782         return m_continueLabelTargets[labelIndex];
 783     }
 784
 785     void buildSwitch(int, const Vector<int64_t>& cases, Vector<JumpTarget>& targets, JumpTarget defaultTarget)
 786     {
 787         load32(temporaryAddress(m_tempStackTop - 1), GPRInfo::regT0);
 788         m_tempStackTop--;
 789         BinarySwitch binarySwitch(GPRInfo::regT0, cases, BinarySwitch::Int32);
 790         while (binarySwitch.advance(*this)) {
 791             unsigned index = binarySwitch.caseIndex();
 792             jump(targets[index].label);
 793         }
 794         binarySwitch.fallThrough().linkTo(defaultTarget.label, this);
 795     }
 796
 797 private:
 798     union StackSlot {
 799         int32_t intValue;
 800         float floatValue;
 801         double doubleValue;
 802     };
 803
 804     Address localAddress(unsigned localIndex) const
 805     {
 806         ASSERT(localIndex < m_numberOfLocals);
 807         return Address(GPRInfo::callFrameRegister, -(localIndex + 1) * sizeof(StackSlot));
 808     }
 809
 810     Address temporaryAddress(unsigned temporaryIndex) const
 811     {
 812         ASSERT(m_numberOfLocals + temporaryIndex < m_stackHeight);
 813         return Address(GPRInfo::callFrameRegister, -(m_numberOfLocals + temporaryIndex + 1) * sizeof(StackSlot));
 814     }
 815
 816     void appendCall(const FunctionPtr& function)
 817     {
 818         m_calls.append(std::make_pair(call(), function.value()));
 819     }
 820
 821     void appendCallWithExceptionCheck(const FunctionPtr& function)
 822     {
 823         appendCall(function);
 824         m_exceptionChecks.append(emitExceptionCheck());
 825     }
 826
 827     Call emitNakedCall(CodePtr function)
 828     {
 829         Call nakedCall = nearCall();
 830         m_calls.append(std::make_pair(nakedCall, function.executableAddress()));
 831         return nakedCall;
 832     }
 833
 834     void appendCallSetResult(const FunctionPtr& function, GPRReg result)
 835     {
 836         appendCall(function);
 837         move(GPRInfo::returnValueGPR, result);
 838     }
 839
 840 #if CPU(X86)
 841     void appendCallSetResult(const FunctionPtr& function, FPRReg result)
 842     {
 843         appendCall(function);
 844         m_assembler.fstpl(0, stackPointerRegister);
 845         loadDouble(stackPointerRegister, result);
 846     }
 847 #elif CPU(ARM) && !CPU(ARM_HARDFP)
 848     void appendCallSetResult(const FunctionPtr& function, FPRReg result)
 849     {
 850         appendCall(function);
 851         m_assembler.vmov(result, GPRInfo::returnValueGPR, GPRInfo::returnValueGPR2);
 852     }
 853 #else // CPU(X86_64) || (CPU(ARM) && CPU(ARM_HARDFP)) || CPU(ARM64) || CPU(MIPS) || CPU(SH4)
 854     void appendCallSetResult(const FunctionPtr& function, FPRReg result)
 855     {
 856         appendCall(function);
 857         moveDouble(FPRInfo::returnValueFPR, result);
 858     }
 859 #endif
 860
 861 #if USE(JSVALUE64)
 862     void callOperation(Z_JITOperation_EJ operation, GPRReg src, GPRReg dst)
 863     {
 864         setupArgumentsWithExecState(src);
 865         appendCallSetResult(operation, dst);
 866     }
 867
 868     void callOperation(D_JITOperation_EJ operation, GPRReg src, FPRReg dst)
 869     {
 870         setupArgumentsWithExecState(src);
 871         appendCallSetResult(operation, dst);
 872     }
 873 #else
 874     // EncodedJSValue in JSVALUE32_64 is a 64-bit integer. When being compiled in ARM EABI, it must be aligned even-numbered register (r0, r2 or [sp]).
 875     // To avoid assemblies from using wrong registers, let's occupy r1 or r3 with a dummy argument when necessary.
 876 #if (COMPILER_SUPPORTS(EABI) && CPU(ARM)) || CPU(MIPS)
 877 #define EABI_32BIT_DUMMY_ARG      TrustedImm32(0),
 878 #else
 879 #define EABI_32BIT_DUMMY_ARG
 880 #endif
 881
 882     void callOperation(Z_JITOperation_EJ operation, GPRReg srcTag, GPRReg srcPayload, GPRReg dst)
 883     {
 884         setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG srcPayload, srcTag);
 885         appendCallSetResult(operation, dst);
 886     }
 887
 888     void callOperation(D_JITOperation_EJ operation, GPRReg srcTag, GPRReg srcPayload, FPRReg dst)
 889     {
 890         setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG srcPayload, srcTag);
 891         appendCallSetResult(operation, dst);
 892     }
 893 #endif
 894
 895     void callOperation(float JIT_OPERATION (*operation)(float), FPRegisterID src, FPRegisterID dst)
 896     {
 897         setupArguments(src);
 898         appendCallSetResult(operation, dst);
 899     }
 900
 901     void callOperation(int32_t JIT_OPERATION (*operation)(int32_t, int32_t), GPRReg src1, GPRReg src2, GPRReg dst)
 902     {
 903         setupArguments(src1, src2);
 904         appendCallSetResult(operation, dst);
 905     }
 906
 907     void callOperation(uint32_t JIT_OPERATION (*operation)(uint32_t, uint32_t), GPRReg src1, GPRReg src2, GPRReg dst)
 908     {
 909         setupArguments(src1, src2);
 910         appendCallSetResult(operation, dst);
 911     }
 912
 913     void boxArgumentsAndAdjustStackPointer(const Vector<WASMType>& arguments)
 914     {
 915         size_t argumentCount = arguments.size();
 916         int stackOffset = -WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_numberOfLocals + m_tempStackTop + argumentCount + 1 + JSStack::CallFrameHeaderSize);
 917
 918         storeTrustedValue(jsUndefined(), Address(GPRInfo::callFrameRegister, (stackOffset + CallFrame::thisArgumentOffset()) * sizeof(Register)));
 919
 920         for (size_t i = 0; i < argumentCount; ++i) {
 921             Address address(GPRInfo::callFrameRegister, (stackOffset + CallFrame::argumentOffset(i)) * sizeof(Register));
 922             switch (arguments[i]) {
 923             case WASMType::I32:
 924                 load32(temporaryAddress(m_tempStackTop - argumentCount + i), GPRInfo::regT0);
 925 #if USE(JSVALUE64)
 926                 or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
 927                 store64(GPRInfo::regT0, address);
 928 #else
 929                 store32(GPRInfo::regT0, address.withOffset(PayloadOffset));
 930                 store32(TrustedImm32(JSValue::Int32Tag), address.withOffset(TagOffset));
 931 #endif
 932                 break;
 933             default:
 934                 ASSERT_NOT_REACHED();
 935             }
 936         }
 937         m_tempStackTop -= argumentCount;
 938
 939         addPtr(TrustedImm32(stackOffset * sizeof(Register) + sizeof(CallerFrameAndPC)), GPRInfo::callFrameRegister, stackPointerRegister);
 940         store32(TrustedImm32(argumentCount + 1), Address(stackPointerRegister, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)));
 941     }
 942
 943     void callAndUnboxResult(WASMExpressionType returnType)
 944     {
 945         // regT0 holds callee.
 946 #if USE(JSVALUE64)
 947         store64(GPRInfo::regT0, Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) - sizeof(CallerFrameAndPC)));
 948 #else
 949         store32(GPRInfo::regT0, Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)));
 950         store32(TrustedImm32(JSValue::CellTag), Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) + TagOffset - sizeof(CallerFrameAndPC)));
 951 #endif
 952
 953         DataLabelPtr addressOfLinkedFunctionCheck;
 954         Jump slowCase = branchPtrWithPatch(NotEqual, GPRInfo::regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
 955
 956         CallLinkInfo* info = m_codeBlock->addCallLinkInfo();
 957         info->setUpCall(CallLinkInfo::Call, CodeOrigin(), GPRInfo::regT0);
 958         m_callCompilationInfo.append(CallCompilationInfo());
 959         m_callCompilationInfo.last().hotPathBegin = addressOfLinkedFunctionCheck;
 960         m_callCompilationInfo.last().callLinkInfo = info;
 961         m_callCompilationInfo.last().hotPathOther = nearCall();
 962         Jump end = jump();
 963
 964         slowCase.link(this);
 965         move(TrustedImmPtr(info), GPRInfo::regT2);
 966         m_callCompilationInfo.last().callReturnLocation = emitNakedCall(m_vm->getCTIStub(linkCallThunkGenerator).code());
 967
 968         end.link(this);
 969         addPtr(TrustedImm32(-WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_stackHeight) * sizeof(StackSlot) - maxFrameExtentForSlowPathCall), GPRInfo::callFrameRegister, stackPointerRegister);
 970         checkStackPointerAlignment();
 971
 972         switch (returnType) {
 973         case WASMExpressionType::I32:
 974             store32(GPRInfo::returnValueGPR, temporaryAddress(m_tempStackTop++));
 975             break;
 976         case WASMExpressionType::Void:
 977             break;
 978         default:
 979             ASSERT_NOT_REACHED();
 980         }
 981     }
 982
 983 #if USE(JSVALUE64)
 984     void loadValueAndConvertToInt32(Address address, GPRReg dst)
 985     {
 986         JSValueRegs tempRegs(dst);
 987         loadValue(address, tempRegs);
 988         Jump checkJSInt32 = branchIfInt32(tempRegs);
 989
 990         callOperation(operationConvertJSValueToInt32, dst, dst);
 991
 992         checkJSInt32.link(this);
 993     }
 994
 995     void loadValueAndConvertToDouble(Address address, FPRReg dst, GPRReg scratch1, GPRReg scratch2)
 996     {
 997         JSValueRegs tempRegs(scratch1);
 998         loadValue(address, tempRegs);
 999         Jump checkJSInt32 = branchIfInt32(tempRegs);
1000         Jump checkJSNumber = branchIfNumber(tempRegs, scratch2);
1001         JumpList end;
1002
1003         callOperation(operationConvertJSValueToDouble, tempRegs.gpr(), dst);
1004         end.append(jump());
1005
1006         checkJSInt32.link(this);
1007         convertInt32ToDouble(tempRegs.gpr(), dst);
1008         end.append(jump());
1009
1010         checkJSNumber.link(this);
1011         unboxDoubleWithoutAssertions(tempRegs.gpr(), dst);
1012         end.link(this);
1013     }
1014 #else
1015     void loadValueAndConvertToInt32(Address address, GPRReg dst, GPRReg scratch)
1016     {
1017         JSValueRegs tempRegs(scratch, dst);
1018         loadValue(address, tempRegs);
1019         Jump checkJSInt32 = branchIfInt32(tempRegs);
1020
1021         callOperation(operationConvertJSValueToInt32, tempRegs.tagGPR(), tempRegs.payloadGPR(), dst);
1022
1023         checkJSInt32.link(this);
1024     }
1025
1026     void loadValueAndConvertToDouble(Address address, FPRReg dst, GPRReg scratch1, GPRReg scratch2, GPRReg scratch3, FPRReg fpScratch)
1027     {
1028         JSValueRegs tempRegs(scratch2, scratch1);
1029         loadValue(address, tempRegs);
1030         Jump checkJSInt32 = branchIfInt32(tempRegs);
1031         Jump checkJSNumber = branchIfNumber(tempRegs, scratch3);
1032         JumpList end;
1033
1034         callOperation(operationConvertJSValueToDouble, tempRegs.tagGPR(), tempRegs.payloadGPR(), dst);
1035         end.append(jump());
1036
1037         checkJSInt32.link(this);
1038         convertInt32ToDouble(tempRegs.payloadGPR(), dst);
1039         end.append(jump());
1040
1041         checkJSNumber.link(this);
1042         unboxDouble(tempRegs.tagGPR(), tempRegs.payloadGPR(), dst, fpScratch);
1043         end.link(this);
1044     }
1045 #endif
1046
1047     JSWASMModule* m_module;
1048     unsigned m_stackHeight;
1049     unsigned m_numberOfLocals;
1050     unsigned m_tempStackTop { 0 };
1051
1052     Vector<JumpTarget> m_breakTargets;
1053     Vector<JumpTarget> m_continueTargets;
1054     Vector<JumpTarget> m_breakLabelTargets;
1055     Vector<JumpTarget> m_continueLabelTargets;
1056
1057     Label m_beginLabel;
1058     Jump m_stackOverflow;
1059     JumpList m_divideErrorJumpList;
1060     JumpList m_exceptionChecks;
1061
1062     Vector<std::pair<Call, void*>> m_calls;
1063     Vector<CallCompilationInfo> m_callCompilationInfo;
1064 };
1065
1066 } // namespace JSC
1067
1068 #endif // ENABLE(WEBASSEMBLY)
1069
1070 #endif // WASMFunctionCompiler_h