Source/JavaScriptCore/jit/ThunkGenerators.cpp

   1 /*
   2  * Copyright (C) 2010, 2012, 2013, 2014 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  23  * THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "ThunkGenerators.h"
  28
  29 #include "CodeBlock.h"
  30 #include "DFGSpeculativeJIT.h"
  31 #include "JITOperations.h"
  32 #include "JSArray.h"
  33 #include "JSArrayIterator.h"
  34 #include "JSStack.h"
  35 #include "MathCommon.h"
  36 #include "MaxFrameExtentForSlowPathCall.h"
  37 #include "JSCInlines.h"
  38 #include "SpecializedThunkJIT.h"
  39 #include <wtf/InlineASM.h>
  40 #include <wtf/StringPrintStream.h>
  41 #include <wtf/text/StringImpl.h>
  42
  43 #if ENABLE(JIT)
  44
  45 namespace JSC {
  46
  47 inline void emitPointerValidation(CCallHelpers& jit, GPRReg pointerGPR)
  48 {
  49     if (ASSERT_DISABLED)
  50         return;
  51     CCallHelpers::Jump isNonZero = jit.branchTestPtr(CCallHelpers::NonZero, pointerGPR);
  52     jit.abortWithReason(TGInvalidPointer);
  53     isNonZero.link(&jit);
  54     jit.pushToSave(pointerGPR);
  55     jit.load8(pointerGPR, pointerGPR);
  56     jit.popToRestore(pointerGPR);
  57 }
  58
  59 // We will jump here if the JIT code tries to make a call, but the
  60 // linking helper (C++ code) decides to throw an exception instead.
  61 MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(VM* vm)
  62 {
  63     CCallHelpers jit(vm);
  64
  65     // The call pushed a return address, so we need to pop it back off to re-align the stack,
  66     // even though we won't use it.
  67     jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
  68
  69     jit.copyCalleeSavesToVMCalleeSavesBuffer();
  70
  71     jit.setupArguments(CCallHelpers::TrustedImmPtr(vm), GPRInfo::callFrameRegister);
  72     jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(lookupExceptionHandler)), GPRInfo::nonArgGPR0);
  73     emitPointerValidation(jit, GPRInfo::nonArgGPR0);
  74     jit.call(GPRInfo::nonArgGPR0);
  75     jit.jumpToExceptionHandler();
  76
  77     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
  78     return FINALIZE_CODE(patchBuffer, ("Throw exception from call slow path thunk"));
  79 }
  80
  81 static void slowPathFor(
  82     CCallHelpers& jit, VM* vm, Sprt_JITOperation_ECli slowPathFunction)
  83 {
  84     jit.emitFunctionPrologue();
  85     jit.storePtr(GPRInfo::callFrameRegister, &vm->topCallFrame);
  86 #if OS(WINDOWS) && CPU(X86_64)
  87     // Windows X86_64 needs some space pointed to by arg0 for return types larger than 64 bits.
  88     // Other argument values are shift by 1. Use space on the stack for our two return values.
  89     // Moving the stack down maxFrameExtentForSlowPathCall bytes gives us room for our 3 arguments
  90     // and space for the 16 byte return area.
  91     jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
  92     jit.move(GPRInfo::regT2, GPRInfo::argumentGPR2);
  93     jit.addPtr(CCallHelpers::TrustedImm32(32), CCallHelpers::stackPointerRegister, GPRInfo::argumentGPR0);
  94     jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
  95     jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0);
  96     emitPointerValidation(jit, GPRInfo::nonArgGPR0);
  97     jit.call(GPRInfo::nonArgGPR0);
  98     jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR, 8), GPRInfo::returnValueGPR2);
  99     jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR), GPRInfo::returnValueGPR);
 100     jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
 101 #else
 102     if (maxFrameExtentForSlowPathCall)
 103         jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
 104     jit.setupArgumentsWithExecState(GPRInfo::regT2);
 105     jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0);
 106     emitPointerValidation(jit, GPRInfo::nonArgGPR0);
 107     jit.call(GPRInfo::nonArgGPR0);
 108     if (maxFrameExtentForSlowPathCall)
 109         jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
 110 #endif
 111
 112     // This slow call will return the address of one of the following:
 113     // 1) Exception throwing thunk.
 114     // 2) Host call return value returner thingy.
 115     // 3) The function to call.
 116     // The second return value GPR will hold a non-zero value for tail calls.
 117
 118     emitPointerValidation(jit, GPRInfo::returnValueGPR);
 119     jit.emitFunctionEpilogue();
 120
 121     RELEASE_ASSERT(reinterpret_cast<void*>(KeepTheFrame) == reinterpret_cast<void*>(0));
 122     CCallHelpers::Jump doNotTrash = jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::returnValueGPR2);
 123
 124     jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
 125     jit.prepareForTailCallSlow(GPRInfo::returnValueGPR);
 126
 127     doNotTrash.link(&jit);
 128     jit.jump(GPRInfo::returnValueGPR);
 129 }
 130
 131 MacroAssemblerCodeRef linkCallThunkGenerator(VM* vm)
 132 {
 133     // The return address is on the stack or in the link register. We will hence
 134     // save the return address to the call frame while we make a C++ function call
 135     // to perform linking and lazy compilation if necessary. We expect the callee
 136     // to be in regT0/regT1 (payload/tag), the CallFrame to have already
 137     // been adjusted, and all other registers to be available for use.
 138     CCallHelpers jit(vm);
 139
 140     slowPathFor(jit, vm, operationLinkCall);
 141
 142     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
 143     return FINALIZE_CODE(patchBuffer, ("Link call slow path thunk"));
 144 }
 145
 146 // For closure optimizations, we only include calls, since if you're using closures for
 147 // object construction then you're going to lose big time anyway.
 148 MacroAssemblerCodeRef linkPolymorphicCallThunkGenerator(VM* vm)
 149 {
 150     CCallHelpers jit(vm);
 151
 152     slowPathFor(jit, vm, operationLinkPolymorphicCall);
 153
 154     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
 155     return FINALIZE_CODE(patchBuffer, ("Link polymorphic call slow path thunk"));
 156 }
 157
 158 // FIXME: We should distinguish between a megamorphic virtual call vs. a slow
 159 // path virtual call so that we can enable fast tail calls for megamorphic
 160 // virtual calls by using the shuffler.
 161 // https://bugs.webkit.org/show_bug.cgi?id=148831
 162 MacroAssemblerCodeRef virtualThunkFor(VM* vm, CallLinkInfo& callLinkInfo)
 163 {
 164     // The callee is in regT0 (for JSVALUE32_64, the tag is in regT1).
 165     // The return address is on the stack, or in the link register. We will hence
 166     // jump to the callee, or save the return address to the call frame while we
 167     // make a C++ function call to the appropriate JIT operation.
 168
 169     CCallHelpers jit(vm);
 170
 171     CCallHelpers::JumpList slowCase;
 172
 173     // This is a slow path execution, and regT2 contains the CallLinkInfo. Count the
 174     // slow path execution for the profiler.
 175     jit.add32(
 176         CCallHelpers::TrustedImm32(1),
 177         CCallHelpers::Address(GPRInfo::regT2, CallLinkInfo::offsetOfSlowPathCount()));
 178
 179     // FIXME: we should have a story for eliminating these checks. In many cases,
 180     // the DFG knows that the value is definitely a cell, or definitely a function.
 181
 182 #if USE(JSVALUE64)
 183     slowCase.append(
 184         jit.branchTest64(
 185             CCallHelpers::NonZero, GPRInfo::regT0, GPRInfo::tagMaskRegister));
 186 #else
 187     slowCase.append(
 188         jit.branch32(
 189             CCallHelpers::NotEqual, GPRInfo::regT1,
 190             CCallHelpers::TrustedImm32(JSValue::CellTag)));
 191 #endif
 192     jit.emitLoadStructure(GPRInfo::regT0, GPRInfo::regT4, GPRInfo::regT1);
 193     slowCase.append(
 194         jit.branchPtr(
 195             CCallHelpers::NotEqual,
 196             CCallHelpers::Address(GPRInfo::regT4, Structure::classInfoOffset()),
 197             CCallHelpers::TrustedImmPtr(JSFunction::info())));
 198
 199     // Now we know we have a JSFunction.
 200
 201     jit.loadPtr(
 202         CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfExecutable()),
 203         GPRInfo::regT4);
 204     jit.loadPtr(
 205         CCallHelpers::Address(
 206             GPRInfo::regT4, ExecutableBase::offsetOfJITCodeWithArityCheckFor(
 207                 callLinkInfo.specializationKind())),
 208         GPRInfo::regT4);
 209     slowCase.append(jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT4));
 210
 211     // Now we know that we have a CodeBlock, and we're committed to making a fast
 212     // call.
 213
 214     // Make a tail call. This will return back to JIT code.
 215     emitPointerValidation(jit, GPRInfo::regT4);
 216     if (callLinkInfo.isTailCall()) {
 217         jit.preserveReturnAddressAfterCall(GPRInfo::regT0);
 218         jit.prepareForTailCallSlow(GPRInfo::regT4);
 219     }
 220     jit.jump(GPRInfo::regT4);
 221
 222     slowCase.link(&jit);
 223
 224     // Here we don't know anything, so revert to the full slow path.
 225
 226     slowPathFor(jit, vm, operationVirtualCall);
 227
 228     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
 229     return FINALIZE_CODE(
 230         patchBuffer,
 231         ("Virtual %s slow path thunk",
 232         callLinkInfo.callMode() == CallMode::Regular ? "call" : callLinkInfo.callMode() == CallMode::Tail ? "tail call" : "construct"));
 233 }
 234
 235 enum ThunkEntryType { EnterViaCall, EnterViaJump };
 236
 237 static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall)
 238 {
 239     // FIXME: This should be able to log ShadowChicken prologue packets.
 240     // https://bugs.webkit.org/show_bug.cgi?id=155689
 241
 242     int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
 243
 244     JSInterfaceJIT jit(vm);
 245
 246     if (entryType == EnterViaCall)
 247         jit.emitFunctionPrologue();
 248 #if USE(JSVALUE64)
 249     else if (entryType == EnterViaJump) {
 250         // We're coming from a specialized thunk that has saved the prior tag registers' contents.
 251         // Restore them now.
 252 #if CPU(ARM64)
 253         jit.popPair(JSInterfaceJIT::tagTypeNumberRegister, JSInterfaceJIT::tagMaskRegister);
 254 #else
 255         jit.pop(JSInterfaceJIT::tagMaskRegister);
 256         jit.pop(JSInterfaceJIT::tagTypeNumberRegister);
 257 #endif
 258     }
 259 #endif
 260
 261     jit.emitPutToCallFrameHeader(0, JSStack::CodeBlock);
 262     jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
 263
 264 #if CPU(X86)
 265     // Calling convention:      f(ecx, edx, ...);
 266     // Host function signature: f(ExecState*);
 267     jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
 268
 269     jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue.
 270
 271     // call the function
 272     jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
 273     jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
 274     jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
 275
 276     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister);
 277
 278 #elif CPU(X86_64)
 279 #if !OS(WINDOWS)
 280     // Calling convention:      f(edi, esi, edx, ecx, ...);
 281     // Host function signature: f(ExecState*);
 282     jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
 283
 284     jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
 285     jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
 286     jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
 287
 288 #else
 289     // Calling convention:      f(ecx, edx, r8, r9, ...);
 290     // Host function signature: f(ExecState*);
 291     jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
 292
 293     // Leave space for the callee parameter home addresses.
 294     // At this point the stack is aligned to 16 bytes, but if this changes at some point, we need to emit code to align it.
 295     jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
 296
 297     jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx);
 298     jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
 299     jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
 300
 301     jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
 302 #endif
 303
 304 #elif CPU(ARM64)
 305     COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0);
 306     COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1);
 307     COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2);
 308
 309     // Host function signature: f(ExecState*);
 310     jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0);
 311
 312     jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1);
 313     jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2);
 314     jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction));
 315 #elif CPU(ARM) || CPU(SH4) || CPU(MIPS)
 316 #if CPU(MIPS)
 317     // Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments.
 318     jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
 319 #endif
 320
 321     // Calling convention is f(argumentGPR0, argumentGPR1, ...).
 322     // Host function signature is f(ExecState*).
 323     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
 324
 325     jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1);
 326     jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
 327     jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
 328
 329 #if CPU(MIPS)
 330     // Restore stack space
 331     jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
 332 #endif
 333 #else
 334 #error "JIT not supported on this platform."
 335     UNUSED_PARAM(executableOffsetToFunction);
 336     abortWithReason(TGNotSupported);
 337 #endif
 338
 339     // Check for an exception
 340 #if USE(JSVALUE64)
 341     jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2);
 342     JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
 343 #else
 344     JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
 345         JSInterfaceJIT::NotEqual,
 346         JSInterfaceJIT::AbsoluteAddress(vm->addressOfException()),
 347         JSInterfaceJIT::TrustedImm32(0));
 348 #endif
 349
 350     jit.emitFunctionEpilogue();
 351     // Return.
 352     jit.ret();
 353
 354     // Handle an exception
 355     exceptionHandler.link(&jit);
 356
 357     jit.copyCalleeSavesToVMCalleeSavesBuffer();
 358     jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
 359
 360 #if CPU(X86) && USE(JSVALUE32_64)
 361     jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister);
 362     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT0);
 363     jit.push(JSInterfaceJIT::regT0);
 364 #else
 365 #if OS(WINDOWS)
 366     // Allocate space on stack for the 4 parameter registers.
 367     jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
 368 #endif
 369     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
 370 #endif
 371     jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3);
 372     jit.call(JSInterfaceJIT::regT3);
 373 #if CPU(X86) && USE(JSVALUE32_64)
 374     jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
 375 #elif OS(WINDOWS)
 376     jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
 377 #endif
 378
 379     jit.jumpToExceptionHandler();
 380
 381     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
 382     return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJump ? "Tail " : "", toCString(kind).data()));
 383 }
 384
 385 MacroAssemblerCodeRef nativeCallGenerator(VM* vm)
 386 {
 387     return nativeForGenerator(vm, CodeForCall);
 388 }
 389
 390 MacroAssemblerCodeRef nativeTailCallGenerator(VM* vm)
 391 {
 392     return nativeForGenerator(vm, CodeForCall, EnterViaJump);
 393 }
 394
 395 MacroAssemblerCodeRef nativeConstructGenerator(VM* vm)
 396 {
 397     return nativeForGenerator(vm, CodeForConstruct);
 398 }
 399
 400 MacroAssemblerCodeRef arityFixupGenerator(VM* vm)
 401 {
 402     JSInterfaceJIT jit(vm);
 403
 404     // We enter with fixup count in argumentGPR0
 405     // We have the guarantee that a0, a1, a2, t3, t4 and t5 (or t0 for Windows) are all distinct :-)
 406 #if USE(JSVALUE64)
 407 #if OS(WINDOWS)
 408     const GPRReg extraTemp = JSInterfaceJIT::regT0;
 409 #else
 410     const GPRReg extraTemp = JSInterfaceJIT::regT5;
 411 #endif
 412 #  if CPU(X86_64)
 413     jit.pop(JSInterfaceJIT::regT4);
 414 #  endif
 415     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
 416     jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
 417     jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
 418
 419     // Check to see if we have extra slots we can use
 420     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
 421     jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
 422     JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
 423     jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
 424     JSInterfaceJIT::Label fillExtraSlots(jit.label());
 425     jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight));
 426     jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
 427     jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
 428     jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
 429     JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
 430     noExtraSlot.link(&jit);
 431
 432     jit.neg64(JSInterfaceJIT::argumentGPR0);
 433
 434     // Move current frame down argumentGPR0 number of slots
 435     JSInterfaceJIT::Label copyLoop(jit.label());
 436     jit.load64(JSInterfaceJIT::regT3, extraTemp);
 437     jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
 438     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
 439     jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
 440
 441     // Fill in argumentGPR0 missing arg slots with undefined
 442     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
 443     jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
 444     JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
 445     jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
 446     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
 447     jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
 448
 449     // Adjust call frame register and stack pointer to account for missing args
 450     jit.move(JSInterfaceJIT::argumentGPR0, extraTemp);
 451     jit.lshift64(JSInterfaceJIT::TrustedImm32(3), extraTemp);
 452     jit.addPtr(extraTemp, JSInterfaceJIT::callFrameRegister);
 453     jit.addPtr(extraTemp, JSInterfaceJIT::stackPointerRegister);
 454
 455     done.link(&jit);
 456
 457 #  if CPU(X86_64)
 458     jit.push(JSInterfaceJIT::regT4);
 459 #  endif
 460     jit.ret();
 461 #else
 462 #  if CPU(X86)
 463     jit.pop(JSInterfaceJIT::regT4);
 464 #  endif
 465     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
 466     jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
 467     jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
 468
 469     // Check to see if we have extra slots we can use
 470     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
 471     jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
 472     JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
 473     JSInterfaceJIT::Label fillExtraSlots(jit.label());
 474     jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
 475     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, PayloadOffset));
 476     jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
 477     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, TagOffset));
 478     jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
 479     jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
 480     jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
 481     JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
 482     noExtraSlot.link(&jit);
 483
 484     jit.neg32(JSInterfaceJIT::argumentGPR0);
 485
 486     // Move current frame down argumentGPR0 number of slots
 487     JSInterfaceJIT::Label copyLoop(jit.label());
 488     jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, PayloadOffset), JSInterfaceJIT::regT5);
 489     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
 490     jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, TagOffset), JSInterfaceJIT::regT5);
 491     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
 492     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
 493     jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
 494
 495     // Fill in argumentGPR0 missing arg slots with undefined
 496     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
 497     JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
 498     jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
 499     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
 500     jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
 501     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
 502
 503     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
 504     jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
 505
 506     // Adjust call frame register and stack pointer to account for missing args
 507     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::regT5);
 508     jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT5);
 509     jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::callFrameRegister);
 510     jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::stackPointerRegister);
 511
 512     done.link(&jit);
 513
 514 #  if CPU(X86)
 515     jit.push(JSInterfaceJIT::regT4);
 516 #  endif
 517     jit.ret();
 518 #endif
 519
 520     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
 521     return FINALIZE_CODE(patchBuffer, ("fixup arity"));
 522 }
 523
 524 MacroAssemblerCodeRef unreachableGenerator(VM* vm)
 525 {
 526     JSInterfaceJIT jit(vm);
 527
 528     jit.breakpoint();
 529
 530     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
 531     return FINALIZE_CODE(patchBuffer, ("unreachable thunk"));
 532 }
 533
 534 static void stringCharLoad(SpecializedThunkJIT& jit, VM* vm)
 535 {
 536     // load string
 537     jit.loadJSStringArgument(*vm, SpecializedThunkJIT::ThisArgument, SpecializedThunkJIT::regT0);
 538
 539     // Load string length to regT2, and start the process of loading the data pointer into regT0
 540     jit.load32(MacroAssembler::Address(SpecializedThunkJIT::regT0, ThunkHelpers::jsStringLengthOffset()), SpecializedThunkJIT::regT2);
 541     jit.loadPtr(MacroAssembler::Address(SpecializedThunkJIT::regT0, ThunkHelpers::jsStringValueOffset()), SpecializedThunkJIT::regT0);
 542     jit.appendFailure(jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0));
 543
 544     // load index
 545     jit.loadInt32Argument(0, SpecializedThunkJIT::regT1); // regT1 contains the index
 546
 547     // Do an unsigned compare to simultaneously filter negative indices as well as indices that are too large
 548     jit.appendFailure(jit.branch32(MacroAssembler::AboveOrEqual, SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT2));
 549
 550     // Load the character
 551     SpecializedThunkJIT::JumpList is16Bit;
 552     SpecializedThunkJIT::JumpList cont8Bit;
 553     // Load the string flags
 554     jit.loadPtr(MacroAssembler::Address(SpecializedThunkJIT::regT0, StringImpl::flagsOffset()), SpecializedThunkJIT::regT2);
 555     jit.loadPtr(MacroAssembler::Address(SpecializedThunkJIT::regT0, StringImpl::dataOffset()), SpecializedThunkJIT::regT0);
 556     is16Bit.append(jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT2, MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
 557     jit.load8(MacroAssembler::BaseIndex(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, MacroAssembler::TimesOne, 0), SpecializedThunkJIT::regT0);
 558     cont8Bit.append(jit.jump());
 559     is16Bit.link(&jit);
 560     jit.load16(MacroAssembler::BaseIndex(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, MacroAssembler::TimesTwo, 0), SpecializedThunkJIT::regT0);
 561     cont8Bit.link(&jit);
 562 }
 563
 564 static void charToString(SpecializedThunkJIT& jit, VM* vm, MacroAssembler::RegisterID src, MacroAssembler::RegisterID dst, MacroAssembler::RegisterID scratch)
 565 {
 566     jit.appendFailure(jit.branch32(MacroAssembler::AboveOrEqual, src, MacroAssembler::TrustedImm32(0x100)));
 567     jit.move(MacroAssembler::TrustedImmPtr(vm->smallStrings.singleCharacterStrings()), scratch);
 568     jit.loadPtr(MacroAssembler::BaseIndex(scratch, src, MacroAssembler::ScalePtr, 0), dst);
 569     jit.appendFailure(jit.branchTestPtr(MacroAssembler::Zero, dst));
 570 }
 571
 572 MacroAssemblerCodeRef charCodeAtThunkGenerator(VM* vm)
 573 {
 574     SpecializedThunkJIT jit(vm, 1);
 575     stringCharLoad(jit, vm);
 576     jit.returnInt32(SpecializedThunkJIT::regT0);
 577     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "charCodeAt");
 578 }
 579
 580 MacroAssemblerCodeRef charAtThunkGenerator(VM* vm)
 581 {
 582     SpecializedThunkJIT jit(vm, 1);
 583     stringCharLoad(jit, vm);
 584     charToString(jit, vm, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
 585     jit.returnJSCell(SpecializedThunkJIT::regT0);
 586     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "charAt");
 587 }
 588
 589 MacroAssemblerCodeRef fromCharCodeThunkGenerator(VM* vm)
 590 {
 591     SpecializedThunkJIT jit(vm, 1);
 592     // load char code
 593     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0);
 594     charToString(jit, vm, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
 595     jit.returnJSCell(SpecializedThunkJIT::regT0);
 596     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "fromCharCode");
 597 }
 598
 599 MacroAssemblerCodeRef clz32ThunkGenerator(VM* vm)
 600 {
 601     SpecializedThunkJIT jit(vm, 1);
 602     MacroAssembler::Jump nonIntArgJump;
 603     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntArgJump);
 604
 605     SpecializedThunkJIT::Label convertedArgumentReentry(&jit);
 606     jit.countLeadingZeros32(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
 607     jit.returnInt32(SpecializedThunkJIT::regT1);
 608
 609     if (jit.supportsFloatingPointTruncate()) {
 610         nonIntArgJump.link(&jit);
 611         jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 612         jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(convertedArgumentReentry, &jit);
 613         jit.appendFailure(jit.jump());
 614     } else
 615         jit.appendFailure(nonIntArgJump);
 616
 617     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "clz32");
 618 }
 619
 620 MacroAssemblerCodeRef sqrtThunkGenerator(VM* vm)
 621 {
 622     SpecializedThunkJIT jit(vm, 1);
 623     if (!jit.supportsFloatingPointSqrt())
 624         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 625
 626     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 627     jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
 628     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
 629     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "sqrt");
 630 }
 631
 632
 633 #define UnaryDoubleOpWrapper(function) function##Wrapper
 634 enum MathThunkCallingConvention { };
 635 typedef MathThunkCallingConvention(*MathThunk)(MathThunkCallingConvention);
 636
 637 #if CPU(X86_64) && COMPILER(GCC_OR_CLANG) && (OS(DARWIN) || OS(LINUX))
 638
 639 #define defineUnaryDoubleOpWrapper(function) \
 640     asm( \
 641         ".text\n" \
 642         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
 643         HIDE_SYMBOL(function##Thunk) "\n" \
 644         SYMBOL_STRING(function##Thunk) ":" "\n" \
 645         "pushq %rax\n" \
 646         "call " GLOBAL_REFERENCE(function) "\n" \
 647         "popq %rcx\n" \
 648         "ret\n" \
 649     );\
 650     extern "C" { \
 651         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
 652     } \
 653     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
 654
 655 #elif CPU(X86) && COMPILER(GCC_OR_CLANG) && OS(LINUX) && defined(__PIC__)
 656 #define defineUnaryDoubleOpWrapper(function) \
 657     asm( \
 658         ".text\n" \
 659         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
 660         HIDE_SYMBOL(function##Thunk) "\n" \
 661         SYMBOL_STRING(function##Thunk) ":" "\n" \
 662         "pushl %ebx\n" \
 663         "subl $20, %esp\n" \
 664         "movsd %xmm0, (%esp) \n" \
 665         "call __x86.get_pc_thunk.bx\n" \
 666         "addl $_GLOBAL_OFFSET_TABLE_, %ebx\n" \
 667         "call " GLOBAL_REFERENCE(function) "\n" \
 668         "fstpl (%esp) \n" \
 669         "movsd (%esp), %xmm0 \n" \
 670         "addl $20, %esp\n" \
 671         "popl %ebx\n" \
 672         "ret\n" \
 673     );\
 674     extern "C" { \
 675         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
 676     } \
 677     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
 678
 679 #elif CPU(X86) && COMPILER(GCC_OR_CLANG) && (OS(DARWIN) || OS(LINUX))
 680 #define defineUnaryDoubleOpWrapper(function) \
 681     asm( \
 682         ".text\n" \
 683         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
 684         HIDE_SYMBOL(function##Thunk) "\n" \
 685         SYMBOL_STRING(function##Thunk) ":" "\n" \
 686         "subl $20, %esp\n" \
 687         "movsd %xmm0, (%esp) \n" \
 688         "call " GLOBAL_REFERENCE(function) "\n" \
 689         "fstpl (%esp) \n" \
 690         "movsd (%esp), %xmm0 \n" \
 691         "addl $20, %esp\n" \
 692         "ret\n" \
 693     );\
 694     extern "C" { \
 695         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
 696     } \
 697     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
 698
 699 #elif CPU(ARM_THUMB2) && COMPILER(GCC_OR_CLANG) && PLATFORM(IOS)
 700
 701 #define defineUnaryDoubleOpWrapper(function) \
 702     asm( \
 703         ".text\n" \
 704         ".align 2\n" \
 705         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
 706         HIDE_SYMBOL(function##Thunk) "\n" \
 707         ".thumb\n" \
 708         ".thumb_func " THUMB_FUNC_PARAM(function##Thunk) "\n" \
 709         SYMBOL_STRING(function##Thunk) ":" "\n" \
 710         "push {lr}\n" \
 711         "vmov r0, r1, d0\n" \
 712         "blx " GLOBAL_REFERENCE(function) "\n" \
 713         "vmov d0, r0, r1\n" \
 714         "pop {lr}\n" \
 715         "bx lr\n" \
 716     ); \
 717     extern "C" { \
 718         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
 719     } \
 720     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
 721
 722 #elif CPU(ARM64)
 723
 724 #define defineUnaryDoubleOpWrapper(function) \
 725     asm( \
 726         ".text\n" \
 727         ".align 2\n" \
 728         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
 729         HIDE_SYMBOL(function##Thunk) "\n" \
 730         SYMBOL_STRING(function##Thunk) ":" "\n" \
 731         "b " GLOBAL_REFERENCE(function) "\n" \
 732         ".previous" \
 733     ); \
 734     extern "C" { \
 735         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
 736     } \
 737     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
 738
 739 #elif CPU(X86) && COMPILER(MSVC) && OS(WINDOWS)
 740
 741 // MSVC does not accept floor, etc, to be called directly from inline assembly, so we need to wrap these functions.
 742 static double (_cdecl *floorFunction)(double) = floor;
 743 static double (_cdecl *ceilFunction)(double) = ceil;
 744 static double (_cdecl *truncFunction)(double) = trunc;
 745 static double (_cdecl *expFunction)(double) = exp;
 746 static double (_cdecl *logFunction)(double) = log;
 747 static double (_cdecl *jsRoundFunction)(double) = jsRound;
 748
 749 #define defineUnaryDoubleOpWrapper(function) \
 750     extern "C" __declspec(naked) MathThunkCallingConvention function##Thunk(MathThunkCallingConvention) \
 751     { \
 752         __asm \
 753         { \
 754         __asm sub esp, 20 \
 755         __asm movsd mmword ptr [esp], xmm0  \
 756         __asm call function##Function \
 757         __asm fstp qword ptr [esp] \
 758         __asm movsd xmm0, mmword ptr [esp] \
 759         __asm add esp, 20 \
 760         __asm ret \
 761         } \
 762     } \
 763     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
 764
 765 #else
 766
 767 #define defineUnaryDoubleOpWrapper(function) \
 768     static MathThunk UnaryDoubleOpWrapper(function) = 0
 769 #endif
 770
 771 defineUnaryDoubleOpWrapper(jsRound);
 772 defineUnaryDoubleOpWrapper(exp);
 773 defineUnaryDoubleOpWrapper(log);
 774 defineUnaryDoubleOpWrapper(floor);
 775 defineUnaryDoubleOpWrapper(ceil);
 776 defineUnaryDoubleOpWrapper(trunc);
 777
 778 static const double oneConstant = 1.0;
 779 static const double negativeHalfConstant = -0.5;
 780 static const double halfConstant = 0.5;
 781
 782 MacroAssemblerCodeRef floorThunkGenerator(VM* vm)
 783 {
 784     SpecializedThunkJIT jit(vm, 1);
 785     MacroAssembler::Jump nonIntJump;
 786     if (!UnaryDoubleOpWrapper(floor) || !jit.supportsFloatingPoint())
 787         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 788     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
 789     jit.returnInt32(SpecializedThunkJIT::regT0);
 790     nonIntJump.link(&jit);
 791     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 792
 793     if (jit.supportsFloatingPointRounding()) {
 794         SpecializedThunkJIT::JumpList doubleResult;
 795         jit.floorDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
 796         jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
 797         jit.returnInt32(SpecializedThunkJIT::regT0);
 798         doubleResult.link(&jit);
 799         jit.returnDouble(SpecializedThunkJIT::fpRegT0);
 800         return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "floor");
 801     }
 802
 803     SpecializedThunkJIT::Jump intResult;
 804     SpecializedThunkJIT::JumpList doubleResult;
 805     if (jit.supportsFloatingPointTruncate()) {
 806         jit.moveZeroToDouble(SpecializedThunkJIT::fpRegT1);
 807         doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
 808         SpecializedThunkJIT::JumpList slowPath;
 809         // Handle the negative doubles in the slow path for now.
 810         slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
 811         slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0));
 812         intResult = jit.jump();
 813         slowPath.link(&jit);
 814     }
 815     jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(floor));
 816     jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
 817     if (jit.supportsFloatingPointTruncate())
 818         intResult.link(&jit);
 819     jit.returnInt32(SpecializedThunkJIT::regT0);
 820     doubleResult.link(&jit);
 821     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
 822     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "floor");
 823 }
 824
 825 MacroAssemblerCodeRef ceilThunkGenerator(VM* vm)
 826 {
 827     SpecializedThunkJIT jit(vm, 1);
 828     if (!UnaryDoubleOpWrapper(ceil) || !jit.supportsFloatingPoint())
 829         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 830     MacroAssembler::Jump nonIntJump;
 831     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
 832     jit.returnInt32(SpecializedThunkJIT::regT0);
 833     nonIntJump.link(&jit);
 834     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 835     if (jit.supportsFloatingPointRounding())
 836         jit.ceilDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
 837     else
 838         jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(ceil));
 839
 840     SpecializedThunkJIT::JumpList doubleResult;
 841     jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
 842     jit.returnInt32(SpecializedThunkJIT::regT0);
 843     doubleResult.link(&jit);
 844     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
 845     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "ceil");
 846 }
 847
 848 MacroAssemblerCodeRef truncThunkGenerator(VM* vm)
 849 {
 850     SpecializedThunkJIT jit(vm, 1);
 851     if (!UnaryDoubleOpWrapper(trunc) || !jit.supportsFloatingPoint())
 852         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 853     MacroAssembler::Jump nonIntJump;
 854     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
 855     jit.returnInt32(SpecializedThunkJIT::regT0);
 856     nonIntJump.link(&jit);
 857     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 858     if (jit.supportsFloatingPointRounding())
 859         jit.roundTowardZeroDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
 860     else
 861         jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(trunc));
 862
 863     SpecializedThunkJIT::JumpList doubleResult;
 864     jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
 865     jit.returnInt32(SpecializedThunkJIT::regT0);
 866     doubleResult.link(&jit);
 867     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
 868     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "trunc");
 869 }
 870
 871 MacroAssemblerCodeRef roundThunkGenerator(VM* vm)
 872 {
 873     SpecializedThunkJIT jit(vm, 1);
 874     if (!UnaryDoubleOpWrapper(jsRound) || !jit.supportsFloatingPoint())
 875         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 876     MacroAssembler::Jump nonIntJump;
 877     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
 878     jit.returnInt32(SpecializedThunkJIT::regT0);
 879     nonIntJump.link(&jit);
 880     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 881     SpecializedThunkJIT::Jump intResult;
 882     SpecializedThunkJIT::JumpList doubleResult;
 883     if (jit.supportsFloatingPointTruncate()) {
 884         jit.moveZeroToDouble(SpecializedThunkJIT::fpRegT1);
 885         doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
 886         SpecializedThunkJIT::JumpList slowPath;
 887         // Handle the negative doubles in the slow path for now.
 888         slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
 889         jit.loadDouble(MacroAssembler::TrustedImmPtr(&halfConstant), SpecializedThunkJIT::fpRegT1);
 890         jit.addDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
 891         slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0));
 892         intResult = jit.jump();
 893         slowPath.link(&jit);
 894     }
 895     jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(jsRound));
 896     jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
 897     if (jit.supportsFloatingPointTruncate())
 898         intResult.link(&jit);
 899     jit.returnInt32(SpecializedThunkJIT::regT0);
 900     doubleResult.link(&jit);
 901     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
 902     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "round");
 903 }
 904
 905 MacroAssemblerCodeRef expThunkGenerator(VM* vm)
 906 {
 907     if (!UnaryDoubleOpWrapper(exp))
 908         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 909     SpecializedThunkJIT jit(vm, 1);
 910     if (!jit.supportsFloatingPoint())
 911         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 912     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 913     jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(exp));
 914     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
 915     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "exp");
 916 }
 917
 918 MacroAssemblerCodeRef logThunkGenerator(VM* vm)
 919 {
 920     if (!UnaryDoubleOpWrapper(log))
 921         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 922     SpecializedThunkJIT jit(vm, 1);
 923     if (!jit.supportsFloatingPoint())
 924         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 925     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 926     jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(log));
 927     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
 928     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "log");
 929 }
 930
 931 MacroAssemblerCodeRef absThunkGenerator(VM* vm)
 932 {
 933     SpecializedThunkJIT jit(vm, 1);
 934     if (!jit.supportsFloatingPointAbs())
 935         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 936     MacroAssembler::Jump nonIntJump;
 937     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
 938     jit.rshift32(SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(31), SpecializedThunkJIT::regT1);
 939     jit.add32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
 940     jit.xor32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
 941     jit.appendFailure(jit.branchTest32(MacroAssembler::Signed, SpecializedThunkJIT::regT0));
 942     jit.returnInt32(SpecializedThunkJIT::regT0);
 943     nonIntJump.link(&jit);
 944     // Shame about the double int conversion here.
 945     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 946     jit.absDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
 947     jit.returnDouble(SpecializedThunkJIT::fpRegT1);
 948     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "abs");
 949 }
 950
 951 MacroAssemblerCodeRef powThunkGenerator(VM* vm)
 952 {
 953     SpecializedThunkJIT jit(vm, 2);
 954     if (!jit.supportsFloatingPoint())
 955         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
 956
 957     jit.loadDouble(MacroAssembler::TrustedImmPtr(&oneConstant), SpecializedThunkJIT::fpRegT1);
 958     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
 959     MacroAssembler::Jump nonIntExponent;
 960     jit.loadInt32Argument(1, SpecializedThunkJIT::regT0, nonIntExponent);
 961     jit.appendFailure(jit.branch32(MacroAssembler::LessThan, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(0)));
 962
 963     MacroAssembler::Jump exponentIsZero = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0);
 964     MacroAssembler::Label startLoop(jit.label());
 965
 966     MacroAssembler::Jump exponentIsEven = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1));
 967     jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
 968     exponentIsEven.link(&jit);
 969     jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
 970     jit.rshift32(MacroAssembler::TrustedImm32(1), SpecializedThunkJIT::regT0);
 971     jit.branchTest32(MacroAssembler::NonZero, SpecializedThunkJIT::regT0).linkTo(startLoop, &jit);
 972
 973     exponentIsZero.link(&jit);
 974
 975     {
 976         SpecializedThunkJIT::JumpList doubleResult;
 977         jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0);
 978         jit.returnInt32(SpecializedThunkJIT::regT0);
 979         doubleResult.link(&jit);
 980         jit.returnDouble(SpecializedThunkJIT::fpRegT1);
 981     }
 982
 983     if (jit.supportsFloatingPointSqrt()) {
 984         nonIntExponent.link(&jit);
 985         jit.loadDouble(MacroAssembler::TrustedImmPtr(&negativeHalfConstant), SpecializedThunkJIT::fpRegT3);
 986         jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::regT0);
 987         jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleLessThanOrEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
 988         jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::fpRegT3));
 989         jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
 990         jit.divDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
 991
 992         SpecializedThunkJIT::JumpList doubleResult;
 993         jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0);
 994         jit.returnInt32(SpecializedThunkJIT::regT0);
 995         doubleResult.link(&jit);
 996         jit.returnDouble(SpecializedThunkJIT::fpRegT1);
 997     } else
 998         jit.appendFailure(nonIntExponent);
 999
1000     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "pow");
1001 }
1002
1003 MacroAssemblerCodeRef imulThunkGenerator(VM* vm)
1004 {
1005     SpecializedThunkJIT jit(vm, 2);
1006     MacroAssembler::Jump nonIntArg0Jump;
1007     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntArg0Jump);
1008     SpecializedThunkJIT::Label doneLoadingArg0(&jit);
1009     MacroAssembler::Jump nonIntArg1Jump;
1010     jit.loadInt32Argument(1, SpecializedThunkJIT::regT1, nonIntArg1Jump);
1011     SpecializedThunkJIT::Label doneLoadingArg1(&jit);
1012     jit.mul32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
1013     jit.returnInt32(SpecializedThunkJIT::regT0);
1014
1015     if (jit.supportsFloatingPointTruncate()) {
1016         nonIntArg0Jump.link(&jit);
1017         jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
1018         jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(doneLoadingArg0, &jit);
1019         jit.appendFailure(jit.jump());
1020     } else
1021         jit.appendFailure(nonIntArg0Jump);
1022
1023     if (jit.supportsFloatingPointTruncate()) {
1024         nonIntArg1Jump.link(&jit);
1025         jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT1);
1026         jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT1, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(doneLoadingArg1, &jit);
1027         jit.appendFailure(jit.jump());
1028     } else
1029         jit.appendFailure(nonIntArg1Jump);
1030
1031     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "imul");
1032 }
1033
1034 MacroAssemblerCodeRef randomThunkGenerator(VM* vm)
1035 {
1036     SpecializedThunkJIT jit(vm, 0);
1037     if (!jit.supportsFloatingPoint())
1038         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
1039
1040 #if USE(JSVALUE64)
1041     jit.emitRandomThunk(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT3, SpecializedThunkJIT::fpRegT0);
1042     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
1043
1044     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "random");
1045 #else
1046     return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
1047 #endif
1048 }
1049
1050 }
1051
1052 #endif // ENABLE(JIT)