AssemblyHelpers should not have a VM field
[WebKit-https.git] / Source / JavaScriptCore / jit / ThunkGenerators.cpp
1 /*
2  * Copyright (C) 2010, 2012-2014, 2016-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ThunkGenerators.h"
28
29 #include "CodeBlock.h"
30 #include "DFGSpeculativeJIT.h"
31 #include "JITExceptions.h"
32 #include "JITOperations.h"
33 #include "JSArray.h"
34 #include "JSBoundFunction.h"
35 #include "JSCInlines.h"
36 #include "JSWebAssemblyInstance.h"
37 #include "JSWebAssemblyRuntimeError.h"
38 #include "MathCommon.h"
39 #include "MaxFrameExtentForSlowPathCall.h"
40 #include "SpecializedThunkJIT.h"
41 #include "WasmExceptionType.h"
42 #include <wtf/InlineASM.h>
43 #include <wtf/StringPrintStream.h>
44 #include <wtf/text/StringImpl.h>
45
46 #if ENABLE(JIT)
47
48 namespace JSC {
49
50 inline void emitPointerValidation(CCallHelpers& jit, GPRReg pointerGPR)
51 {
52     if (ASSERT_DISABLED)
53         return;
54     CCallHelpers::Jump isNonZero = jit.branchTestPtr(CCallHelpers::NonZero, pointerGPR);
55     jit.abortWithReason(TGInvalidPointer);
56     isNonZero.link(&jit);
57     jit.pushToSave(pointerGPR);
58     jit.load8(pointerGPR, pointerGPR);
59     jit.popToRestore(pointerGPR);
60 }
61
62 // We will jump here if the JIT code tries to make a call, but the
63 // linking helper (C++ code) decides to throw an exception instead.
64 MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(VM* vm)
65 {
66     CCallHelpers jit;
67     
68     // The call pushed a return address, so we need to pop it back off to re-align the stack,
69     // even though we won't use it.
70     jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
71
72     jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(*vm);
73
74     jit.setupArguments(CCallHelpers::TrustedImmPtr(vm), GPRInfo::callFrameRegister);
75     jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(lookupExceptionHandler)), GPRInfo::nonArgGPR0);
76     emitPointerValidation(jit, GPRInfo::nonArgGPR0);
77     jit.call(GPRInfo::nonArgGPR0);
78     jit.jumpToExceptionHandler(*vm);
79
80     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
81     return FINALIZE_CODE(patchBuffer, ("Throw exception from call slow path thunk"));
82 }
83
84 static void slowPathFor(
85     CCallHelpers& jit, VM* vm, Sprt_JITOperation_ECli slowPathFunction)
86 {
87     jit.emitFunctionPrologue();
88     jit.storePtr(GPRInfo::callFrameRegister, &vm->topCallFrame);
89 #if OS(WINDOWS) && CPU(X86_64)
90     // Windows X86_64 needs some space pointed to by arg0 for return types larger than 64 bits.
91     // Other argument values are shift by 1. Use space on the stack for our two return values.
92     // Moving the stack down maxFrameExtentForSlowPathCall bytes gives us room for our 3 arguments
93     // and space for the 16 byte return area.
94     jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
95     jit.move(GPRInfo::regT2, GPRInfo::argumentGPR2);
96     jit.addPtr(CCallHelpers::TrustedImm32(32), CCallHelpers::stackPointerRegister, GPRInfo::argumentGPR0);
97     jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
98     jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0);
99     emitPointerValidation(jit, GPRInfo::nonArgGPR0);
100     jit.call(GPRInfo::nonArgGPR0);
101     jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR, 8), GPRInfo::returnValueGPR2);
102     jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR), GPRInfo::returnValueGPR);
103     jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
104 #else
105     if (maxFrameExtentForSlowPathCall)
106         jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
107     jit.setupArgumentsWithExecState(GPRInfo::regT2);
108     jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0);
109     emitPointerValidation(jit, GPRInfo::nonArgGPR0);
110     jit.call(GPRInfo::nonArgGPR0);
111     if (maxFrameExtentForSlowPathCall)
112         jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister);
113 #endif
114
115     // This slow call will return the address of one of the following:
116     // 1) Exception throwing thunk.
117     // 2) Host call return value returner thingy.
118     // 3) The function to call.
119     // The second return value GPR will hold a non-zero value for tail calls.
120
121     emitPointerValidation(jit, GPRInfo::returnValueGPR);
122     jit.emitFunctionEpilogue();
123
124     RELEASE_ASSERT(reinterpret_cast<void*>(KeepTheFrame) == reinterpret_cast<void*>(0));
125     CCallHelpers::Jump doNotTrash = jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::returnValueGPR2);
126
127     jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
128     jit.prepareForTailCallSlow(GPRInfo::returnValueGPR);
129
130     doNotTrash.link(&jit);
131     jit.jump(GPRInfo::returnValueGPR);
132 }
133
134 MacroAssemblerCodeRef linkCallThunkGenerator(VM* vm)
135 {
136     // The return address is on the stack or in the link register. We will hence
137     // save the return address to the call frame while we make a C++ function call
138     // to perform linking and lazy compilation if necessary. We expect the callee
139     // to be in regT0/regT1 (payload/tag), the CallFrame to have already
140     // been adjusted, and all other registers to be available for use.
141     CCallHelpers jit;
142     
143     slowPathFor(jit, vm, operationLinkCall);
144     
145     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
146     return FINALIZE_CODE(patchBuffer, ("Link call slow path thunk"));
147 }
148
149 // For closure optimizations, we only include calls, since if you're using closures for
150 // object construction then you're going to lose big time anyway.
151 MacroAssemblerCodeRef linkPolymorphicCallThunkGenerator(VM* vm)
152 {
153     CCallHelpers jit;
154     
155     slowPathFor(jit, vm, operationLinkPolymorphicCall);
156     
157     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
158     return FINALIZE_CODE(patchBuffer, ("Link polymorphic call slow path thunk"));
159 }
160
161 // FIXME: We should distinguish between a megamorphic virtual call vs. a slow
162 // path virtual call so that we can enable fast tail calls for megamorphic
163 // virtual calls by using the shuffler.
164 // https://bugs.webkit.org/show_bug.cgi?id=148831
165 MacroAssemblerCodeRef virtualThunkFor(VM* vm, CallLinkInfo& callLinkInfo)
166 {
167     // The callee is in regT0 (for JSVALUE32_64, the tag is in regT1).
168     // The return address is on the stack, or in the link register. We will hence
169     // jump to the callee, or save the return address to the call frame while we
170     // make a C++ function call to the appropriate JIT operation.
171
172     CCallHelpers jit;
173     
174     CCallHelpers::JumpList slowCase;
175     
176     // This is a slow path execution, and regT2 contains the CallLinkInfo. Count the
177     // slow path execution for the profiler.
178     jit.add32(
179         CCallHelpers::TrustedImm32(1),
180         CCallHelpers::Address(GPRInfo::regT2, CallLinkInfo::offsetOfSlowPathCount()));
181
182     // FIXME: we should have a story for eliminating these checks. In many cases,
183     // the DFG knows that the value is definitely a cell, or definitely a function.
184     
185 #if USE(JSVALUE64)
186     slowCase.append(
187         jit.branchTest64(
188             CCallHelpers::NonZero, GPRInfo::regT0, GPRInfo::tagMaskRegister));
189 #else
190     slowCase.append(
191         jit.branch32(
192             CCallHelpers::NotEqual, GPRInfo::regT1,
193             CCallHelpers::TrustedImm32(JSValue::CellTag)));
194 #endif
195     slowCase.append(jit.branchIfNotType(GPRInfo::regT0, JSFunctionType));
196     
197     // Now we know we have a JSFunction.
198     
199     jit.loadPtr(
200         CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfExecutable()),
201         GPRInfo::regT4);
202     jit.loadPtr(
203         CCallHelpers::Address(
204             GPRInfo::regT4, ExecutableBase::offsetOfJITCodeWithArityCheckFor(
205                 callLinkInfo.specializationKind())),
206         GPRInfo::regT4);
207     slowCase.append(jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT4));
208     
209     // Now we know that we have a CodeBlock, and we're committed to making a fast
210     // call.
211     
212     // Make a tail call. This will return back to JIT code.
213     emitPointerValidation(jit, GPRInfo::regT4);
214     if (callLinkInfo.isTailCall()) {
215         jit.preserveReturnAddressAfterCall(GPRInfo::regT0);
216         jit.prepareForTailCallSlow(GPRInfo::regT4);
217     }
218     jit.jump(GPRInfo::regT4);
219
220     slowCase.link(&jit);
221     
222     // Here we don't know anything, so revert to the full slow path.
223     
224     slowPathFor(jit, vm, operationVirtualCall);
225     
226     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
227     return FINALIZE_CODE(
228         patchBuffer,
229         ("Virtual %s slow path thunk",
230         callLinkInfo.callMode() == CallMode::Regular ? "call" : callLinkInfo.callMode() == CallMode::Tail ? "tail call" : "construct"));
231 }
232
233 enum ThunkEntryType { EnterViaCall, EnterViaJumpWithSavedTags, EnterViaJumpWithoutSavedTags };
234
235 static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall)
236 {
237     // FIXME: This should be able to log ShadowChicken prologue packets.
238     // https://bugs.webkit.org/show_bug.cgi?id=155689
239     
240     int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
241     
242     JSInterfaceJIT jit(vm);
243
244     switch (entryType) {
245     case EnterViaCall:
246         jit.emitFunctionPrologue();
247         break;
248     case EnterViaJumpWithSavedTags:
249 #if USE(JSVALUE64)
250         // We're coming from a specialized thunk that has saved the prior tag registers' contents.
251         // Restore them now.
252 #if CPU(ARM64)
253         jit.popPair(JSInterfaceJIT::tagTypeNumberRegister, JSInterfaceJIT::tagMaskRegister);
254 #else
255         jit.pop(JSInterfaceJIT::tagMaskRegister);
256         jit.pop(JSInterfaceJIT::tagTypeNumberRegister);
257 #endif
258 #endif
259         break;
260     case EnterViaJumpWithoutSavedTags:
261         jit.move(JSInterfaceJIT::framePointerRegister, JSInterfaceJIT::stackPointerRegister);
262         break;
263     }
264
265     jit.emitPutToCallFrameHeader(0, CallFrameSlot::codeBlock);
266     jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
267
268 #if CPU(X86)
269     // Calling convention:      f(ecx, edx, ...);
270     // Host function signature: f(ExecState*);
271     jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
272
273     jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue.
274
275     // call the function
276     jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, JSInterfaceJIT::regT1);
277     jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
278     jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
279
280     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister);
281
282 #elif CPU(X86_64)
283 #if !OS(WINDOWS)
284     // Calling convention:      f(edi, esi, edx, ecx, ...);
285     // Host function signature: f(ExecState*);
286     jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
287
288     jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, X86Registers::esi);
289     jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
290     jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
291
292 #else
293     // Calling convention:      f(ecx, edx, r8, r9, ...);
294     // Host function signature: f(ExecState*);
295     jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
296
297     // Leave space for the callee parameter home addresses.
298     // At this point the stack is aligned to 16 bytes, but if this changes at some point, we need to emit code to align it.
299     jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
300
301     jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, X86Registers::edx);
302     jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
303     jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
304
305     jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
306 #endif
307
308 #elif CPU(ARM64)
309     COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0);
310     COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1);
311     COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2);
312
313     // Host function signature: f(ExecState*);
314     jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0);
315
316     jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, ARM64Registers::x1);
317     jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2);
318     jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction));
319 #elif CPU(ARM) || CPU(MIPS)
320 #if CPU(MIPS)
321     // Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments.
322     jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
323 #endif
324
325     // Calling convention is f(argumentGPR0, argumentGPR1, ...).
326     // Host function signature is f(ExecState*).
327     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
328
329     jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, JSInterfaceJIT::argumentGPR1);
330     jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
331     jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
332
333 #if CPU(MIPS)
334     // Restore stack space
335     jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
336 #endif
337 #else
338 #error "JIT not supported on this platform."
339     UNUSED_PARAM(executableOffsetToFunction);
340     abortWithReason(TGNotSupported);
341 #endif
342
343     // Check for an exception
344 #if USE(JSVALUE64)
345     jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2);
346     JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
347 #else
348     JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
349         JSInterfaceJIT::NotEqual,
350         JSInterfaceJIT::AbsoluteAddress(vm->addressOfException()),
351         JSInterfaceJIT::TrustedImm32(0));
352 #endif
353
354     jit.emitFunctionEpilogue();
355     // Return.
356     jit.ret();
357
358     // Handle an exception
359     exceptionHandler.link(&jit);
360
361     jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(*vm);
362     jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
363
364 #if CPU(X86) && USE(JSVALUE32_64)
365     jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister);
366     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT0);
367     jit.push(JSInterfaceJIT::regT0);
368 #else
369 #if OS(WINDOWS)
370     // Allocate space on stack for the 4 parameter registers.
371     jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
372 #endif
373     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
374 #endif
375     jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3);
376     jit.call(JSInterfaceJIT::regT3);
377 #if CPU(X86) && USE(JSVALUE32_64)
378     jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
379 #elif OS(WINDOWS)
380     jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
381 #endif
382
383     jit.jumpToExceptionHandler(*vm);
384
385     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
386     return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJumpWithSavedTags ? "Tail With Saved Tags " : entryType == EnterViaJumpWithoutSavedTags ? "Tail Without Saved Tags " : "", toCString(kind).data()));
387 }
388
389 MacroAssemblerCodeRef nativeCallGenerator(VM* vm)
390 {
391     return nativeForGenerator(vm, CodeForCall);
392 }
393
394 MacroAssemblerCodeRef nativeTailCallGenerator(VM* vm)
395 {
396     return nativeForGenerator(vm, CodeForCall, EnterViaJumpWithSavedTags);
397 }
398
399 MacroAssemblerCodeRef nativeTailCallWithoutSavedTagsGenerator(VM* vm)
400 {
401     return nativeForGenerator(vm, CodeForCall, EnterViaJumpWithoutSavedTags);
402 }
403
404 MacroAssemblerCodeRef nativeConstructGenerator(VM* vm)
405 {
406     return nativeForGenerator(vm, CodeForConstruct);
407 }
408
409 MacroAssemblerCodeRef arityFixupGenerator(VM* vm)
410 {
411     JSInterfaceJIT jit(vm);
412
413     // We enter with fixup count in argumentGPR0
414     // We have the guarantee that a0, a1, a2, t3, t4 and t5 (or t0 for Windows) are all distinct :-)
415 #if USE(JSVALUE64)
416 #if OS(WINDOWS)
417     const GPRReg extraTemp = JSInterfaceJIT::regT0;
418 #else
419     const GPRReg extraTemp = JSInterfaceJIT::regT5;
420 #endif
421 #  if CPU(X86_64)
422     jit.pop(JSInterfaceJIT::regT4);
423 #  endif
424     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
425     jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrameSlot::argumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
426     jit.add32(JSInterfaceJIT::TrustedImm32(CallFrame::headerSizeInRegisters), JSInterfaceJIT::argumentGPR2);
427
428     // Check to see if we have extra slots we can use
429     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
430     jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
431     JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
432     jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
433     JSInterfaceJIT::Label fillExtraSlots(jit.label());
434     jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight));
435     jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
436     jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
437     jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
438     JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
439     noExtraSlot.link(&jit);
440
441     jit.neg64(JSInterfaceJIT::argumentGPR0);
442
443     // Adjust call frame register and stack pointer to account for missing args.
444     // We need to change the stack pointer first before performing copy/fill loops.
445     // This stack space below the stack pointer is considered unsed by OS. Therefore,
446     // OS may corrupt this space when constructing a signal stack.
447     jit.move(JSInterfaceJIT::argumentGPR0, extraTemp);
448     jit.lshift64(JSInterfaceJIT::TrustedImm32(3), extraTemp);
449     jit.addPtr(extraTemp, JSInterfaceJIT::callFrameRegister);
450     jit.addPtr(extraTemp, JSInterfaceJIT::stackPointerRegister);
451
452     // Move current frame down argumentGPR0 number of slots
453     JSInterfaceJIT::Label copyLoop(jit.label());
454     jit.load64(JSInterfaceJIT::regT3, extraTemp);
455     jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
456     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
457     jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
458
459     // Fill in argumentGPR0 missing arg slots with undefined
460     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
461     jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
462     JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
463     jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
464     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
465     jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
466     
467     done.link(&jit);
468
469 #  if CPU(X86_64)
470     jit.push(JSInterfaceJIT::regT4);
471 #  endif
472     jit.ret();
473 #else
474 #  if CPU(X86)
475     jit.pop(JSInterfaceJIT::regT4);
476 #  endif
477     jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
478     jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrameSlot::argumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
479     jit.add32(JSInterfaceJIT::TrustedImm32(CallFrame::headerSizeInRegisters), JSInterfaceJIT::argumentGPR2);
480
481     // Check to see if we have extra slots we can use
482     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
483     jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
484     JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
485     JSInterfaceJIT::Label fillExtraSlots(jit.label());
486     jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
487     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, PayloadOffset));
488     jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
489     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, TagOffset));
490     jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
491     jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
492     jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
493     JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
494     noExtraSlot.link(&jit);
495
496     jit.neg32(JSInterfaceJIT::argumentGPR0);
497
498     // Move current frame down argumentGPR0 number of slots
499     JSInterfaceJIT::Label copyLoop(jit.label());
500     jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, PayloadOffset), JSInterfaceJIT::regT5);
501     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
502     jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, TagOffset), JSInterfaceJIT::regT5);
503     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
504     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
505     jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
506
507     // Fill in argumentGPR0 missing arg slots with undefined
508     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
509     JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
510     jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
511     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
512     jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
513     jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
514
515     jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
516     jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
517
518     // Adjust call frame register and stack pointer to account for missing args
519     jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::regT5);
520     jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT5);
521     jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::callFrameRegister);
522     jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::stackPointerRegister);
523
524     done.link(&jit);
525
526 #  if CPU(X86)
527     jit.push(JSInterfaceJIT::regT4);
528 #  endif
529     jit.ret();
530 #endif
531
532     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
533     return FINALIZE_CODE(patchBuffer, ("fixup arity"));
534 }
535
536 MacroAssemblerCodeRef unreachableGenerator(VM* vm)
537 {
538     JSInterfaceJIT jit(vm);
539
540     jit.breakpoint();
541
542     LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
543     return FINALIZE_CODE(patchBuffer, ("unreachable thunk"));
544 }
545
546 static void stringCharLoad(SpecializedThunkJIT& jit, VM* vm)
547 {
548     // load string
549     jit.loadJSStringArgument(*vm, SpecializedThunkJIT::ThisArgument, SpecializedThunkJIT::regT0);
550
551     // Load string length to regT2, and start the process of loading the data pointer into regT0
552     jit.load32(MacroAssembler::Address(SpecializedThunkJIT::regT0, ThunkHelpers::jsStringLengthOffset()), SpecializedThunkJIT::regT2);
553     jit.loadPtr(MacroAssembler::Address(SpecializedThunkJIT::regT0, ThunkHelpers::jsStringValueOffset()), SpecializedThunkJIT::regT0);
554     jit.appendFailure(jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0));
555
556     // load index
557     jit.loadInt32Argument(0, SpecializedThunkJIT::regT1); // regT1 contains the index
558
559     // Do an unsigned compare to simultaneously filter negative indices as well as indices that are too large
560     jit.appendFailure(jit.branch32(MacroAssembler::AboveOrEqual, SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT2));
561
562     // Load the character
563     SpecializedThunkJIT::JumpList is16Bit;
564     SpecializedThunkJIT::JumpList cont8Bit;
565     // Load the string flags
566     jit.loadPtr(MacroAssembler::Address(SpecializedThunkJIT::regT0, StringImpl::flagsOffset()), SpecializedThunkJIT::regT2);
567     jit.loadPtr(MacroAssembler::Address(SpecializedThunkJIT::regT0, StringImpl::dataOffset()), SpecializedThunkJIT::regT0);
568     is16Bit.append(jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT2, MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
569     jit.load8(MacroAssembler::BaseIndex(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, MacroAssembler::TimesOne, 0), SpecializedThunkJIT::regT0);
570     cont8Bit.append(jit.jump());
571     is16Bit.link(&jit);
572     jit.load16(MacroAssembler::BaseIndex(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, MacroAssembler::TimesTwo, 0), SpecializedThunkJIT::regT0);
573     cont8Bit.link(&jit);
574 }
575
576 static void charToString(SpecializedThunkJIT& jit, VM* vm, MacroAssembler::RegisterID src, MacroAssembler::RegisterID dst, MacroAssembler::RegisterID scratch)
577 {
578     jit.appendFailure(jit.branch32(MacroAssembler::AboveOrEqual, src, MacroAssembler::TrustedImm32(0x100)));
579     jit.move(MacroAssembler::TrustedImmPtr(vm->smallStrings.singleCharacterStrings()), scratch);
580     jit.loadPtr(MacroAssembler::BaseIndex(scratch, src, MacroAssembler::ScalePtr, 0), dst);
581     jit.appendFailure(jit.branchTestPtr(MacroAssembler::Zero, dst));
582 }
583
584 MacroAssemblerCodeRef charCodeAtThunkGenerator(VM* vm)
585 {
586     SpecializedThunkJIT jit(vm, 1);
587     stringCharLoad(jit, vm);
588     jit.returnInt32(SpecializedThunkJIT::regT0);
589     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "charCodeAt");
590 }
591
592 MacroAssemblerCodeRef charAtThunkGenerator(VM* vm)
593 {
594     SpecializedThunkJIT jit(vm, 1);
595     stringCharLoad(jit, vm);
596     charToString(jit, vm, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
597     jit.returnJSCell(SpecializedThunkJIT::regT0);
598     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "charAt");
599 }
600
601 MacroAssemblerCodeRef fromCharCodeThunkGenerator(VM* vm)
602 {
603     SpecializedThunkJIT jit(vm, 1);
604     // load char code
605     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0);
606     charToString(jit, vm, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
607     jit.returnJSCell(SpecializedThunkJIT::regT0);
608     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "fromCharCode");
609 }
610
611 MacroAssemblerCodeRef clz32ThunkGenerator(VM* vm)
612 {
613     SpecializedThunkJIT jit(vm, 1);
614     MacroAssembler::Jump nonIntArgJump;
615     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntArgJump);
616
617     SpecializedThunkJIT::Label convertedArgumentReentry(&jit);
618     jit.countLeadingZeros32(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
619     jit.returnInt32(SpecializedThunkJIT::regT1);
620
621     if (jit.supportsFloatingPointTruncate()) {
622         nonIntArgJump.link(&jit);
623         jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
624         jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(convertedArgumentReentry, &jit);
625         jit.appendFailure(jit.jump());
626     } else
627         jit.appendFailure(nonIntArgJump);
628
629     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "clz32");
630 }
631
632 MacroAssemblerCodeRef sqrtThunkGenerator(VM* vm)
633 {
634     SpecializedThunkJIT jit(vm, 1);
635     if (!jit.supportsFloatingPointSqrt())
636         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
637
638     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
639     jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
640     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
641     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "sqrt");
642 }
643
644
645 #define UnaryDoubleOpWrapper(function) function##Wrapper
646 enum MathThunkCallingConvention { };
647 typedef MathThunkCallingConvention(*MathThunk)(MathThunkCallingConvention);
648
649 #if CPU(X86_64) && COMPILER(GCC_OR_CLANG) && (OS(DARWIN) || OS(LINUX))
650
651 #define defineUnaryDoubleOpWrapper(function) \
652     asm( \
653         ".text\n" \
654         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
655         HIDE_SYMBOL(function##Thunk) "\n" \
656         SYMBOL_STRING(function##Thunk) ":" "\n" \
657         "pushq %rax\n" \
658         "call " GLOBAL_REFERENCE(function) "\n" \
659         "popq %rcx\n" \
660         "ret\n" \
661     );\
662     extern "C" { \
663         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
664     } \
665     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
666
667 #elif CPU(X86) && COMPILER(GCC_OR_CLANG) && OS(LINUX) && defined(__PIC__)
668 #define defineUnaryDoubleOpWrapper(function) \
669     asm( \
670         ".text\n" \
671         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
672         HIDE_SYMBOL(function##Thunk) "\n" \
673         SYMBOL_STRING(function##Thunk) ":" "\n" \
674         "pushl %ebx\n" \
675         "subl $20, %esp\n" \
676         "movsd %xmm0, (%esp) \n" \
677         "call __x86.get_pc_thunk.bx\n" \
678         "addl $_GLOBAL_OFFSET_TABLE_, %ebx\n" \
679         "call " GLOBAL_REFERENCE(function) "\n" \
680         "fstpl (%esp) \n" \
681         "movsd (%esp), %xmm0 \n" \
682         "addl $20, %esp\n" \
683         "popl %ebx\n" \
684         "ret\n" \
685     );\
686     extern "C" { \
687         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
688     } \
689     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
690
691 #elif CPU(X86) && COMPILER(GCC_OR_CLANG) && (OS(DARWIN) || OS(LINUX))
692 #define defineUnaryDoubleOpWrapper(function) \
693     asm( \
694         ".text\n" \
695         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
696         HIDE_SYMBOL(function##Thunk) "\n" \
697         SYMBOL_STRING(function##Thunk) ":" "\n" \
698         "subl $20, %esp\n" \
699         "movsd %xmm0, (%esp) \n" \
700         "call " GLOBAL_REFERENCE(function) "\n" \
701         "fstpl (%esp) \n" \
702         "movsd (%esp), %xmm0 \n" \
703         "addl $20, %esp\n" \
704         "ret\n" \
705     );\
706     extern "C" { \
707         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
708     } \
709     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
710
711 #elif CPU(ARM_THUMB2) && COMPILER(GCC_OR_CLANG) && PLATFORM(IOS)
712
713 #define defineUnaryDoubleOpWrapper(function) \
714     asm( \
715         ".text\n" \
716         ".align 2\n" \
717         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
718         HIDE_SYMBOL(function##Thunk) "\n" \
719         ".thumb\n" \
720         ".thumb_func " THUMB_FUNC_PARAM(function##Thunk) "\n" \
721         SYMBOL_STRING(function##Thunk) ":" "\n" \
722         "push {lr}\n" \
723         "vmov r0, r1, d0\n" \
724         "blx " GLOBAL_REFERENCE(function) "\n" \
725         "vmov d0, r0, r1\n" \
726         "pop {lr}\n" \
727         "bx lr\n" \
728     ); \
729     extern "C" { \
730         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
731     } \
732     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
733
734 #elif CPU(ARM64)
735
736 #define defineUnaryDoubleOpWrapper(function) \
737     asm( \
738         ".text\n" \
739         ".align 2\n" \
740         ".globl " SYMBOL_STRING(function##Thunk) "\n" \
741         HIDE_SYMBOL(function##Thunk) "\n" \
742         SYMBOL_STRING(function##Thunk) ":" "\n" \
743         "b " GLOBAL_REFERENCE(function) "\n" \
744         ".previous" \
745     ); \
746     extern "C" { \
747         MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \
748     } \
749     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
750
751 #elif CPU(X86) && COMPILER(MSVC) && OS(WINDOWS)
752
753 // MSVC does not accept floor, etc, to be called directly from inline assembly, so we need to wrap these functions.
754 static double (_cdecl *floorFunction)(double) = floor;
755 static double (_cdecl *ceilFunction)(double) = ceil;
756 static double (_cdecl *truncFunction)(double) = trunc;
757 static double (_cdecl *expFunction)(double) = exp;
758 static double (_cdecl *logFunction)(double) = log;
759 static double (_cdecl *jsRoundFunction)(double) = jsRound;
760
761 #define defineUnaryDoubleOpWrapper(function) \
762     extern "C" __declspec(naked) MathThunkCallingConvention function##Thunk(MathThunkCallingConvention) \
763     { \
764         __asm \
765         { \
766         __asm sub esp, 20 \
767         __asm movsd mmword ptr [esp], xmm0  \
768         __asm call function##Function \
769         __asm fstp qword ptr [esp] \
770         __asm movsd xmm0, mmword ptr [esp] \
771         __asm add esp, 20 \
772         __asm ret \
773         } \
774     } \
775     static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk;
776
777 #else
778
779 #define defineUnaryDoubleOpWrapper(function) \
780     static MathThunk UnaryDoubleOpWrapper(function) = 0
781 #endif
782
783 defineUnaryDoubleOpWrapper(jsRound);
784 defineUnaryDoubleOpWrapper(exp);
785 defineUnaryDoubleOpWrapper(log);
786 defineUnaryDoubleOpWrapper(floor);
787 defineUnaryDoubleOpWrapper(ceil);
788 defineUnaryDoubleOpWrapper(trunc);
789
790 static const double halfConstant = 0.5;
791     
792 MacroAssemblerCodeRef floorThunkGenerator(VM* vm)
793 {
794     SpecializedThunkJIT jit(vm, 1);
795     MacroAssembler::Jump nonIntJump;
796     if (!UnaryDoubleOpWrapper(floor) || !jit.supportsFloatingPoint())
797         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
798     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
799     jit.returnInt32(SpecializedThunkJIT::regT0);
800     nonIntJump.link(&jit);
801     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
802
803     if (jit.supportsFloatingPointRounding()) {
804         SpecializedThunkJIT::JumpList doubleResult;
805         jit.floorDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
806         jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
807         jit.returnInt32(SpecializedThunkJIT::regT0);
808         doubleResult.link(&jit);
809         jit.returnDouble(SpecializedThunkJIT::fpRegT0);
810         return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "floor");
811     }
812
813     SpecializedThunkJIT::Jump intResult;
814     SpecializedThunkJIT::JumpList doubleResult;
815     if (jit.supportsFloatingPointTruncate()) {
816         jit.moveZeroToDouble(SpecializedThunkJIT::fpRegT1);
817         doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
818         SpecializedThunkJIT::JumpList slowPath;
819         // Handle the negative doubles in the slow path for now.
820         slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
821         slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0));
822         intResult = jit.jump();
823         slowPath.link(&jit);
824     }
825     jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(floor));
826     jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
827     if (jit.supportsFloatingPointTruncate())
828         intResult.link(&jit);
829     jit.returnInt32(SpecializedThunkJIT::regT0);
830     doubleResult.link(&jit);
831     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
832     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "floor");
833 }
834
835 MacroAssemblerCodeRef ceilThunkGenerator(VM* vm)
836 {
837     SpecializedThunkJIT jit(vm, 1);
838     if (!UnaryDoubleOpWrapper(ceil) || !jit.supportsFloatingPoint())
839         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
840     MacroAssembler::Jump nonIntJump;
841     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
842     jit.returnInt32(SpecializedThunkJIT::regT0);
843     nonIntJump.link(&jit);
844     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
845     if (jit.supportsFloatingPointRounding())
846         jit.ceilDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
847     else
848         jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(ceil));
849
850     SpecializedThunkJIT::JumpList doubleResult;
851     jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
852     jit.returnInt32(SpecializedThunkJIT::regT0);
853     doubleResult.link(&jit);
854     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
855     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "ceil");
856 }
857
858 MacroAssemblerCodeRef truncThunkGenerator(VM* vm)
859 {
860     SpecializedThunkJIT jit(vm, 1);
861     if (!UnaryDoubleOpWrapper(trunc) || !jit.supportsFloatingPoint())
862         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
863     MacroAssembler::Jump nonIntJump;
864     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
865     jit.returnInt32(SpecializedThunkJIT::regT0);
866     nonIntJump.link(&jit);
867     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
868     if (jit.supportsFloatingPointRounding())
869         jit.roundTowardZeroDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
870     else
871         jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(trunc));
872
873     SpecializedThunkJIT::JumpList doubleResult;
874     jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
875     jit.returnInt32(SpecializedThunkJIT::regT0);
876     doubleResult.link(&jit);
877     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
878     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "trunc");
879 }
880
881 MacroAssemblerCodeRef roundThunkGenerator(VM* vm)
882 {
883     SpecializedThunkJIT jit(vm, 1);
884     if (!UnaryDoubleOpWrapper(jsRound) || !jit.supportsFloatingPoint())
885         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
886     MacroAssembler::Jump nonIntJump;
887     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
888     jit.returnInt32(SpecializedThunkJIT::regT0);
889     nonIntJump.link(&jit);
890     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
891     SpecializedThunkJIT::Jump intResult;
892     SpecializedThunkJIT::JumpList doubleResult;
893     if (jit.supportsFloatingPointTruncate()) {
894         jit.moveZeroToDouble(SpecializedThunkJIT::fpRegT1);
895         doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
896         SpecializedThunkJIT::JumpList slowPath;
897         // Handle the negative doubles in the slow path for now.
898         slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
899         jit.loadDouble(MacroAssembler::TrustedImmPtr(&halfConstant), SpecializedThunkJIT::fpRegT1);
900         jit.addDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
901         slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0));
902         intResult = jit.jump();
903         slowPath.link(&jit);
904     }
905     jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(jsRound));
906     jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
907     if (jit.supportsFloatingPointTruncate())
908         intResult.link(&jit);
909     jit.returnInt32(SpecializedThunkJIT::regT0);
910     doubleResult.link(&jit);
911     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
912     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "round");
913 }
914
915 MacroAssemblerCodeRef expThunkGenerator(VM* vm)
916 {
917     if (!UnaryDoubleOpWrapper(exp))
918         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
919     SpecializedThunkJIT jit(vm, 1);
920     if (!jit.supportsFloatingPoint())
921         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
922     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
923     jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(exp));
924     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
925     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "exp");
926 }
927
928 MacroAssemblerCodeRef logThunkGenerator(VM* vm)
929 {
930     if (!UnaryDoubleOpWrapper(log))
931         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
932     SpecializedThunkJIT jit(vm, 1);
933     if (!jit.supportsFloatingPoint())
934         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
935     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
936     jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(log));
937     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
938     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "log");
939 }
940
941 MacroAssemblerCodeRef absThunkGenerator(VM* vm)
942 {
943     SpecializedThunkJIT jit(vm, 1);
944     if (!jit.supportsFloatingPointAbs())
945         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
946
947 #if USE(JSVALUE64)
948     unsigned virtualRegisterIndex = CallFrame::argumentOffset(0);
949     jit.load64(AssemblyHelpers::addressFor(virtualRegisterIndex), GPRInfo::regT0);
950     MacroAssembler::Jump notInteger = jit.branch64(MacroAssembler::Below, GPRInfo::regT0, GPRInfo::tagTypeNumberRegister);
951
952     // Abs Int32.
953     jit.rshift32(GPRInfo::regT0, MacroAssembler::TrustedImm32(31), GPRInfo::regT1);
954     jit.add32(GPRInfo::regT1, GPRInfo::regT0);
955     jit.xor32(GPRInfo::regT1, GPRInfo::regT0);
956
957     // IntMin cannot be inverted.
958     MacroAssembler::Jump integerIsIntMin = jit.branchTest32(MacroAssembler::Signed, GPRInfo::regT0);
959
960     // Box and finish.
961     jit.or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
962     MacroAssembler::Jump doneWithIntegers = jit.jump();
963
964     // Handle Doubles.
965     notInteger.link(&jit);
966     jit.appendFailure(jit.branchTest64(MacroAssembler::Zero, GPRInfo::regT0, GPRInfo::tagTypeNumberRegister));
967     jit.unboxDoubleWithoutAssertions(GPRInfo::regT0, GPRInfo::regT0, FPRInfo::fpRegT0);
968     MacroAssembler::Label absFPR0Label = jit.label();
969     jit.absDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT1);
970     jit.boxDouble(FPRInfo::fpRegT1, GPRInfo::regT0);
971
972     // Tail.
973     doneWithIntegers.link(&jit);
974     jit.returnJSValue(GPRInfo::regT0);
975
976     // We know the value of regT0 is IntMin. We could load that value from memory but
977     // it is simpler to just convert it.
978     integerIsIntMin.link(&jit);
979     jit.convertInt32ToDouble(GPRInfo::regT0, FPRInfo::fpRegT0);
980     jit.jump().linkTo(absFPR0Label, &jit);
981 #else
982     MacroAssembler::Jump nonIntJump;
983     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
984     jit.rshift32(SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(31), SpecializedThunkJIT::regT1);
985     jit.add32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
986     jit.xor32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
987     jit.appendFailure(jit.branchTest32(MacroAssembler::Signed, SpecializedThunkJIT::regT0));
988     jit.returnInt32(SpecializedThunkJIT::regT0);
989     nonIntJump.link(&jit);
990     // Shame about the double int conversion here.
991     jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
992     jit.absDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
993     jit.returnDouble(SpecializedThunkJIT::fpRegT1);
994 #endif
995     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "abs");
996 }
997
998 MacroAssemblerCodeRef imulThunkGenerator(VM* vm)
999 {
1000     SpecializedThunkJIT jit(vm, 2);
1001     MacroAssembler::Jump nonIntArg0Jump;
1002     jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntArg0Jump);
1003     SpecializedThunkJIT::Label doneLoadingArg0(&jit);
1004     MacroAssembler::Jump nonIntArg1Jump;
1005     jit.loadInt32Argument(1, SpecializedThunkJIT::regT1, nonIntArg1Jump);
1006     SpecializedThunkJIT::Label doneLoadingArg1(&jit);
1007     jit.mul32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
1008     jit.returnInt32(SpecializedThunkJIT::regT0);
1009
1010     if (jit.supportsFloatingPointTruncate()) {
1011         nonIntArg0Jump.link(&jit);
1012         jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
1013         jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(doneLoadingArg0, &jit);
1014         jit.appendFailure(jit.jump());
1015     } else
1016         jit.appendFailure(nonIntArg0Jump);
1017
1018     if (jit.supportsFloatingPointTruncate()) {
1019         nonIntArg1Jump.link(&jit);
1020         jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT1);
1021         jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT1, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(doneLoadingArg1, &jit);
1022         jit.appendFailure(jit.jump());
1023     } else
1024         jit.appendFailure(nonIntArg1Jump);
1025
1026     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "imul");
1027 }
1028
1029 MacroAssemblerCodeRef randomThunkGenerator(VM* vm)
1030 {
1031     SpecializedThunkJIT jit(vm, 0);
1032     if (!jit.supportsFloatingPoint())
1033         return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
1034
1035 #if USE(JSVALUE64)
1036     jit.emitRandomThunk(*vm, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT3, SpecializedThunkJIT::fpRegT0);
1037     jit.returnDouble(SpecializedThunkJIT::fpRegT0);
1038
1039     return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "random");
1040 #else
1041     return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
1042 #endif
1043 }
1044
1045 MacroAssemblerCodeRef boundThisNoArgsFunctionCallGenerator(VM* vm)
1046 {
1047     CCallHelpers jit;
1048     
1049     jit.emitFunctionPrologue();
1050     
1051     // Set up our call frame.
1052     jit.storePtr(CCallHelpers::TrustedImmPtr(nullptr), CCallHelpers::addressFor(CallFrameSlot::codeBlock));
1053     jit.store32(CCallHelpers::TrustedImm32(0), CCallHelpers::tagFor(CallFrameSlot::argumentCount));
1054
1055     unsigned extraStackNeeded = 0;
1056     if (unsigned stackMisalignment = sizeof(CallerFrameAndPC) % stackAlignmentBytes())
1057         extraStackNeeded = stackAlignmentBytes() - stackMisalignment;
1058     
1059     // We need to forward all of the arguments that we were passed. We aren't allowed to do a tail
1060     // call here as far as I can tell. At least not so long as the generic path doesn't do a tail
1061     // call, since that would be way too weird.
1062     
1063     // The formula for the number of stack bytes needed given some number of parameters (including
1064     // this) is:
1065     //
1066     //     stackAlign((numParams + CallFrameHeaderSize) * sizeof(Register) - sizeof(CallerFrameAndPC))
1067     //
1068     // Probably we want to write this as:
1069     //
1070     //     stackAlign((numParams + (CallFrameHeaderSize - CallerFrameAndPCSize)) * sizeof(Register))
1071     //
1072     // That's really all there is to this. We have all the registers we need to do it.
1073     
1074     jit.load32(CCallHelpers::payloadFor(CallFrameSlot::argumentCount), GPRInfo::regT1);
1075     jit.add32(CCallHelpers::TrustedImm32(CallFrame::headerSizeInRegisters - CallerFrameAndPC::sizeInRegisters), GPRInfo::regT1, GPRInfo::regT2);
1076     jit.lshift32(CCallHelpers::TrustedImm32(3), GPRInfo::regT2);
1077     jit.add32(CCallHelpers::TrustedImm32(stackAlignmentBytes() - 1), GPRInfo::regT2);
1078     jit.and32(CCallHelpers::TrustedImm32(-stackAlignmentBytes()), GPRInfo::regT2);
1079     
1080     if (extraStackNeeded)
1081         jit.add32(CCallHelpers::TrustedImm32(extraStackNeeded), GPRInfo::regT2);
1082     
1083     // At this point regT1 has the actual argument count and regT2 has the amount of stack we will
1084     // need.
1085     
1086     jit.subPtr(GPRInfo::regT2, CCallHelpers::stackPointerRegister);
1087
1088     // Do basic callee frame setup, including 'this'.
1089     
1090     jit.loadCell(CCallHelpers::addressFor(CallFrameSlot::callee), GPRInfo::regT3);
1091
1092     jit.store32(GPRInfo::regT1, CCallHelpers::calleeFramePayloadSlot(CallFrameSlot::argumentCount));
1093     
1094     JSValueRegs valueRegs = JSValueRegs::withTwoAvailableRegs(GPRInfo::regT0, GPRInfo::regT2);
1095     jit.loadValue(CCallHelpers::Address(GPRInfo::regT3, JSBoundFunction::offsetOfBoundThis()), valueRegs);
1096     jit.storeValue(valueRegs, CCallHelpers::calleeArgumentSlot(0));
1097
1098     jit.loadPtr(CCallHelpers::Address(GPRInfo::regT3, JSBoundFunction::offsetOfTargetFunction()), GPRInfo::regT3);
1099     jit.storeCell(GPRInfo::regT3, CCallHelpers::calleeFrameSlot(CallFrameSlot::callee));
1100     
1101     // OK, now we can start copying. This is a simple matter of copying parameters from the caller's
1102     // frame to the callee's frame. Note that we know that regT1 (the argument count) must be at
1103     // least 1.
1104     jit.sub32(CCallHelpers::TrustedImm32(1), GPRInfo::regT1);
1105     CCallHelpers::Jump done = jit.branchTest32(CCallHelpers::Zero, GPRInfo::regT1);
1106     
1107     CCallHelpers::Label loop = jit.label();
1108     jit.sub32(CCallHelpers::TrustedImm32(1), GPRInfo::regT1);
1109     jit.loadValue(CCallHelpers::addressFor(virtualRegisterForArgument(1)).indexedBy(GPRInfo::regT1, CCallHelpers::TimesEight), valueRegs);
1110     jit.storeValue(valueRegs, CCallHelpers::calleeArgumentSlot(1).indexedBy(GPRInfo::regT1, CCallHelpers::TimesEight));
1111     jit.branchTest32(CCallHelpers::NonZero, GPRInfo::regT1).linkTo(loop, &jit);
1112     
1113     done.link(&jit);
1114     
1115     jit.loadPtr(
1116         CCallHelpers::Address(GPRInfo::regT3, JSFunction::offsetOfExecutable()),
1117         GPRInfo::regT0);
1118     jit.loadPtr(
1119         CCallHelpers::Address(
1120             GPRInfo::regT0, ExecutableBase::offsetOfJITCodeWithArityCheckFor(CodeForCall)),
1121         GPRInfo::regT0);
1122     CCallHelpers::Jump noCode = jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT0);
1123     
1124     emitPointerValidation(jit, GPRInfo::regT0);
1125     jit.call(GPRInfo::regT0);
1126     
1127     jit.emitFunctionEpilogue();
1128     jit.ret();
1129     
1130     LinkBuffer linkBuffer(*vm, jit, GLOBAL_THUNK_ID);
1131     linkBuffer.link(noCode, CodeLocationLabel(vm->jitStubs->ctiNativeTailCallWithoutSavedTags(vm)));
1132     return FINALIZE_CODE(
1133         linkBuffer, ("Specialized thunk for bound function calls with no arguments"));
1134 }
1135
1136 #if ENABLE(WEBASSEMBLY)
1137 MacroAssemblerCodeRef throwExceptionFromWasmThunkGenerator(VM* vm)
1138 {
1139     CCallHelpers jit;
1140
1141     // The thing that jumps here must move ExceptionType into the argumentGPR1 and jump here.
1142     // We're allowed to use temp registers here, but not callee saves.
1143     {
1144         RegisterSet usedRegisters = RegisterSet::stubUnavailableRegisters();
1145         usedRegisters.set(GPRInfo::argumentGPR1);
1146         jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(*vm, usedRegisters);
1147     }
1148
1149     jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
1150     jit.loadWasmContext(GPRInfo::argumentGPR2);
1151     CCallHelpers::Call call = jit.call();
1152     jit.jumpToExceptionHandler(*vm);
1153
1154     void (*throwWasmException)(ExecState*, Wasm::ExceptionType, JSWebAssemblyInstance*) = [] (ExecState* exec, Wasm::ExceptionType type, JSWebAssemblyInstance* wasmContext) {
1155         VM* vm = &exec->vm();
1156         NativeCallFrameTracer tracer(vm, exec);
1157
1158         {
1159             auto throwScope = DECLARE_THROW_SCOPE(*vm);
1160             JSGlobalObject* globalObject = wasmContext->globalObject();
1161
1162             JSWebAssemblyRuntimeError* error = JSWebAssemblyRuntimeError::create(exec, *vm, globalObject->WebAssemblyRuntimeErrorStructure(), Wasm::errorMessageForExceptionType(type));
1163             throwException(exec, throwScope, error);
1164         }
1165
1166         genericUnwind(vm, exec);
1167         ASSERT(!!vm->callFrameForCatch);
1168     };
1169
1170     LinkBuffer linkBuffer(*vm, jit, GLOBAL_THUNK_ID);
1171     linkBuffer.link(call, throwWasmException);
1172     return FINALIZE_CODE(
1173         linkBuffer, ("Throw exception from Wasm"));
1174 }
1175 #endif // ENABLE(WEBASSEMBLY)
1176
1177 } // namespace JSC
1178
1179 #endif // ENABLE(JIT)