op_add/ValueAdd should be an IC in all JIT tiers
[WebKit.git] / Source / JavaScriptCore / jit / JITInlines.h
1 /*
2  * Copyright (C) 2008, 2012-2013, 2015-2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #ifndef JITInlines_h
27 #define JITInlines_h
28
29 #if ENABLE(JIT)
30
31 #include "JSCInlines.h"
32
33 namespace JSC {
34
35 #if USE(JSVALUE64)
36 inline MacroAssembler::JumpList JIT::emitDoubleGetByVal(Instruction* instruction, PatchableJump& badType)
37 {
38     JumpList slowCases = emitDoubleLoad(instruction, badType);
39     moveDoubleTo64(fpRegT0, regT0);
40     sub64(tagTypeNumberRegister, regT0);
41     return slowCases;
42 }
43 #else
44 inline MacroAssembler::JumpList JIT::emitDoubleGetByVal(Instruction* instruction, PatchableJump& badType)
45 {
46     JumpList slowCases = emitDoubleLoad(instruction, badType);
47     moveDoubleToInts(fpRegT0, regT0, regT1);
48     return slowCases;
49 }
50 #endif // USE(JSVALUE64)
51
52 ALWAYS_INLINE MacroAssembler::JumpList JIT::emitLoadForArrayMode(Instruction* currentInstruction, JITArrayMode arrayMode, PatchableJump& badType)
53 {
54     switch (arrayMode) {
55     case JITInt32:
56         return emitInt32Load(currentInstruction, badType);
57     case JITDouble:
58         return emitDoubleLoad(currentInstruction, badType);
59     case JITContiguous:
60         return emitContiguousLoad(currentInstruction, badType);
61     case JITArrayStorage:
62         return emitArrayStorageLoad(currentInstruction, badType);
63     default:
64         break;
65     }
66     RELEASE_ASSERT_NOT_REACHED();
67     return MacroAssembler::JumpList();
68 }
69
70 inline MacroAssembler::JumpList JIT::emitContiguousGetByVal(Instruction* instruction, PatchableJump& badType, IndexingType expectedShape)
71 {
72     return emitContiguousLoad(instruction, badType, expectedShape);
73 }
74
75 inline MacroAssembler::JumpList JIT::emitArrayStorageGetByVal(Instruction* instruction, PatchableJump& badType)
76 {
77     return emitArrayStorageLoad(instruction, badType);
78 }
79
80 ALWAYS_INLINE bool JIT::isOperandConstantDouble(int src)
81 {
82     return m_codeBlock->isConstantRegisterIndex(src) && getConstantOperand(src).isDouble();
83 }
84
85 ALWAYS_INLINE JSValue JIT::getConstantOperand(int src)
86 {
87     ASSERT(m_codeBlock->isConstantRegisterIndex(src));
88     return m_codeBlock->getConstant(src);
89 }
90
91 ALWAYS_INLINE void JIT::emitPutIntToCallFrameHeader(RegisterID from, int entry)
92 {
93 #if USE(JSVALUE32_64)
94     store32(TrustedImm32(Int32Tag), intTagFor(entry, callFrameRegister));
95     store32(from, intPayloadFor(entry, callFrameRegister));
96 #else
97     store64(from, addressFor(entry, callFrameRegister));
98 #endif
99 }
100
101 ALWAYS_INLINE void JIT::emitLoadCharacterString(RegisterID src, RegisterID dst, JumpList& failures)
102 {
103     failures.append(branchStructure(NotEqual, Address(src, JSCell::structureIDOffset()), m_vm->stringStructure.get()));
104     failures.append(branch32(NotEqual, MacroAssembler::Address(src, ThunkHelpers::jsStringLengthOffset()), TrustedImm32(1)));
105     loadPtr(MacroAssembler::Address(src, ThunkHelpers::jsStringValueOffset()), dst);
106     failures.append(branchTest32(Zero, dst));
107     loadPtr(MacroAssembler::Address(dst, StringImpl::flagsOffset()), regT1);
108     loadPtr(MacroAssembler::Address(dst, StringImpl::dataOffset()), dst);
109
110     JumpList is16Bit;
111     JumpList cont8Bit;
112     is16Bit.append(branchTest32(Zero, regT1, TrustedImm32(StringImpl::flagIs8Bit())));
113     load8(MacroAssembler::Address(dst, 0), dst);
114     cont8Bit.append(jump());
115     is16Bit.link(this);
116     load16(MacroAssembler::Address(dst, 0), dst);
117     cont8Bit.link(this);
118 }
119
120 ALWAYS_INLINE JIT::Call JIT::emitNakedCall(CodePtr function)
121 {
122     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
123     Call nakedCall = nearCall();
124     m_calls.append(CallRecord(nakedCall, m_bytecodeOffset, function.executableAddress()));
125     return nakedCall;
126 }
127
128 ALWAYS_INLINE JIT::Call JIT::emitNakedTailCall(CodePtr function)
129 {
130     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
131     Call nakedCall = nearTailCall();
132     m_calls.append(CallRecord(nakedCall, m_bytecodeOffset, function.executableAddress()));
133     return nakedCall;
134 }
135
136 ALWAYS_INLINE void JIT::updateTopCallFrame()
137 {
138     ASSERT(static_cast<int>(m_bytecodeOffset) >= 0);
139 #if USE(JSVALUE32_64)
140     Instruction* instruction = m_codeBlock->instructions().begin() + m_bytecodeOffset + 1; 
141     uint32_t locationBits = CallSiteIndex(instruction).bits();
142 #else
143     uint32_t locationBits = CallSiteIndex(m_bytecodeOffset + 1).bits();
144 #endif
145     store32(TrustedImm32(locationBits), intTagFor(CallFrameSlot::argumentCount));
146     
147     // FIXME: It's not clear that this is needed. JITOperations tend to update the top call frame on
148     // the C++ side.
149     // https://bugs.webkit.org/show_bug.cgi?id=155693
150     storePtr(callFrameRegister, &m_vm->topCallFrame);
151 }
152
153 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithExceptionCheck(const FunctionPtr& function)
154 {
155     updateTopCallFrame();
156     MacroAssembler::Call call = appendCall(function);
157     exceptionCheck();
158     return call;
159 }
160
161 #if OS(WINDOWS) && CPU(X86_64)
162 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithExceptionCheckAndSlowPathReturnType(const FunctionPtr& function)
163 {
164     updateTopCallFrame();
165     MacroAssembler::Call call = appendCallWithSlowPathReturnType(function);
166     exceptionCheck();
167     return call;
168 }
169 #endif
170
171 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithCallFrameRollbackOnException(const FunctionPtr& function)
172 {
173     updateTopCallFrame(); // The callee is responsible for setting topCallFrame to their caller
174     MacroAssembler::Call call = appendCall(function);
175     exceptionCheckWithCallFrameRollback();
176     return call;
177 }
178
179 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithExceptionCheckSetJSValueResult(const FunctionPtr& function, int dst)
180 {
181     MacroAssembler::Call call = appendCallWithExceptionCheck(function);
182 #if USE(JSVALUE64)
183     emitPutVirtualRegister(dst, returnValueGPR);
184 #else
185     emitStore(dst, returnValueGPR2, returnValueGPR);
186 #endif
187     return call;
188 }
189
190 ALWAYS_INLINE MacroAssembler::Call JIT::appendCallWithExceptionCheckSetJSValueResultWithProfile(const FunctionPtr& function, int dst)
191 {
192     MacroAssembler::Call call = appendCallWithExceptionCheck(function);
193     emitValueProfilingSite();
194 #if USE(JSVALUE64)
195     emitPutVirtualRegister(dst, returnValueGPR);
196 #else
197     emitStore(dst, returnValueGPR2, returnValueGPR);
198 #endif
199     return call;
200 }
201
202 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(P_JITOperation_E operation)
203 {
204     setupArgumentsExecState();
205     return appendCallWithExceptionCheck(operation);
206 }
207
208 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationNoExceptionCheck(Z_JITOperation_E operation)
209 {
210     setupArgumentsExecState();
211     updateTopCallFrame();
212     return appendCall(operation);
213 }
214
215 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_E operation)
216 {
217     setupArgumentsExecState();
218     return appendCallWithExceptionCheck(operation);
219 }
220
221 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EJsc operation, GPRReg arg1)
222 {
223     setupArgumentsWithExecState(arg1);
224     return appendCallWithExceptionCheck(operation);
225 }
226
227 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EJscZ operation, GPRReg arg1, int32_t arg2)
228 {
229     setupArgumentsWithExecState(arg1, TrustedImm32(arg2));
230     return appendCallWithExceptionCheck(operation);
231 }
232
233 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EL operation, GPRReg arg1)
234 {
235     setupArgumentsWithExecState(arg1);
236     return appendCallWithExceptionCheck(operation);
237 }
238     
239 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EL operation, TrustedImmPtr arg1)
240 {
241     setupArgumentsWithExecState(arg1);
242     return appendCallWithExceptionCheck(operation);
243 }
244     
245 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EO operation, GPRReg arg)
246 {
247     setupArgumentsWithExecState(arg);
248     return appendCallWithExceptionCheck(operation);
249 }
250
251 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_ESt operation, Structure* structure)
252 {
253     setupArgumentsWithExecState(TrustedImmPtr(structure));
254     return appendCallWithExceptionCheck(operation);
255 }
256
257 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(C_JITOperation_EZ operation, int32_t arg)
258 {
259     setupArgumentsWithExecState(TrustedImm32(arg));
260     return appendCallWithExceptionCheck(operation);
261 }
262
263 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_E operation, int dst)
264 {
265     setupArgumentsExecState();
266     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
267 }
268
269 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EAapJcpZ operation, int dst, ArrayAllocationProfile* arg1, GPRReg arg2, int32_t arg3)
270 {
271     setupArgumentsWithExecState(TrustedImmPtr(arg1), arg2, TrustedImm32(arg3));
272     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
273 }
274
275 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EAapJcpZ operation, int dst, ArrayAllocationProfile* arg1, const JSValue* arg2, int32_t arg3)
276 {
277     setupArgumentsWithExecState(TrustedImmPtr(arg1), TrustedImmPtr(arg2), TrustedImm32(arg3));
278     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
279 }
280
281 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EC operation, int dst, JSCell* cell)
282 {
283     setupArgumentsWithExecState(TrustedImmPtr(cell));
284     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
285 }
286
287 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EC operation, JSCell* cell)
288 {
289     setupArgumentsWithExecState(TrustedImmPtr(cell));
290     return appendCallWithExceptionCheck(operation);
291 }
292
293 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJscC operation, int dst, GPRReg arg1, JSCell* cell)
294 {
295     setupArgumentsWithExecState(arg1, TrustedImmPtr(cell));
296     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
297 }
298
299 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJscCJ operation, int dst, GPRReg arg1, JSCell* cell, GPRReg arg2)
300 {
301     setupArgumentsWithExecState(arg1, TrustedImmPtr(cell), arg2);
302     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
303 }
304     
305 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EP operation, int dst, void* pointer)
306 {
307     setupArgumentsWithExecState(TrustedImmPtr(pointer));
308     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
309 }
310
311 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(WithProfileTag, J_JITOperation_EPc operation, int dst, Instruction* bytecodePC)
312 {
313     setupArgumentsWithExecState(TrustedImmPtr(bytecodePC));
314     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
315 }
316
317 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EPc operation, int dst, Instruction* bytecodePC)
318 {
319     setupArgumentsWithExecState(TrustedImmPtr(bytecodePC));
320     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
321 }
322
323 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EZ operation, int dst, int32_t arg)
324 {
325     setupArgumentsWithExecState(TrustedImm32(arg));
326     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
327 }
328
329 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EZZ operation, int dst, int32_t arg1, int32_t arg2)
330 {
331     setupArgumentsWithExecState(TrustedImm32(arg1), TrustedImm32(arg2));
332     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
333 }
334
335 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_ECC operation, RegisterID regOp1, RegisterID regOp2)
336 {
337     setupArgumentsWithExecState(regOp1, regOp2);
338     return appendCallWithExceptionCheck(operation);
339 }
340
341 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EOJss operation, RegisterID regOp1, RegisterID regOp2)
342 {
343     setupArgumentsWithExecState(regOp1, regOp2);
344     return appendCallWithExceptionCheck(operation);
345 }
346
347 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(Sprt_JITOperation_EZ operation, int32_t op)
348 {
349 #if OS(WINDOWS) && CPU(X86_64)
350     setupArgumentsWithExecStateForCallWithSlowPathReturnType(TrustedImm32(op));
351     return appendCallWithExceptionCheckAndSlowPathReturnType(operation);
352 #else
353     setupArgumentsWithExecState(TrustedImm32(op));
354     return appendCallWithExceptionCheck(operation);
355 #endif
356 }
357
358 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_E operation)
359 {
360     setupArgumentsExecState();
361     return appendCallWithExceptionCheck(operation);
362 }
363
364 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EC operation, RegisterID regOp)
365 {
366     setupArgumentsWithExecState(regOp);
367     return appendCallWithExceptionCheck(operation);
368 }
369
370 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECC operation, RegisterID regOp1, RegisterID regOp2)
371 {
372     setupArgumentsWithExecState(regOp1, regOp2);
373     return appendCallWithExceptionCheck(operation);
374 }
375
376 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EE operation, RegisterID regOp)
377 {
378     setupArgumentsWithExecState(regOp);
379     updateTopCallFrame();
380     return appendCallWithExceptionCheck(operation);
381 }
382
383 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EPc operation, Instruction* bytecodePC)
384 {
385     setupArgumentsWithExecState(TrustedImmPtr(bytecodePC));
386     return appendCallWithExceptionCheck(operation);
387 }
388
389 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZ operation, int32_t op)
390 {
391     setupArgumentsWithExecState(TrustedImm32(op));
392     return appendCallWithExceptionCheck(operation);
393 }
394
395 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationWithCallFrameRollbackOnException(J_JITOperation_E operation)
396 {
397     setupArgumentsExecState();
398     return appendCallWithCallFrameRollbackOnException(operation);
399 }
400
401 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationWithCallFrameRollbackOnException(V_JITOperation_ECb operation, CodeBlock* pointer)
402 {
403     setupArgumentsWithExecState(TrustedImmPtr(pointer));
404     return appendCallWithCallFrameRollbackOnException(operation);
405 }
406
407 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationWithCallFrameRollbackOnException(Z_JITOperation_E operation)
408 {
409     setupArgumentsExecState();
410     return appendCallWithCallFrameRollbackOnException(operation);
411 }
412
413 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECIZC operation, RegisterID regOp1, UniquedStringImpl* identOp2, int32_t op3, RegisterID regOp4)
414 {
415     setupArgumentsWithExecState(regOp1, TrustedImmPtr(identOp2), TrustedImm32(op3), regOp4);
416     return appendCallWithExceptionCheck(operation);
417 }
418
419 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJ operation, JSValueRegs result, JSValueRegs arg1, JSValueRegs arg2)
420 {
421     setupArgumentsWithExecState(arg1, arg2);
422     Call call = appendCallWithExceptionCheck(operation);
423     setupResults(result);
424     return call;
425 }
426
427 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJArp operation, JSValueRegs result, JSValueRegs arg1, JSValueRegs arg2, ArithProfile* arithProfile)
428 {
429     setupArgumentsWithExecState(arg1, arg2, TrustedImmPtr(arithProfile));
430     Call call = appendCallWithExceptionCheck(operation);
431     setupResults(result);
432     return call;
433 }
434
435 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJArpJaic operation, JSValueRegs result, JSValueRegs arg1, JSValueRegs arg2, ArithProfile* arithProfile, JITAddIC* addIC)
436 {
437     setupArgumentsWithExecState(arg1, arg2, TrustedImmPtr(arithProfile), TrustedImmPtr(addIC));
438     Call call = appendCallWithExceptionCheck(operation);
439     setupResults(result);
440     return call;
441 }
442
443 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJJaic operation, JSValueRegs result, JSValueRegs arg1, JSValueRegs arg2, JITAddIC* addIC)
444 {
445     setupArgumentsWithExecState(arg1, arg2, TrustedImmPtr(addIC));
446     Call call = appendCallWithExceptionCheck(operation);
447     setupResults(result);
448     return call;
449 }
450
451 #if USE(JSVALUE64)
452 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(Z_JITOperation_EJZZ operation, GPRReg arg1, int32_t arg2, int32_t arg3)
453 {
454     setupArgumentsWithExecState(arg1, TrustedImm32(arg2), TrustedImm32(arg3));
455     return appendCallWithExceptionCheck(operation);
456 }
457
458 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(F_JITOperation_EFJZZ operation, GPRReg arg1, GPRReg arg2, int32_t arg3, GPRReg arg4)
459 {
460     setupArgumentsWithExecState(arg1, arg2, TrustedImm32(arg3), arg4);
461     return appendCallWithExceptionCheck(operation);
462 }
463
464 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ESsiJJI operation, StructureStubInfo* stubInfo, RegisterID regOp1, RegisterID regOp2, UniquedStringImpl* uid)
465 {
466     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), regOp1, regOp2, TrustedImmPtr(uid));
467     return appendCallWithExceptionCheck(operation);
468 }
469
470 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJ operation, RegisterID regOp1, RegisterID regOp2, RegisterID regOp3)
471 {
472     setupArgumentsWithExecState(regOp1, regOp2, regOp3);
473     return appendCallWithExceptionCheck(operation);
474 }
475
476 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJAp operation, RegisterID regOp1, RegisterID regOp2, RegisterID regOp3, ArrayProfile* arrayProfile)
477 {
478     setupArgumentsWithExecState(regOp1, regOp2, regOp3, TrustedImmPtr(arrayProfile));
479     return appendCallWithExceptionCheck(operation);
480 }
481
482 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJBy operation, RegisterID regOp1, RegisterID regOp2, RegisterID regOp3, ByValInfo* byValInfo)
483 {
484     setupArgumentsWithExecState(regOp1, regOp2, regOp3, TrustedImmPtr(byValInfo));
485     return appendCallWithExceptionCheck(operation);
486 }
487
488 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZJ operation, int dst, GPRReg arg)
489 {
490     setupArgumentsWithExecState(TrustedImm32(dst), arg);
491     return appendCallWithExceptionCheck(operation);
492 }
493
494 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_ESsiJI operation, int dst, StructureStubInfo* stubInfo, GPRReg arg1, UniquedStringImpl* uid)
495 {
496     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1, TrustedImmPtr(uid));
497     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
498 }
499
500 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(JIT::WithProfileTag, J_JITOperation_ESsiJI operation, int dst, StructureStubInfo* stubInfo, GPRReg arg1, UniquedStringImpl* uid)
501 {
502     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1, TrustedImmPtr(uid));
503     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
504 }
505
506 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(JIT::WithProfileTag, J_JITOperation_EJJ operation, int dst, GPRReg arg1, GPRReg arg2)
507 {
508     setupArgumentsWithExecState(arg1, arg2);
509     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
510 }
511
512 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EAapJ operation, int dst, ArrayAllocationProfile* arg1, GPRReg arg2)
513 {
514     setupArgumentsWithExecState(TrustedImmPtr(arg1), arg2);
515     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
516 }
517
518 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJ operation, int dst, GPRReg arg1)
519 {
520     setupArgumentsWithExecState(arg1);
521     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
522 }
523
524 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJI operation, int dst, GPRReg arg1, UniquedStringImpl* arg2)
525 {
526     setupArgumentsWithExecState(arg1, TrustedImmPtr(arg2));
527     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
528 }
529
530 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJ operation, int dst, GPRReg arg1, GPRReg arg2)
531 {
532     setupArgumentsWithExecState(arg1, arg2);
533     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
534 }
535
536 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJAp operation, int dst, GPRReg arg1, GPRReg arg2, ArrayProfile* arrayProfile)
537 {
538     setupArgumentsWithExecState(arg1, arg2, TrustedImmPtr(arrayProfile));
539     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
540 }
541
542 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJBy operation, int dst, GPRReg arg1, GPRReg arg2, ByValInfo* byValInfo)
543 {
544     setupArgumentsWithExecState(arg1, arg2, TrustedImmPtr(byValInfo));
545     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
546 }
547
548 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(Z_JITOperation_EJOJ operation, GPRReg arg1, GPRReg arg2, GPRReg arg3)
549 {
550     setupArgumentsWithExecState(arg1, arg2, arg3);
551     return appendCallWithExceptionCheck(operation);
552 }
553
554 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationNoExceptionCheck(V_JITOperation_EJ operation, GPRReg arg1)
555 {
556     setupArgumentsWithExecState(arg1);
557     updateTopCallFrame();
558     return appendCall(operation);
559 }
560
561 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(P_JITOperation_EJS operation, GPRReg arg1, size_t arg2)
562 {
563     setupArgumentsWithExecState(arg1, TrustedImmPtr(arg2));
564     return appendCallWithExceptionCheck(operation);
565 }
566
567 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJ operation, RegisterID regOp)
568 {
569     setupArgumentsWithExecState(regOp);
570     return appendCallWithExceptionCheck(operation);
571 }
572
573
574 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJI operation, GPRReg arg1, UniquedStringImpl* arg2)
575 {
576     setupArgumentsWithExecState(arg1, TrustedImmPtr(arg2));
577     return appendCallWithExceptionCheck(operation);
578 }
579
580
581 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJJ operation, RegisterID regOp1, RegisterID regOp2)
582 {
583     setupArgumentsWithExecState(regOp1, regOp2);
584     return appendCallWithExceptionCheck(operation);
585 }
586
587 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZSymtabJ operation, int op1, SymbolTable* symbolTable, RegisterID regOp3)
588 {
589     setupArgumentsWithExecState(TrustedImm32(op1), TrustedImmPtr(symbolTable), regOp3);
590     return appendCallWithExceptionCheck(operation);
591 }
592
593 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EZSymtabJ operation, int op1, SymbolTable* symbolTable, RegisterID regOp3)
594 {
595     setupArgumentsWithExecState(TrustedImm32(op1), TrustedImmPtr(symbolTable), regOp3);
596     return appendCallWithExceptionCheck(operation);
597 }
598
599 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJ operation, RegisterID regOp)
600 {
601     setupArgumentsWithExecState(regOp);
602     return appendCallWithExceptionCheck(operation);
603 }
604
605 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECIZJJ operation, RegisterID regOp1, UniquedStringImpl* identOp2, int32_t op3, RegisterID regOp4, RegisterID regOp5)
606 {
607     setupArgumentsWithExecState(regOp1, TrustedImmPtr(identOp2), TrustedImm32(op3), regOp4, regOp5);
608     return appendCallWithExceptionCheck(operation);
609 }
610
611 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECJ operation, RegisterID arg1, RegisterID arg2)
612 {
613     setupArgumentsWithExecState(arg1, arg2);
614     return appendCallWithExceptionCheck(operation);
615 }
616
617 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECJZC operation, RegisterID regOp1, RegisterID regOp2, int32_t op3, RegisterID regOp4)
618 {
619     setupArgumentsWithExecState(regOp1, regOp2, TrustedImm32(op3), regOp4);
620     return appendCallWithExceptionCheck(operation);
621 }
622
623 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJZ operation, RegisterID regOp1, int32_t op2)
624 {
625     setupArgumentsWithExecState(regOp1, TrustedImm32(op2));
626     return appendCallWithExceptionCheck(operation);
627 }
628
629 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJZJ operation, RegisterID regOp1, int32_t op2, RegisterID regOp3)
630 {
631     setupArgumentsWithExecState(regOp1, TrustedImm32(op2), regOp3);
632     return appendCallWithExceptionCheck(operation);
633 }
634
635 #else // USE(JSVALUE32_64)
636
637 ALWAYS_INLINE MacroAssembler::Call JIT::callOperationNoExceptionCheck(V_JITOperation_EJ operation, GPRReg arg1Tag, GPRReg arg1Payload)
638 {
639     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag);
640     updateTopCallFrame();
641     return appendCall(operation);
642 }
643
644 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(Z_JITOperation_EJOJ operation, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2, GPRReg arg3Tag, GPRReg arg3Payload)
645 {
646     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, arg2, EABI_32BIT_DUMMY_ARG arg3Payload, arg3Tag);
647     return appendCallWithExceptionCheck(operation);
648 }
649
650 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(Z_JITOperation_EJZZ operation, GPRReg arg1Tag, GPRReg arg1Payload, int32_t arg2, int32_t arg3)
651 {
652     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, TrustedImm32(arg2), TrustedImm32(arg3));
653     return appendCallWithExceptionCheck(operation);
654 }
655
656 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(F_JITOperation_EFJZZ operation, GPRReg arg1, GPRReg arg2Tag, GPRReg arg2Payload, int32_t arg3, GPRReg arg4)
657 {
658     setupArgumentsWithExecState(arg1, arg2Payload, arg2Tag, TrustedImm32(arg3), arg4);
659     return appendCallWithExceptionCheck(operation);
660 }
661     
662 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EAapJ operation, int dst, ArrayAllocationProfile* arg1, GPRReg arg2Tag, GPRReg arg2Payload)
663 {
664     setupArgumentsWithExecState(TrustedImmPtr(arg1), arg2Payload, arg2Tag);
665     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
666 }
667
668 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJ operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload)
669 {
670     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag);
671     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
672 }
673
674 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_ESsiJI operation, int dst, StructureStubInfo* stubInfo, GPRReg arg1Tag, GPRReg arg1Payload, UniquedStringImpl* uid)
675 {
676     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1Payload, arg1Tag, TrustedImmPtr(uid));
677     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
678 }
679
680 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(JIT::WithProfileTag, J_JITOperation_ESsiJI operation, int dst, StructureStubInfo* stubInfo, GPRReg arg1Tag, GPRReg arg1Payload, UniquedStringImpl* uid)
681 {
682     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), arg1Payload, arg1Tag, TrustedImmPtr(uid));
683     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
684 }
685
686 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJI operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, UniquedStringImpl* arg2)
687 {
688     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, TrustedImmPtr(arg2));
689     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
690 }
691
692 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJ operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload)
693 {
694     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
695     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
696 }
697
698 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJAp operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload, ArrayProfile* arrayProfile)
699 {
700     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag, TrustedImmPtr(arrayProfile));
701     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
702 }
703
704 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJJBy operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload, ByValInfo* byValInfo)
705 {
706     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag, TrustedImmPtr(byValInfo));
707     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
708 }
709
710 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(JIT::WithProfileTag, J_JITOperation_EJJ operation, int dst, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload)
711 {
712     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
713     return appendCallWithExceptionCheckSetJSValueResultWithProfile(operation, dst);
714 }
715
716 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(P_JITOperation_EJS operation, GPRReg arg1Tag, GPRReg arg1Payload, size_t arg2)
717 {
718     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, TrustedImmPtr(arg2));
719     return appendCallWithExceptionCheck(operation);
720 }
721
722 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJ operation, RegisterID argTag, RegisterID argPayload)
723 {
724     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG argPayload, argTag);
725     return appendCallWithExceptionCheck(operation);
726 }
727
728 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJI operation, GPRReg arg1Tag, GPRReg arg1Payload, UniquedStringImpl* arg2)
729 {
730     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, TrustedImmPtr(arg2));
731     return appendCallWithExceptionCheck(operation);
732 }
733
734 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(S_JITOperation_EJJ operation, RegisterID arg1Tag, RegisterID arg1Payload, RegisterID arg2Tag, RegisterID arg2Payload)
735 {
736     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG arg1Payload, arg1Tag, SH4_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
737     return appendCallWithExceptionCheck(operation);
738 }
739
740 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECIZCC operation, RegisterID regOp1, UniquedStringImpl* identOp2, int32_t op3, RegisterID regOp4, RegisterID regOp5)
741 {
742     setupArgumentsWithExecState(regOp1, TrustedImmPtr(identOp2), TrustedImm32(op3), regOp4, regOp5);
743     return appendCallWithExceptionCheck(operation);
744 }
745
746 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECJ operation, RegisterID arg1, RegisterID arg2Tag, RegisterID arg2Payload)
747 {
748     setupArgumentsWithExecState(arg1, arg2Payload, arg2Tag);
749     return appendCallWithExceptionCheck(operation);
750 }
751
752 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ECJZC operation, RegisterID arg1, RegisterID arg2Tag, RegisterID arg2Payload, int32_t arg3, RegisterID arg4)
753 {
754     setupArgumentsWithExecState(arg1, arg2Payload, arg2Tag, TrustedImm32(arg3), arg4);
755     return appendCallWithExceptionCheck(operation);
756 }
757
758 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJ operation, RegisterID regOp1Tag, RegisterID regOp1Payload)
759 {
760     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag);
761     return appendCallWithExceptionCheck(operation);
762 }
763
764 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZSymtabJ operation, int32_t op1, SymbolTable* symbolTable, RegisterID regOp3Tag, RegisterID regOp3Payload)
765 {
766     setupArgumentsWithExecState(TrustedImm32(op1), TrustedImmPtr(symbolTable), EABI_32BIT_DUMMY_ARG regOp3Payload, regOp3Tag);
767     return appendCallWithExceptionCheck(operation);
768 }
769
770 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_ESsiJJI operation, StructureStubInfo* stubInfo, RegisterID regOp1Tag, RegisterID regOp1Payload, RegisterID regOp2Tag, RegisterID regOp2Payload, UniquedStringImpl* uid)
771 {
772     setupArgumentsWithExecState(TrustedImmPtr(stubInfo), regOp1Payload, regOp1Tag, regOp2Payload, regOp2Tag, TrustedImmPtr(uid));
773     return appendCallWithExceptionCheck(operation);
774 }
775
776 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJ operation, RegisterID regOp1Tag, RegisterID regOp1Payload, RegisterID regOp2Tag, RegisterID regOp2Payload, RegisterID regOp3Tag, RegisterID regOp3Payload)
777 {
778     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, SH4_32BIT_DUMMY_ARG regOp2Payload, regOp2Tag, regOp3Payload, regOp3Tag);
779     return appendCallWithExceptionCheck(operation);
780 }
781
782 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJAp operation, RegisterID regOp1Tag, RegisterID regOp1Payload, RegisterID regOp2Tag, RegisterID regOp2Payload, RegisterID regOp3Tag, RegisterID regOp3Payload, ArrayProfile* arrayProfile)
783 {
784     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, SH4_32BIT_DUMMY_ARG regOp2Payload, regOp2Tag, regOp3Payload, regOp3Tag, TrustedImmPtr(arrayProfile));
785     return appendCallWithExceptionCheck(operation);
786 }
787
788 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJJJBy operation, RegisterID regOp1Tag, RegisterID regOp1Payload, RegisterID regOp2Tag, RegisterID regOp2Payload, RegisterID regOp3Tag, RegisterID regOp3Payload, ByValInfo* byValInfo)
789 {
790     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, SH4_32BIT_DUMMY_ARG regOp2Payload, regOp2Tag, regOp3Payload, regOp3Tag, TrustedImmPtr(byValInfo));
791     return appendCallWithExceptionCheck(operation);
792 }
793
794 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EZJ operation, int dst, RegisterID regOp1Tag, RegisterID regOp1Payload)
795 {
796     setupArgumentsWithExecState(TrustedImm32(dst), regOp1Payload, regOp1Tag);
797     return appendCallWithExceptionCheck(operation);
798 }
799
800 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJZ operation, RegisterID regOp1Tag, RegisterID regOp1Payload, int32_t op2)
801 {
802     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, TrustedImm32(op2));
803     return appendCallWithExceptionCheck(operation);
804 }
805
806 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(V_JITOperation_EJZJ operation, RegisterID regOp1Tag, RegisterID regOp1Payload, int32_t op2, RegisterID regOp3Tag, RegisterID regOp3Payload)
807 {
808     setupArgumentsWithExecState(EABI_32BIT_DUMMY_ARG regOp1Payload, regOp1Tag, TrustedImm32(op2), EABI_32BIT_DUMMY_ARG regOp3Payload, regOp3Tag);
809     return appendCallWithExceptionCheck(operation);
810 }
811     
812 ALWAYS_INLINE MacroAssembler::Call JIT::callOperation(J_JITOperation_EJscCJ operation, int dst, GPRReg arg1, JSCell* cell, GPRReg arg2Tag, GPRReg arg2Payload)
813 {
814     setupArgumentsWithExecState(arg1, TrustedImmPtr(cell), EABI_32BIT_DUMMY_ARG arg2Payload, arg2Tag);
815     return appendCallWithExceptionCheckSetJSValueResult(operation, dst);
816 }
817
818 #endif // USE(JSVALUE32_64)
819
820 ALWAYS_INLINE JIT::Jump JIT::checkStructure(RegisterID reg, Structure* structure)
821 {
822     return branchStructure(NotEqual, Address(reg, JSCell::structureIDOffset()), structure);
823 }
824
825 ALWAYS_INLINE void JIT::linkSlowCaseIfNotJSCell(Vector<SlowCaseEntry>::iterator& iter, int vReg)
826 {
827     if (!m_codeBlock->isKnownNotImmediate(vReg))
828         linkSlowCase(iter);
829 }
830
831 ALWAYS_INLINE void JIT::linkAllSlowCasesForBytecodeOffset(Vector<SlowCaseEntry>& slowCases, Vector<SlowCaseEntry>::iterator& iter, unsigned bytecodeOffset)
832 {
833     while (iter != slowCases.end() && iter->to == bytecodeOffset) {
834         iter->from.link(this);
835         ++iter;
836     }
837 }
838
839 ALWAYS_INLINE void JIT::addSlowCase(Jump jump)
840 {
841     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
842
843     m_slowCases.append(SlowCaseEntry(jump, m_bytecodeOffset));
844 }
845
846 ALWAYS_INLINE void JIT::addSlowCase(const JumpList& jumpList)
847 {
848     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
849
850     for (const Jump& jump : jumpList.jumps())
851         m_slowCases.append(SlowCaseEntry(jump, m_bytecodeOffset));
852 }
853
854 ALWAYS_INLINE void JIT::addSlowCase()
855 {
856     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
857     
858     Jump emptyJump; // Doing it this way to make Windows happy.
859     m_slowCases.append(SlowCaseEntry(emptyJump, m_bytecodeOffset));
860 }
861
862 ALWAYS_INLINE void JIT::addJump(Jump jump, int relativeOffset)
863 {
864     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
865
866     m_jmpTable.append(JumpTable(jump, m_bytecodeOffset + relativeOffset));
867 }
868
869 ALWAYS_INLINE void JIT::emitJumpSlowToHot(Jump jump, int relativeOffset)
870 {
871     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
872
873     jump.linkTo(m_labels[m_bytecodeOffset + relativeOffset], this);
874 }
875
876 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfCellObject(RegisterID cellReg)
877 {
878     return branch8(AboveOrEqual, Address(cellReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ObjectType));
879 }
880
881 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfCellNotObject(RegisterID cellReg)
882 {
883     return branch8(Below, Address(cellReg, JSCell::typeInfoTypeOffset()), TrustedImm32(ObjectType));
884 }
885
886 #if ENABLE(SAMPLING_FLAGS)
887 ALWAYS_INLINE void JIT::setSamplingFlag(int32_t flag)
888 {
889     ASSERT(flag >= 1);
890     ASSERT(flag <= 32);
891     or32(TrustedImm32(1u << (flag - 1)), AbsoluteAddress(SamplingFlags::addressOfFlags()));
892 }
893
894 ALWAYS_INLINE void JIT::clearSamplingFlag(int32_t flag)
895 {
896     ASSERT(flag >= 1);
897     ASSERT(flag <= 32);
898     and32(TrustedImm32(~(1u << (flag - 1))), AbsoluteAddress(SamplingFlags::addressOfFlags()));
899 }
900 #endif
901
902 #if ENABLE(SAMPLING_COUNTERS)
903 ALWAYS_INLINE void JIT::emitCount(AbstractSamplingCounter& counter, int32_t count)
904 {
905     add64(TrustedImm32(count), AbsoluteAddress(counter.addressOfCounter()));
906 }
907 #endif
908
909 #if ENABLE(OPCODE_SAMPLING)
910 #if CPU(X86_64)
911 ALWAYS_INLINE void JIT::sampleInstruction(Instruction* instruction, bool inHostFunction)
912 {
913     move(TrustedImmPtr(m_interpreter->sampler()->sampleSlot()), X86Registers::ecx);
914     storePtr(TrustedImmPtr(m_interpreter->sampler()->encodeSample(instruction, inHostFunction)), X86Registers::ecx);
915 }
916 #else
917 ALWAYS_INLINE void JIT::sampleInstruction(Instruction* instruction, bool inHostFunction)
918 {
919     storePtr(TrustedImmPtr(m_interpreter->sampler()->encodeSample(instruction, inHostFunction)), m_interpreter->sampler()->sampleSlot());
920 }
921 #endif
922 #endif
923
924 #if ENABLE(CODEBLOCK_SAMPLING)
925 #if CPU(X86_64)
926 ALWAYS_INLINE void JIT::sampleCodeBlock(CodeBlock* codeBlock)
927 {
928     move(TrustedImmPtr(m_interpreter->sampler()->codeBlockSlot()), X86Registers::ecx);
929     storePtr(TrustedImmPtr(codeBlock), X86Registers::ecx);
930 }
931 #else
932 ALWAYS_INLINE void JIT::sampleCodeBlock(CodeBlock* codeBlock)
933 {
934     storePtr(TrustedImmPtr(codeBlock), m_interpreter->sampler()->codeBlockSlot());
935 }
936 #endif
937 #endif
938
939 ALWAYS_INLINE bool JIT::isOperandConstantChar(int src)
940 {
941     return m_codeBlock->isConstantRegisterIndex(src) && getConstantOperand(src).isString() && asString(getConstantOperand(src).asCell())->length() == 1;
942 }
943
944 template<typename StructureType>
945 inline void JIT::emitAllocateJSObject(RegisterID allocator, StructureType structure, RegisterID result, RegisterID scratch)
946 {
947     if (Options::forceGCSlowPaths())
948         addSlowCase(jump());
949     else {
950         loadPtr(Address(allocator, MarkedAllocator::offsetOfFreeListHead()), result);
951         addSlowCase(branchTestPtr(Zero, result));
952     }
953
954     // remove the object from the free list
955     loadPtr(Address(result), scratch);
956     storePtr(scratch, Address(allocator, MarkedAllocator::offsetOfFreeListHead()));
957
958     // initialize the object's property storage pointer
959     storePtr(TrustedImmPtr(0), Address(result, JSObject::butterflyOffset()));
960
961     // initialize the object's structure
962     emitStoreStructureWithTypeInfo(structure, result, scratch);
963 }
964
965 inline void JIT::emitValueProfilingSite(ValueProfile* valueProfile)
966 {
967     ASSERT(shouldEmitProfiling());
968     ASSERT(valueProfile);
969
970     const RegisterID value = regT0;
971 #if USE(JSVALUE32_64)
972     const RegisterID valueTag = regT1;
973 #endif
974     
975     // We're in a simple configuration: only one bucket, so we can just do a direct
976     // store.
977 #if USE(JSVALUE64)
978     store64(value, valueProfile->m_buckets);
979 #else
980     EncodedValueDescriptor* descriptor = bitwise_cast<EncodedValueDescriptor*>(valueProfile->m_buckets);
981     store32(value, &descriptor->asBits.payload);
982     store32(valueTag, &descriptor->asBits.tag);
983 #endif
984 }
985
986 inline void JIT::emitValueProfilingSite(unsigned bytecodeOffset)
987 {
988     if (!shouldEmitProfiling())
989         return;
990     emitValueProfilingSite(m_codeBlock->valueProfileForBytecodeOffset(bytecodeOffset));
991 }
992
993 inline void JIT::emitValueProfilingSite()
994 {
995     emitValueProfilingSite(m_bytecodeOffset);
996 }
997
998 inline void JIT::emitArrayProfilingSiteWithCell(RegisterID cell, RegisterID indexingType, ArrayProfile* arrayProfile)
999 {
1000     if (shouldEmitProfiling()) {
1001         load32(MacroAssembler::Address(cell, JSCell::structureIDOffset()), indexingType);
1002         store32(indexingType, arrayProfile->addressOfLastSeenStructureID());
1003     }
1004
1005     load8(Address(cell, JSCell::indexingTypeOffset()), indexingType);
1006 }
1007
1008 inline void JIT::emitArrayProfilingSiteForBytecodeIndexWithCell(RegisterID cell, RegisterID indexingType, unsigned bytecodeIndex)
1009 {
1010     emitArrayProfilingSiteWithCell(cell, indexingType, m_codeBlock->getOrAddArrayProfile(bytecodeIndex));
1011 }
1012
1013 inline void JIT::emitArrayProfileStoreToHoleSpecialCase(ArrayProfile* arrayProfile)
1014 {
1015     store8(TrustedImm32(1), arrayProfile->addressOfMayStoreToHole());
1016 }
1017
1018 inline void JIT::emitArrayProfileOutOfBoundsSpecialCase(ArrayProfile* arrayProfile)
1019 {
1020     store8(TrustedImm32(1), arrayProfile->addressOfOutOfBounds());
1021 }
1022
1023 static inline bool arrayProfileSaw(ArrayModes arrayModes, IndexingType capability)
1024 {
1025     return arrayModesInclude(arrayModes, capability);
1026 }
1027
1028 inline JITArrayMode JIT::chooseArrayMode(ArrayProfile* profile)
1029 {
1030     ConcurrentJITLocker locker(m_codeBlock->m_lock);
1031     profile->computeUpdatedPrediction(locker, m_codeBlock);
1032     ArrayModes arrayModes = profile->observedArrayModes(locker);
1033     if (arrayProfileSaw(arrayModes, DoubleShape))
1034         return JITDouble;
1035     if (arrayProfileSaw(arrayModes, Int32Shape))
1036         return JITInt32;
1037     if (arrayProfileSaw(arrayModes, ArrayStorageShape))
1038         return JITArrayStorage;
1039     return JITContiguous;
1040 }
1041
1042 ALWAYS_INLINE int32_t JIT::getOperandConstantInt(int src)
1043 {
1044     return getConstantOperand(src).asInt32();
1045 }
1046
1047 ALWAYS_INLINE double JIT::getOperandConstantDouble(int src)
1048 {
1049     return getConstantOperand(src).asDouble();
1050 }
1051
1052 #if USE(JSVALUE32_64)
1053
1054 inline void JIT::emitLoadTag(int index, RegisterID tag)
1055 {
1056     if (m_codeBlock->isConstantRegisterIndex(index)) {
1057         move(Imm32(getConstantOperand(index).tag()), tag);
1058         return;
1059     }
1060
1061     load32(tagFor(index), tag);
1062 }
1063
1064 inline void JIT::emitLoadPayload(int index, RegisterID payload)
1065 {
1066     if (m_codeBlock->isConstantRegisterIndex(index)) {
1067         move(Imm32(getConstantOperand(index).payload()), payload);
1068         return;
1069     }
1070
1071     load32(payloadFor(index), payload);
1072 }
1073
1074 inline void JIT::emitLoad(const JSValue& v, RegisterID tag, RegisterID payload)
1075 {
1076     move(Imm32(v.payload()), payload);
1077     move(Imm32(v.tag()), tag);
1078 }
1079
1080 ALWAYS_INLINE void JIT::emitGetVirtualRegister(int src, JSValueRegs dst)
1081 {
1082     emitLoad(src, dst.tagGPR(), dst.payloadGPR());
1083 }
1084
1085 ALWAYS_INLINE void JIT::emitPutVirtualRegister(int dst, JSValueRegs from)
1086 {
1087     emitStore(dst, from.tagGPR(), from.payloadGPR());
1088 }
1089
1090 inline void JIT::emitLoad(int index, RegisterID tag, RegisterID payload, RegisterID base)
1091 {
1092     RELEASE_ASSERT(tag != payload);
1093
1094     if (base == callFrameRegister) {
1095         RELEASE_ASSERT(payload != base);
1096         emitLoadPayload(index, payload);
1097         emitLoadTag(index, tag);
1098         return;
1099     }
1100
1101     if (payload == base) { // avoid stomping base
1102         load32(tagFor(index, base), tag);
1103         load32(payloadFor(index, base), payload);
1104         return;
1105     }
1106
1107     load32(payloadFor(index, base), payload);
1108     load32(tagFor(index, base), tag);
1109 }
1110
1111 inline void JIT::emitLoad2(int index1, RegisterID tag1, RegisterID payload1, int index2, RegisterID tag2, RegisterID payload2)
1112 {
1113     emitLoad(index2, tag2, payload2);
1114     emitLoad(index1, tag1, payload1);
1115 }
1116
1117 inline void JIT::emitLoadDouble(int index, FPRegisterID value)
1118 {
1119     if (m_codeBlock->isConstantRegisterIndex(index)) {
1120         WriteBarrier<Unknown>& inConstantPool = m_codeBlock->constantRegister(index);
1121         loadDouble(TrustedImmPtr(&inConstantPool), value);
1122     } else
1123         loadDouble(addressFor(index), value);
1124 }
1125
1126 inline void JIT::emitLoadInt32ToDouble(int index, FPRegisterID value)
1127 {
1128     if (m_codeBlock->isConstantRegisterIndex(index)) {
1129         WriteBarrier<Unknown>& inConstantPool = m_codeBlock->constantRegister(index);
1130         char* bytePointer = reinterpret_cast<char*>(&inConstantPool);
1131         convertInt32ToDouble(AbsoluteAddress(bytePointer + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), value);
1132     } else
1133         convertInt32ToDouble(payloadFor(index), value);
1134 }
1135
1136 inline void JIT::emitStore(int index, RegisterID tag, RegisterID payload, RegisterID base)
1137 {
1138     store32(payload, payloadFor(index, base));
1139     store32(tag, tagFor(index, base));
1140 }
1141
1142 inline void JIT::emitStoreInt32(int index, RegisterID payload, bool indexIsInt32)
1143 {
1144     store32(payload, payloadFor(index, callFrameRegister));
1145     if (!indexIsInt32)
1146         store32(TrustedImm32(JSValue::Int32Tag), tagFor(index, callFrameRegister));
1147 }
1148
1149 inline void JIT::emitStoreInt32(int index, TrustedImm32 payload, bool indexIsInt32)
1150 {
1151     store32(payload, payloadFor(index, callFrameRegister));
1152     if (!indexIsInt32)
1153         store32(TrustedImm32(JSValue::Int32Tag), tagFor(index, callFrameRegister));
1154 }
1155
1156 inline void JIT::emitStoreCell(int index, RegisterID payload, bool indexIsCell)
1157 {
1158     store32(payload, payloadFor(index, callFrameRegister));
1159     if (!indexIsCell)
1160         store32(TrustedImm32(JSValue::CellTag), tagFor(index, callFrameRegister));
1161 }
1162
1163 inline void JIT::emitStoreBool(int index, RegisterID payload, bool indexIsBool)
1164 {
1165     store32(payload, payloadFor(index, callFrameRegister));
1166     if (!indexIsBool)
1167         store32(TrustedImm32(JSValue::BooleanTag), tagFor(index, callFrameRegister));
1168 }
1169
1170 inline void JIT::emitStoreDouble(int index, FPRegisterID value)
1171 {
1172     storeDouble(value, addressFor(index));
1173 }
1174
1175 inline void JIT::emitStore(int index, const JSValue constant, RegisterID base)
1176 {
1177     store32(Imm32(constant.payload()), payloadFor(index, base));
1178     store32(Imm32(constant.tag()), tagFor(index, base));
1179 }
1180
1181 ALWAYS_INLINE void JIT::emitInitRegister(int dst)
1182 {
1183     emitStore(dst, jsUndefined());
1184 }
1185
1186 inline void JIT::emitJumpSlowCaseIfNotJSCell(int virtualRegisterIndex)
1187 {
1188     if (!m_codeBlock->isKnownNotImmediate(virtualRegisterIndex)) {
1189         if (m_codeBlock->isConstantRegisterIndex(virtualRegisterIndex))
1190             addSlowCase(jump());
1191         else
1192             addSlowCase(emitJumpIfNotJSCell(virtualRegisterIndex));
1193     }
1194 }
1195
1196 inline void JIT::emitJumpSlowCaseIfNotJSCell(int virtualRegisterIndex, RegisterID tag)
1197 {
1198     if (!m_codeBlock->isKnownNotImmediate(virtualRegisterIndex)) {
1199         if (m_codeBlock->isConstantRegisterIndex(virtualRegisterIndex))
1200             addSlowCase(jump());
1201         else
1202             addSlowCase(branch32(NotEqual, tag, TrustedImm32(JSValue::CellTag)));
1203     }
1204 }
1205
1206 ALWAYS_INLINE bool JIT::isOperandConstantInt(int src)
1207 {
1208     return m_codeBlock->isConstantRegisterIndex(src) && getConstantOperand(src).isInt32();
1209 }
1210
1211 ALWAYS_INLINE bool JIT::getOperandConstantInt(int op1, int op2, int& op, int32_t& constant)
1212 {
1213     if (isOperandConstantInt(op1)) {
1214         constant = getConstantOperand(op1).asInt32();
1215         op = op2;
1216         return true;
1217     }
1218
1219     if (isOperandConstantInt(op2)) {
1220         constant = getConstantOperand(op2).asInt32();
1221         op = op1;
1222         return true;
1223     }
1224     
1225     return false;
1226 }
1227
1228 #else // USE(JSVALUE32_64)
1229
1230 // get arg puts an arg from the SF register array into a h/w register
1231 ALWAYS_INLINE void JIT::emitGetVirtualRegister(int src, RegisterID dst)
1232 {
1233     ASSERT(m_bytecodeOffset != std::numeric_limits<unsigned>::max()); // This method should only be called during hot/cold path generation, so that m_bytecodeOffset is set.
1234
1235     // TODO: we want to reuse values that are already in registers if we can - add a register allocator!
1236     if (m_codeBlock->isConstantRegisterIndex(src)) {
1237         JSValue value = m_codeBlock->getConstant(src);
1238         if (!value.isNumber())
1239             move(TrustedImm64(JSValue::encode(value)), dst);
1240         else
1241             move(Imm64(JSValue::encode(value)), dst);
1242         return;
1243     }
1244
1245     load64(Address(callFrameRegister, src * sizeof(Register)), dst);
1246 }
1247
1248 ALWAYS_INLINE void JIT::emitGetVirtualRegister(int src, JSValueRegs dst)
1249 {
1250     emitGetVirtualRegister(src, dst.payloadGPR());
1251 }
1252
1253 ALWAYS_INLINE void JIT::emitGetVirtualRegister(VirtualRegister src, RegisterID dst)
1254 {
1255     emitGetVirtualRegister(src.offset(), dst);
1256 }
1257
1258 ALWAYS_INLINE void JIT::emitGetVirtualRegisters(int src1, RegisterID dst1, int src2, RegisterID dst2)
1259 {
1260     emitGetVirtualRegister(src1, dst1);
1261     emitGetVirtualRegister(src2, dst2);
1262 }
1263
1264 ALWAYS_INLINE void JIT::emitGetVirtualRegisters(VirtualRegister src1, RegisterID dst1, VirtualRegister src2, RegisterID dst2)
1265 {
1266     emitGetVirtualRegisters(src1.offset(), dst1, src2.offset(), dst2);
1267 }
1268
1269 ALWAYS_INLINE bool JIT::isOperandConstantInt(int src)
1270 {
1271     return m_codeBlock->isConstantRegisterIndex(src) && getConstantOperand(src).isInt32();
1272 }
1273
1274 ALWAYS_INLINE void JIT::emitPutVirtualRegister(int dst, RegisterID from)
1275 {
1276     store64(from, Address(callFrameRegister, dst * sizeof(Register)));
1277 }
1278
1279 ALWAYS_INLINE void JIT::emitPutVirtualRegister(int dst, JSValueRegs from)
1280 {
1281     emitPutVirtualRegister(dst, from.payloadGPR());
1282 }
1283
1284 ALWAYS_INLINE void JIT::emitPutVirtualRegister(VirtualRegister dst, RegisterID from)
1285 {
1286     emitPutVirtualRegister(dst.offset(), from);
1287 }
1288
1289 ALWAYS_INLINE void JIT::emitInitRegister(int dst)
1290 {
1291     store64(TrustedImm64(JSValue::encode(jsUndefined())), Address(callFrameRegister, dst * sizeof(Register)));
1292 }
1293
1294 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfJSCell(RegisterID reg)
1295 {
1296     return branchTest64(Zero, reg, tagMaskRegister);
1297 }
1298
1299 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfBothJSCells(RegisterID reg1, RegisterID reg2, RegisterID scratch)
1300 {
1301     move(reg1, scratch);
1302     or64(reg2, scratch);
1303     return emitJumpIfJSCell(scratch);
1304 }
1305
1306 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfJSCell(RegisterID reg)
1307 {
1308     addSlowCase(emitJumpIfJSCell(reg));
1309 }
1310
1311 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotJSCell(RegisterID reg)
1312 {
1313     addSlowCase(emitJumpIfNotJSCell(reg));
1314 }
1315
1316 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotJSCell(RegisterID reg, int vReg)
1317 {
1318     if (!m_codeBlock->isKnownNotImmediate(vReg))
1319         emitJumpSlowCaseIfNotJSCell(reg);
1320 }
1321
1322 inline void JIT::emitLoadDouble(int index, FPRegisterID value)
1323 {
1324     if (m_codeBlock->isConstantRegisterIndex(index)) {
1325         WriteBarrier<Unknown>& inConstantPool = m_codeBlock->constantRegister(index);
1326         loadDouble(TrustedImmPtr(&inConstantPool), value);
1327     } else
1328         loadDouble(addressFor(index), value);
1329 }
1330
1331 inline void JIT::emitLoadInt32ToDouble(int index, FPRegisterID value)
1332 {
1333     if (m_codeBlock->isConstantRegisterIndex(index)) {
1334         ASSERT(isOperandConstantInt(index));
1335         convertInt32ToDouble(Imm32(getConstantOperand(index).asInt32()), value);
1336     } else
1337         convertInt32ToDouble(addressFor(index), value);
1338 }
1339
1340 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfInt(RegisterID reg)
1341 {
1342     return branch64(AboveOrEqual, reg, tagTypeNumberRegister);
1343 }
1344
1345 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfNotInt(RegisterID reg)
1346 {
1347     return branch64(Below, reg, tagTypeNumberRegister);
1348 }
1349
1350 ALWAYS_INLINE JIT::PatchableJump JIT::emitPatchableJumpIfNotInt(RegisterID reg)
1351 {
1352     return patchableBranch64(Below, reg, tagTypeNumberRegister);
1353 }
1354
1355 ALWAYS_INLINE JIT::Jump JIT::emitJumpIfNotInt(RegisterID reg1, RegisterID reg2, RegisterID scratch)
1356 {
1357     move(reg1, scratch);
1358     and64(reg2, scratch);
1359     return emitJumpIfNotInt(scratch);
1360 }
1361
1362 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotInt(RegisterID reg)
1363 {
1364     addSlowCase(emitJumpIfNotInt(reg));
1365 }
1366
1367 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotInt(RegisterID reg1, RegisterID reg2, RegisterID scratch)
1368 {
1369     addSlowCase(emitJumpIfNotInt(reg1, reg2, scratch));
1370 }
1371
1372 ALWAYS_INLINE void JIT::emitJumpSlowCaseIfNotNumber(RegisterID reg)
1373 {
1374     addSlowCase(emitJumpIfNotNumber(reg));
1375 }
1376
1377 ALWAYS_INLINE void JIT::emitTagBool(RegisterID reg)
1378 {
1379     or32(TrustedImm32(static_cast<int32_t>(ValueFalse)), reg);
1380 }
1381
1382 inline Instruction* JIT::copiedInstruction(Instruction* inst)
1383 {
1384     ASSERT(inst >= m_codeBlock->instructions().begin() && inst < m_codeBlock->instructions().end());
1385     return m_instructions.begin() + (inst - m_codeBlock->instructions().begin());
1386 }
1387
1388 #endif // USE(JSVALUE32_64)
1389
1390 } // namespace JSC
1391
1392 #endif // ENABLE(JIT)
1393
1394 #endif // JITInlines_h
1395